From 013eab2991f69eed6e5dad40ea1ea45c596f6c87 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 12:46:50 +0200 Subject: [PATCH 01/15] feat(stack): assemble the Caddyfile from stacklet snippets Domain mode documented a Caddyfile assembled from each stacklet's `caddy.snippet`, but nothing wrote it. The snippets were never read and the proxy container mounted a file that did not exist. In domain mode, `stack up`, `stack down` and `stack destroy` now write `{data_dir}/infra/Caddyfile` from the snippets of the stacklets that are up, and have a running Caddy reload it once the containers have changed. - A crash-looping stacklet keeps its routes. - A host no running stacklet claims gets a 404 from a `*.` site. - Without TLS every site is served as plain HTTP on port 80. - A reload Caddy rejects leaves it on its previous routes and prints a warning; the command itself still succeeds. - Port mode writes nothing and reloads nothing. Snippets now use `{$STACK_DOMAIN}`, the variable the Caddy container is given. `{$FAMSTACK_DOMAIN}` was never set there. --- docs/stack-reference.md | 59 +++++-- lib/stack/caddy.py | 57 +++++++ lib/stack/cli.py | 27 +++- lib/stack/docker.py | 9 ++ lib/stack/stack.py | 53 ++++++ stacklets/chatai/caddy.snippet | 2 +- stacklets/core/caddy.snippet | 4 +- stacklets/docs/caddy.snippet | 2 +- stacklets/memory/caddy.snippet | 4 +- stacklets/messages/caddy.snippet | 4 +- stacklets/photos/caddy.snippet | 4 +- tests/integration/test_caddy.py | 95 +++++++++++ tests/unit/framework/test_caddy.py | 250 +++++++++++++++++++++++++++++ 13 files changed, 543 insertions(+), 27 deletions(-) create mode 100644 lib/stack/caddy.py create mode 100644 tests/integration/test_caddy.py create mode 100644 tests/unit/framework/test_caddy.py diff --git a/docs/stack-reference.md b/docs/stack-reference.md index fce23544..e7236395 100644 --- a/docs/stack-reference.md +++ b/docs/stack-reference.md @@ -512,15 +512,15 @@ effect. b. hooks/on_install.sh — create dirs, install deps, build 4. Write .env to stacklet directory 5. Bot runner discovers bots (if stacklet has bot/bot.toml) - 6. Assemble Caddyfile (domain mode) - 7. Build or pull Docker images - 8. hooks/on_start.sh — start native services (host stacklets) + 6. hooks/on_start.sh — start native services (host stacklets) + 7. Write the Caddyfile, counting this stacklet as up (domain mode) + 8. Build or pull Docker images 9. Start containers (docker compose up -d) -10. Wait for health check -11. First run only: +10. Reload Caddy (domain mode) +11. Wait for health check +12. First run only: a. hooks/on_install_success.py — obtain tokens, seed data -12. hooks/on_start_ready.py — service is healthy, seed data, sync accounts -13. Reload Caddy (domain mode) +13. hooks/on_start_ready.py — service is healthy, seed data, sync accounts 14. Show welcome screen with URL, login, hints ``` @@ -546,7 +546,9 @@ dependency order (dependents first, deps last). ``` 1. hooks/on_stop.sh — stop native services (host stacklets only) -2. docker compose stop — pause containers +2. Write the Caddyfile without this stacklet (domain mode) +3. docker compose stop — pause containers +4. Reload Caddy (domain mode) ``` ### `stack destroy ` @@ -563,7 +565,7 @@ secrets, config. Requires confirmation. 6. Delete stacklet secrets from secrets.toml ({id}__*) 7. Delete setup-done marker 8. Delete data directory (~/{data_dir}/{id}/) -9. Reassemble Caddyfile (domain mode) +9. Write the Caddyfile without this stacklet, reload Caddy (domain mode) ``` Global secrets (`global__ADMIN_PASSWORD`) survive destroy. The user's @@ -947,23 +949,49 @@ The runtime operates in one of two modes based on `stack.toml`: **Domain mode** (`domain = "home.internal"`): - Services bind to `127.0.0.1:` (only Caddy reaches them) - URLs are `http://photos.home.internal` -- Caddy assembles routes from `caddy.snippet` files +- The runtime assembles the Caddyfile from `caddy.snippet` files - Requires wildcard DNS on router ### Caddy Snippets -Each stacklet can include a `caddy.snippet` file. The runtime assembles -all snippets into a single Caddyfile on every `stack up`. +A stacklet that serves something in a browser ships its routes in a +`caddy.snippet`: ``` # stacklets/docs/caddy.snippet -docs.{$FAMSTACK_DOMAIN} { +docs.{$STACK_DOMAIN} { reverse_proxy stack-docs-paperless:8000 } ``` -The `{$FAMSTACK_DOMAIN}` variable is set by the runtime in Caddy's -environment. +The domain stays the `{$STACK_DOMAIN}` placeholder. Caddy substitutes it +from its own environment when it loads the file; the runtime never does. +Backends are container names on the `stack` network. + +In domain mode the runtime assembles the snippets of every stacklet that +is up into `{data_dir}/infra/Caddyfile`, in stacklet order, and the infra +stacklet's Caddy container mounts that file. "Up" is read from Docker like +every other state: the stacklet has containers running, starting, or +restarting. A crash-looping stacklet keeps its routes, because its other +containers are usually still serving. + +`stack up`, `stack down` and `stack destroy` write the file, each counting +the stacklet it acts on as already in its new state, and have a running +Caddy load it with `caddy reload` once the containers have changed. The +file is written before a starting stacklet's containers exist, because +infra's own Caddy reads it the moment it starts. A reload Caddy rejects +leaves it serving its previous routes, and the command prints a warning. +Nothing is written in port mode, or while infra is not up. + +Around the snippets the assembler (`lib/stack/caddy.py`) emits: + +- A global options block. Without TLS it serves every site as plain HTTP + on port 80. +- A `*.{$STACK_DOMAIN}` site that answers 404 for any host no running + stacklet claims, a stopped stacklet's included. + +Snippets describe routes only. Anything that applies to every site belongs +in the assembler, not in a snippet. --- @@ -1090,7 +1118,6 @@ Gitignored. Created by `stack init`. Contains: |---|---| | `secrets.toml` | Auto-generated credentials (passwords, API tokens). | | `*.setup-done` | Marker files. Gates once-only hooks (`on_install`, `on_install_success`). | -| `caddy/conf.d/*.snippet` | Assembled Caddy snippets (domain mode). | No `enabled` file — stacklet state is derived from Docker containers and the filesystem. See [States](#states). diff --git a/lib/stack/caddy.py b/lib/stack/caddy.py new file mode 100644 index 00000000..15b3f40d --- /dev/null +++ b/lib/stack/caddy.py @@ -0,0 +1,57 @@ +"""The Caddyfile for domain mode, assembled from stacklet snippets. + +In domain mode every service is reached by name (`photos.`) +through one reverse proxy, the Caddy container of the proxy stacklet. +A stacklet that serves something ships its routes as a `caddy.snippet`; +this module joins the snippets of the stacklets that are up into the one +Caddyfile that container mounts. + +Snippets describe routes only. Everything that applies to the proxy as a +whole, which today is the catch-all for hosts no stacklet claims, is +emitted here, so a snippet never has to know about any other. + +The domain is left as the `{$STACK_DOMAIN}` placeholder, in snippets and +here alike. Caddy fills it in from its own environment when it loads the +file, so the file is a function of which stacklets are up and nothing +else. +""" +from __future__ import annotations + +# The stacklet that runs the proxy, and where it expects its config. +STACKLET = "infra" +CONTAINER = "stack-infra-caddy" +CONFIG_IN_CONTAINER = "/etc/caddy/Caddyfile" + +RELOAD_COMMAND = ("caddy", "reload", "--config", CONFIG_IN_CONTAINER) + +_HEADER = """\ +# Assembled by the stack CLI from the caddy.snippet of every stacklet that +# is up. It is rewritten whenever a stacklet starts or stops, so an edit +# here does not survive: change the stacklet's snippet instead. +""" + +# Sites without a scheme listen on the HTTPS port, TLS or not. With no +# certificates to serve, that port is moved to 80, so every snippet is +# served as plain HTTP without having to spell out `http://` itself. +_PLAIN_HTTP = """\ +{ +\tauto_https off +\thttps_port 80 +} +""" + +# A host that no running stacklet claims, including one whose stacklet is +# stopped, gets an answer instead of a connection error. +_UNKNOWN_HOSTS = """\ +*.{$STACK_DOMAIN} { +\trespond "No service at {host}" 404 +} +""" + + +def assemble(snippets: list[tuple[str, str]]) -> str: + """The Caddyfile for these `(stacklet_id, snippet)` pairs, in order.""" + parts = [_HEADER, _PLAIN_HTTP, _UNKNOWN_HOSTS] + for stacklet_id, snippet in snippets: + parts.append(f"# ── {stacklet_id} ──\n\n{snippet.strip()}\n") + return "\n".join(parts) diff --git a/lib/stack/cli.py b/lib/stack/cli.py index 2531ba6d..a18c69da 100644 --- a/lib/stack/cli.py +++ b/lib/stack/cli.py @@ -21,6 +21,7 @@ import tomllib from pathlib import Path +from . import caddy from . import docker from . import doctor from .commands import COMMANDS @@ -255,6 +256,8 @@ def up(self, stacklet_id: str) -> dict: if code != 0: return {"error": "Failed to start services", "output": err} + self._reload_proxy() + if compose_file: # Health checks run first — hooks can assume the service is healthy template_vars = self.stack._build_template_vars() @@ -330,6 +333,7 @@ def down(self, stacklet_id: str) -> dict: except ValueError: pass code, output = docker.compose_stop(compose_file) + self._reload_proxy() result = {"stacklet": stacklet_id, "action": "down", "success": code == 0, "output": output} if code != 0: @@ -337,6 +341,7 @@ def down(self, stacklet_id: str) -> dict: # "unknown error", which hid the real cause. result["error"] = output or "docker compose stop failed" return result + self._reload_proxy() return {"stacklet": stacklet_id, "action": "down", "success": True} def _down_all(self) -> dict: @@ -440,7 +445,27 @@ def destroy(self, stacklet_id: str) -> dict: return {"error": "Failed to stop containers", "output": output} self.stack.output.step("Containers removed") - return self.stack.destroy(stacklet_id) + result = self.stack.destroy(stacklet_id) + self._reload_proxy() + return result + + def _reload_proxy(self) -> None: + """Have the running proxy load the Caddyfile the framework just wrote. + + Called after every change to which stacklets are up. Port mode has + no proxy, and a proxy that is not up reads the file when it starts. + A rejected reload leaves Caddy serving its previous config, so the + step that asked for it still succeeded; the admin is told the + routes did not change. + """ + if not self.stack._cfg("core", "domain"): + return + if caddy.STACKLET not in self.stack.serving_ids(): + return + code, err = docker.exec_in(caddy.CONTAINER, *caddy.RELOAD_COMMAND) + if code != 0: + reason = err.strip().splitlines()[-1] if err.strip() else f"exit {code}" + self.stack.output.warn(f"Caddy kept its previous routes: {reason}") # ── Stacklet id arguments ───────────────────────────────────────────────── diff --git a/lib/stack/docker.py b/lib/stack/docker.py index bfffd739..7ff8d07d 100644 --- a/lib/stack/docker.py +++ b/lib/stack/docker.py @@ -109,6 +109,15 @@ def compose_build(compose_file: str | Path, env: dict | None = None) -> None: ) +def exec_in(container: str, *cmd: str) -> tuple[int, str]: + """Run a command in a running container. Returns (exit_code, error_output).""" + result = _docker( + "exec", container, *cmd, + capture_output=True, text=True, timeout=60, + ) + return result.returncode, result.stderr + + def find_compose_file(stacklet_dir: Path) -> Path | None: """Find docker-compose.yml for a stacklet. Returns path or None.""" compose = stacklet_dir / "docker-compose.yml" diff --git a/lib/stack/stack.py b/lib/stack/stack.py index b6dd7bca..31e16a09 100644 --- a/lib/stack/stack.py +++ b/lib/stack/stack.py @@ -20,6 +20,7 @@ import tomllib from pathlib import Path +from . import caddy from .docker import running_project_ids from .hooks import HookResolver, build_hook_ctx from .output import SilentOutput @@ -866,6 +867,51 @@ def _running_project_ids(self) -> set[str]: """Query Docker for stacklet IDs with running containers.""" return running_project_ids() + # ── Reverse proxy (domain mode) ─────────────────────────────────── + + def serving_ids(self) -> set[str]: + """Stacklets with containers that are up, crash-looping ones included. + + This is what the proxy routes to. A stacklet with one container + restarting usually has others still serving, so it keeps its routes. + """ + from . import docker + return {sid for sid, state in docker.project_states().items() + if state in ("running", "starting", "failing")} + + def write_caddyfile(self, starting: str = "", stopping: str = "") -> Path | None: + """Write the proxy's Caddyfile from the snippets of the stacklets up. + + `starting` and `stopping` name a stacklet that is changing state as + this runs, so its routes follow where it is going rather than where + its containers are right now. The file is written before a starting + stacklet's containers exist: the proxy's own container reads it the + moment it starts. + + Returns the path written, or None when there is nothing to write: + port mode has no proxy, and with the proxy not up there is nobody to + read it. + """ + if not self._cfg("core", "domain"): + return None + up = (self.serving_ids() | {starting}) - {stopping, ""} + if caddy.STACKLET not in up: + return None + + snippets = [] + for s in self.discover(): + snippet = Path(s["path"]) / "caddy.snippet" + if s["id"] in up and snippet.is_file(): + snippets.append((s["id"], snippet.read_text())) + + path = self.data / caddy.STACKLET / "Caddyfile" + path.parent.mkdir(parents=True, exist_ok=True) + # Written in place. The file is bind-mounted into the proxy + # container, and a single-file bind mount follows the inode, so a + # file replaced by rename would leave Caddy reading the old one. + path.write_text(caddy.assemble(snippets)) + return path + # ── Secrets (delegated to SecretStore) ──────────────────────────── def secret(self, stacklet_id: str, name: str) -> str | None: @@ -1101,6 +1147,9 @@ def _step(msg): self.output.error("on_start hook failed") return {"error": "on_start hook failed", "steps": steps} + if self.write_caddyfile(starting=stacklet_id): + self.output.step("Writing Caddyfile") + # Render manifest hints with template vars so credentials are visible template_vars = self._build_template_vars() template_vars["url"] = self._public_url(stacklet_id, s.get("port", 0)) @@ -1238,6 +1287,8 @@ def down(self, stacklet_id: str) -> dict: ctx = build_hook_ctx(stacklet_id, env={}, step_fn=lambda msg: None, stack=self) resolver.run("on_stop", ctx) + self.write_caddyfile(stopping=stacklet_id) + return {"ok": True, "stacklet": stacklet_id, "path": s["path"]} def destroy(self, stacklet_id: str) -> dict: @@ -1273,4 +1324,6 @@ def destroy(self, stacklet_id: str) -> dict: if env_file.exists(): env_file.unlink() + self.write_caddyfile(stopping=stacklet_id) + return {"ok": True, "stacklet": stacklet_id} diff --git a/stacklets/chatai/caddy.snippet b/stacklets/chatai/caddy.snippet index 28f0d4a5..8160b0db 100644 --- a/stacklets/chatai/caddy.snippet +++ b/stacklets/chatai/caddy.snippet @@ -3,6 +3,6 @@ # Assembled into the Caddyfile by 'stack up'. Caddy runs in Docker on the # stack network, so backends are referenced by container name. -chatai.{$FAMSTACK_DOMAIN} { +chatai.{$STACK_DOMAIN} { reverse_proxy chatai:8080 } diff --git a/stacklets/core/caddy.snippet b/stacklets/core/caddy.snippet index e10c1e72..714f4569 100644 --- a/stacklets/core/caddy.snippet +++ b/stacklets/core/caddy.snippet @@ -1,6 +1,6 @@ # stacklets/core/caddy.snippet # -# core owns its home — the bare {$FAMSTACK_DOMAIN} — and serves the +# core owns its home — the bare {$STACK_DOMAIN} — and serves the # persistent-link resolver under /go/. A dashboard / welcome page can take # over `/` later; for now home just answers a friendly placeholder. # @@ -12,7 +12,7 @@ # The /go prefix mirrors core's LINK_PREFIX env (default "go"); if you move # the namespace, change both. -{$FAMSTACK_DOMAIN} { +{$STACK_DOMAIN} { handle /go/* { reverse_proxy stack-core-tools:8000 } diff --git a/stacklets/docs/caddy.snippet b/stacklets/docs/caddy.snippet index 95069a56..0d1fa749 100644 --- a/stacklets/docs/caddy.snippet +++ b/stacklets/docs/caddy.snippet @@ -3,6 +3,6 @@ # Assembled into the Caddyfile by 'stack up'. Caddy runs in Docker on the # stack network, so backends are referenced by container name. -docs.{$FAMSTACK_DOMAIN} { +docs.{$STACK_DOMAIN} { reverse_proxy stack-docs-paperless:8000 } diff --git a/stacklets/memory/caddy.snippet b/stacklets/memory/caddy.snippet index 79a11d5e..72b4661d 100644 --- a/stacklets/memory/caddy.snippet +++ b/stacklets/memory/caddy.snippet @@ -1,7 +1,7 @@ # stacklets/memory/caddy.snippet # # One route. The family wiki — Quartz rendering the family vault — -# lives at wiki.{$FAMSTACK_DOMAIN}. Caddy runs on the stack network +# lives at wiki.{$STACK_DOMAIN}. Caddy runs on the stack network # so the backend is the container name; the in-container port is # Quartz's preview server default (8080), not the published host port. # @@ -9,6 +9,6 @@ # (photos, docs, messages). The LAN is the perimeter. Add `basicauth` # inside this block if you ever want to gate the vault separately. -wiki.{$FAMSTACK_DOMAIN} { +wiki.{$STACK_DOMAIN} { reverse_proxy stack-memory-wiki:8080 } diff --git a/stacklets/messages/caddy.snippet b/stacklets/messages/caddy.snippet index 842dbc82..39717229 100644 --- a/stacklets/messages/caddy.snippet +++ b/stacklets/messages/caddy.snippet @@ -3,10 +3,10 @@ # Two routes: Synapse API and Element Web client. # Assembled into the Caddyfile by 'stack up'. -messages.{$FAMSTACK_DOMAIN} { +messages.{$STACK_DOMAIN} { reverse_proxy stack-messages-synapse:8008 } -element.{$FAMSTACK_DOMAIN} { +element.{$STACK_DOMAIN} { reverse_proxy stack-messages-element:80 } diff --git a/stacklets/photos/caddy.snippet b/stacklets/photos/caddy.snippet index 3da5e374..ddc403bf 100644 --- a/stacklets/photos/caddy.snippet +++ b/stacklets/photos/caddy.snippet @@ -2,8 +2,8 @@ # # Assembled into the Caddyfile by 'stack up'. Caddy runs in Docker on the # stack network, so backends are referenced by container name. -# {$FAMSTACK_DOMAIN} is set as an env var when Caddy starts. +# {$STACK_DOMAIN} is set as an env var when Caddy starts. -photos.{$FAMSTACK_DOMAIN} { +photos.{$STACK_DOMAIN} { reverse_proxy immich_server:2283 } diff --git a/tests/integration/test_caddy.py b/tests/integration/test_caddy.py new file mode 100644 index 00000000..3e87140b --- /dev/null +++ b/tests/integration/test_caddy.py @@ -0,0 +1,95 @@ +"""The assembled Caddyfile, as a real Caddy reads it. + +The unit tests pin which snippets the framework puts in the file and when. +Whether Caddy accepts the result, and what it serves from it, only Caddy +can say, so these hand it the snippets that ship in the repo and read the +JSON config it adapts them to. +""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parent.parent.parent +sys.path.insert(0, str(REPO_ROOT / "lib")) + +from stack.caddy import assemble # noqa: E402 + +try: + subprocess.run(["docker", "info"], capture_output=True, timeout=5, check=True) + HAS_DOCKER = True +except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): + HAS_DOCKER = False + +pytestmark = pytest.mark.skipif(not HAS_DOCKER, reason="Docker not available") + +STOCK_CADDY = "caddy:2" +DOMAIN = "home.example.family" + + +def _shipped_snippets() -> list[tuple[str, str]]: + return [(path.parent.name, path.read_text()) + for path in sorted(REPO_ROOT.glob("stacklets/*/caddy.snippet"))] + + +def _caddy(image: str, caddyfile: Path, *args: str) -> subprocess.CompletedProcess: + """Run `caddy ` against this Caddyfile, as the proxy container would.""" + return subprocess.run( + ["docker", "run", "--rm", + "-e", f"STACK_DOMAIN={DOMAIN}", + "-v", f"{caddyfile}:/etc/caddy/Caddyfile:ro", + image, "caddy", *args, "--config", "/etc/caddy/Caddyfile"], + capture_output=True, text=True, timeout=180, + ) + + +def _adapt(image: str, caddyfile: Path) -> dict: + result = _caddy(image, caddyfile, "adapt") + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +def _hosts(config: dict) -> set[str]: + return { + host + for server in config["apps"]["http"]["servers"].values() + for route in server.get("routes", []) + for match in route.get("match", []) + for host in match.get("host", []) + } + + +def _listen(config: dict) -> set[str]: + return {addr + for server in config["apps"]["http"]["servers"].values() + for addr in server["listen"]} + + +@pytest.fixture +def plain_caddyfile(tmp_path) -> Path: + path = tmp_path / "Caddyfile" + path.write_text(assemble(_shipped_snippets())) + return path + + +def test_every_shipped_snippet_assembles_into_a_valid_config(plain_caddyfile): + result = _caddy(STOCK_CADDY, plain_caddyfile, "validate") + assert result.returncode == 0, result.stderr + + +def test_the_bare_domain_and_the_wildcard_are_both_served(plain_caddyfile): + """The bare domain is core's home and `/go` links. The wildcard answers + every other name, and covers neither the bare name's DNS record nor its + certificate, so both have to be sites of their own.""" + hosts = _hosts(_adapt(STOCK_CADDY, plain_caddyfile)) + assert DOMAIN in hosts + assert f"*.{DOMAIN}" in hosts + + +def test_without_tls_every_site_is_plain_http_on_port_80(plain_caddyfile): + config = _adapt(STOCK_CADDY, plain_caddyfile) + assert _listen(config) == {":80"} + assert "tls" not in config.get("apps", {}) diff --git a/tests/unit/framework/test_caddy.py b/tests/unit/framework/test_caddy.py new file mode 100644 index 00000000..5e9c6eb0 --- /dev/null +++ b/tests/unit/framework/test_caddy.py @@ -0,0 +1,250 @@ +"""Domain mode: one Caddyfile, assembled from the stacklets that are up. + +Each stacklet that serves something in a browser ships a `caddy.snippet` +with its routes. In domain mode the framework assembles the snippets of +every stacklet that is up into `{data_dir}/infra/Caddyfile`, which the +infra stacklet's Caddy container mounts, and has that Caddy reload it +whenever a stacklet starts or stops. Port mode has no proxy, so none of +this happens there. + +Whether the result is a Caddyfile Caddy accepts, and what Caddy does with +it, is checked against a real Caddy in tests/integration/test_caddy.py. +These tests pin what the framework promises around it: which snippets go +in, when the file is written, and when Caddy is told to reload. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "lib")) + +from stack.caddy import assemble # noqa: E402 + + +PHOTOS = """\ +photos.{$STACK_DOMAIN} { + reverse_proxy stack-photos-server:2283 +} +""" + +DOCS = """\ +docs.{$STACK_DOMAIN} { + reverse_proxy stack-docs-paperless:8000 +} +""" + +CORE = """\ +{$STACK_DOMAIN} { + respond "home" 200 +} +""" + + +def _site_addresses(caddyfile: str) -> list[str]: + """The site addresses of every top-level block, in order. + + A top-level block opens on a line with no indentation that ends in + `{`. The global options block has no address and is left out. + """ + addresses = [] + for line in caddyfile.splitlines(): + if line[:1] in ("", " ", "\t", "#", "}"): + continue + if line.rstrip().endswith("{") and line.strip() != "{": + addresses.append(line.rstrip()[:-1].strip()) + return addresses + + +# ── The assembler ───────────────────────────────────────────────────────── + + +class TestAssemble: + + def test_each_snippet_becomes_a_site_in_stacklet_order(self): + caddyfile = assemble([("core", CORE), ("docs", DOCS), ("photos", PHOTOS)]) + + sites = _site_addresses(caddyfile) + assert sites.index("{$STACK_DOMAIN}") < sites.index("docs.{$STACK_DOMAIN}") + assert sites.index("docs.{$STACK_DOMAIN}") < sites.index("photos.{$STACK_DOMAIN}") + + def test_the_wildcard_is_always_a_site(self): + """`*.` answers hosts no running stacklet claims, a stopped + stacklet among them, and is the site Caddy's wildcard certificate + belongs to. It is there even when no snippet is.""" + assert "*.{$STACK_DOMAIN}" in _site_addresses(assemble([])) + + def test_the_domain_stays_a_placeholder_for_caddy(self): + """Caddy substitutes `{$STACK_DOMAIN}` from its own environment, so + the file does not change when the domain does and never needs to be + regenerated for it.""" + caddyfile = assemble([("photos", PHOTOS)]) + assert "photos.{$STACK_DOMAIN}" in _site_addresses(caddyfile) + + def test_a_snippet_is_copied_verbatim(self): + caddyfile = assemble([("photos", PHOTOS)]) + assert PHOTOS.strip() in caddyfile + + +# ── Lifecycle ───────────────────────────────────────────────────────────── + + +@pytest.fixture +def docker(monkeypatch): + """Docker at its boundary: which compose projects are up, and every + command run inside a container.""" + + class Docker: + states: dict[str, str] = {} + execs: list[tuple] = [] + + fake = Docker() + fake.states = {} + fake.execs = [] + monkeypatch.setattr("stack.docker.project_states", lambda: dict(fake.states)) + monkeypatch.setattr("stack.docker.running_project_ids", + lambda: {s for s, st in fake.states.items() if st == "running"}) + monkeypatch.setattr("stack.docker.ensure_network", lambda: ("stack", None)) + + def exec_in(container, *cmd): + fake.execs.append((container, *cmd)) + return 0, "" + + monkeypatch.setattr("stack.docker.exec_in", exec_in) + return fake + + +def _make_cli(tmp_path, domain="home.example.family"): + """A stack with infra (the proxy) and two stacklets that have routes. + + None of them has a compose file, so the CLI runs the framework side of + each lifecycle step and no container is started. + """ + from stack import Stack + from stack.cli import CLI + from stack.output import CollectorOutput + + (tmp_path / "stack.toml").write_text( + f'[core]\ndomain = "{domain}"\ntimezone = "Europe/Berlin"\n') + (tmp_path / ".stack").mkdir() + (tmp_path / ".stack" / "secrets.toml").write_text('global__ADMIN_PASSWORD = "test"\n') + + for sid, snippet in (("infra", ""), ("docs", DOCS), ("photos", PHOTOS)): + sdir = tmp_path / "stacklets" / sid + sdir.mkdir(parents=True) + (sdir / "stacklet.toml").write_text(f'id = "{sid}"\nname = "{sid.title()}"\n') + if snippet: + (sdir / "caddy.snippet").write_text(snippet) + + output = CollectorOutput() + return CLI(Stack(root=tmp_path, data=tmp_path / "data", output=output)), output + + +def _caddyfile(tmp_path) -> Path: + return tmp_path / "data" / "infra" / "Caddyfile" + + +RELOAD = ("stack-infra-caddy", "caddy", "reload", "--config", "/etc/caddy/Caddyfile") + + +class TestLifecycle: + + def test_up_routes_the_stacklet_it_starts(self, tmp_path, docker): + """The Caddyfile is written before the stacklet's containers start, + so it already counts the stacklet being started as up.""" + cli, _ = _make_cli(tmp_path) + docker.states = {"infra": "running", "docs": "running"} + + assert cli.up("photos")["ok"] + + sites = _site_addresses(_caddyfile(tmp_path).read_text()) + assert "photos.{$STACK_DOMAIN}" in sites + assert "docs.{$STACK_DOMAIN}" in sites + assert docker.execs == [RELOAD] + + def test_a_stacklet_that_is_not_up_gets_no_route(self, tmp_path, docker): + cli, _ = _make_cli(tmp_path) + docker.states = {"infra": "running", "docs": "stopped"} + + cli.up("photos") + + assert "docs.{$STACK_DOMAIN}" not in _site_addresses(_caddyfile(tmp_path).read_text()) + + def test_a_crash_looping_stacklet_keeps_its_route(self, tmp_path, docker): + """A stacklet with one container restarting still has the others + serving, and a proxy that dropped it would hide the rest too.""" + cli, _ = _make_cli(tmp_path) + docker.states = {"infra": "running", "docs": "failing"} + + cli.up("photos") + + assert "docs.{$STACK_DOMAIN}" in _site_addresses(_caddyfile(tmp_path).read_text()) + + def test_down_removes_the_route(self, tmp_path, docker): + cli, _ = _make_cli(tmp_path) + docker.states = {"infra": "running", "photos": "running"} + cli.up("photos") + docker.execs.clear() + + cli.down("photos") + + assert "photos.{$STACK_DOMAIN}" not in _site_addresses(_caddyfile(tmp_path).read_text()) + assert docker.execs == [RELOAD] + + def test_destroy_removes_the_route(self, tmp_path, docker): + cli, _ = _make_cli(tmp_path) + docker.states = {"infra": "running", "photos": "running"} + cli.up("photos") + docker.states = {"infra": "running"} + docker.execs.clear() + + cli.destroy("photos") + + assert "photos.{$STACK_DOMAIN}" not in _site_addresses(_caddyfile(tmp_path).read_text()) + assert docker.execs == [RELOAD] + + def test_starting_infra_writes_the_caddyfile_its_container_mounts(self, tmp_path, docker): + """Caddy reads the file the moment its container starts. A bind + mount whose source is missing is created as a directory, so the + file has to exist before that.""" + cli, _ = _make_cli(tmp_path) + docker.states = {"docs": "running"} + + cli.up("infra") + + assert "docs.{$STACK_DOMAIN}" in _site_addresses(_caddyfile(tmp_path).read_text()) + + def test_without_infra_up_nothing_is_written_or_reloaded(self, tmp_path, docker): + """With no proxy running there is nobody to read the file, and + writing it would create infra's data directory for a stacklet that + was never started.""" + cli, _ = _make_cli(tmp_path) + docker.states = {"docs": "running"} + + cli.up("photos") + + assert not _caddyfile(tmp_path).exists() + assert docker.execs == [] + + def test_port_mode_has_no_proxy(self, tmp_path, docker): + cli, _ = _make_cli(tmp_path, domain="") + docker.states = {"infra": "running", "docs": "running"} + + cli.up("photos") + cli.down("photos") + + assert not _caddyfile(tmp_path).exists() + assert docker.execs == [] + + def test_a_failed_reload_is_reported_and_does_not_fail_the_step(self, tmp_path, docker, monkeypatch): + """Caddy keeps serving its previous config when a reload is + rejected, so the stacklet is up either way. The admin needs to + know the routes did not change.""" + cli, output = _make_cli(tmp_path) + docker.states = {"infra": "running"} + monkeypatch.setattr("stack.docker.exec_in", + lambda *a: (1, "Error: adapting config: bad directive\n")) + + assert cli.up("photos")["ok"] + assert any("bad directive" in w for w in output.warnings) From f12948882b1b05c0c5d3efd1412acc0cec5d904a Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 12:51:27 +0200 Subject: [PATCH 02/15] feat(infra): serve HTTPS with certificates from the DNS provider Domain mode could only serve plain HTTP. With `[core] dns_provider` set to `hetzner` or `cloudflare`, Caddy gets two certificates from Let's Encrypt: a wildcard for `*.`, which every stacklet's subdomain uses, and one for the bare domain, which a wildcard does not cover. It proves ownership with a DNS-01 challenge through the provider's API, so nothing on the LAN has to be reachable from the internet. - `stack infra dns-token` stores the provider's API token as a secret, and replaces it. The first `stack up infra` asks for it in a terminal. The token is never printed, only its length. - `stack up infra` refuses an unsupported provider or a missing token, and names the command that fixes it. - An unsupported provider leaves the current Caddyfile in place for every other stacklet, with a warning. - The URLs the stack renders (`{url}`, `{_url}`, `{home_url}`) are https once a provider is set. Hetzner needs a Hetzner Cloud API token with Read & Write access, Cloudflare a token with Zone.Zone:Read and Zone.DNS:Edit on the zone. Caddy needs the provider's plugin compiled in, which the infra image gains in a separate change. --- docs/stack-reference.md | 48 +++++++++- lib/stack/caddy.py | 49 +++++++++-- lib/stack/stack.py | 33 ++++--- stack.example.toml | 8 ++ stacklets/infra/cli/dns-token.py | 58 ++++++++++++ stacklets/infra/docker-compose.yml | 2 + stacklets/infra/hooks/on_configure.py | 19 ++++ stacklets/infra/hooks/on_start.py | 26 ++++++ stacklets/infra/stacklet.toml | 3 + tests/unit/framework/test_caddy.py | 86 ++++++++++++++++++ tests/unit/stacklets/test_infra_dns.py | 117 +++++++++++++++++++++++++ 11 files changed, 431 insertions(+), 18 deletions(-) create mode 100644 stacklets/infra/cli/dns-token.py create mode 100644 stacklets/infra/hooks/on_configure.py create mode 100644 stacklets/infra/hooks/on_start.py create mode 100644 tests/unit/stacklets/test_infra_dns.py diff --git a/docs/stack-reference.md b/docs/stack-reference.md index e7236395..470e2f0e 100644 --- a/docs/stack-reference.md +++ b/docs/stack-reference.md @@ -202,6 +202,8 @@ Available template variables: | `{ai_default_model}` | `stack.toml` → `[ai].default` | | `{ai_tts_voice}` | Derived from `[ai].language` | | `{messages_server_name}` | `stack.toml` → `[messages].server_name` | +| `{url}`, `{_url}`, `{home_url}` | Public URLs: `http://:` in port mode; `https://.` in domain mode once `[core].dns_provider` is set, `http://` before that | +| `{__}` | A secret from `secrets.toml`, e.g. `{docs__API_TOKEN}`, `{infra__DNS_API_TOKEN}` | ### Hints @@ -948,7 +950,8 @@ The runtime operates in one of two modes based on `stack.toml`: **Domain mode** (`domain = "home.internal"`): - Services bind to `127.0.0.1:` (only Caddy reaches them) -- URLs are `http://photos.home.internal` +- URLs are `http://photos.home.internal`, or `https://` once + `dns_provider` is set (see TLS below) - The runtime assembles the Caddyfile from `caddy.snippet` files - Requires wildcard DNS on router @@ -985,14 +988,47 @@ Nothing is written in port mode, or while infra is not up. Around the snippets the assembler (`lib/stack/caddy.py`) emits: -- A global options block. Without TLS it serves every site as plain HTTP - on port 80. +- A global options block: how certificates are obtained, or, without TLS, + every site served as plain HTTP on port 80. - A `*.{$STACK_DOMAIN}` site that answers 404 for any host no running stacklet claims, a stopped stacklet's included. Snippets describe routes only. Anything that applies to every site belongs in the assembler, not in a snippet. +### TLS + +`[core] dns_provider` (`"hetzner"` or `"cloudflare"`) turns on HTTPS. The +global block then makes Let's Encrypt the issuer for every certificate, +through a DNS-01 challenge at that provider: + +``` +{ + cert_issuer acme { + dns hetzner {env.DNS_API_TOKEN} + propagation_delay 30s + propagation_timeout 5m + } +} +``` + +Cloudflare's block has the `dns` line only. DNS-01 needs nothing on the +LAN to be reachable from the internet, and it is the only challenge that +issues a wildcard. Caddy obtains two certificates: `*.`, for the +catch-all site, which every subdomain site then uses, and `` +itself, which a wildcard does not cover. + +The token is the infra stacklet's secret `DNS_API_TOKEN`. `stack infra +dns-token` stores or replaces it, and the first `stack up infra` asks for +it in a terminal. It reaches Caddy as an environment variable and is read +when a certificate is requested, so it appears neither in the Caddyfile +nor in the config Caddy serves on its admin endpoint. infra's `on_start` +refuses a provider the image has no plugin for, or a missing token. + +With `dns_provider` set, the URLs the runtime renders (`{url}`, +`{_url}`, `{home_url}`) are `https://`. `[core] https = true` does the +same for a reverse proxy in front of the stack that is not this one. + --- ## Secrets @@ -1064,6 +1100,12 @@ One file, committed to the repo. User edits it directly. ```toml [core] domain = "" # empty = port mode +dns_provider = "" # domain mode: "hetzner" or + # "cloudflare" serves HTTPS with + # certificates via DNS-01; empty + # serves plain HTTP. The API token + # is a secret (`stack infra + # dns-token`), never in this file. host = "" # port-mode host override. # Empty = auto-detect LAN IP. # Set to "localhost" for diff --git a/lib/stack/caddy.py b/lib/stack/caddy.py index 15b3f40d..e261fef6 100644 --- a/lib/stack/caddy.py +++ b/lib/stack/caddy.py @@ -7,8 +7,18 @@ Caddyfile that container mounts. Snippets describe routes only. Everything that applies to the proxy as a -whole, which today is the catch-all for hosts no stacklet claims, is -emitted here, so a snippet never has to know about any other. +whole, the catch-all for hosts no stacklet claims and how certificates +are obtained, is emitted here, so a snippet never has to know about any +other. + +With `[core] dns_provider` set, every certificate comes from Let's +Encrypt through a DNS-01 challenge at that provider: Caddy writes a TXT +record through the provider's API, so nothing on the LAN has to be +reachable from the internet. That is also the only challenge that can +issue the wildcard certificate. The `*.` site makes Caddy manage +one, and every subdomain site uses it rather than getting its own. The +bare domain is not covered by the wildcard, and gets a second +certificate the same way. The domain is left as the `{$STACK_DOMAIN}` placeholder, in snippets and here alike. Caddy fills it in from its own environment when it loads the @@ -40,6 +50,19 @@ } """ +# The DNS providers whose Caddy plugin is compiled into the infra image, +# and the settings each one's plugin asks for beyond the token. Hetzner's +# README asks for a delay before Caddy starts checking for the record. +DNS_PROVIDERS = { + "hetzner": ("propagation_delay 30s", "propagation_timeout 5m"), + "cloudflare": (), +} + +# The token is read from Caddy's environment when a certificate is +# requested, so it never appears in this file or in the config Caddy +# serves on its admin endpoint. +_TOKEN = "{env.DNS_API_TOKEN}" + # A host that no running stacklet claims, including one whose stacklet is # stopped, gets an answer instead of a connection error. _UNKNOWN_HOSTS = """\ @@ -49,9 +72,25 @@ """ -def assemble(snippets: list[tuple[str, str]]) -> str: - """The Caddyfile for these `(stacklet_id, snippet)` pairs, in order.""" - parts = [_HEADER, _PLAIN_HTTP, _UNKNOWN_HOSTS] +def _global_options(dns_provider: str) -> str: + """Global options: certificates through the DNS provider, or none.""" + if not dns_provider: + return _PLAIN_HTTP + if dns_provider not in DNS_PROVIDERS: + raise ValueError( + f"dns_provider '{dns_provider}' is not supported; " + f"use {' or '.join(DNS_PROVIDERS)}, or leave it empty for plain HTTP") + lines = [f"dns {dns_provider} {_TOKEN}", *DNS_PROVIDERS[dns_provider]] + body = "".join(f"\t\t{line}\n" for line in lines) + return f"{{\n\tcert_issuer acme {{\n{body}\t}}\n}}\n" + + +def assemble(snippets: list[tuple[str, str]], dns_provider: str = "") -> str: + """The Caddyfile for these `(stacklet_id, snippet)` pairs, in order. + + Raises ValueError for a DNS provider the infra image has no plugin for. + """ + parts = [_HEADER, _global_options(dns_provider), _UNKNOWN_HOSTS] for stacklet_id, snippet in snippets: parts.append(f"# ── {stacklet_id} ──\n\n{snippet.strip()}\n") return "\n".join(parts) diff --git a/lib/stack/stack.py b/lib/stack/stack.py index 31e16a09..574fb214 100644 --- a/lib/stack/stack.py +++ b/lib/stack/stack.py @@ -484,18 +484,25 @@ def _lan_ip(self) -> str: def _public_url(self, stacklet_id: str, port: int) -> str: """Build the URL a user would click to reach a service. - In domain mode: http(s)://{stacklet}.{domain} + In domain mode: http(s)://{stacklet}.{domain}, see `_domain_scheme` In port mode: http://{ip}:{port} (LAN-reachable) - - Set [core].https = true when a reverse proxy terminates TLS in front - of the stack. """ domain = self._cfg("core", "domain") if domain: - scheme = "https" if self._cfg("core", "https") else "http" - return f"{scheme}://{stacklet_id}.{domain}" + return f"{self._domain_scheme()}://{stacklet_id}.{domain}" return f"http://{self._lan_ip()}:{port}" + def _domain_scheme(self) -> str: + """https once the proxy serves certificates, http until then. + + The proxy has certificates when `[core] dns_provider` is set. Set + `[core] https = true` instead when some other reverse proxy in + front of the stack terminates TLS. + """ + if self._cfg("core", "dns_provider") or self._cfg("core", "https"): + return "https" + return "http" + def _home_url(self) -> str: """Browser-facing base of core's home — where `/go` links live. @@ -510,8 +517,7 @@ def _home_url(self) -> str: """ domain = self._cfg("core", "domain") if domain: - scheme = "https" if self._cfg("core", "https") else "http" - return f"{scheme}://{domain}" + return f"{self._domain_scheme()}://{domain}" return f"http://{self._lan_ip()}:42000" def env(self, stacklet_id: str) -> dict: @@ -890,7 +896,9 @@ def write_caddyfile(self, starting: str = "", stopping: str = "") -> Path | None Returns the path written, or None when there is nothing to write: port mode has no proxy, and with the proxy not up there is nobody to - read it. + read it. An unsupported `[core] dns_provider` also writes nothing: + the proxy keeps the config it has, which still works, and the + warning names the value. Infra's on_start refuses it outright. """ if not self._cfg("core", "domain"): return None @@ -903,13 +911,18 @@ def write_caddyfile(self, starting: str = "", stopping: str = "") -> Path | None snippet = Path(s["path"]) / "caddy.snippet" if s["id"] in up and snippet.is_file(): snippets.append((s["id"], snippet.read_text())) + try: + caddyfile = caddy.assemble(snippets, self._cfg("core", "dns_provider")) + except ValueError as e: + self.output.warn(f"Caddyfile not updated: {e}") + return None path = self.data / caddy.STACKLET / "Caddyfile" path.parent.mkdir(parents=True, exist_ok=True) # Written in place. The file is bind-mounted into the proxy # container, and a single-file bind mount follows the inode, so a # file replaced by rename would leave Caddy reading the old one. - path.write_text(caddy.assemble(snippets)) + path.write_text(caddyfile) return path # ── Secrets (delegated to SecretStore) ──────────────────────────── diff --git a/stack.example.toml b/stack.example.toml index 88e1a62d..2da8f24d 100644 --- a/stack.example.toml +++ b/stack.example.toml @@ -16,6 +16,14 @@ name = "famstack" # Requires wildcard DNS on your router: *.home.internal → server IP domain = "" +# With a domain, HTTPS for every service: one wildcard certificate plus one +# for the bare domain, from Let's Encrypt. Caddy proves you own the domain by +# writing a DNS record through your DNS provider's API, so nothing has to be +# reachable from the internet. The domain must be one you own at the provider. +# "hetzner" or "cloudflare". Empty serves plain HTTP. +# `stack up infra` asks for the API token; `stack infra dns-token` replaces it. +dns_provider = "" + # Absolute path where all persistent data lives: databases, uploads, config. # This directory is NOT managed by git — back it up separately. # The ~ shorthand is expanded by the stack CLI. diff --git a/stacklets/infra/cli/dns-token.py b/stacklets/infra/cli/dns-token.py new file mode 100644 index 00000000..38e3192f --- /dev/null +++ b/stacklets/infra/cli/dns-token.py @@ -0,0 +1,58 @@ +""" +stack infra dns-token — store the API token of the DNS provider. + +With `[core] dns_provider` set, Caddy obtains its certificates by writing +a TXT record through the provider's API, and needs a token for it. The +token is kept in the secret store (`.stack/secrets.toml`), never in +stack.toml, and reaches the Caddy container as `DNS_API_TOKEN` on the +next `stack up infra`. + +In a terminal it asks for the token without echoing it. Piped, it reads +the first line of stdin. Run it again to replace the token, then +`stack up infra`. + + stack infra dns-token +""" + +HELP = "Store or replace the DNS provider API token used for HTTPS" + +import getpass +import sys +from pathlib import Path + +from stack.secrets import TomlSecretStore + + +# What each provider's Caddy plugin authenticates with. Hetzner's plugin +# talks to the Hetzner Cloud API, so a token from the older DNS Console +# does not work. +_TOKEN_KIND = { + "hetzner": "a Hetzner Cloud API token with Read & Write access, created in " + "the Console project that holds the zone (Security > API tokens)", + "cloudflare": "a Cloudflare API token with Zone.Zone:Read and Zone.DNS:Edit " + "on the zone", +} + + +def _read_token(provider: str) -> str: + if sys.stdin.isatty(): + if provider in _TOKEN_KIND: + print(f" {provider} needs {_TOKEN_KIND[provider]}.", file=sys.stderr) + return getpass.getpass(" ▸ DNS API token (input hidden): ") + return sys.stdin.readline() + + +def run(args, stacklet, config): + provider = config["stack"].get("core", {}).get("dns_provider", "") + token = _read_token(provider).strip() + if not token: + return {"error": "No token given. The stored token, if any, is unchanged."} + + store = TomlSecretStore(Path(config["instance_dir"]) / ".stack" / "secrets.toml") + store.set(stacklet["id"], "DNS_API_TOKEN", token) + # The token itself is never printed. Its length tells a complete paste + # from a truncated one. + return { + "stored": f"DNS API token ({len(token)} characters)", + "next": "stack up infra", + } diff --git a/stacklets/infra/docker-compose.yml b/stacklets/infra/docker-compose.yml index ff27dc75..a34054f3 100644 --- a/stacklets/infra/docker-compose.yml +++ b/stacklets/infra/docker-compose.yml @@ -55,6 +55,8 @@ services: - "${PORT_BIND_IP:-0.0.0.0}:443:443" environment: - STACK_DOMAIN=${STACK_DOMAIN:-home.internal} + # Read by the Caddyfile's DNS challenge as {env.DNS_API_TOKEN}. + - DNS_API_TOKEN=${DNS_API_TOKEN:-} volumes: - ${CADDYFILE_PATH}:/etc/caddy/Caddyfile:ro - ${CADDY_DATA_DIR}:/data diff --git a/stacklets/infra/hooks/on_configure.py b/stacklets/infra/hooks/on_configure.py new file mode 100644 index 00000000..4b020752 --- /dev/null +++ b/stacklets/infra/hooks/on_configure.py @@ -0,0 +1,19 @@ +"""First run: ask for the DNS provider's API token when HTTPS is configured. + +Only in a terminal. Anywhere else on_start stops the start and names the +command that stores the token, which is also how it is replaced later. +""" + +import sys + + +def run(ctx): + if not ctx.stack._cfg("core", "dns_provider") or ctx.secret("DNS_API_TOKEN"): + return + if not sys.stdin.isatty(): + return + + result = ctx.stack.run_cli_command("infra", "dns-token") or {} + if "error" in result: + raise RuntimeError(result["error"]) + ctx.step(result["stored"]) diff --git a/stacklets/infra/hooks/on_start.py b/stacklets/infra/hooks/on_start.py new file mode 100644 index 00000000..35402314 --- /dev/null +++ b/stacklets/infra/hooks/on_start.py @@ -0,0 +1,26 @@ +"""Refuse to start a proxy that could not get its certificates. + +With `[core] dns_provider` set, Caddy needs a plugin for that provider, +compiled into the infra image, and an API token for it. Without either it +would start, fail every certificate request, and serve nothing over +HTTPS, with the reason only in its logs. Stopping here puts the fix in +front of the admin instead. +""" + +from stack.caddy import DNS_PROVIDERS + + +def run(ctx): + provider = ctx.stack._cfg("core", "dns_provider") + if not provider: + return + + if provider not in DNS_PROVIDERS: + raise RuntimeError( + f"[core] dns_provider = '{provider}' is not supported. " + f"Use {' or '.join(DNS_PROVIDERS)}, or leave it empty for plain HTTP.") + + if not ctx.secret("DNS_API_TOKEN"): + raise RuntimeError( + f"HTTPS through {provider} needs an API token. " + "Store it with `stack infra dns-token`, then run `stack up infra` again.") diff --git a/stacklets/infra/stacklet.toml b/stacklets/infra/stacklet.toml index 013741a6..d623d08f 100644 --- a/stacklets/infra/stacklet.toml +++ b/stacklets/infra/stacklet.toml @@ -32,3 +32,6 @@ STACK_DOMAIN = "{domain}" ADGUARD_DATA_DIR = "{data_dir}/infra/adguard" CADDYFILE_PATH = "{data_dir}/infra/Caddyfile" CADDY_DATA_DIR = "{data_dir}/infra/caddy" +# API token of [core] dns_provider, stored by `stack infra dns-token`. +# Empty while the proxy serves plain HTTP. +DNS_API_TOKEN = "{infra__DNS_API_TOKEN}" diff --git a/tests/unit/framework/test_caddy.py b/tests/unit/framework/test_caddy.py index 5e9c6eb0..eed4633b 100644 --- a/tests/unit/framework/test_caddy.py +++ b/tests/unit/framework/test_caddy.py @@ -87,6 +87,76 @@ def test_a_snippet_is_copied_verbatim(self): assert PHOTOS.strip() in caddyfile +# ── TLS through a DNS provider ──────────────────────────────────────────── + + +class TestDnsProvider: + """`[core] dns_provider` turns on HTTPS. Caddy obtains every certificate + through a DNS-01 challenge at that provider, so nothing on the LAN has + to be reachable from the internet. The provider's plugin is compiled + into the infra image; the token reaches Caddy as an environment + variable and never appears in the file.""" + + @pytest.mark.parametrize("provider", ["hetzner", "cloudflare"]) + def test_every_certificate_comes_through_the_providers_dns(self, provider): + caddyfile = assemble([("photos", PHOTOS)], dns_provider=provider) + assert f"dns {provider} {{env.DNS_API_TOKEN}}" in caddyfile + assert "auto_https off" not in caddyfile + + def test_hetzner_waits_before_checking_propagation(self): + """The Hetzner plugin's README asks for a 30 second delay before + Caddy starts checking for the TXT record.""" + caddyfile = assemble([], dns_provider="hetzner") + assert "propagation_delay 30s" in caddyfile + + def test_without_a_provider_no_certificate_is_requested(self): + assert "cert_issuer" not in assemble([("photos", PHOTOS)]) + + def test_an_unknown_provider_is_refused_by_name(self): + """Caddy would reject a provider it has no plugin for, and only + once the file is already in place.""" + with pytest.raises(ValueError, match="hetzner.*cloudflare"): + assemble([], dns_provider="route53") + + +def _stack(tmp_path, core: str, stacklets: dict[str, str] | None = None): + """A Stack over throwaway stacklets whose manifests are given in full.""" + from stack import Stack + from stack.output import CollectorOutput + + (tmp_path / "stack.toml").write_text( + f'[core]\n{core}\nextension_dirs = ["{tmp_path / "ext"}"]\n') + (tmp_path / ".stack").mkdir(exist_ok=True) + (tmp_path / ".stack" / "secrets.toml").write_text('global__ADMIN_PASSWORD = "test"\n') + for sid, manifest in (stacklets or {}).items(): + sdir = tmp_path / "stacklets" / sid + sdir.mkdir(parents=True) + (sdir / "stacklet.toml").write_text(f'id = "{sid}"\n{manifest}') + return Stack(root=tmp_path, data=tmp_path / "data", output=CollectorOutput()) + + +class TestPublicUrls: + """Browsers, phones and Element are handed the URLs the stack renders + (`{url}`, `{photos_url}`, `{home_url}`). Once the proxy has + certificates those have to be https: Element loaded over https refuses + to call an http homeserver.""" + + PHOTOS = 'port = 42010\n[env.defaults]\nURL = "{url}"\nHOME = "{home_url}"\n' + + def test_with_a_dns_provider_urls_are_https(self, tmp_path): + stck = _stack(tmp_path, 'domain = "home.example.family"\ndns_provider = "hetzner"', + {"photos": self.PHOTOS}) + env = stck.env("photos") + assert env["URL"] == "https://photos.home.example.family" + assert env["HOME"] == "https://home.example.family" + + def test_without_one_they_stay_http(self, tmp_path): + stck = _stack(tmp_path, 'domain = "home.example.family"', {"photos": self.PHOTOS}) + env = stck.env("photos") + assert env["URL"] == "http://photos.home.example.family" + assert env["HOME"] == "http://home.example.family" + + # ── Lifecycle ───────────────────────────────────────────────────────────── @@ -248,3 +318,19 @@ def test_a_failed_reload_is_reported_and_does_not_fail_the_step(self, tmp_path, assert cli.up("photos")["ok"] assert any("bad directive" in w for w in output.warnings) + + def test_an_unknown_dns_provider_leaves_the_caddyfile_as_it_was(self, tmp_path, docker): + """A typo in stack.toml must not replace a working proxy config + with one Caddy cannot load. Infra's own start refuses the value + outright; for every other stacklet the routes stay as they were.""" + cli, output = _make_cli(tmp_path) + docker.states = {"infra": "running", "docs": "running"} + cli.up("photos") + before = _caddyfile(tmp_path).read_text() + + config = tmp_path / "stack.toml" + config.write_text(config.read_text() + 'dns_provider = "route53"\n') + cli.down("docs") + + assert _caddyfile(tmp_path).read_text() == before + assert any("route53" in w for w in output.warnings) diff --git a/tests/unit/stacklets/test_infra_dns.py b/tests/unit/stacklets/test_infra_dns.py new file mode 100644 index 00000000..197c2dba --- /dev/null +++ b/tests/unit/stacklets/test_infra_dns.py @@ -0,0 +1,117 @@ +"""The DNS provider token: stored once, checked on every start, never shown. + +With `[core] dns_provider` set, the infra stacklet's Caddy obtains its +certificates through that provider's API, which needs a token. It lives in +the secret store, never in stack.toml, and `stack infra dns-token` is the +one way in, for the first run and for every rotation after it. `stack up +infra` refuses to start a proxy that would fail to get certificates, and +says how to fix it. + +The tests drive the real infra stacklet: its CLI command through +`Stack.run_cli_command`, the same path as `stack infra dns-token`, and its +start hook the way the framework calls it. +""" + +from __future__ import annotations + +import importlib.util +import io +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent.parent.parent +sys.path.insert(0, str(REPO / "lib")) + +from stack import Stack # noqa: E402 +from stack.hooks import StackContext # noqa: E402 + +TOKEN = "hk_test_0123456789abcdefghijklmnopqrstuvwxyz" + + +@pytest.fixture +def stack(tmp_path): + """The repo's stacklets over a throwaway instance.""" + def _make(dns_provider: str = "hetzner") -> Stack: + (tmp_path / "stack.toml").write_text( + '[core]\ndomain = "home.example.family"\n' + f'dns_provider = "{dns_provider}"\n' + f'extension_dirs = ["{tmp_path / "ext"}"]\n') + (tmp_path / ".stack").mkdir(exist_ok=True) + return Stack(root=REPO, data=tmp_path / "data", instance_dir=tmp_path) + return _make + + +def _dns_token(stck: Stack, stdin: str, monkeypatch) -> dict: + """`stack infra dns-token` with the token piped in, as a script would.""" + monkeypatch.setattr(sys, "stdin", io.StringIO(stdin)) + result = stck.run_cli_command("infra", "dns-token") + assert result is not None + return result + + +def _start(stck: Stack) -> None: + path = REPO / "stacklets" / "infra" / "hooks" / "on_start.py" + spec = importlib.util.spec_from_file_location("infra_on_start", path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.run(StackContext(stck, "infra", {})) + + +# ── stack infra dns-token ───────────────────────────────────────────────── + + +def test_the_token_is_kept_in_the_secret_store(stack, monkeypatch): + stck = stack() + _dns_token(stck, TOKEN + "\n", monkeypatch) + assert stck.secrets.get("infra", "DNS_API_TOKEN") == TOKEN + + +def test_the_token_is_never_echoed(stack, monkeypatch): + """Command output ends up in terminals, logs and chat. Its length is + enough to tell a pasted token from a truncated one.""" + result = _dns_token(stack(), TOKEN + "\n", monkeypatch) + assert TOKEN not in repr(result) + assert str(len(TOKEN)) in repr(result) + + +def test_a_new_token_replaces_the_old_one(stack, monkeypatch): + stck = stack() + _dns_token(stck, "old-token\n", monkeypatch) + _dns_token(stck, TOKEN + "\n", monkeypatch) + assert stck.secrets.get("infra", "DNS_API_TOKEN") == TOKEN + + +def test_an_empty_token_is_refused(stack, monkeypatch): + stck = stack() + _dns_token(stck, TOKEN + "\n", monkeypatch) + result = _dns_token(stck, "\n", monkeypatch) + assert "error" in result + assert stck.secrets.get("infra", "DNS_API_TOKEN") == TOKEN + + +# ── stack up infra ──────────────────────────────────────────────────────── + + +def test_start_needs_a_token_once_a_provider_is_set(stack): + with pytest.raises(RuntimeError, match="stack infra dns-token"): + _start(stack()) + + +def test_start_goes_ahead_once_the_token_is_stored(stack, monkeypatch): + stck = stack() + _dns_token(stck, TOKEN + "\n", monkeypatch) + _start(stck) + + +def test_plain_http_needs_no_token(stack): + _start(stack(dns_provider="")) + + +def test_start_refuses_a_provider_the_image_has_no_plugin_for(stack, monkeypatch): + stck = stack(dns_provider="route53") + _dns_token(stck, TOKEN + "\n", monkeypatch) + with pytest.raises(RuntimeError, match="route53"): + _start(stck) From 7c0b5db1df5df0152101be58fe5838deeea31d47 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:04:57 +0200 Subject: [PATCH 03/15] test: serve valid HTTP from the Alpine test stacklet The fixture's command was a string, which compose splits itself, and the splitting dropped the backslashes of `\r\n`. The response arrived as one line with no header end, which the health probe tolerated and a real HTTP client such as Caddy's reverse proxy rejects. The command is now a list, passed to the container as written, and the response closes the connection after the body. --- .../framework/fixtures/test/docker-compose.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/unit/framework/fixtures/test/docker-compose.yml b/tests/unit/framework/fixtures/test/docker-compose.yml index 29c0afb9..2574d8ad 100644 --- a/tests/unit/framework/fixtures/test/docker-compose.yml +++ b/tests/unit/framework/fixtures/test/docker-compose.yml @@ -7,11 +7,15 @@ services: stack-test: container_name: stack-test image: alpine:latest - command: > - sh -c "while true; do - echo -e 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok' | - nc -l -p 8080; - done" + # A list, not a string: compose splits a string command itself and + # drops the backslashes, which left the response without line breaks. + command: + - sh + - -c + - | + while true; do + printf 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok' | nc -l -p 8080 + done labels: - "com.centurylinklabs.watchtower.enable=false" networks: From 892e716c224ebe5a080c570b4875bb85a0554bc1 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:05:42 +0200 Subject: [PATCH 04/15] feat(infra): build Caddy with the Hetzner and Cloudflare DNS plugins A certificate through a DNS-01 challenge needs the DNS provider's plugin compiled into Caddy, and the stock image has none, so the proxy could not obtain certificates for `[core] dns_provider`. The infra stacklet now builds its Caddy image with xcaddy on the Caddy 2 line, with both plugins, so switching providers is a config change and not a rebuild. The Hetzner plugin is v2, which uses the Hetzner Cloud API; v1 used the deprecated DNS Console API. `stack up infra` rebuilds the image, and Watchtower leaves it alone because there is nothing to pull. --- stacklets/infra/Dockerfile | 20 +++++++ stacklets/infra/docker-compose.yml | 13 ++++- stacklets/infra/stacklet.toml | 2 + tests/integration/test_caddy.py | 93 +++++++++++++++++++++++++++++- 4 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 stacklets/infra/Dockerfile diff --git a/stacklets/infra/Dockerfile b/stacklets/infra/Dockerfile new file mode 100644 index 00000000..5141a54a --- /dev/null +++ b/stacklets/infra/Dockerfile @@ -0,0 +1,20 @@ +# Caddy with the DNS provider plugins `[core] dns_provider` can name. +# +# A certificate through a DNS-01 challenge, and so the wildcard, needs the +# provider's plugin compiled into the binary; the stock image has none. +# One image carries every supported provider, so switching providers is a +# config change, not a rebuild. Supported names: lib/stack/caddy.py. +# +# The hetzner module is v2, which talks to the Hetzner Cloud API. v1 used +# the DNS Console API, which Hetzner has deprecated. +# +# Pinned to the Caddy 2 line. Every subdomain reusing the wildcard +# certificate needs 2.10 or later. + +FROM caddy:2-builder AS builder +RUN xcaddy build \ + --with github.com/caddy-dns/hetzner/v2 \ + --with github.com/caddy-dns/cloudflare + +FROM caddy:2 +COPY --from=builder /usr/bin/caddy /usr/bin/caddy diff --git a/stacklets/infra/docker-compose.yml b/stacklets/infra/docker-compose.yml index a34054f3..a2c4becf 100644 --- a/stacklets/infra/docker-compose.yml +++ b/stacklets/infra/docker-compose.yml @@ -40,13 +40,20 @@ services: TZ: ${TZ} # ── Caddy ────────────────────────────────────────────────────────────────── - # Reverse proxy. Routes requests by hostname to the right container. + # Reverse proxy. Routes requests by hostname to the right container, from + # the Caddyfile the stack CLI assembles out of every running stacklet's + # caddy.snippet. Built with the DNS provider plugins (see Dockerfile). stack-infra-caddy: - image: caddy:2-alpine + build: + context: . + dockerfile: Dockerfile + image: stack-infra-caddy:local container_name: stack-infra-caddy restart: unless-stopped + # Locally-built image, nothing for Watchtower to pull. `stack up infra` + # rebuilds it. labels: - - "com.centurylinklabs.watchtower.enable=true" + - "com.centurylinklabs.watchtower.enable=false" - "stack.commit=${STACK_COMMIT:-unknown}" networks: - stack diff --git a/stacklets/infra/stacklet.toml b/stacklets/infra/stacklet.toml index d623d08f..c217104e 100644 --- a/stacklets/infra/stacklet.toml +++ b/stacklets/infra/stacklet.toml @@ -13,6 +13,8 @@ description = "DNS + reverse proxy for domain mode" version = "0.2.1" category = "infrastructure" always_on = true +# Caddy is built locally with the DNS provider plugins (Dockerfile). +build = true [ports] dns = 53 diff --git a/tests/integration/test_caddy.py b/tests/integration/test_caddy.py index 3e87140b..601b29a8 100644 --- a/tests/integration/test_caddy.py +++ b/tests/integration/test_caddy.py @@ -9,6 +9,7 @@ import json import subprocess import sys +import time from pathlib import Path import pytest @@ -16,7 +17,7 @@ REPO_ROOT = Path(__file__).parent.parent.parent sys.path.insert(0, str(REPO_ROOT / "lib")) -from stack.caddy import assemble # noqa: E402 +from stack.caddy import DNS_PROVIDERS, assemble # noqa: E402 try: subprocess.run(["docker", "info"], capture_output=True, timeout=5, check=True) @@ -35,11 +36,18 @@ def _shipped_snippets() -> list[tuple[str, str]]: for path in sorted(REPO_ROOT.glob("stacklets/*/caddy.snippet"))] +# Never a real credential: nothing here reaches a DNS provider. Shaped +# like a Cloudflare token, because its plugin refuses to load a config +# whose token does not look like one. +FAKE_TOKEN = "fake" + "0" * 36 + + def _caddy(image: str, caddyfile: Path, *args: str) -> subprocess.CompletedProcess: """Run `caddy ` against this Caddyfile, as the proxy container would.""" return subprocess.run( ["docker", "run", "--rm", "-e", f"STACK_DOMAIN={DOMAIN}", + "-e", f"DNS_API_TOKEN={FAKE_TOKEN}", "-v", f"{caddyfile}:/etc/caddy/Caddyfile:ro", image, "caddy", *args, "--config", "/etc/caddy/Caddyfile"], capture_output=True, text=True, timeout=180, @@ -93,3 +101,86 @@ def test_without_tls_every_site_is_plain_http_on_port_80(plain_caddyfile): config = _adapt(STOCK_CADDY, plain_caddyfile) assert _listen(config) == {":80"} assert "tls" not in config.get("apps", {}) + + +# ── The infra image: TLS through each DNS provider ──────────────────────── + +INFRA_IMAGE = "stack-infra-caddy:local" + + +@pytest.fixture(scope="module") +def infra_image() -> str: + """The image `stack up infra` builds, under the tag its compose file uses.""" + result = subprocess.run( + ["docker", "build", "-q", "-t", INFRA_IMAGE, str(REPO_ROOT / "stacklets" / "infra")], + capture_output=True, text=True, timeout=900, + ) + assert result.returncode == 0, result.stderr + return INFRA_IMAGE + + +def _tls_caddyfile(tmp_path: Path, provider: str) -> Path: + path = tmp_path / "Caddyfile" + path.write_text(assemble(_shipped_snippets(), dns_provider=provider)) + return path + + +@pytest.mark.parametrize("provider", sorted(DNS_PROVIDERS)) +def test_the_image_has_a_plugin_for_every_supported_provider(infra_image, tmp_path, provider): + result = _caddy(infra_image, _tls_caddyfile(tmp_path, provider), "validate") + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("provider", sorted(DNS_PROVIDERS)) +def test_every_certificate_is_requested_through_the_providers_dns(infra_image, tmp_path, provider): + config = _adapt(infra_image, _tls_caddyfile(tmp_path, provider)) + challenges = [issuer["challenges"]["dns"] + for policy in config["apps"]["tls"]["automation"]["policies"] + for issuer in policy["issuers"]] + assert challenges + assert {c["provider"]["name"] for c in challenges} == {provider} + # Still the placeholder: the token is read from the environment at + # request time, so it is not in the config Caddy's admin endpoint serves. + assert FAKE_TOKEN not in json.dumps(config) + + +def test_caddy_asks_for_the_wildcard_and_the_bare_domain_and_nothing_else(infra_image, tmp_path): + """Two certificates cover every site: subdomain sites reuse the + wildcard, and the bare domain, which the wildcard does not cover, gets + its own. Anything more would put every subdomain in public + Certificate Transparency logs and spend Let's Encrypt's rate limit. + + The ACME directory is pointed at a closed local port, so the requests + Caddy starts fail at once and nothing leaves this machine. + """ + caddyfile = _tls_caddyfile(tmp_path, "cloudflare") + caddyfile.write_text(caddyfile.read_text().replace( + "cert_issuer acme {", "cert_issuer acme http://127.0.0.1:9/directory {")) + + name = "stack-test-caddy-certs" + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) + subprocess.run( + ["docker", "run", "-d", "--name", name, + "-e", f"STACK_DOMAIN={DOMAIN}", "-e", f"DNS_API_TOKEN={FAKE_TOKEN}", + "-v", f"{caddyfile}:/etc/caddy/Caddyfile:ro", infra_image], + check=True, capture_output=True, timeout=60, + ) + try: + deadline = time.monotonic() + 20 + while len(_certificates_requested(name)) < 2 and time.monotonic() < deadline: + time.sleep(1) + time.sleep(2) # room for a third request, if Caddy were to make one + requested = _certificates_requested(name) + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) + + assert requested == {DOMAIN, f"*.{DOMAIN}"} + + +def _certificates_requested(container: str) -> set[str]: + """Names Caddy has started obtaining a certificate for, from its log.""" + logs = subprocess.run(["docker", "logs", container], + capture_output=True, text=True, timeout=30).stderr + entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")] + return {e["identifier"] for e in entries if e.get("msg") == "obtaining certificate"} + From f7173da0a57cb63cfec5a556e8ea54f6499078cb Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:06:03 +0200 Subject: [PATCH 05/15] fix(infra): answer DNS and web requests from the LAN in domain mode The infra stacklet published DNS on 53 and the proxy on 80 and 443 through `PORT_BIND_IP`, which the framework sets to 127.0.0.1 in domain mode, the only mode infra runs in. No other device could resolve a name through AdGuard or reach a service through Caddy. These three ports now bind all interfaces. `PORT_BIND_IP` still keeps every service behind the proxy, and AdGuard's own admin pages, on the loopback. --- stacklets/infra/docker-compose.yml | 15 ++-- tests/integration/test_caddy.py | 107 ++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/stacklets/infra/docker-compose.yml b/stacklets/infra/docker-compose.yml index a2c4becf..2af580bb 100644 --- a/stacklets/infra/docker-compose.yml +++ b/stacklets/infra/docker-compose.yml @@ -26,9 +26,11 @@ services: networks: - stack ports: - # DNS (must be 53 for network clients) - - "${PORT_BIND_IP:-0.0.0.0}:53:53/udp" - - "${PORT_BIND_IP:-0.0.0.0}:53:53/tcp" + # DNS (must be 53 for network clients). Every device on the LAN asks + # here, so it binds all interfaces. PORT_BIND_IP is 127.0.0.1 in + # domain mode, the one mode this stacklet runs in. + - "0.0.0.0:53:53/udp" + - "0.0.0.0:53:53/tcp" # Setup wizard (first run) - "${PORT_BIND_IP:-0.0.0.0}:3000:3000" # Admin panel (after setup) @@ -58,8 +60,11 @@ services: networks: - stack ports: - - "${PORT_BIND_IP:-0.0.0.0}:80:80" - - "${PORT_BIND_IP:-0.0.0.0}:443:443" + # The front door for every device on the LAN, so it binds all + # interfaces. PORT_BIND_IP (127.0.0.1 in domain mode) keeps the + # services behind the proxy off the LAN, not the proxy itself. + - "0.0.0.0:80:80" + - "0.0.0.0:443:443" environment: - STACK_DOMAIN=${STACK_DOMAIN:-home.internal} # Read by the Caddyfile's DNS challenge as {env.DNS_API_TOKEN}. diff --git a/tests/integration/test_caddy.py b/tests/integration/test_caddy.py index 601b29a8..5441ceec 100644 --- a/tests/integration/test_caddy.py +++ b/tests/integration/test_caddy.py @@ -1,20 +1,27 @@ -"""The assembled Caddyfile, as a real Caddy reads it. +"""The assembled Caddyfile, as a real Caddy reads it, and the proxy running. The unit tests pin which snippets the framework puts in the file and when. Whether Caddy accepts the result, and what it serves from it, only Caddy can say, so these hand it the snippets that ship in the repo and read the -JSON config it adapts them to. +JSON config it adapts them to. The last part runs `stack up infra` on a +throwaway instance and talks to the proxy the way a device on the LAN +would. """ import json +import os +import shutil import subprocess import sys import time +import urllib.error +import urllib.request from pathlib import Path import pytest REPO_ROOT = Path(__file__).parent.parent.parent +FIXTURES_DIR = REPO_ROOT / "tests" / "unit" / "framework" / "fixtures" sys.path.insert(0, str(REPO_ROOT / "lib")) from stack.caddy import DNS_PROVIDERS, assemble # noqa: E402 @@ -184,3 +191,99 @@ def _certificates_requested(container: str) -> set[str]: entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")] return {e["identifier"] for e in entries if e.get("msg") == "obtaining certificate"} + +# ── stack up infra ──────────────────────────────────────────────────────── + + +class Instance: + """A throwaway instance: the repo's infra, the Alpine test stacklet + with a route, domain mode, no TLS.""" + + def __init__(self, root: Path): + self.root = root + + def stack(self, *args: str, timeout: int = 900) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-m", "stack", *args], + cwd=str(self.root), capture_output=True, text=True, timeout=timeout, + env={**os.environ, "PYTHONPATH": str(self.root / "lib")}, + ) + + def ok(self, *args: str) -> None: + result = self.stack(*args) + assert result.returncode == 0, ( + f"`stack {' '.join(args)}` exited {result.returncode}\n" + f"--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}") + + +@pytest.fixture(scope="module") +def instance(tmp_path_factory): + root = tmp_path_factory.mktemp("domain") / "stack" + (root / "stacklets").mkdir(parents=True) + (root / "lib").symlink_to(REPO_ROOT / "lib") + shutil.copytree(REPO_ROOT / "stacklets" / "infra", root / "stacklets" / "infra") + shutil.copytree(FIXTURES_DIR / "test", root / "stacklets" / "test") + (root / "stacklets" / "test" / "caddy.snippet").write_text( + "test.{$STACK_DOMAIN} {\n\treverse_proxy stack-test:8080\n}\n") + + (root / "stack.toml").write_text( + f'[core]\nname = "teststack"\ndomain = "{DOMAIN}"\ndns_provider = ""\n' + f'data_dir = "{root.parent / "data"}"\n' + f'extension_dirs = ["{root / "extensions"}"]\ntimezone = "Europe/Berlin"\n') + (root / "users.toml").write_text( + '[[users]]\nname = "Test Admin"\nemail = "admin@test.local"\n' + 'password = "testpass"\nrole = "admin"\n') + (root / ".stack").mkdir() + (root / ".stack" / "secrets.toml").write_text('global__ADMIN_PASSWORD = "testpass"\n') + + inst = Instance(root) + yield inst + + inst.stack("destroy", "test", "--yes") + inst.stack("destroy", "infra", "--yes") + # `destroy` skips containers of a stacklet whose first `up` failed, so + # a failed run would otherwise leave these holding their ports. + subprocess.run(["docker", "rm", "-f", "stack-test", "stack-infra-caddy", + "stack-infra-adguard"], capture_output=True, timeout=60) + + +def _get(host: str) -> tuple[int, str]: + """GET / through the proxy on the host's port 80, as `host`.""" + request = urllib.request.Request("http://127.0.0.1/", headers={"Host": host}) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, response.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.read().decode() + + +def _host_ips(container: str) -> dict[str, str]: + """Published container port -> host address it is bound to.""" + result = subprocess.run( + ["docker", "inspect", "--format", "{{json .HostConfig.PortBindings}}", container], + capture_output=True, text=True, timeout=30, check=True) + return {port: bindings[0]["HostIp"] + for port, bindings in json.loads(result.stdout).items()} + + +@pytest.mark.container_lifecycle +def test_the_proxy_and_dns_face_the_lan_and_nothing_else_does(instance): + instance.ok("up", "infra") + + assert _host_ips("stack-infra-caddy") == {"80/tcp": "0.0.0.0", "443/tcp": "0.0.0.0"} + adguard = _host_ips("stack-infra-adguard") + assert adguard["53/udp"] == adguard["53/tcp"] == "0.0.0.0" + assert adguard["80/tcp"] == adguard["3000/tcp"] == "127.0.0.1" + + +@pytest.mark.container_lifecycle +def test_a_stacklet_is_served_from_the_moment_it_is_up_until_it_is_down(instance): + instance.ok("up", "infra") + assert _get(f"test.{DOMAIN}")[0] == 404 + + instance.ok("up", "test") + assert _get(f"test.{DOMAIN}") == (200, "ok") + + instance.ok("down", "test") + status, body = _get(f"test.{DOMAIN}") + assert (status, body) == (404, f"No service at test.{DOMAIN}") From 6915647035e0e75ec17a2475c6c81c9cb4ec1f92 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:06:40 +0200 Subject: [PATCH 06/15] fix(infra): move the AdGuard setup wizard to port 42081 AdGuard's first-run wizard was published on host port 3000, where development servers live. With one running, `stack up infra` failed on a port conflict that named the port and not the stacklet. The wizard is now on 42081, next to the admin panel on 42080, inside the 42xxx range every other stacklet uses. A new test checks that range for every published port, and that no two stacklets publish the same one. An instance that has finished the wizard is not affected. --- stacklets/infra/docker-compose.yml | 4 +- stacklets/infra/stacklet.toml | 2 +- tests/unit/framework/test_compose_ports.py | 58 ++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 tests/unit/framework/test_compose_ports.py diff --git a/stacklets/infra/docker-compose.yml b/stacklets/infra/docker-compose.yml index 2af580bb..4f1da96e 100644 --- a/stacklets/infra/docker-compose.yml +++ b/stacklets/infra/docker-compose.yml @@ -14,7 +14,7 @@ services: # ── AdGuard Home ───────────────────────────────────────────────────────────── # DNS server with ad-blocking. Provides wildcard DNS for domain mode. - # First run: complete setup wizard at http://ip:3000 + # First run: complete setup wizard at http://localhost:42081 # After setup: admin panel at http://ip:42080 stack-infra-adguard: image: adguard/adguardhome:latest @@ -32,7 +32,7 @@ services: - "0.0.0.0:53:53/udp" - "0.0.0.0:53:53/tcp" # Setup wizard (first run) - - "${PORT_BIND_IP:-0.0.0.0}:3000:3000" + - "${PORT_BIND_IP:-0.0.0.0}:42081:3000" # Admin panel (after setup) - "${PORT_BIND_IP:-0.0.0.0}:42080:80" volumes: diff --git a/stacklets/infra/stacklet.toml b/stacklets/infra/stacklet.toml index c217104e..f227f0f2 100644 --- a/stacklets/infra/stacklet.toml +++ b/stacklets/infra/stacklet.toml @@ -18,7 +18,7 @@ build = true [ports] dns = 53 -dns_setup = 3000 +dns_setup = 42081 dns_admin = 42080 http = 80 https = 443 diff --git a/tests/unit/framework/test_compose_ports.py b/tests/unit/framework/test_compose_ports.py new file mode 100644 index 00000000..527a80bf --- /dev/null +++ b/tests/unit/framework/test_compose_ports.py @@ -0,0 +1,58 @@ +"""Host ports follow the 42xxx convention and never collide (adr-007). + +A Mac running famstack usually runs other things too, and 3000, 5000 and +8080 are where development servers live. A stacklet that publishes one of +those fails to start the day its owner opens a Node project, with an +error that names the port and not the stacklet. The same goes for two +stacklets claiming one port: each works alone, and the second one up +fails. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent + +# Ports a protocol fixes, which no convention can move: DNS clients ask on +# 53, browsers on 80 and 443. Forgejo's SSH sits on 222 because macOS +# keeps 22 for itself. +PROTOCOL_PORTS = {53, 80, 443, 222} + +# `- "[ip:]host:container[/proto]"`, where ip and host may be `${VAR:-default}`. +_PORT_RE = re.compile( + r'^\s*-\s*"(?:(?:\$\{[^}]+\}|[\d.]+):)?' + r'(?:\$\{\w+:-(?P\d+)\}|(?P\d+))' + r':\d+(?:/(?Ptcp|udp))?"', + re.MULTILINE, +) + + +def _published() -> list[tuple[str, int, str]]: + """(stacklet, host port, protocol) for every port any stacklet publishes.""" + found = [] + for compose in sorted(REPO_ROOT.glob("stacklets/*/docker-compose*.yml")): + for m in _PORT_RE.finditer(compose.read_text()): + port = int(m["host"] or m["default"]) + found.append((compose.parent.name, port, m["proto"] or "tcp")) + return found + + +def test_published_ports_are_found(): + # Guard the guard: a pattern that matched nothing would pass both tests. + assert len(_published()) > 10 + + +def test_every_host_port_is_in_the_42xxx_range_or_fixed_by_its_protocol(): + offenders = [(sid, port) for sid, port, _ in _published() + if port not in PROTOCOL_PORTS and not 42000 <= port <= 42999] + assert not offenders, f"outside 42000-42999 (adr-007): {offenders}" + + +def test_no_two_stacklets_publish_the_same_host_port(): + owners: dict[tuple[int, str], set[str]] = {} + for sid, port, proto in _published(): + owners.setdefault((port, proto), set()).add(sid) + clashes = {key: sids for key, sids in owners.items() if len(sids) > 1} + assert not clashes, f"host ports claimed by more than one stacklet: {clashes}" From e8f3265380807feebc4df088521402a68f4c812d Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:06:58 +0200 Subject: [PATCH 07/15] fix(infra): pin AdGuard Home to v0.107.79 AdGuard ran `:latest`, which Watchtower replaces with whatever is published, a new minor line included. AdGuard publishes exact versions only, with no line tag to follow, so the image is pinned to the current release and moves with a bump in the compose file. --- stacklets/infra/docker-compose.yml | 4 +++- tests/unit/framework/test_compose_pins.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/stacklets/infra/docker-compose.yml b/stacklets/infra/docker-compose.yml index 4f1da96e..b05d731e 100644 --- a/stacklets/infra/docker-compose.yml +++ b/stacklets/infra/docker-compose.yml @@ -17,7 +17,9 @@ services: # First run: complete setup wizard at http://localhost:42081 # After setup: admin panel at http://ip:42080 stack-infra-adguard: - image: adguard/adguardhome:latest + # AdGuard publishes exact versions only, no line tag Watchtower could + # follow, so a new release is a bump here. + image: adguard/adguardhome:v0.107.79 container_name: stack-infra-adguard restart: unless-stopped labels: diff --git a/tests/unit/framework/test_compose_pins.py b/tests/unit/framework/test_compose_pins.py index 4e2c9411..c8283fc9 100644 --- a/tests/unit/framework/test_compose_pins.py +++ b/tests/unit/framework/test_compose_pins.py @@ -31,7 +31,6 @@ "ghcr.io/matatonic/openedai-speech-min", # no tag at all "nickfedor/watchtower:latest", "apache/tika:latest", - "adguard/adguardhome:latest", "matrixdotorg/synapse:latest", "vectorim/element-web:latest", } From fef359337d30ba8a2f4515d208cc9dcd3e394571 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:09:55 +0200 Subject: [PATCH 08/15] fix: serve each stacklet at the URL the stack hands out In domain mode the stack gives every stacklet the URL `.`, in hints, welcome messages and links other stacklets build, but several snippets routed elsewhere: - photos and chatai proxied to container names no compose file uses (`immich_server`, `chatai`), so both answered 502. - messages served Synapse at `messages.` and Element at `element.`, while the stack sends browsers and Element's homeserver setting to `messages.`. It is now one host: Element for browsers, Synapse under `/_matrix` and `/_synapse`, and `/.well-known/matrix/client` and `/server` built from the request so they are right with and without TLS. - memory served the wiki at `wiki.`, while its hint and core's `/go` links point at `memory.`. The wiki's own absolute links move with it. - code had no snippet at all. A new test checks every snippet's upstreams against the container names in the compose files, and that every stacklet showing `{url}` has a site at `.`. --- stacklets/chatai/caddy.snippet | 2 +- stacklets/code/caddy.snippet | 8 ++ stacklets/memory/caddy.snippet | 9 ++- stacklets/memory/quartz/quartz.config.ts | 4 +- stacklets/memory/stacklet.toml | 2 +- stacklets/messages/caddy.snippet | 20 +++-- stacklets/photos/caddy.snippet | 7 +- tests/unit/framework/test_caddy_snippets.py | 85 +++++++++++++++++++++ 8 files changed, 123 insertions(+), 14 deletions(-) create mode 100644 stacklets/code/caddy.snippet create mode 100644 tests/unit/framework/test_caddy_snippets.py diff --git a/stacklets/chatai/caddy.snippet b/stacklets/chatai/caddy.snippet index 8160b0db..c6273e33 100644 --- a/stacklets/chatai/caddy.snippet +++ b/stacklets/chatai/caddy.snippet @@ -4,5 +4,5 @@ # stack network, so backends are referenced by container name. chatai.{$STACK_DOMAIN} { - reverse_proxy chatai:8080 + reverse_proxy stack-chatai:8080 } diff --git a/stacklets/code/caddy.snippet b/stacklets/code/caddy.snippet new file mode 100644 index 00000000..7c3f7807 --- /dev/null +++ b/stacklets/code/caddy.snippet @@ -0,0 +1,8 @@ +# stacklets/code/caddy.snippet +# +# Forgejo's web UI and its git-over-HTTPS. Git over SSH keeps its own +# port (222) and does not pass through the proxy. + +code.{$STACK_DOMAIN} { + reverse_proxy stack-code:3000 +} diff --git a/stacklets/memory/caddy.snippet b/stacklets/memory/caddy.snippet index 72b4661d..f4ee7cff 100644 --- a/stacklets/memory/caddy.snippet +++ b/stacklets/memory/caddy.snippet @@ -1,14 +1,15 @@ # stacklets/memory/caddy.snippet # -# One route. The family wiki — Quartz rendering the family vault — -# lives at wiki.{$STACK_DOMAIN}. Caddy runs on the stack network -# so the backend is the container name; the in-container port is +# One route. The family wiki (Quartz rendering the family vault) lives at +# memory.{$STACK_DOMAIN}, the URL the stack hands out for this stacklet: +# the wiki hint, and core's /go/topic links. Caddy runs on the stack +# network so the backend is the container name; the in-container port is # Quartz's preview server default (8080), not the published host port. # # No auth here, matching the rest of the family-facing stacklets # (photos, docs, messages). The LAN is the perimeter. Add `basicauth` # inside this block if you ever want to gate the vault separately. -wiki.{$STACK_DOMAIN} { +memory.{$STACK_DOMAIN} { reverse_proxy stack-memory-wiki:8080 } diff --git a/stacklets/memory/quartz/quartz.config.ts b/stacklets/memory/quartz/quartz.config.ts index 1c375622..2a457272 100644 --- a/stacklets/memory/quartz/quartz.config.ts +++ b/stacklets/memory/quartz/quartz.config.ts @@ -14,8 +14,8 @@ import { QuartzConfig } from "./quartz/cfg" import * as Plugin from "./quartz/plugins" -// Runtime env. `WIKI_HOST` is "wiki." in domain mode and -// the empty-domain rendering "wiki." in port mode — we treat the +// Runtime env. `WIKI_HOST` is "memory." in domain mode and +// the empty-domain rendering "memory." in port mode — we treat the // trailing-dot form as missing and fall back to the LAN IP so // absolute URLs in the sitemap stay reachable in either mode. const wikiHost = process.env.WIKI_HOST?.replace(/^https?:\/\//, "") || "" diff --git a/stacklets/memory/stacklet.toml b/stacklets/memory/stacklet.toml index 91f6171f..4dc72420 100644 --- a/stacklets/memory/stacklet.toml +++ b/stacklets/memory/stacklet.toml @@ -132,7 +132,7 @@ WIKI_NIGHTLY = "{wiki_nightly}" # Public hostname the wiki renders into absolute URLs (sitemap, # social cards, etc). Empty domain (port mode) falls back to the # auto-detected LAN IP via `{ip}` so the site stays reachable. -WIKI_HOST = "wiki.{domain}" +WIKI_HOST = "memory.{domain}" WIKI_IP = "{ip}" # What the wiki calls itself, in the sidebar and the browser tab: the diff --git a/stacklets/messages/caddy.snippet b/stacklets/messages/caddy.snippet index 39717229..16724a1f 100644 --- a/stacklets/messages/caddy.snippet +++ b/stacklets/messages/caddy.snippet @@ -1,12 +1,22 @@ # stacklets/messages/caddy.snippet # -# Two routes: Synapse API and Element Web client. -# Assembled into the Caddyfile by 'stack up'. +# One host for the family's chat. A browser gets Element; Element and the +# phone apps reach Synapse under /_matrix and /_synapse on the same host, +# which is the address the stack hands out for both ({messages_url}, +# {messages_synapse_url}). +# +# Matrix apps look up the homeserver at /.well-known/matrix/client on the +# host the user types in. The answer is built from the request, so it is +# right with and without TLS. The JSON braces are escaped so Caddy does +# not read them as placeholders; {scheme} and {host} are. messages.{$STACK_DOMAIN} { - reverse_proxy stack-messages-synapse:8008 -} + header /.well-known/matrix/* Content-Type application/json + header /.well-known/matrix/* Access-Control-Allow-Origin * + respond /.well-known/matrix/client `\{"m.homeserver":\{"base_url":"{scheme}://{host}"\}\}` + respond /.well-known/matrix/server `\{"m.server":"{host}:443"\}` -element.{$STACK_DOMAIN} { + reverse_proxy /_matrix/* stack-messages-synapse:8008 + reverse_proxy /_synapse/* stack-messages-synapse:8008 reverse_proxy stack-messages-element:80 } diff --git a/stacklets/photos/caddy.snippet b/stacklets/photos/caddy.snippet index ddc403bf..3178b245 100644 --- a/stacklets/photos/caddy.snippet +++ b/stacklets/photos/caddy.snippet @@ -3,7 +3,12 @@ # Assembled into the Caddyfile by 'stack up'. Caddy runs in Docker on the # stack network, so backends are referenced by container name. # {$STACK_DOMAIN} is set as an env var when Caddy starts. +# +# flush_interval -1 passes the response through as it arrives instead of +# buffering it, which Immich's video streaming needs. photos.{$STACK_DOMAIN} { - reverse_proxy immich_server:2283 + reverse_proxy stack-photos-server:2283 { + flush_interval -1 + } } diff --git a/tests/unit/framework/test_caddy_snippets.py b/tests/unit/framework/test_caddy_snippets.py new file mode 100644 index 00000000..32b7dfa8 --- /dev/null +++ b/tests/unit/framework/test_caddy_snippets.py @@ -0,0 +1,85 @@ +"""Every caddy.snippet points at something that exists. + +A snippet names its backends by container name and its sites by host +name, and nothing checks either until a browser gets a 502 or a 404. +Two snippets targeted container names that no compose file used +(`immich_server`, `chatai`), and two stacklets were served on a host +other than the one the stack hands out as their URL. These checks read +the repo's own files, so they fail the moment a container is renamed or +a stacklet starts advertising a URL no snippet serves. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent + +# Native services on the Mac, reached from the proxy container through +# Docker's host gateway, not by container name. +HOST_GATEWAY = "host.docker.internal" + + +def _snippets() -> dict[str, str]: + return {p.parent.name: p.read_text() + for p in sorted(REPO_ROOT.glob("stacklets/*/caddy.snippet"))} + + +def _container_names() -> set[str]: + return {name + for compose in REPO_ROOT.glob("stacklets/*/docker-compose*.yml") + for name in re.findall(r"^\s*container_name:\s*(\S+)", compose.read_text(), re.M)} + + +def _upstreams(snippet: str) -> list[str]: + """Hosts of every `reverse_proxy` upstream. Matchers (`/path`, `@name`) + and the block brace are skipped.""" + hosts = [] + for line in snippet.splitlines(): + tokens = line.split("#", 1)[0].split() + if not tokens or tokens[0] != "reverse_proxy": + continue + for token in tokens[1:]: + if token.startswith(("/", "@", "{")): + continue + hosts.append(token.rsplit(":", 1)[0]) + return hosts + + +def _site_addresses(snippet: str) -> set[str]: + """Addresses of the top-level site blocks: unindented lines ending in `{`.""" + addresses = set() + for line in snippet.splitlines(): + if line[:1] in ("", " ", "\t", "#") or not line.rstrip().endswith("{"): + continue + addresses.update(a.strip() for a in line.rstrip()[:-1].split(",") if a.strip()) + return addresses + + +def test_snippets_are_found(): + # Guard the guard: an empty glob would make every check below pass. + assert len(_snippets()) >= 5 + + +def test_every_upstream_is_a_container_some_stacklet_runs(): + containers = _container_names() + unknown = {sid: host for sid, snippet in _snippets().items() + for host in _upstreams(snippet) + if host != HOST_GATEWAY and host not in containers} + assert not unknown, f"reverse_proxy targets no compose file names: {unknown}" + + +def test_the_url_a_stacklet_hands_out_is_served_by_its_snippet(): + """In domain mode `{url}` renders as `.`. A stacklet that + shows it to the family (a hint, a welcome message, a link another + stacklet builds) needs a site at exactly that host.""" + snippets = _snippets() + unserved = [] + for manifest_path in sorted(REPO_ROOT.glob("stacklets/*/stacklet.toml")): + sid = manifest_path.parent.name + if "{url}" not in manifest_path.read_text(): + continue + if f"{sid}.{{$STACK_DOMAIN}}" not in _site_addresses(snippets.get(sid, "")): + unserved.append(sid) + assert not unserved, f"no site at .{{$STACK_DOMAIN}} for: {unserved}" From 4b35fa3ef1f14b96da27807e56bbebc9a2b99eeb Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:10:03 +0200 Subject: [PATCH 09/15] feat(ai): serve speech and the LLM API by name in domain mode Clients on the LAN can reach the AI stacklet's endpoints through the proxy: `stt.` for speech to text, `tts.` for text to speech and `openai.` for the OpenAI-compatible LLM API. whisper-server and oMLX stay bound to the Mac's loopback. The proxy reaches them through Docker's host gateway, which the Caddy container now maps as `host.docker.internal`, as every other container that talks to the host already does. --- stacklets/ai/caddy.snippet | 22 ++++++++++++++++++++++ stacklets/infra/docker-compose.yml | 4 ++++ 2 files changed, 26 insertions(+) create mode 100644 stacklets/ai/caddy.snippet diff --git a/stacklets/ai/caddy.snippet b/stacklets/ai/caddy.snippet new file mode 100644 index 00000000..f0674fb7 --- /dev/null +++ b/stacklets/ai/caddy.snippet @@ -0,0 +1,22 @@ +# stacklets/ai/caddy.snippet +# +# The AI endpoints by name, for clients on the LAN: speech to text, text +# to speech, and the OpenAI-compatible LLM API. +# +# whisper-server and oMLX run natively on the Mac. In domain mode they +# listen on its loopback only, like every service behind the proxy, and +# the proxy reaches them through Docker's host gateway +# (host.docker.internal), the one route in from the LAN. Piper's speech +# server is a container on the stack network. + +stt.{$STACK_DOMAIN} { + reverse_proxy host.docker.internal:42062 +} + +tts.{$STACK_DOMAIN} { + reverse_proxy stack-ai-speech:8000 +} + +openai.{$STACK_DOMAIN} { + reverse_proxy host.docker.internal:42060 +} diff --git a/stacklets/infra/docker-compose.yml b/stacklets/infra/docker-compose.yml index b05d731e..25be9035 100644 --- a/stacklets/infra/docker-compose.yml +++ b/stacklets/infra/docker-compose.yml @@ -61,6 +61,10 @@ services: - "stack.commit=${STACK_COMMIT:-unknown}" networks: - stack + extra_hosts: + # Native services on the Mac (whisper-server, oMLX) listen on the + # host's loopback, and this is how a container reaches them. + - "host.docker.internal:host-gateway" ports: # The front door for every device on the LAN, so it binds all # interfaces. PORT_BIND_IP (127.0.0.1 in domain mode) keeps the From 0e2536b0e4bb16c08a43e9d28ab9826eaca6d664 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:10:03 +0200 Subject: [PATCH 10/15] fix(infra): serve the AdGuard admin pages at dns. The infra README pointed at `dns.` for AdGuard's admin pages, but no snippet served it. The pages are there once the setup wizard has run; before that the host answers 502. --- stacklets/infra/caddy.snippet | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 stacklets/infra/caddy.snippet diff --git a/stacklets/infra/caddy.snippet b/stacklets/infra/caddy.snippet new file mode 100644 index 00000000..22d1a2a8 --- /dev/null +++ b/stacklets/infra/caddy.snippet @@ -0,0 +1,9 @@ +# stacklets/infra/caddy.snippet +# +# AdGuard's admin pages. They listen on port 80 inside the container once +# the setup wizard has run; until then this answers 502, and the wizard +# is at http://localhost:42081 on the Mac. + +dns.{$STACK_DOMAIN} { + reverse_proxy stack-infra-adguard:80 +} From 247ec0fc62157c7d474710baa04d36adfe0097af Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:10:55 +0200 Subject: [PATCH 11/15] feat(stack): serve the admin's own sites from Caddyfile.local Services that are not stacklets, such as a dashboard or a Docker UI, had no place in domain mode. Only one proxy can own ports 80 and 443, so a hand-written Caddyfile cannot run beside the one the stack assembles. `{data_dir}/infra/Caddyfile.local` is now appended after the stacklet snippets every time the Caddyfile is written. The stack reads it and never writes it; `stack up infra` applies an edit. Its sites get the wildcard certificate like any other subdomain. A host that a running stacklet also serves makes Caddy reject the reload, with a warning. --- docs/stack-reference.md | 10 ++++++++++ lib/stack/caddy.py | 17 ++++++++++++----- lib/stack/stack.py | 10 +++++++++- tests/unit/framework/test_caddy.py | 17 +++++++++++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/docs/stack-reference.md b/docs/stack-reference.md index 470e2f0e..db659208 100644 --- a/docs/stack-reference.md +++ b/docs/stack-reference.md @@ -996,6 +996,16 @@ Around the snippets the assembler (`lib/stack/caddy.py`) emits: Snippets describe routes only. Anything that applies to every site belongs in the assembler, not in a snippet. +Sites for services that are not stacklets go in +`{data_dir}/infra/Caddyfile.local`. The admin writes it and the runtime +only reads it, appending it after the snippets every time it writes the +Caddyfile; `stack up infra` applies an edit. Only one proxy can own ports +80 and 443, so a hand-written Caddyfile cannot run beside this one. A local +site gets the wildcard certificate like any other subdomain. A host that a +running stacklet also serves makes Caddy reject the reload, and the +command prints the warning. `stack destroy infra` deletes the file with +the rest of infra's data. + ### TLS `[core] dns_provider` (`"hetzner"` or `"cloudflare"`) turns on HTTPS. The diff --git a/lib/stack/caddy.py b/lib/stack/caddy.py index e261fef6..27168325 100644 --- a/lib/stack/caddy.py +++ b/lib/stack/caddy.py @@ -34,10 +34,15 @@ RELOAD_COMMAND = ("caddy", "reload", "--config", CONFIG_IN_CONTAINER) +# The admin's own sites, next to the assembled Caddyfile, appended after +# the stacklets' snippets. +LOCAL_FILE = "Caddyfile.local" + _HEADER = """\ # Assembled by the stack CLI from the caddy.snippet of every stacklet that -# is up. It is rewritten whenever a stacklet starts or stops, so an edit -# here does not survive: change the stacklet's snippet instead. +# is up, and Caddyfile.local next to this file. It is rewritten whenever a +# stacklet starts or stops, so an edit here does not survive: put your own +# sites in Caddyfile.local, or change the stacklet's snippet. """ # Sites without a scheme listen on the HTTPS port, TLS or not. With no @@ -86,11 +91,13 @@ def _global_options(dns_provider: str) -> str: def assemble(snippets: list[tuple[str, str]], dns_provider: str = "") -> str: - """The Caddyfile for these `(stacklet_id, snippet)` pairs, in order. + """The Caddyfile for these `(source, snippet)` pairs, in order. + + A source is the stacklet a snippet came from, or `LOCAL_FILE`. Raises ValueError for a DNS provider the infra image has no plugin for. """ parts = [_HEADER, _global_options(dns_provider), _UNKNOWN_HOSTS] - for stacklet_id, snippet in snippets: - parts.append(f"# ── {stacklet_id} ──\n\n{snippet.strip()}\n") + for source, snippet in snippets: + parts.append(f"# ── {source} ──\n\n{snippet.strip()}\n") return "\n".join(parts) diff --git a/lib/stack/stack.py b/lib/stack/stack.py index 574fb214..86f7a02c 100644 --- a/lib/stack/stack.py +++ b/lib/stack/stack.py @@ -911,13 +911,21 @@ def write_caddyfile(self, starting: str = "", stopping: str = "") -> Path | None snippet = Path(s["path"]) / "caddy.snippet" if s["id"] in up and snippet.is_file(): snippets.append((s["id"], snippet.read_text())) + + # Sites for services that are not stacklets. Only one proxy can own + # ports 80 and 443, so they go in this Caddyfile too. The file is + # the admin's, read and never written. + path = self.data / caddy.STACKLET / "Caddyfile" + local = path.with_name(caddy.LOCAL_FILE) + if local.is_file(): + snippets.append((caddy.LOCAL_FILE, local.read_text())) + try: caddyfile = caddy.assemble(snippets, self._cfg("core", "dns_provider")) except ValueError as e: self.output.warn(f"Caddyfile not updated: {e}") return None - path = self.data / caddy.STACKLET / "Caddyfile" path.parent.mkdir(parents=True, exist_ok=True) # Written in place. The file is bind-mounted into the proxy # container, and a single-file bind mount follows the inode, so a diff --git a/tests/unit/framework/test_caddy.py b/tests/unit/framework/test_caddy.py index eed4633b..411204d4 100644 --- a/tests/unit/framework/test_caddy.py +++ b/tests/unit/framework/test_caddy.py @@ -285,6 +285,23 @@ def test_starting_infra_writes_the_caddyfile_its_container_mounts(self, tmp_path assert "docs.{$STACK_DOMAIN}" in _site_addresses(_caddyfile(tmp_path).read_text()) + def test_the_admins_own_routes_come_after_the_stacklets(self, tmp_path, docker): + """Services that are not stacklets (a dashboard, a Docker UI) get + their sites from `Caddyfile.local` next to the assembled file. Only + one proxy can own ports 80 and 443, so they have to live in this + one, and the file is the admin's: it is read, never written.""" + cli, _ = _make_cli(tmp_path) + docker.states = {"infra": "running"} + local = tmp_path / "data" / "infra" / "Caddyfile.local" + local.parent.mkdir(parents=True) + local.write_text("status.{$STACK_DOMAIN} {\n\treverse_proxy homepage:3000\n}\n") + + cli.up("photos") + + sites = _site_addresses(_caddyfile(tmp_path).read_text()) + assert sites.index("photos.{$STACK_DOMAIN}") < sites.index("status.{$STACK_DOMAIN}") + assert local.read_text() == "status.{$STACK_DOMAIN} {\n\treverse_proxy homepage:3000\n}\n" + def test_without_infra_up_nothing_is_written_or_reloaded(self, tmp_path, docker): """With no proxy running there is nobody to read the file, and writing it would create infra's data directory for a stacklet that From c5550af31c9c8831404a78440ebc992959127b94 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 14:12:49 +0200 Subject: [PATCH 12/15] docs(infra): describe setting up domain mode with HTTPS The docs said domain mode needs "wildcard DNS on your router". It needs two records to the Mac's LAN IP, `*.` and ``, because a wildcard covers neither the bare name's DNS nor its certificate. They belong in AdGuard's DNS rewrites, and at the DNS provider for devices that bypass AdGuard. - The infra README covers enabling domain mode, the DNS records, TLS (which token each provider needs, where it is kept, how to rotate it), `Caddyfile.local`, and the wizard on 42081. - The admin guide explains `dns_provider` and the wiki's new host. - The operator reference lists infra's ports and `stack infra dns-token`. --- docs/admin-guide.md | 5 +- docs/agent/ops.md | 6 ++- docs/stack-reference.md | 3 +- stack.example.toml | 5 +- stacklets/infra/README.md | 88 ++++++++++++++++++++++++++++---- stacklets/infra/cli/dns-token.py | 2 +- 6 files changed, 93 insertions(+), 16 deletions(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 5027286f..02092292 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -547,7 +547,7 @@ The wiki maintains itself. A curator sidecar (`stack-memory-curator`) watches th | | | |---|---| -| Wiki port | `42070` (domain mode: `wiki.`) | +| Wiki port | `42070` (domain mode: `memory.`) | | Data | `~/famstack-data/memory/vault/` (a checkout of Forgejo `family/memory`) | Useful commands: @@ -592,7 +592,8 @@ wiki_nightly = "03:30" # nightly full rebuild, local time ("" disables Key things to know: -- **domain**: leave empty to start. Services are reachable via `:`. Set a domain later for pretty URLs like `photos.home.internal` (requires wildcard DNS on your router). +- **domain**: leave empty to start. Services are reachable via `:`. Set a domain later for pretty URLs like `photos.home.example.family`, served by the infra stacklet's proxy. It needs two DNS records pointing at the Mac's LAN IP, `*.` and `` itself, because the wildcard covers neither the bare name's DNS nor its certificate. Add both as DNS rewrites in AdGuard, and at your DNS provider for devices that bypass AdGuard. Details: [stacklets/infra/README.md](../stacklets/infra/README.md). +- **dns_provider**: with a domain, `"hetzner"` or `"cloudflare"` serves everything over HTTPS, with certificates Caddy obtains through the provider's API. Empty serves plain HTTP. The API token is stored with `./stack infra dns-token`, never in `stack.toml`. - **host**: the hostname used in port-mode URLs and hints. Empty = auto-detect the LAN IP (right for households: phones need it). Set to `localhost` for a single-machine setup, or to a fixed name like `mac-mini.local` if the LAN IP keeps shifting. - **data_dir**: where all persistent data lives. Back this up. Outside the git repo. - **language**: detected from your timezone. Controls which document categories get seeded (German or English). Change it and run `./stack restart docs` to seed missing tags. diff --git a/docs/agent/ops.md b/docs/agent/ops.md index 24e074f5..902b8e96 100644 --- a/docs/agent/ops.md +++ b/docs/agent/ops.md @@ -43,6 +43,7 @@ If a precondition is missing, `./stack` prints exactly what to do. Don't improvi | `./stack ai models` | yes | no | lists installed AI models | | `./stack ai connect ` | yes | no | points `[ai]` at an AI server elsewhere, or `local`; checks it, installs nothing, restarts nothing. Warns when the address is outside the home network | | `./stack setup ai` | yes | no | re-runs AI install (model swap path) | +| `./stack infra dns-token` | yes | no | stores or replaces the DNS provider API token (HTTPS in domain mode); `./stack up infra` applies it | **Output contract:** every command returns JSON when piped or when `--json` is passed. Force human output with `--pretty`. Exit code 0 == success. @@ -72,7 +73,7 @@ Refuse without explicit, scoped human approval: | Editing `.stack/secrets.toml` | Breaks every stacklet that depends on the changed secret. | | Moving `data_dir` while stacklets are running | Bind mounts break. `stack down all` first, then move, then `stack up`. | | Changing `stack.toml [core] language` | Re-seeds Paperless taxonomy. Orphan tags require manual cleanup. | -| Switching `[core] domain` empty ↔ non-empty | Switches port mode ↔ domain mode. Requires wildcard DNS + Caddy understanding. | +| Switching `[core] domain` empty ↔ non-empty | Switches port mode ↔ domain mode. Needs `*.` and `` DNS records to the Mac, and the infra stacklet up. | ## Ports (42xxx range) @@ -88,6 +89,9 @@ Refuse without explicit, scoped human approval: | 42060 | ai | oMLX | | 42062 | ai | Whisper | | 42070 | memory | Family wiki (Quartz) | +| 42080 | infra | AdGuard admin (domain mode, Mac only) | +| 42081 | infra | AdGuard setup wizard, first run (domain mode, Mac only) | +| 53, 80, 443 | infra | DNS and the Caddy proxy (domain mode, whole LAN) | Port collisions: do not silently rebind. Surface them. The user's fix is "stop the offender" or switch to domain mode. diff --git a/docs/stack-reference.md b/docs/stack-reference.md index db659208..5f7255dd 100644 --- a/docs/stack-reference.md +++ b/docs/stack-reference.md @@ -953,7 +953,8 @@ The runtime operates in one of two modes based on `stack.toml`: - URLs are `http://photos.home.internal`, or `https://` once `dns_provider` is set (see TLS below) - The runtime assembles the Caddyfile from `caddy.snippet` files -- Requires wildcard DNS on router +- Needs two DNS records to the server's LAN IP, `*.` and + ``, because a wildcard does not cover the bare name ### Caddy Snippets diff --git a/stack.example.toml b/stack.example.toml index 2da8f24d..399e32e4 100644 --- a/stack.example.toml +++ b/stack.example.toml @@ -11,9 +11,10 @@ name = "famstack" # Leave empty for port mode (zero setup, access via hostname:port from any device). -# Set to enable pretty URLs via Caddy reverse proxy: +# Set to enable pretty URLs via Caddy reverse proxy (the infra stacklet): # domain = "home.internal" → photos.home.internal, docs.home.internal -# Requires wildcard DNS on your router: *.home.internal → server IP +# Needs two DNS records to the server's LAN IP, *.home.internal and +# home.internal itself: a wildcard does not cover the bare name. domain = "" # With a domain, HTTPS for every service: one wildcard certificate plus one diff --git a/stacklets/infra/README.md b/stacklets/infra/README.md index e9f2d401..bf1c9877 100644 --- a/stacklets/infra/README.md +++ b/stacklets/infra/README.md @@ -5,19 +5,86 @@ DNS server with ad-blocking, and a reverse proxy that gives every stacklet a pre ## What it runs - `stack-infra-adguard`: DNS server with ad-blocking. Resolves `*.{domain}` to the server IP and filters ads, trackers, and malicious domains for every device on the LAN. -- `stack-infra-caddy`: reverse proxy. Routes incoming requests by hostname to the right container, terminates TLS, and (optionally) provisions Let's Encrypt certificates if you use a real domain. +- `stack-infra-caddy`: reverse proxy. Routes incoming requests by hostname to the right container and, with a DNS provider configured, serves them over HTTPS with certificates from Let's Encrypt. ## Enable +Set a domain in `stack.toml`, plus a DNS provider if you want HTTPS: + +```toml +[core] +domain = "home.example.family" +dns_provider = "hetzner" # or "cloudflare"; empty serves plain HTTP +``` + +Then: + ```bash stack up infra ``` -`stack up` creates the data directories, renders the Caddyfile from your `stack.toml`, starts both containers, and prints next-step hints. AdGuard runs its first-run wizard on port 3000. +`stack up infra` builds the Caddy image with the DNS provider plugins (a minute or two the first time). If HTTPS is on and no token is stored yet, it asks for the provider's API token (see [TLS](#tls)). Then it writes the Caddyfile and starts both containers. + +From then on the Caddyfile follows the stacklets. Every `stack up`, `stack down` and `stack destroy` rewrites `~/famstack-data/infra/Caddyfile` from the `caddy.snippet` of each stacklet that is up, and reloads Caddy. Each stacklet is served at `.` (`photos.home.example.family`), and core's home and `/go` links at the bare domain. A name no running stacklet claims answers 404. + +### DNS records + +Every device has to resolve the bare domain and every name under it to the Mac. That takes two records, because a wildcard does not cover the bare name: + +| Name | Type | Value | +|---|---|---| +| `*.home.example.family` | A | `192.0.2.10` (the Mac's LAN IP) | +| `home.example.family` | A | `192.0.2.10` | + +Add both as DNS rewrites in AdGuard (Filters > DNS rewrites). That is what the LAN asks once the router points at AdGuard. Add them at the DNS provider too, for devices that bypass AdGuard, such as a browser with its own secure DNS or iCloud Private Relay. Some routers drop public DNS answers that point at a private address (DNS rebind protection); the AdGuard rewrites are not affected. + +Give the Mac a fixed LAN IP with a DHCP reservation on the router, or every record goes stale when it changes. + +## TLS + +With `dns_provider` set, Caddy serves every service over HTTPS with two certificates from Let's Encrypt: a wildcard for `*.`, which every stacklet's subdomain uses, and one for the bare domain. It proves you own the domain by writing a TXT record through the provider's API (a DNS-01 challenge), so nothing on your LAN has to be reachable from the internet. The domain has to be registered, with its DNS zone hosted at that provider. + +### Provider and token + +| `dns_provider` | Token | +|---|---| +| `hetzner` | A Hetzner Cloud API token with Read & Write access, created in the Hetzner Console project that holds the zone (Security > API tokens). A token from the old DNS Console at dns.hetzner.com does not work. | +| `cloudflare` | A Cloudflare API token with `Zone.Zone:Read` and `Zone.DNS:Edit`, limited to this zone. | + +### Where the token lives + +In `.stack/secrets.toml` as `infra__DNS_API_TOKEN`, never in `stack.toml`. It reaches the Caddy container as the environment variable `DNS_API_TOKEN` and does not appear in the Caddyfile. Nothing prints it; `stack infra dns-token` reports its length only. + +The first `stack up infra` asks for it in a terminal. Without a token, or with a provider the image has no plugin for, `stack up infra` stops and says what to do. + +### Rotating it + +Create the new token at the provider, then: + +```bash +stack infra dns-token # paste the new token; input is hidden +stack up infra # restarts Caddy with it +``` + +Revoke the old token at the provider afterwards. Certificates Caddy already holds stay valid; the token is only used when a certificate is issued or renewed. Switching provider works the same way: change `dns_provider`, store the new provider's token, `stack up infra`. + +Certificates and the ACME account live in `~/famstack-data/infra/caddy/`. Keep that directory. Without it Caddy requests every certificate again, and Let's Encrypt limits how many it issues per domain per week. + +## Your own sites + +Services that are not stacklets (a dashboard, a Docker UI) go in `~/famstack-data/infra/Caddyfile.local`. Only one proxy can own ports 80 and 443, so they cannot keep a Caddyfile of their own. The file is yours: the stack appends it after the stacklets' sites and never writes it. + +``` +status.{$STACK_DOMAIN} { + reverse_proxy homepage:3000 +} +``` + +Backends are container names on the `stack` network, or `host.docker.internal:` for something running on the Mac itself. Run `stack up infra` after an edit. A name that a running stacklet also serves makes Caddy refuse the new config and keep the old one, and the command prints a warning. ## First run -Open `http://:3000` and walk through the AdGuard wizard. +AdGuard's setup wizard listens on the Mac only. Open `http://localhost:42081` there, or from another computer tunnel to it first with `ssh -L 42081:localhost:42081 you@` and open the same address. Walk through the wizard. 1. **Admin user**: pick any username and password. You will only use this when changing AdGuard settings, not for daily browsing. 2. **Network interfaces**: accept the defaults (`0.0.0.0` for both web and DNS, port 53). @@ -126,9 +193,11 @@ docker logs -f stack-infra-adguard | grep -iE "error|timeout" ## Access -- Setup wizard (first run only): `http://:3000` -- AdGuard admin (after setup): `http://:42080` or `http://dns.{domain}` -- Every other stacklet: `http://.{domain}` +- Setup wizard (first run only): `http://localhost:42081` on the Mac +- AdGuard admin (after setup): `https://dns.`, or `http://localhost:42080` on the Mac +- Every other stacklet: `https://.` + +Without a DNS provider, every address is `http://` instead. ## Point your router @@ -142,8 +211,9 @@ Stored under `${ADGUARD_DATA_DIR}` and `${CADDY_DATA_DIR}` (defaults to `~/famst - `adguard/work/`: query logs and stats. Volatile, fine to wipe. - `adguard/conf/AdGuardHome.yaml`: DNS configuration, filter lists, custom rules. Back this up. -- `Caddyfile`: reverse-proxy routes. Regenerated from `stack.toml` on every `stack up`. -- `caddy/`: TLS certificates and ACME state. Back this up if you use a real domain. +- `Caddyfile`: reverse-proxy routes, rewritten from the stacklets' snippets whenever a stacklet starts or stops. Do not edit it. +- `Caddyfile.local`: your own sites, if you have any. Back this up. +- `caddy/`: TLS certificates and ACME state. Back this up if you use HTTPS. ## Updating @@ -159,4 +229,4 @@ stack restart infra stack destroy infra ``` -This stops both containers, removes state, and deletes everything under `~/famstack-data/infra/`. Your router's DNS server setting does not change automatically; update it back to whatever it was before, or to your ISP's DNS, to keep the LAN online. +This stops both containers, removes state, and deletes everything under `~/famstack-data/infra/`, including certificates and `Caddyfile.local`. The stored DNS API token is removed too; revoke it at the provider. Your router's DNS server setting does not change automatically; update it back to whatever it was before, or to your ISP's DNS, to keep the LAN online. diff --git a/stacklets/infra/cli/dns-token.py b/stacklets/infra/cli/dns-token.py index 38e3192f..4c8127a7 100644 --- a/stacklets/infra/cli/dns-token.py +++ b/stacklets/infra/cli/dns-token.py @@ -1,5 +1,5 @@ """ -stack infra dns-token — store the API token of the DNS provider. +stack infra dns-token: store the API token of the DNS provider. With `[core] dns_provider` set, Caddy obtains its certificates by writing a TXT record through the provider's API, and needs a token for it. The From 844971a62491a6f353b1c8d9765480c6ab082ca5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 15:21:31 +0200 Subject: [PATCH 13/15] chore(infra): mark the infra stacklet as beta Domain mode now assembles its routes and can serve HTTPS, but HTTPS has not yet run against a real DNS provider, and how TLS and the admin's own sites are configured can still change. `stack list` now marks infra as beta and `stack up infra` says so, with a reminder to back up first. --- stacklets/infra/README.md | 2 ++ stacklets/infra/stacklet.toml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/stacklets/infra/README.md b/stacklets/infra/README.md index bf1c9877..2ee7ef63 100644 --- a/stacklets/infra/README.md +++ b/stacklets/infra/README.md @@ -2,6 +2,8 @@ DNS server with ad-blocking, and a reverse proxy that gives every stacklet a pretty URL. This is what makes `photos.home.internal` resolve to your Mac and serve the right service. +**Beta.** It works, but how HTTPS and your own sites are configured can still change between releases. Back up `~/famstack-data/infra/` before you upgrade. + ## What it runs - `stack-infra-adguard`: DNS server with ad-blocking. Resolves `*.{domain}` to the server IP and filters ads, trackers, and malicious domains for every device on the LAN. diff --git a/stacklets/infra/stacklet.toml b/stacklets/infra/stacklet.toml index f227f0f2..099c1b90 100644 --- a/stacklets/infra/stacklet.toml +++ b/stacklets/infra/stacklet.toml @@ -13,6 +13,9 @@ description = "DNS + reverse proxy for domain mode" version = "0.2.1" category = "infrastructure" always_on = true +# Beta until HTTPS has run on a real instance: how TLS and the admin's own +# sites are configured can still change between releases. +stage = "beta" # Caddy is built locally with the DNS provider plugins (Dockerfile). build = true From 0dd264e7799e9933b7eda9093016ac2b09890898 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 15:39:47 +0200 Subject: [PATCH 14/15] feat(memory): redirect wiki. to the family wiki The family wiki is served at `memory.`, the address the stack hands out for the memory stacklet, but `wiki.` is the name people type. It now redirects to the same page on the memory host, path and query included. The redirect is temporary (302). Browsers keep a permanent one indefinitely, which would get in the way if the family-facing name changes again. --- docs/admin-guide.md | 2 +- stacklets/memory/caddy.snippet | 8 ++++++++ tests/integration/test_caddy.py | 27 +++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 02092292..97403419 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -547,7 +547,7 @@ The wiki maintains itself. A curator sidecar (`stack-memory-curator`) watches th | | | |---|---| -| Wiki port | `42070` (domain mode: `memory.`) | +| Wiki port | `42070` (domain mode: `memory.`; `wiki.` redirects there) | | Data | `~/famstack-data/memory/vault/` (a checkout of Forgejo `family/memory`) | Useful commands: diff --git a/stacklets/memory/caddy.snippet b/stacklets/memory/caddy.snippet index f4ee7cff..42844c36 100644 --- a/stacklets/memory/caddy.snippet +++ b/stacklets/memory/caddy.snippet @@ -13,3 +13,11 @@ memory.{$STACK_DOMAIN} { reverse_proxy stack-memory-wiki:8080 } + +# wiki.{$STACK_DOMAIN} is what people type for the family wiki. It sends +# the browser to the same page here. The redirect is temporary (302): a +# permanent one is kept by browsers for good, which would get in the way +# if the names change again. +wiki.{$STACK_DOMAIN} { + redir {scheme}://memory.{$STACK_DOMAIN}{uri} +} diff --git a/tests/integration/test_caddy.py b/tests/integration/test_caddy.py index 5441ceec..4e0de371 100644 --- a/tests/integration/test_caddy.py +++ b/tests/integration/test_caddy.py @@ -110,6 +110,33 @@ def test_without_tls_every_site_is_plain_http_on_port_80(plain_caddyfile): assert "tls" not in config.get("apps", {}) +def test_wiki_sends_the_browser_to_the_same_page_on_the_memory_host(plain_caddyfile): + """The family wiki is served at memory., and wiki. is + what people type. The redirect is temporary: browsers keep a permanent + one for good, which would get in the way if the names change again.""" + name = "stack-test-caddy-wiki" + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) + subprocess.run( + ["docker", "run", "-d", "--name", name, "-e", f"STACK_DOMAIN={DOMAIN}", + "-v", f"{plain_caddyfile}:/etc/caddy/Caddyfile:ro", STOCK_CADDY], + check=True, capture_output=True, timeout=60, + ) + try: + time.sleep(1) + # busybox wget follows the redirect and fails to resolve the target; + # -S has printed the first response's headers by then. + headers = subprocess.run( + ["docker", "exec", name, "wget", "-S", "-O", "/dev/null", + "--header", f"Host: wiki.{DOMAIN}", "http://127.0.0.1/family/camping?view=todo"], + capture_output=True, text=True, timeout=30, + ).stderr + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) + + assert "HTTP/1.1 302" in headers + assert f"Location: http://memory.{DOMAIN}/family/camping?view=todo" in headers + + # ── The infra image: TLS through each DNS provider ──────────────────────── INFRA_IMAGE = "stack-infra-caddy:local" From dbb5b8daa034ee729f69c2b00f74bcc8a6b22f64 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 15:57:00 +0200 Subject: [PATCH 15/15] feat(infra): warn that infra is DNS for the whole network Pointing the router's DNS at the Mac makes every device in the house depend on it, and setting domain mode up takes networking knowledge no other stacklet asks for. Nothing said so. - The first `stack up infra` now warns, once, before anything is set up: what depends on it, what knowledge it takes, and to note the router's current DNS setting first. - The infra README and the admin guide say the same up front, and suggest port mode to anyone for whom this is new. - The operator reference lists changing the router's DNS, and stopping or destroying infra, as danger zones for agents. --- docs/admin-guide.md | 2 +- docs/agent/ops.md | 2 + stacklets/infra/README.md | 2 + stacklets/infra/hooks/on_configure.py | 23 ++++++++-- tests/unit/stacklets/test_infra_first_run.py | 44 ++++++++++++++++++++ 5 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 tests/unit/stacklets/test_infra_first_run.py diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 97403419..7dd65e40 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -592,7 +592,7 @@ wiki_nightly = "03:30" # nightly full rebuild, local time ("" disables Key things to know: -- **domain**: leave empty to start. Services are reachable via `:`. Set a domain later for pretty URLs like `photos.home.example.family`, served by the infra stacklet's proxy. It needs two DNS records pointing at the Mac's LAN IP, `*.` and `` itself, because the wildcard covers neither the bare name's DNS nor its certificate. Add both as DNS rewrites in AdGuard, and at your DNS provider for devices that bypass AdGuard. Details: [stacklets/infra/README.md](../stacklets/infra/README.md). +- **domain**: leave empty to start. Services are reachable via `:`. Set a domain later for pretty URLs like `photos.home.example.family`, served by the infra stacklet's proxy. It needs two DNS records pointing at the Mac's LAN IP, `*.` and `` itself, because the wildcard covers neither the bare name's DNS nor its certificate. Add both as DNS rewrites in AdGuard, and at your DNS provider for devices that bypass AdGuard. Domain mode makes the Mac the DNS server of your whole network and takes some networking knowledge to set up; read [stacklets/infra/README.md](../stacklets/infra/README.md) first. - **dns_provider**: with a domain, `"hetzner"` or `"cloudflare"` serves everything over HTTPS, with certificates Caddy obtains through the provider's API. Empty serves plain HTTP. The API token is stored with `./stack infra dns-token`, never in `stack.toml`. - **host**: the hostname used in port-mode URLs and hints. Empty = auto-detect the LAN IP (right for households: phones need it). Set to `localhost` for a single-machine setup, or to a fixed name like `mac-mini.local` if the LAN IP keeps shifting. - **data_dir**: where all persistent data lives. Back this up. Outside the git repo. diff --git a/docs/agent/ops.md b/docs/agent/ops.md index 902b8e96..45204451 100644 --- a/docs/agent/ops.md +++ b/docs/agent/ops.md @@ -74,6 +74,8 @@ Refuse without explicit, scoped human approval: | Moving `data_dir` while stacklets are running | Bind mounts break. `stack down all` first, then move, then `stack up`. | | Changing `stack.toml [core] language` | Re-seeds Paperless taxonomy. Orphan tags require manual cleanup. | | Switching `[core] domain` empty ↔ non-empty | Switches port mode ↔ domain mode. Needs `*.` and `` DNS records to the Mac, and the infra stacklet up. | +| Pointing the router's DNS at the Mac (infra) | Every device on the LAN then resolves through AdGuard; the Mac or AdGuard going down takes name resolution for the whole house with it. The admin changes the router, never an agent; note the previous DNS setting first. | +| `./stack down infra` / `./stack destroy infra` | With the router pointing at the Mac, the house loses DNS until infra is back or the router is switched back. | ## Ports (42xxx range) diff --git a/stacklets/infra/README.md b/stacklets/infra/README.md index 2ee7ef63..4b64f2b0 100644 --- a/stacklets/infra/README.md +++ b/stacklets/infra/README.md @@ -4,6 +4,8 @@ DNS server with ad-blocking, and a reverse proxy that gives every stacklet a pre **Beta.** It works, but how HTTPS and your own sites are configured can still change between releases. Back up `~/famstack-data/infra/` before you upgrade. +**Sensitive infrastructure.** This stacklet is DNS and the entry point for your whole home network. Once your router hands out the Mac as its DNS server, every device depends on it: when the Mac is off, asleep or rebooting, or AdGuard stops, nothing on the network resolves names, not even sites on the internet. Setting it up takes some networking knowledge: your router's DHCP and DNS settings, DNS records, and a domain at a DNS provider if you want HTTPS. Note your router's current DNS setting before you change it, so you can go back. If this is new territory, stay in port mode, which needs none of it. + ## What it runs - `stack-infra-adguard`: DNS server with ad-blocking. Resolves `*.{domain}` to the server IP and filters ads, trackers, and malicious domains for every device on the LAN. diff --git a/stacklets/infra/hooks/on_configure.py b/stacklets/infra/hooks/on_configure.py index 4b020752..892bb722 100644 --- a/stacklets/infra/hooks/on_configure.py +++ b/stacklets/infra/hooks/on_configure.py @@ -1,13 +1,30 @@ -"""First run: ask for the DNS provider's API token when HTTPS is configured. +"""First run: say what infra takes on, then ask for the DNS token if needed. -Only in a terminal. Anywhere else on_start stops the start and names the -command that stores the token, which is also how it is replaced later. +Infra is DNS and the entry point for the whole home network, which no +other stacklet is, so its first `stack up` says so once, through the +same warning channel as a stacklet's stage. + +The DNS provider's API token is asked for only in a terminal. Anywhere +else on_start stops the start and names the command that stores the +token, which is also how it is replaced later. """ import sys +NETWORK_WARNING = ( + "Infrastructure runs DNS and the entry point for your whole network. " + "Once your router hands out this Mac as its DNS server, every device " + "depends on it: when the Mac or AdGuard is down, nothing on the network " + "resolves names. Setting it up takes some networking knowledge (your " + "router's DHCP and DNS settings, DNS records, a domain at a DNS provider " + "for HTTPS). Read stacklets/infra/README.md first, and note your router's " + "current DNS setting so you can go back to it." +) + def run(ctx): + ctx.warn(NETWORK_WARNING) + if not ctx.stack._cfg("core", "dns_provider") or ctx.secret("DNS_API_TOKEN"): return if not sys.stdin.isatty(): diff --git a/tests/unit/stacklets/test_infra_first_run.py b/tests/unit/stacklets/test_infra_first_run.py new file mode 100644 index 00000000..59c96f83 --- /dev/null +++ b/tests/unit/stacklets/test_infra_first_run.py @@ -0,0 +1,44 @@ +"""The first `stack up infra` says what the admin is taking on. + +Infra is DNS and the entry point for the whole home network. Once the +router hands out the Mac as its DNS server, every device depends on it, +and setting it up takes networking knowledge most stacklets do not ask +for. That is said once, when the stacklet is first set up, through the +same warning channel the framework uses for a stacklet's stage. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent.parent +sys.path.insert(0, str(REPO / "lib")) + +from stack import Stack # noqa: E402 +from stack.hooks import StackContext # noqa: E402 +from stack.output import CollectorOutput # noqa: E402 + + +def _configure(tmp_path) -> CollectorOutput: + (tmp_path / "stack.toml").write_text( + '[core]\ndomain = "home.example.family"\n' + f'extension_dirs = ["{tmp_path / "ext"}"]\n') + output = CollectorOutput() + stck = Stack(root=REPO, data=tmp_path / "data", instance_dir=tmp_path, output=output) + + path = REPO / "stacklets" / "infra" / "hooks" / "on_configure.py" + spec = importlib.util.spec_from_file_location("infra_on_configure", path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.run(StackContext(stck, "infra", {})) + return output + + +def test_setting_up_infra_warns_that_the_whole_network_depends_on_it(tmp_path): + warnings = " ".join(_configure(tmp_path).warnings) + assert "every device" in warnings + assert "networking knowledge" in warnings + assert "stacklets/infra/README.md" in warnings