fix: repair seven lifecycle and domain mode bugs

docs/stack-reference.md

@@ -203,6 +203,7 @@ Available template variables:
 | `{ai_tts_voice}` | Derived from `[ai].language` |
 | `{messages_server_name}` | `stack.toml` → `[messages].server_name` |
 | `{url}`, `{<id>_url}`, `{home_url}` | Public URLs: `http://<ip>:<port>` in port mode; `https://<id>.<domain>` in domain mode once `[core].dns_provider` is set, `http://` before that |
+| `{url_host}` | The host of `{url}`: `<id>.<domain>` in domain mode, `{ip}` in port mode. For a protocol the stacklet serves under the same name besides HTTP, such as Forgejo's SSH |
 | `{<id>__<NAME>}` | A secret from `secrets.toml`, e.g. `{docs__API_TOKEN}`, `{infra__DNS_API_TOKEN}` |
 
 ### Hints
@@ -822,6 +823,10 @@ The `PORT_BIND_IP` variable controls access scope. The runtime sets it:
 - Port mode: `0.0.0.0` (reachable from the LAN)
 - Domain mode: `127.0.0.1` (only Caddy reaches it)
 
+A port the LAN needs in domain mode too is bound to `0.0.0.0` literally
+instead: infra's DNS (53) and proxy (80, 443), and Forgejo's SSH, which
+does not go through Caddy and is encrypted and authenticated by itself.
+
 The host port is the stacklet's declared `port` from `stacklet.toml`.
 The container port is whatever the upstream service uses internally.
 

lib/stack/cli.py

@@ -639,6 +639,13 @@ def print_error(result: dict) -> None:
     """Print an error with stack colors."""
     error = result.get("error", "unknown error")
     print(f"\n  {RED}✗{RESET}  {error}\n", file=sys.stderr)
+    # The tool's own output says why (compose names the port, the mount,
+    # the image). Its last lines carry the error; the rest is progress.
+    output = (result.get("output") or "").strip()
+    if output and output != str(error).strip():
+        for line in [ln for ln in output.splitlines() if ln.strip()][-5:]:
+            print(f"      {DIM}{line.strip()}{RESET}", file=sys.stderr)
+        print(file=sys.stderr)
     for p in result.get("problems", result.get("dependents", [])):
         print(f"      {p}", file=sys.stderr)
     if result.get("hint"):
@@ -978,7 +985,10 @@ def _destroy_one(stck, args):
     name = stacklet.get("name", args.stacklet)
     data_path = stck.data / args.stacklet
 
-    if not stck.is_installed(args.stacklet):
+    # Not installed and no containers: only data a failed setup left behind.
+    # A first `up` that failed after starting containers leaves those too,
+    # holding their names and ports, so that case takes the full path.
+    if not stck.is_installed(args.stacklet) and args.stacklet not in docker.all_project_ids():
         if data_path.exists():
             import shutil
             shutil.rmtree(data_path)

lib/stack/docker.py

@@ -101,12 +101,24 @@ def compose_pull(compose_file: str | Path, env: dict | None = None) -> None:
 
 
 def compose_build(compose_file: str | Path, env: dict | None = None) -> None:
-    """Build images for a compose file. Streams output."""
+    """Build images for a compose file, on current base images. Streams output.
+
+    Without `--pull` the build cache keeps the base image it first pulled
+    for good, and Watchtower does not update built images, so the base
+    would never be patched. With it, a new base means a rebuild and an
+    unchanged one costs a registry lookup. Offline the pull fails, and
+    the build runs again from what is cached so the stacklet still starts.
+    """
     full_env = {**__import__("os").environ, **(env or {})}
-    _docker(
-        "compose", "-f", str(compose_file), "build",
+    pulled = _docker(
+        "compose", "-f", str(compose_file), "build", "--pull",
         timeout=600, env=full_env,
     )
+    if pulled.returncode != 0:
+        _docker(
+            "compose", "-f", str(compose_file), "build",
+            timeout=600, env=full_env,
+        )
 
 
 def exec_in(container: str, *cmd: str) -> tuple[int, str]:

lib/stack/stack.py

@@ -492,14 +492,27 @@ def _public_url(self, stacklet_id: str, port: int) -> str:
             return f"{self._domain_scheme()}://{stacklet_id}.{domain}"
         return f"http://{self._lan_ip()}:{port}"
 
+    def _own_template_vars(self, stacklet_id: str, port: int) -> dict:
+        """The template variables whose value depends on the stacklet.
+
+        `{url_host}` is the host of `{url}`: `<id>.<domain>` in domain
+        mode, the address `{ip}` names in port mode. It is for a stacklet
+        that serves another protocol under the same name, such as SSH.
+        """
+        from urllib.parse import urlsplit
+        url = self._public_url(stacklet_id, port)
+        return {"url": url, "url_host": urlsplit(url).hostname or "", "ip": self._lan_ip()}
+
     def _domain_scheme(self) -> str:
         """https once the proxy serves certificates, http until then.
 
         The proxy has certificates when `[core] dns_provider` is set. Set
         `[core] https = true` instead when some other reverse proxy in
-        front of the stack terminates TLS.
+        front of the stack terminates TLS. `_cfg` returns the TOML boolean
+        as the string "True" or "False", and "False" is truthy, so the
+        value is compared rather than tested.
         """
-        if self._cfg("core", "dns_provider") or self._cfg("core", "https"):
+        if self._cfg("core", "dns_provider") or self._cfg("core", "https").lower() == "true":
             return "https"
         return "http"
 
@@ -547,8 +560,7 @@ def env(self, stacklet_id: str) -> dict:
 
         template_vars = self._build_template_vars()
         template_vars["stacklet_id"] = stacklet_id
-        template_vars["url"] = self._public_url(stacklet_id, s.get("port", 0))
-        template_vars["ip"] = self._lan_ip()
+        template_vars.update(self._own_template_vars(stacklet_id, s.get("port", 0)))
 
         # Render templates — warn on missing vars (typos cause silent failures)
         import re
@@ -1173,8 +1185,7 @@ def _step(msg):
 
         # Render manifest hints with template vars so credentials are visible
         template_vars = self._build_template_vars()
-        template_vars["url"] = self._public_url(stacklet_id, s.get("port", 0))
-        template_vars["ip"] = self._lan_ip()
+        template_vars.update(self._own_template_vars(stacklet_id, s.get("port", 0)))
         raw_hints = manifest.get("hints", [])
         hints = []
         for h in raw_hints:

stacklets/code/docker-compose.yml

@@ -1,7 +1,7 @@
 # Forgejo — lightweight self-hosted Git server
 #
-# Web UI:  http://<ip>:42040
-# SSH:     ssh://git@<ip>:222/user/repo.git
+# Web UI:  http://<ip>:42040, or https://code.<domain> in domain mode
+# SSH:     ssh://git@<ip>:222/user/repo.git, or code.<domain> in place of <ip>
 #
 # First user to register becomes admin.
 # All data stored in CODE_DATA_DIR (~/famstack-data/code/).
@@ -20,7 +20,10 @@ services:
       - stack
     ports:
       - "${PORT_BIND_IP:-0.0.0.0}:42040:3000"
-      - "${PORT_BIND_IP:-0.0.0.0}:${CODE_SSH_PORT:-222}:22"
+      # SSH does not go through the proxy, so it faces the LAN in both
+      # modes. It is encrypted and authenticated by itself, which is what
+      # keeping plain-HTTP services on the loopback in domain mode is for.
+      - "0.0.0.0:${CODE_SSH_PORT:-222}:22"
     volumes:
       - ${CODE_DATA_DIR}:/data
       - /etc/localtime:/etc/localtime:ro

stacklets/code/stacklet.toml

@@ -18,7 +18,7 @@ login_field = "username"
 
 hints = [
     "Open {url}",
-    "SSH clone: ssh://git@{ip}:222/user/repo.git",
+    "SSH clone: ssh://git@{url_host}:222/user/repo.git",
 ]
 
 [upstream]
@@ -34,7 +34,9 @@ admin_email    = "{admin_email}"
 admin_password = "{admin_password}"
 FORGEJO__server__ROOT_URL              = "{code_url}/"
 FORGEJO__server__DOMAIN                = "{ip}"
-FORGEJO__server__SSH_DOMAIN            = "{ip}"
+# The host in the SSH clone URLs Forgejo shows: the web UI's host
+# (code.<domain>, or the Mac's address in port mode).
+FORGEJO__server__SSH_DOMAIN            = "{url_host}"
 FORGEJO__server__SSH_PORT              = "222"
 FORGEJO__database__DB_TYPE             = "sqlite3"
 FORGEJO__security__INSTALL_LOCK        = "true"

stacklets/infra/stacklet.toml

@@ -19,18 +19,18 @@ stage       = "beta"
 # Caddy is built locally with the DNS provider plugins (Dockerfile).
 build       = true
 
+hints = [
+    "Point your router's DNS to {ip}",
+    "Services are now available at *.{domain}",
+]
+
 [ports]
 dns         = 53
 dns_setup   = 42081
 dns_admin   = 42080
 http        = 80
 https       = 443
 
-hints = [
-    "Point your router's DNS to {ip}",
-    "Services are now available at *.{domain}",
-]
-
 [env.defaults]
 TZ                  = "{timezone}"
 STACK_DOMAIN        = "{domain}"

stacklets/messages/hooks/on_start.py

@@ -1,15 +1,23 @@
-"""Keep the rate limits famstack relaxes in Synapse's config.
-
-Synapse's defaults are sized for a public homeserver with strangers on
-it. On a family server the ones that bite are the per-account limits:
-an account may receive five invites in a burst and then one every 333
-seconds, so a bot that sets up a sixth room for the same person waits
-five and a half minutes before the invite goes through.
-
-`homeserver.yaml` is generated once, at install, and never again, so a
-limit added to the installer would reach new servers only. This hook
-runs on every start and adds whichever of these keys the file lacks.
-A key already present is the admin's and is left exactly as it is.
+"""Keep the config files the installer writes once in step with the stack.
+
+`homeserver.yaml` and `element-config.json` are generated at install and
+never again, so a change made later reaches neither. This hook runs on
+every start and brings two things up to date.
+
+Rate limits. Synapse's defaults are sized for a public homeserver with
+strangers on it. On a family server the ones that bite are the
+per-account limits: an account may receive five invites in a burst and
+then one every 333 seconds, so a bot that sets up a sixth room for the
+same person waits five and a half minutes before the invite goes
+through. Whichever of the relaxed keys the file lacks is added. A key
+already present is the admin's and is left exactly as it is.
+
+Element's homeserver. Element finds Synapse through `base_url`, which is
+the URL the stack hands out for Synapse and changes with it: a move to
+domain mode, or HTTPS turned on. An Element served over https that still
+calls an http homeserver is blocked by the browser. `base_url` follows
+the rendered `SYNAPSE_PUBLIC_URL`; the rest of the file stays the
+admin's.
 """
 
 from __future__ import annotations
@@ -34,21 +42,46 @@
 
 
 def run(ctx):
-    conf = Path(ctx.stack.data) / "messages" / "synapse" / "homeserver.yaml"
+    synapse_dir = Path(ctx.stack.data) / "messages" / "synapse"
+    notes = [note for note in (
+        _add_rate_limits(synapse_dir / "homeserver.yaml"),
+        _follow_homeserver_url(synapse_dir / "element-config.json",
+                               ctx.env.get("SYNAPSE_PUBLIC_URL", "")),
+    ) if note]
+    return {"ok": True, **({"message": "; ".join(notes)} if notes else {})}
+
+
+def _add_rate_limits(conf: Path) -> str:
     if not conf.exists():
-        return {"ok": True}
+        return ""
 
     # The installer writes JSON, which Synapse reads as YAML. A file an
     # admin rewrote as YAML cannot be parsed without a YAML library, and
     # it is theirs to maintain, so it is left as it is.
     try:
         config = json.loads(conf.read_text())
     except ValueError:
-        return {"ok": True, "message": f"{conf} is not JSON; rate limits left as they are"}
+        return f"{conf} is not JSON; rate limits left as they are"
 
     missing = {k: v for k, v in RATE_LIMITS.items() if k not in config}
     if not missing:
-        return {"ok": True}
+        return ""
 
     conf.write_text(json.dumps({**config, **missing}, indent=2))
-    return {"ok": True, "message": f"added Synapse rate limits: {', '.join(missing)}"}
+    return f"added Synapse rate limits: {', '.join(missing)}"
+
+
+def _follow_homeserver_url(conf: Path, synapse_url: str) -> str:
+    if not synapse_url or not conf.exists():
+        return ""
+    try:
+        config = json.loads(conf.read_text())
+    except ValueError:
+        return f"{conf} is not JSON; Element's homeserver left as it is"
+
+    homeserver = config.setdefault("default_server_config", {}).setdefault("m.homeserver", {})
+    if homeserver.get("base_url") == synapse_url:
+        return ""
+    homeserver["base_url"] = synapse_url
+    conf.write_text(json.dumps(config, indent=2))
+    return f"Element now connects to {synapse_url}"

stacklets/messages/stacklet.toml

@@ -29,7 +29,7 @@ login_field = "username"
 
 hints = [
     "Open {url} in your browser",
-    "On phones: install Element, use http://{ip}:42031 as the server",
+    "On phones: install Element, use {messages_synapse_url} as the server",
 ]
 
 # Secondary ports — the main 'port' is Element (web client), but Synapse

tests/unit/framework/test_caddy.py

@@ -156,6 +156,16 @@ def test_without_one_they_stay_http(self, tmp_path):
         assert env["URL"] == "http://photos.home.example.family"
         assert env["HOME"] == "http://home.example.family"
 
+    def test_https_true_is_for_a_proxy_in_front_of_the_stack(self, tmp_path):
+        stck = _stack(tmp_path, 'domain = "home.example.family"\nhttps = true',
+                      {"photos": self.PHOTOS})
+        assert stck.env("photos")["URL"] == "https://photos.home.example.family"
+
+    def test_https_false_means_http(self, tmp_path):
+        stck = _stack(tmp_path, 'domain = "home.example.family"\nhttps = false',
+                      {"photos": self.PHOTOS})
+        assert stck.env("photos")["URL"] == "http://photos.home.example.family"
+
 
 # ── Lifecycle ─────────────────────────────────────────────────────────────
 

tests/unit/framework/test_cli_class.py

@@ -370,6 +370,71 @@ def test_destroy_with_an_unknown_id_destroys_nothing(self, tmp_path, monkeypatch
         assert cli.stack.is_installed("a")
 
 
+class TestUpFailureNamesTheCause:
+    """When compose cannot start a container, the reason is in its output
+    (a port another process holds, a missing mount), and the admin
+    cannot act on "Failed to start services" alone."""
+
+    def test_composes_error_is_shown(self, tmp_path, monkeypatch, capsys):
+        from argparse import Namespace
+        from stack.cli import handle_up
+        cli, _ = _make_cli(tmp_path, {"myapp": {}})
+        (tmp_path / "stacklets" / "myapp" / "docker-compose.yml").write_text(
+            "name: stack-myapp\nservices: {}\n")
+        monkeypatch.setattr("stack.docker.compose_pull", lambda *a, **kw: None)
+        monkeypatch.setattr("stack.docker.compose_up", lambda *a, **kw: (1, (
+            " Container stack-myapp Starting\n"
+            "Error response from daemon: Bind for 0.0.0.0:42080 failed: "
+            "port is already allocated\n")))
+
+        with pytest.raises(SystemExit):
+            handle_up(cli.stack, Namespace(stacklet=["myapp"], no_voice=False))
+
+        assert "Bind for 0.0.0.0:42080 failed: port is already allocated" in capsys.readouterr().err
+
+
+class TestDestroyAfterAFailedFirstUp:
+    """A first `stack up` can fail after compose has started containers,
+    on a port conflict for instance. The stacklet is then not installed
+    (no setup-done marker) but has containers, which keep their names and
+    ports until something removes them. `stack destroy` is that something."""
+
+    def _stacklet_with_compose(self, tmp_path):
+        cli, _ = _make_cli(tmp_path, {"myapp": {}})
+        (tmp_path / "stacklets" / "myapp" / "docker-compose.yml").write_text(
+            "name: stack-myapp\nservices: {}\n")
+        (tmp_path / "data" / "myapp").mkdir(parents=True)
+        return cli
+
+    def test_its_containers_are_removed(self, tmp_path, monkeypatch):
+        from argparse import Namespace
+        from stack.cli import handle_destroy
+        cli = self._stacklet_with_compose(tmp_path)
+        monkeypatch.setattr("stack.docker.project_states", lambda: {"myapp": "failing"})
+        removed = []
+        monkeypatch.setattr("stack.docker.compose_down",
+                            lambda f: (removed.append(Path(f).parent.name), (0, ""))[1])
+
+        handle_destroy(cli.stack, Namespace(stacklet=["myapp"], yes=True))
+
+        assert removed == ["myapp"]
+        assert not (tmp_path / "data" / "myapp").exists()
+
+    def test_without_containers_only_the_leftover_data_goes(self, tmp_path, monkeypatch):
+        from argparse import Namespace
+        from stack.cli import handle_destroy
+        cli = self._stacklet_with_compose(tmp_path)
+        monkeypatch.setattr("stack.docker.project_states", lambda: {})
+        removed = []
+        monkeypatch.setattr("stack.docker.compose_down",
+                            lambda f: (removed.append(f), (0, ""))[1])
+
+        handle_destroy(cli.stack, Namespace(stacklet=["myapp"], yes=True))
+
+        assert removed == []
+        assert not (tmp_path / "data" / "myapp").exists()
+
+
 class TestCLIDestroy:
     """CLI.destroy() runs Docker compose down then Stack.destroy()."""