From 211ee9ca194ccbca0882276bf987d5defc897042 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 22 Sep 2026 17:15:25 +0200 Subject: [PATCH] fix: pin container images to a minor line or an exact version Seven images followed a floating tag (`latest`, `main`, `release` or none), so Watchtower's nightly pull could move a service to a new minor or major version, or to an Open WebUI development build, without a release deciding it. That is how Paperless went from 2.x to 3.0. Each image now names the minor line where upstream publishes one, so Watchtower still delivers patch releases, and the exact version where it does not: - Minor line: Open WebUI `0.11`, Watchtower `1.16`, Forgejo `14.0`, Paperless `3.0`, Gotenberg `8.37`, Valkey `8.1-alpine`, Caddy `2.10` - Major line: Immich `v3`, which keeps to semantic versioning and tags no minor line - Exact version: Synapse `v1.161.0`, Element `v1.12.28`, Tika `4.0.0` - Digest: openedai-speech, which has tagged no release since 0.18.2 - Unchanged: PostgreSQL `16-alpine`, whose minor releases are its patch releases; the Immich database image and AdGuard, already exact The pin test now rejects every tag that is not a version, including one set through a stacklet's `[env.defaults]`. It only rejected `latest` before, which let `main` and `release` through. --- docs/admin-guide.md | 2 +- ...-002-installation-instances-and-updates.md | 4 +- docs/creating-stacklets.md | 4 +- docs/stack-reference.md | 12 ++- stacklets/ai/docker-compose.yml | 4 +- stacklets/chatai/docker-compose.yml | 2 +- stacklets/code/docker-compose.yml | 2 +- stacklets/core/docker-compose.yml | 2 +- stacklets/docs/docker-compose.yml | 12 +-- stacklets/infra/Dockerfile | 6 +- stacklets/messages/docker-compose.yml | 4 +- stacklets/messages/hooks/on_install.sh | 4 +- stacklets/photos/docker-compose.yml | 6 +- stacklets/photos/stacklet.toml | 2 +- tests/integration/test_caddy.py | 2 +- tests/unit/framework/test_compose_pins.py | 90 ++++++++++--------- 16 files changed, 90 insertions(+), 68 deletions(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 7dd65e40..763ad101 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -857,7 +857,7 @@ Stash the tarball on a USB stick or in 1Password. ## Updating -Container images update themselves nightly at 3am via Watchtower. You do nothing for that. +Container images update themselves nightly at 3am via Watchtower. You do nothing for that. Each famstack release pins every service to a version line, so Watchtower brings in bug-fix releases, and a new minor or major version of Immich, Paperless or Synapse arrives with a famstack update, not overnight. The famstack code is a git checkout, and a release is a tag. Updating means moving your checkout to a newer tag. diff --git a/docs/adr/rfc-002-installation-instances-and-updates.md b/docs/adr/rfc-002-installation-instances-and-updates.md index 63424ee0..1f1d7455 100644 --- a/docs/adr/rfc-002-installation-instances-and-updates.md +++ b/docs/adr/rfc-002-installation-instances-and-updates.md @@ -43,7 +43,9 @@ reported, so on a checkout between tags it named a hundred different trees. Every surface now derives it from `git describe --tags --dirty`, which answers by comparing the tree against the commit a tag points at. -**P3. A release does not describe what runs.** Across the stacklets there are +**P3. A release does not describe what runs.** *(narrowed: every image now +names a minor line or an exact version, so installs of one tag differ by patch +releases only)* Across the stacklets there are 19 image references. Exactly one is pinned to an exact upstream version (`paperless-ngx:3.0.4`). Five are `:latest` (element-web, synapse, tika, adguard, watchtower), one is `:main` (open-webui), one carries no tag at all diff --git a/docs/creating-stacklets.md b/docs/creating-stacklets.md index c515cf82..20a349ff 100644 --- a/docs/creating-stacklets.md +++ b/docs/creating-stacklets.md @@ -145,7 +145,7 @@ name: stack-myapp # project name: stack-{id} services: stack-myapp: # container name: stack-{id} container_name: stack-myapp - image: someimage:latest + image: someimage:1.4 # minor line, else the exact version labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" networks: @@ -284,7 +284,7 @@ name: stack-code services: stack-code: container_name: stack-code - image: codeberg.org/forgejo/forgejo:14 + image: codeberg.org/forgejo/forgejo:14.0 networks: - stack ports: diff --git a/docs/stack-reference.md b/docs/stack-reference.md index 3e11f7f5..86f04a9b 100644 --- a/docs/stack-reference.md +++ b/docs/stack-reference.md @@ -857,6 +857,16 @@ Every container gets this label. Watchtower (in the `core` stacklet) monitors labeled containers and pulls new images on the nightly schedule. Set `channel = "none"` in `stacklet.toml` to disable. +Watchtower pulls the tag the compose file names, so the tag decides how far +an image can move. Pin the minor line where upstream publishes one +(`gotenberg:8.37`), otherwise the exact version (`synapse:v1.161.0`), or a +digest when upstream tags no release at all. Watchtower then delivers patch +releases, and a minor or major version changes only with a commit. Immich +keeps to semantic versioning and tags only its major line, so it is pinned +to that (`immich-server:v3`). For PostgreSQL the major tag (`postgres:16-alpine`) is already the patch line: +its minor releases are bug and security fixes. Named tags such as `latest`, +`main` or `release` fail `tests/unit/framework/test_compose_pins.py`. + ### Health Checks ```yaml @@ -903,7 +913,7 @@ name: stack-docs services: stack-docs-paperless: container_name: stack-docs-paperless - image: ghcr.io/paperless-ngx/paperless-ngx:latest + image: ghcr.io/paperless-ngx/paperless-ngx:3.0 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" networks: diff --git a/stacklets/ai/docker-compose.yml b/stacklets/ai/docker-compose.yml index 2f332c1a..e5d85e76 100644 --- a/stacklets/ai/docker-compose.yml +++ b/stacklets/ai/docker-compose.yml @@ -12,7 +12,9 @@ name: stack-ai services: stack-ai-speech: container_name: stack-ai-speech - image: ghcr.io/matatonic/openedai-speech-min + # No version tag after 0.18.2 (2024). `latest` is a later build, so the + # digest is the only name for the image that runs. + image: ghcr.io/matatonic/openedai-speech-min@sha256:e82d719252fb1b861967f4f9abb8513f028ab9c1139451245688dbf34992b060 # Gated by the "voice" compose profile. The stacklet defaults # COMPOSE_PROFILES=voice so this runs in normal use; `STACK_AI_NO_VOICE=1` # clears it so local-dev `stack up ai` skips the speech container. diff --git a/stacklets/chatai/docker-compose.yml b/stacklets/chatai/docker-compose.yml index 31053e3a..fd916ba7 100644 --- a/stacklets/chatai/docker-compose.yml +++ b/stacklets/chatai/docker-compose.yml @@ -12,7 +12,7 @@ name: stack-chatai services: stack-chatai: container_name: stack-chatai - image: ghcr.io/open-webui/open-webui:main + image: ghcr.io/open-webui/open-webui:0.11 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" diff --git a/stacklets/code/docker-compose.yml b/stacklets/code/docker-compose.yml index 526ba553..ffd9d4ee 100644 --- a/stacklets/code/docker-compose.yml +++ b/stacklets/code/docker-compose.yml @@ -12,7 +12,7 @@ name: stack-code services: stack-code: container_name: stack-code - image: codeberg.org/forgejo/forgejo:14 + image: codeberg.org/forgejo/forgejo:14.0 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" diff --git a/stacklets/core/docker-compose.yml b/stacklets/core/docker-compose.yml index a8b37b01..a25cf567 100644 --- a/stacklets/core/docker-compose.yml +++ b/stacklets/core/docker-compose.yml @@ -15,7 +15,7 @@ services: # containrrr/watchtower was archived in 2024 and ships an old Docker # SDK pinned to API v1.25, which modern daemons (incl. OrbStack) # reject — the container crashlooped and poisoned core's health. - image: nickfedor/watchtower:latest + image: nickfedor/watchtower:1.16 container_name: stack-core-watchtower restart: unless-stopped environment: diff --git a/stacklets/docs/docker-compose.yml b/stacklets/docs/docker-compose.yml index 716e5664..65f49215 100644 --- a/stacklets/docs/docker-compose.yml +++ b/stacklets/docs/docker-compose.yml @@ -15,12 +15,12 @@ name: stack-docs services: stack-docs-paperless: container_name: stack-docs-paperless - # Pinned to an exact release, never `:latest`. A watchtower pull of + # Pinned to the 3.0 line, never `:latest`. A watchtower pull of # `:latest` once rolled 2.x straight to 3.0 and broke document filing. # Paperless migrates its database on the first 3.x start and 2.x will # not boot afterwards, so a major bump is one-way: do it deliberately, - # with a backup taken first. Bump within 3.0.x for patches. - image: ghcr.io/paperless-ngx/paperless-ngx:3.0.4 + # with a backup taken first. Watchtower delivers the 3.0.x patches. + image: ghcr.io/paperless-ngx/paperless-ngx:3.0 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" @@ -91,7 +91,7 @@ services: stack-docs-redis: container_name: stack-docs-redis - image: docker.io/valkey/valkey:8-alpine + image: docker.io/valkey/valkey:8.1-alpine labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" @@ -106,7 +106,7 @@ services: stack-docs-gotenberg: container_name: stack-docs-gotenberg - image: gotenberg/gotenberg:8 + image: gotenberg/gotenberg:8.37 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" @@ -120,7 +120,7 @@ services: stack-docs-tika: container_name: stack-docs-tika - image: apache/tika:latest + image: apache/tika:4.0.0 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" diff --git a/stacklets/infra/Dockerfile b/stacklets/infra/Dockerfile index 5141a54a..e6efaf39 100644 --- a/stacklets/infra/Dockerfile +++ b/stacklets/infra/Dockerfile @@ -8,13 +8,13 @@ # The hetzner module is v2, which talks to the Hetzner Cloud API. v1 used # the DNS Console API, which Hetzner has deprecated. # -# Pinned to the Caddy 2 line. Every subdomain reusing the wildcard +# Pinned to the Caddy 2.10 line. Every subdomain reusing the wildcard # certificate needs 2.10 or later. -FROM caddy:2-builder AS builder +FROM caddy:2.10-builder AS builder RUN xcaddy build \ --with github.com/caddy-dns/hetzner/v2 \ --with github.com/caddy-dns/cloudflare -FROM caddy:2 +FROM caddy:2.10 COPY --from=builder /usr/bin/caddy /usr/bin/caddy diff --git a/stacklets/messages/docker-compose.yml b/stacklets/messages/docker-compose.yml index 464db686..818ca70a 100644 --- a/stacklets/messages/docker-compose.yml +++ b/stacklets/messages/docker-compose.yml @@ -14,7 +14,7 @@ name: stack-messages services: stack-messages-synapse: container_name: stack-messages-synapse - image: matrixdotorg/synapse:latest + image: matrixdotorg/synapse:v1.161.0 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" @@ -55,7 +55,7 @@ services: stack-messages-element: container_name: stack-messages-element - image: vectorim/element-web:latest + image: vectorim/element-web:v1.12.28 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" diff --git a/stacklets/messages/hooks/on_install.sh b/stacklets/messages/hooks/on_install.sh index 64d6a96b..77c91702 100755 --- a/stacklets/messages/hooks/on_install.sh +++ b/stacklets/messages/hooks/on_install.sh @@ -39,11 +39,13 @@ chmod 700 "$DB_DIR" if [ ! -f "$SYNAPSE_DIR/homeserver.yaml" ]; then echo "messages: generating Synapse configuration for '$SERVER_NAME'..." + # Same tag as docker-compose.yml, so the config comes from the Synapse + # version that will run it. docker run --rm \ -v "$SYNAPSE_DIR:/data" \ -e SYNAPSE_SERVER_NAME="$SERVER_NAME" \ -e SYNAPSE_REPORT_STATS=no \ - matrixdotorg/synapse:latest generate + matrixdotorg/synapse:v1.161.0 generate # ── Patch homeserver.yaml ──────────────────────────────────────────── # diff --git a/stacklets/photos/docker-compose.yml b/stacklets/photos/docker-compose.yml index 2654b6b5..8f8384e1 100644 --- a/stacklets/photos/docker-compose.yml +++ b/stacklets/photos/docker-compose.yml @@ -12,7 +12,7 @@ name: stack-photos services: stack-photos-server: container_name: stack-photos-server - image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-release} + image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-v3} labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" @@ -36,7 +36,7 @@ services: stack-photos-ml: container_name: stack-photos-ml - image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release} + image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-v3} labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" @@ -57,7 +57,7 @@ services: stack-photos-redis: container_name: stack-photos-redis - image: docker.io/valkey/valkey:8-alpine + image: docker.io/valkey/valkey:8.1-alpine labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" - "stack.commit=${STACK_COMMIT:-unknown}" diff --git a/stacklets/photos/stacklet.toml b/stacklets/photos/stacklet.toml index 899bfd95..203a02b3 100644 --- a/stacklets/photos/stacklet.toml +++ b/stacklets/photos/stacklet.toml @@ -45,7 +45,7 @@ DB_USERNAME = "postgres" DB_DATABASE_NAME = "immich" DB_HOSTNAME = "stack-photos-db" REDIS_HOSTNAME = "stack-photos-redis" -IMMICH_VERSION = "release" +IMMICH_VERSION = "v3" # stack status uses this to confirm the service is actually responding, # not just that the container is running. diff --git a/tests/integration/test_caddy.py b/tests/integration/test_caddy.py index 4e0de371..9c8493bf 100644 --- a/tests/integration/test_caddy.py +++ b/tests/integration/test_caddy.py @@ -34,7 +34,7 @@ pytestmark = pytest.mark.skipif(not HAS_DOCKER, reason="Docker not available") -STOCK_CADDY = "caddy:2" +STOCK_CADDY = "caddy:2.10" DOMAIN = "home.example.family" diff --git a/tests/unit/framework/test_compose_pins.py b/tests/unit/framework/test_compose_pins.py index c8283fc9..8658fa15 100644 --- a/tests/unit/framework/test_compose_pins.py +++ b/tests/unit/framework/test_compose_pins.py @@ -1,9 +1,16 @@ -"""Every container image must name an explicit version. +"""Every container image must name a version. -`:latest` plus watchtower is a scheduled outage. Paperless-ngx rolled from +A floating tag plus watchtower is a scheduled outage. Paperless-ngx rolled from the 2.x line to 3.0.2 unattended and broke document filing across the whole e2e suite; we found out from a red test run, not from a decision. +Images are pinned to the minor line where upstream publishes one +(`gotenberg:8.37`), otherwise to the exact version (`synapse:v1.161.0`), so +watchtower delivers patch releases and nothing larger. Whether a minor tag +exists is a registry question, so this test checks what it can offline: every +tag is a version, never a name such as `latest`, `main` or `release` that +upstream moves to its next build. + This is the audit that would have caught it, as a test instead of a one-time grep — it costs milliseconds and cannot silently stop being true. """ @@ -11,39 +18,54 @@ from __future__ import annotations import re +import tomllib from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent # `image: repo/name:tag` in compose, ignoring commented-out lines. The tag is # optional in the grammar precisely because omitting it means `:latest`, which -# is the case this test exists to reject. +# is one of the cases this test exists to reject. _IMAGE_RE = re.compile(r"^\s*image:\s*[\"']?([^\s\"'#]+)", re.MULTILINE) -# Known-unpinned images, as of 2026-07-31. This is a ratchet, not an allowlist: -# the test blocks any NEW floating tag immediately, while these existing ones -# get pinned deliberately, one at a time, each verified against a running rig. -# Picking a Synapse or Element version is a real decision with a real blast -# radius; making that call to turn a test green would be the wrong order. -# -# Delete entries as they are pinned. An empty set is the goal state. -KNOWN_UNPINNED: set[str] = { - "ghcr.io/matatonic/openedai-speech-min", # no tag at all - "nickfedor/watchtower:latest", - "apache/tika:latest", - "matrixdotorg/synapse:latest", - "vectorim/element-web:latest", -} +# A version starts with a digit, optionally after a `v`: `14.0`, `v1.161.0`, +# `16-alpine`, `14-vectorchord0.4.3-pgvectors0.2.0`. +_VERSION_RE = re.compile(r"^v?\d") + +# A tag taken from the environment, `${IMMICH_VERSION:-v3.2.2}`. +_VAR_RE = re.compile(r"^\$\{(\w+)(?::-([^}]*))?\}$") def _compose_files() -> list[Path]: return sorted(REPO_ROOT.glob("stacklets/*/docker-compose*.yml")) -def _unpinned(text: str) -> list[str]: - """Return image refs carrying no tag, or an explicitly floating one.""" +def _env_defaults(compose: Path) -> dict: + manifest = compose.parent / "stacklet.toml" + if not manifest.exists(): + return {} + data = tomllib.loads(manifest.read_text(encoding="utf-8")) + return data.get("env", {}).get("defaults", {}) + + +def _tag(ref: str, env: dict) -> str: + """The tag `ref` resolves to once the stacklet's `.env` is rendered.""" + # Strip the registry host before looking for the tag separator, so + # `registry:5000/img` is not mistaken for an `img:5000` tag. + last_segment = ref.rsplit("/", 1)[-1] + tag = last_segment.split(":", 1)[1] if ":" in last_segment else "latest" + # `.env` always carries the stacklet's `[env.defaults]` value, so that is + # the pin; the compose fallback only applies to a variable it lacks. + if var := _VAR_RE.match(tag): + tag = str(env.get(var.group(1), var.group(2) or "")) + return tag + + +def _unpinned(compose: Path) -> list[str]: + """Return image refs whose tag is not a version.""" + env = _env_defaults(compose) found = [] - for ref in _IMAGE_RE.findall(text): + for ref in _IMAGE_RE.findall(compose.read_text(encoding="utf-8")): # Environment-substituted refs (${FOO}) are resolved at render time; # the pin lives wherever that variable is defined, not here. if ref.startswith("${"): @@ -51,11 +73,11 @@ def _unpinned(text: str) -> list[str]: # A digest pin (repo@sha256:...) is stricter than a tag. Accept it. if "@sha256:" in ref: continue - # Strip the registry host before looking for the tag separator, so - # `registry:5000/img` is not mistaken for an `img:5000` tag. - last_segment = ref.rsplit("/", 1)[-1] - tag = last_segment.split(":", 1)[1] if ":" in last_segment else "latest" - if tag == "latest" and ref not in KNOWN_UNPINNED: + tag = _tag(ref, env) + # Built from the stacklet's Dockerfile, whose FROM line is the pin. + if tag == "local": + continue + if not _VERSION_RE.match(tag): found.append(ref) return found @@ -66,26 +88,10 @@ def test_compose_files_exist(): assert _compose_files(), "no stacklet compose files found - glob is wrong" -def test_ratchet_has_no_stale_entries(): - # If an image gets pinned but its entry stays behind, the exemption sits - # there covering a name nothing uses - and quietly re-exempts that image - # the day someone reintroduces it. Shrinking the list must be mandatory. - all_refs = { - ref - for path in _compose_files() - for ref in _IMAGE_RE.findall(path.read_text(encoding="utf-8")) - } - stale = KNOWN_UNPINNED - all_refs - assert not stale, ( - "KNOWN_UNPINNED lists images no compose file uses - pinned already? " - f"remove them: {', '.join(sorted(stale))}" - ) - - def test_no_floating_image_tags(): offenders: dict[str, list[str]] = {} for path in _compose_files(): - unpinned = _unpinned(path.read_text(encoding="utf-8")) + unpinned = _unpinned(path) if unpinned: offenders[str(path.relative_to(REPO_ROOT))] = unpinned