diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 870e455e..f11404ee 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -75,5 +75,9 @@ See [README.md](../README.md) for full list of supported environment variables. ## 🔧 CI/CD Workflows - **Dev builds**: All pull requests → `dev-refs-pull--merge` images +- **Test builds**: Push to main → `test-`; PR smoke test → `test--` - **Release builds**: Git tags → versioned images + GitHub releases - **Manual triggers**: Available in GitHub Actions for emergency releases +- **Tag pruning**: Weekly `Prune Docker Tags` workflow deletes `test-*` / `dev-*` + tags older than 14 days (releases and `latest` are never touched). Run it + manually with the `dry_run` input to preview. diff --git a/.github/workflows/prune-docker-tags.yml b/.github/workflows/prune-docker-tags.yml new file mode 100644 index 00000000..6bf9cfed --- /dev/null +++ b/.github/workflows/prune-docker-tags.yml @@ -0,0 +1,54 @@ +name: Prune Docker Tags + +# CI pushes a unique image tag per run (`test-` on main, `test--` +# per PR smoke test) plus a `dev-` tag per branch. Nothing deletes them, so +# the Docker Hub tag list grows without bound. This weekly sweep removes the stale +# ones; releases (`v1.2.3` / `1.2.3`) and `latest` never match a prefix and are +# protected by construction. +on: + schedule: + # Mondays 04:17 UTC — off the :00 mark to avoid the scheduler's top-of-hour rush. + - cron: "17 4 * * 1" + workflow_dispatch: + inputs: + min_age_days: + description: "Delete ephemeral tags not updated in this many days" + required: false + default: "14" + type: string + dry_run: + description: "List what would be deleted without deleting anything" + required: false + default: false + type: boolean + +permissions: + contents: read + +jobs: + prune: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Prune stale ephemeral tags + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO || 'runpod' }} + DOCKERHUB_IMG: ${{ vars.DOCKERHUB_IMG || 'worker-v1-vllm' }} + MIN_AGE_DAYS: ${{ inputs.min_age_days || '14' }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + args=(--repo "${DOCKERHUB_REPO}/${DOCKERHUB_IMG}" --min-age-days "${MIN_AGE_DAYS}") + if [[ "${DRY_RUN}" == "true" ]]; then + args+=(--dry-run) + fi + python scripts/prune_docker_tags.py "${args[@]}" diff --git a/docs/conventions.md b/docs/conventions.md index 290f85ee..a06b766f 100644 --- a/docs/conventions.md +++ b/docs/conventions.md @@ -165,6 +165,12 @@ workers at container start. - **CI/CD**: non-main branches → `runpod/worker-v1-vllm:dev-`; version tags → `runpod/worker-v1-vllm:`. vLLM version bumps are `VLLM_VERSION` changes; CI syncs the README version line from the Dockerfile. +- **Ephemeral CI tags**: `test-` (push to main) and + `test--` (PR smoke test) are unique per run, so they pile + up in Docker Hub. `scripts/prune_docker_tags.py` deletes `test-*` / `dev-*` + tags older than `--min-age-days` (default 14); releases and `latest` match no + prefix and are never candidates. The weekly `prune-docker-tags.yml` + workflow runs it (also manually, with a `dry_run` input). ## Release & Versioning Strategy diff --git a/scripts/prune_docker_tags.py b/scripts/prune_docker_tags.py new file mode 100644 index 00000000..54bebc6f --- /dev/null +++ b/scripts/prune_docker_tags.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Delete stale ephemeral Docker Hub tags for the worker image. + +CI pushes a fresh tag per run — `test--` for each PR smoke +test, `test-` for each push to main — and `dev-` per branch. None +of them are ever cleaned up, so the tag list grows without bound. + +This script deletes only tags whose name starts with one of the allowed +prefixes (`test-` and `dev-` by default) and that are older than +`--min-age-days`. Releases (`v1.2.3` / `1.2.3`) and `latest` are protected by +construction: they match no prefix, so they are never candidates. + +Docker Hub has no native tag-retention policy, hence the API calls below. + +Dry run (listing is public, so no credentials needed): + + python scripts/prune_docker_tags.py --dry-run + +Real run (deletes need a Docker Hub username + PAT): + + DOCKERHUB_USERNAME=... DOCKERHUB_TOKEN=... \\ + python scripts/prune_docker_tags.py --min-age-days 14 + +Exit code is non-zero only on an API/auth failure, never merely because a tag +could not be parsed or nothing was old enough. + +`DOCKERHUB_API_BASE` overrides the Docker Hub API root (used by tests to point +at a stub server; defaults to https://hub.docker.com/v2). +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone + +DEFAULT_API_BASE = "https://hub.docker.com/v2" +DEFAULT_PREFIXES = ("test-", "dev-") +PAGE_SIZE = 100 +DELETE_PAUSE_SECONDS = 0.5 + + +def log(msg: str) -> None: + print(f"[prune_docker_tags] {msg}", flush=True) + + +def api_base() -> str: + """Docker Hub API root. Overridable so tests can point at a stub server.""" + return os.environ.get("DOCKERHUB_API_BASE", DEFAULT_API_BASE).rstrip("/") + + +def _request(method: str, url: str, token: str | None = None, body: dict | None = None): + data = json.dumps(body).encode() if body is not None else None + headers = {"Content-Type": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=60) as resp: + payload = resp.read() + return json.loads(payload) if payload else {} + + +def login(username: str, password: str) -> str: + """Exchange username + PAT for the JWT the delete endpoint expects.""" + try: + response = _request( + "POST", f"{api_base()}/users/login/", body={"username": username, "password": password} + ) + except urllib.error.HTTPError as e: + raise SystemExit(f"Docker Hub login failed ({e.code}): {e.read().decode(errors='replace')}") + token = response.get("token") + if not token: + raise SystemExit("Docker Hub login returned no token") + return token + + +def list_tags(repo: str, token: str | None = None): + """Yield every tag object in the repository, newest page first.""" + page = 1 + while True: + url = f"{api_base()}/repositories/{repo}/tags/?page_size={PAGE_SIZE}&page={page}" + try: + response = _request("GET", url, token=token) + except urllib.error.HTTPError as e: + raise SystemExit(f"listing tags failed ({e.code}): {e.read().decode(errors='replace')}") + results = response.get("results", []) + if not results: + return + yield from results + if not response.get("next"): + return + page += 1 + + +def delete_tag(repo: str, tag: str, token: str) -> None: + quoted = urllib.parse.quote(tag, safe="") + url = f"{api_base()}/repositories/{repo}/tags/{quoted}/" + try: + _request("DELETE", url, token=token) + except urllib.error.HTTPError as e: + raise SystemExit( + f"deleting {tag} failed ({e.code}): {e.read().decode(errors='replace')}" + ) + + +def parse_timestamp(value: str | None) -> float | None: + """Epoch seconds for Docker Hub's ISO-8601 `last_updated`, or None.""" + if not value: + return None + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(text) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.timestamp() + + +def select_stale(tags, prefixes=DEFAULT_PREFIXES, min_age_days=14, now=None): + """Return the tags eligible for deletion. + + A tag qualifies only when its name starts with an allowed prefix AND its + `last_updated` is at (or before) the cutoff. Tags with an unparseable or + missing timestamp are kept — conservative by design, since guessing would + risk deleting a release. + """ + if now is None: + now = time.time() + cutoff = now - min_age_days * 86400 + stale = [] + for tag in tags: + name = tag.get("name", "") + if not any(name.startswith(prefix) for prefix in prefixes): + continue + updated = parse_timestamp(tag.get("last_updated")) + if updated is None: + continue + if updated <= cutoff: + stale.append(tag) + return stale + + +def validate_prefixes(prefixes) -> tuple[str, ...]: + """Reject prefixes that would match everything. + + An empty prefix makes `name.startswith(prefix)` true for every tag — releases + included — so it must never reach the delete loop. + """ + cleaned = tuple(p.strip() for p in prefixes) + if not cleaned: + return DEFAULT_PREFIXES + for prefix in cleaned: + if not prefix: + raise SystemExit("--prefix must not be empty (an empty prefix matches every tag)") + return cleaned + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--repo", default="runpod/worker-v1-vllm", help="Docker Hub repository as namespace/name") + parser.add_argument( + "--prefix", + action="append", + dest="prefixes", + help=f"Tag prefix eligible for deletion (repeatable; default: {' '.join(DEFAULT_PREFIXES)})", + ) + parser.add_argument( + "--min-age-days", + type=float, + default=14, + help="Only delete tags not updated in this many days (default: 14)", + ) + parser.add_argument("--dry-run", action="store_true", help="List what would be deleted; delete nothing") + return parser.parse_args(argv) + + +def main(argv=None) -> int: + args = parse_args(argv) + + prefixes = validate_prefixes(args.prefixes or DEFAULT_PREFIXES) + if "/" not in args.repo: + raise SystemExit(f"--repo must be namespace/name, got {args.repo!r}") + + token = None + if not args.dry_run: + username = os.environ.get("DOCKERHUB_USERNAME") + password = os.environ.get("DOCKERHUB_TOKEN") + if not username or not password: + raise SystemExit("DOCKERHUB_USERNAME and DOCKERHUB_TOKEN are required unless --dry-run") + log(f"logging in to Docker Hub as {username}") + token = login(username, password) + + tags = list(list_tags(args.repo, token=token)) + stale = select_stale(tags, prefixes=prefixes, min_age_days=args.min_age_days) + + log( + f"{args.repo}: {len(tags)} tags, {len(stale)} matching " + f"{'|'.join(prefixes)} older than {args.min_age_days:g}d" + ) + if not stale: + return 0 + + if args.dry_run: + for tag in stale: + log(f" would delete {tag.get('name')} (updated {tag.get('last_updated')})") + log("dry run: nothing deleted") + return 0 + + for tag in stale: + name = tag.get("name") + delete_tag(args.repo, name, token) + log(f" deleted {name} (updated {tag.get('last_updated')})") + time.sleep(DELETE_PAUSE_SECONDS) + log(f"deleted {len(stale)} tags") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_prune_docker_tags.py b/tests/test_prune_docker_tags.py new file mode 100644 index 00000000..f857c06d --- /dev/null +++ b/tests/test_prune_docker_tags.py @@ -0,0 +1,237 @@ +"""Which Docker Hub tags the weekly prune job is allowed to delete. + +The rule that matters: only `test-*` / `dev-*` tags past an age cutoff are +candidates, so releases and `latest` can never be touched. +""" + +import http.server +import json +import sys +import threading +import time +import types +import urllib.parse +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import prune_docker_tags as prune # noqa: E402 + +NOW = datetime(2026, 1, 10, tzinfo=timezone.utc).timestamp() + + +def tag(name, updated): + return {"name": name, "last_updated": updated} + + +OLD = "2025-12-01T00:00:00.000000Z" +RECENT = "2026-01-09T00:00:00.000000Z" + + +def iso_days_ago(days): + """Timestamp relative to the real clock, for tests that run main() end to end. + + The unit tests below inject a fixed `now`, but a real run reads the system + clock, so the stub fixture cannot use hard-coded dates. + """ + return datetime.fromtimestamp(time.time() - days * 86400, timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S.000000Z" + ) + + +def names(tags): + return [t["name"] for t in tags] + + +def test_deletes_old_ephemeral_tags(): + tags = [tag("test-abc123", OLD), tag("dev-my-branch", OLD)] + assert names(prune.select_stale(tags, now=NOW)) == ["test-abc123", "dev-my-branch"] + + +def test_keeps_recent_ephemeral_tags(): + tags = [tag("test-abc123", RECENT), tag("dev-my-branch", RECENT)] + assert prune.select_stale(tags, now=NOW) == [] + + +@pytest.mark.parametrize("name", ["latest", "v2.8.0", "2.8.0", "main"]) +def test_never_deletes_release_or_latest(name): + # Old, but no allowed prefix matches — protected by construction. + assert prune.select_stale([tag(name, OLD)], now=NOW) == [] + + +def test_cutoff_is_inclusive(): + cutoff = NOW - 14 * 86400 + at_cutoff = datetime.fromtimestamp(cutoff, timezone.utc).isoformat().replace("+00:00", "Z") + just_inside = datetime.fromtimestamp(cutoff + 1, timezone.utc).isoformat().replace("+00:00", "Z") + assert names(prune.select_stale([tag("test-a", at_cutoff)], now=NOW)) == ["test-a"] + assert prune.select_stale([tag("test-a", just_inside)], now=NOW) == [] + + +def test_unparseable_timestamp_is_kept(): + # Better to leave a tag than to guess it is old. + assert prune.select_stale([tag("test-a", "not-a-date")], now=NOW) == [] + assert prune.select_stale([tag("test-a", None)], now=NOW) == [] + + +def test_custom_prefixes(): + tags = [tag("test-a", OLD), tag("dev-b", OLD), tag("ci-c", OLD)] + assert names(prune.select_stale(tags, prefixes=("ci-",), now=NOW)) == ["ci-c"] + + +@pytest.mark.parametrize( + "value,expected", + [ + ("2025-12-01T00:00:00.000000Z", datetime(2025, 12, 1, tzinfo=timezone.utc).timestamp()), + ("2025-12-01T00:00:00Z", datetime(2025, 12, 1, tzinfo=timezone.utc).timestamp()), + ("2025-12-01T01:00:00+01:00", datetime(2025, 12, 1, tzinfo=timezone.utc).timestamp()), + ("garbage", None), + (None, None), + ], +) +def test_parse_timestamp(value, expected): + assert prune.parse_timestamp(value) == expected + + +def test_main_dry_run_lists_without_deleting(monkeypatch, capsys): + fixture = [tag("test-abc123", OLD), tag("v2.8.0", OLD)] + monkeypatch.setattr(prune, "list_tags", lambda repo, token=None: iter(fixture)) + monkeypatch.setattr(prune, "delete_tag", lambda *a, **k: pytest.fail("dry run deleted a tag")) + monkeypatch.delenv("DOCKERHUB_USERNAME", raising=False) + monkeypatch.delenv("DOCKERHUB_TOKEN", raising=False) + + assert prune.main(["--repo", "runpod/worker-v1-vllm", "--dry-run"]) == 0 + out = capsys.readouterr().out + assert "test-abc123" in out + assert "v2.8.0" not in out + + +def test_main_requires_credentials_for_real_run(monkeypatch): + monkeypatch.delenv("DOCKERHUB_USERNAME", raising=False) + monkeypatch.delenv("DOCKERHUB_TOKEN", raising=False) + with pytest.raises(SystemExit): + prune.main(["--repo", "runpod/worker-v1-vllm"]) + + +# --- destructive path, exercised against a stub Docker Hub ------------------- + + +def test_validate_prefixes_rejects_empty(): + assert prune.validate_prefixes(["test-"]) == ("test-",) + assert prune.validate_prefixes([]) == prune.DEFAULT_PREFIXES + # An empty prefix would match every tag, releases included. + with pytest.raises(SystemExit): + prune.validate_prefixes([""]) + + +def test_main_refuses_empty_prefix_before_touching_the_network(monkeypatch): + monkeypatch.setattr(prune, "list_tags", lambda *a, **k: pytest.fail("listed tags")) + monkeypatch.setattr(prune, "login", lambda *a, **k: pytest.fail("logged in")) + with pytest.raises(SystemExit): + prune.main(["--repo", "runpod/worker-v1-vllm", "--prefix", "", "--dry-run"]) + + +@pytest.fixture +def stub_hub(): + """A local stand-in for the Docker Hub API that records real HTTP traffic.""" + state = {"deleted": [], "auth": [], "logins": [], "fail_delete": set(), "base_url": None} + tags = [ + tag("test-aaa", iso_days_ago(30)), + tag("dev-x", iso_days_ago(30)), + tag("test-bbb", iso_days_ago(30)), + tag("v2.8.0", iso_days_ago(30)), + tag("latest", iso_days_ago(30)), + tag("2.8.0", iso_days_ago(30)), + tag("test-ccc", iso_days_ago(1)), + ] + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _send(self, code, payload=None): + body = json.dumps(payload).encode() if payload is not None else b"" + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + state["logins"].append(json.loads(self.rfile.read(length) or b"{}")) + self._send(200, {"token": "stub-token"}) + + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + if not parsed.path.endswith("/tags/"): + self._send(404, {"detail": "not found"}) + return + page = int(urllib.parse.parse_qs(parsed.query).get("page", ["1"])[0]) + page_size = 2 # deliberately smaller than the request, to force paging + start = (page - 1) * page_size + chunk = tags[start : start + page_size] + nxt = f"{parsed.path}?page={page + 1}" if start + page_size < len(tags) else None + self._send(200, {"count": len(tags), "next": nxt, "results": chunk}) + + def do_DELETE(self): + name = urllib.parse.unquote(self.path.rstrip("/").rsplit("/", 1)[-1]) + state["auth"].append(self.headers.get("Authorization")) + if name in state["fail_delete"]: + self._send(500, {"detail": "boom"}) + return + state["deleted"].append(name) + self._send(204) + + httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + state["base_url"] = f"http://127.0.0.1:{httpd.server_address[1]}/v2" + try: + yield types.SimpleNamespace(**state) + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + +def _run_against_stub(monkeypatch, stub, *extra): + monkeypatch.setenv("DOCKERHUB_API_BASE", stub.base_url) + monkeypatch.setenv("DOCKERHUB_USERNAME", "user") + monkeypatch.setenv("DOCKERHUB_TOKEN", "pat") + monkeypatch.setattr(prune, "DELETE_PAUSE_SECONDS", 0) + return prune.main(["--repo", "runpod/worker-v1-vllm", *extra]) + + +def test_real_run_deletes_only_stale_ephemeral_tags(stub_hub, monkeypatch): + assert _run_against_stub(monkeypatch, stub_hub) == 0 + # Paginated listing still reaches every page; only old test-/dev- tags go. + assert sorted(stub_hub.deleted) == ["dev-x", "test-aaa", "test-bbb"] + assert "latest" not in stub_hub.deleted + assert "v2.8.0" not in stub_hub.deleted + assert "2.8.0" not in stub_hub.deleted + assert "test-ccc" not in stub_hub.deleted + + +def test_real_run_sends_bearer_token_on_deletes(stub_hub, monkeypatch): + _run_against_stub(monkeypatch, stub_hub) + assert stub_hub.logins == [{"username": "user", "password": "pat"}] + assert stub_hub.auth and all(auth == "Bearer stub-token" for auth in stub_hub.auth) + + +def test_delete_failure_aborts_with_nonzero_exit(stub_hub, monkeypatch): + stub_hub.fail_delete.add("test-bbb") + with pytest.raises(SystemExit): + _run_against_stub(monkeypatch, stub_hub) + + +def test_dry_run_against_stub_deletes_nothing(stub_hub, monkeypatch): + monkeypatch.setenv("DOCKERHUB_API_BASE", stub_hub.base_url) + monkeypatch.delenv("DOCKERHUB_USERNAME", raising=False) + monkeypatch.delenv("DOCKERHUB_TOKEN", raising=False) + assert prune.main(["--repo", "runpod/worker-v1-vllm", "--dry-run"]) == 0 + assert stub_hub.deleted == [] + assert stub_hub.logins == []