From fc4a327fe31a16516d2454605b64c11340459b3a Mon Sep 17 00:00:00 2001 From: tweinmann Date: Wed, 28 Oct 2026 09:37:38 +0000 Subject: [PATCH] Phase 10: branch previews Every branch of a tenant becomes an isolated preview of the whole app, when the operator switches previews on with `shelf app add|credentials --previews[=N]`. The reusable workflow publishes `branch-` for every branch and deletes the tag when the branch is deleted; a provider per app follows those tags, and the platform ResourceSet `previews` runs each one in a namespace `--` with a quota and limits, its own generated secrets, and no exposure. - `shelf build-plan --branch`, `schema.BranchSlug` - chart: `platform.preview` generates secrets with lookup - `shelf app status|open --preview `, previews in `--json` - `shelf app rm` removes the previews too - registry build cache in the reusable workflow - `just smoke-previews`; dev registry allows deletes - tenant skill: wait in the background, watch CI, previews Measured on the dev cluster: 23 s from tag to a Ready first preview, 17 s for a later commit, 30 Mi RAM per preview of examples/hello. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/select-smokes.sh | 7 +- .github/workflows/build.yml | 127 ++++++-- .github/workflows/ci.yml | 2 +- CLAUDE.md | 3 +- README.md | 53 ++- charts/shelf-app/templates/secrets.yaml | 26 ++ charts/shelf-app/values.yaml | 3 + examples/tenant/.claude/skills/shelf/SKILL.md | 76 +++-- examples/tenant/.github/workflows/deploy.yml | 5 +- examples/tenant/CLAUDE.md | 10 +- examples/tenant/README.md | 4 +- hack/cluster-up.sh | 10 + hack/smoke/previews.sh | 171 ++++++++++ internal/chart/chart_test.go | 13 + .../chart/testdata/preview.manifests.yaml | 288 +++++++++++++++++ internal/cli/app.go | 37 ++- internal/cli/app_test.go | 12 + internal/cli/build.go | 17 +- internal/cli/build_test.go | 15 + internal/cli/harness_test.go | 22 +- internal/cli/json.go | 30 +- internal/cli/open.go | 15 +- internal/cli/previews_test.go | 215 +++++++++++++ internal/cli/status.go | 54 +++- internal/cli/tenant_test.go | 4 +- .../cli/testdata/json/appstatus-preview.json | 58 ++++ .../cli/testdata/json/appstatus-previews.json | 78 +++++ internal/cluster/apps.go | 40 +++ internal/cluster/previews.go | 151 +++++++++ internal/cluster/previews_test.go | 103 ++++++ internal/cluster/rollout.go | 3 +- internal/cluster/state.go | 53 ++- internal/cluster/tunnel.go | 24 +- internal/ops/apps.go | 32 +- internal/ops/open.go | 33 +- internal/ops/ops.go | 12 +- internal/ops/ops_test.go | 2 + internal/ops/previews.go | 67 ++++ internal/ops/wait.go | 27 +- internal/schema/branch.go | 47 +++ internal/schema/branch_test.go | 46 +++ justfile | 4 + .../phase-10-branch-previews/.openspec.yaml | 1 - .../phase-10-branch-previews/design.md | 304 ++++++++++++++++++ .../phase-10-branch-previews/proposal.md | 102 +++--- .../specs/agent-interface/spec.md | 55 ++++ .../specs/app-chart/spec.md | 25 ++ .../specs/app-delivery/spec.md | 128 ++++++++ .../specs/app-exposure/spec.md | 16 + .../specs/branch-previews/spec.md | 136 ++++++++ .../specs/cluster-init/spec.md | 15 + .../specs/dev-environment/spec.md | 52 +++ .../changes/phase-10-branch-previews/tasks.md | 52 +++ platform/kustomization.yaml | 1 + platform/previews/kustomization.yaml | 4 + platform/previews/resourceset.yaml | 186 +++++++++++ 56 files changed, 2915 insertions(+), 161 deletions(-) create mode 100644 charts/shelf-app/templates/secrets.yaml create mode 100755 hack/smoke/previews.sh create mode 100644 internal/chart/testdata/preview.manifests.yaml create mode 100644 internal/cli/previews_test.go create mode 100644 internal/cli/testdata/json/appstatus-preview.json create mode 100644 internal/cli/testdata/json/appstatus-previews.json create mode 100644 internal/cluster/previews.go create mode 100644 internal/cluster/previews_test.go create mode 100644 internal/ops/previews.go create mode 100644 internal/schema/branch.go create mode 100644 internal/schema/branch_test.go create mode 100644 openspec/changes/phase-10-branch-previews/design.md create mode 100644 openspec/changes/phase-10-branch-previews/specs/agent-interface/spec.md create mode 100644 openspec/changes/phase-10-branch-previews/specs/app-chart/spec.md create mode 100644 openspec/changes/phase-10-branch-previews/specs/app-delivery/spec.md create mode 100644 openspec/changes/phase-10-branch-previews/specs/app-exposure/spec.md create mode 100644 openspec/changes/phase-10-branch-previews/specs/branch-previews/spec.md create mode 100644 openspec/changes/phase-10-branch-previews/specs/cluster-init/spec.md create mode 100644 openspec/changes/phase-10-branch-previews/specs/dev-environment/spec.md create mode 100644 openspec/changes/phase-10-branch-previews/tasks.md create mode 100644 platform/previews/kustomization.yaml create mode 100644 platform/previews/resourceset.yaml diff --git a/.github/scripts/select-smokes.sh b/.github/scripts/select-smokes.sh index 17c601c..f809571 100755 --- a/.github/scripts/select-smokes.sh +++ b/.github/scripts/select-smokes.sh @@ -7,7 +7,7 @@ # reason to skip. set -euo pipefail -all=(smoke-secrets smoke-chart smoke-init smoke-apps smoke-agent) +all=(smoke-secrets smoke-chart smoke-init smoke-apps smoke-agent smoke-previews) declare -A need=() for_path() { @@ -20,13 +20,14 @@ for_path() { internal/server/* | internal/cloudflare/* | internal/github/*) ;; hack/smoke/secret-expansion*) echo smoke-secrets ;; hack/smoke/chart.sh) echo smoke-chart ;; - charts/* | internal/chart/*) echo smoke-chart smoke-apps smoke-agent ;; + charts/* | internal/chart/*) echo smoke-chart smoke-apps smoke-agent smoke-previews ;; hack/smoke/init.sh | internal/cli/init.go | internal/cli/init_test.go) echo smoke-init ;; hack/smoke/apps.sh) echo smoke-apps ;; hack/smoke/agent.* | internal/cli/json* | internal/cli/open* | internal/cli/wait* | internal/cli/tenant_test.go) echo smoke-agent ;; + hack/smoke/previews.sh | internal/cli/previews* | internal/ops/previews*) echo smoke-previews ;; internal/cli/* | internal/ops/* | internal/progress/* | internal/secrets/* | internal/hostcfg/*) - echo smoke-apps smoke-agent ;; + echo smoke-apps smoke-agent smoke-previews ;; *) echo "${all[@]}" ;; esac } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed1c024..b831c5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,8 +6,9 @@ # # on: # push: -# branches: [main] +# branches: ['**'] # pull_request: +# delete: # permissions: # contents: read # packages: write @@ -17,9 +18,17 @@ # # Everything the workflow publishes lives under the package name, which is app.yaml's `name`: # the deploy artifact is ghcr.io//, and every component with `build: ./web` is -# pushed as ghcr.io///, each with the tags sha- and main. Images are pinned -# to the digest the build reported. On other branches and in pull requests the images are built -# but nothing is pushed, so a change is checked without deploying it. +# pushed as ghcr.io///. Images are pinned to the digest the build +# reported. A push to the default branch publishes everything as sha- and main; a push to +# another branch as sha- and branch- (`shelf build-plan --branch`), which an app with +# previews runs as --; never as main. Deleting the branch deletes its deploy +# artifact, and the preview goes with it. Pull requests build the images but push nothing, so a +# change is checked without deploying it, and a fork can never publish. +# +# Image layers are cached in the registry as :buildcache: every run reads the cache, and +# every run that publishes writes it back. +# +# A caller that keeps `branches: [main]` and no `delete` keeps working; it gets no previews. # # The package is not the app: shelf names an app when it is added (shelf app add ), # so the same artifact can run as several apps. @@ -33,16 +42,19 @@ on: type: string default: app.yaml shelf-version: - description: shelf release to use, e.g. v0.1.0. Defaults to the latest release. + description: >- + shelf release to use, e.g. v0.1.0. Defaults to the latest release. Publishing a branch + needs a release with `shelf build-plan --branch`. type: string default: latest outputs: artifact: - description: The deploy artifact, oci://ghcr.io//:main; empty unless published. + description: The deploy artifact, oci://ghcr.io//:
; empty unless published. value: ${{ jobs.build.outputs.artifact }} jobs: build: + if: github.event_name != 'delete' # The same architecture as the Mac mini, so images build natively. runs-on: ubuntu-24.04-arm timeout-minutes: 30 @@ -53,7 +65,8 @@ jobs: artifact: ${{ steps.publish.outputs.artifact }} env: APP_FILE: ${{ inputs.app-file }} - PUBLISH: ${{ github.event_name == 'push' && github.ref_name == github.event.repository.default_branch }} + PUBLISH: ${{ github.event_name == 'push' }} + DEFAULT_BRANCH: ${{ github.ref_name == github.event.repository.default_branch }} steps: - uses: actions/checkout@v7 @@ -90,19 +103,28 @@ jobs: - name: Build the images of this repository id: images run: | - plan="$(shelf build-plan "$APP_FILE")" + if [[ "$DEFAULT_BRANCH" == true || "$PUBLISH" != true ]]; then + plan="$(shelf build-plan "$APP_FILE")" + channel=main + else + plan="$(shelf build-plan "$APP_FILE" --branch "$GITHUB_REF_NAME")" + channel="$(jq -r .tag <<<"$plan")" + fi package="$(jq -r .package <<<"$plan")" echo "package=$package" >>"$GITHUB_ENV" + echo "channel=$channel" >>"$GITHUB_ENV" images=() while read -r component context; do image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/$package/$component" args=(--platform linux/arm64 - --tag "$image:sha-${GITHUB_SHA::7}" --tag "$image:main" + --tag "$image:sha-${GITHUB_SHA::7}" --tag "$image:$channel" + --cache-from "type=registry,ref=$image:buildcache" --label "org.opencontainers.image.source=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" --label "org.opencontainers.image.revision=$GITHUB_SHA" --metadata-file "$RUNNER_TEMP/$component.json") if [[ "$PUBLISH" == true ]]; then - args+=(--push) + args+=(--push + --cache-to "type=registry,ref=$image:buildcache,mode=max,image-manifest=true,oci-mediatypes=true") fi echo "::group::$image from $context" docker buildx build "${args[@]}" "$context" @@ -137,16 +159,85 @@ jobs: --source "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" \ --revision "$GITHUB_REF_NAME@sha1:$GITHUB_SHA" \ --annotations "dev.shelf.package=$package" - flux tag artifact "oci://$repo:sha-${GITHUB_SHA::7}" --tag main - echo "artifact=oci://$repo:main" >>"$GITHUB_OUTPUT" + flux tag artifact "oci://$repo:sha-${GITHUB_SHA::7}" --tag "$channel" + echo "artifact=oci://$repo:$channel" >>"$GITHUB_OUTPUT" { echo "### Deploy artifact" echo - echo "\`oci://$repo:main\` (also \`sha-${GITHUB_SHA::7}\`)" - echo - echo "Add it as an app once, under a name of your choice:" + echo "\`oci://$repo:$channel\` (also \`sha-${GITHUB_SHA::7}\`)" echo - echo "\`\`\`" - echo "shelf app add $package oci://$repo:main" - echo "\`\`\`" + if [[ "$channel" == main ]]; then + echo "Add it as an app once, under a name of your choice:" + echo + echo "\`\`\`" + echo "shelf app add $package oci://$repo:main" + echo "\`\`\`" + else + echo "An app with previews (\`shelf app credentials --previews\`) runs it as a preview:" + echo + echo "\`\`\`" + echo "shelf app status --preview $GITHUB_REF_NAME --wait --commit $GITHUB_SHA" + echo "shelf app open --preview $GITHUB_REF_NAME" + echo "\`\`\`" + fi } >>"$GITHUB_STEP_SUMMARY" + + # A deleted branch takes its deploy artifact with it, and with it its preview: the platform + # follows the tags of the registry. GHCR deletes a tag only as the package version that carries + # it, through the GitHub API; the cluster never calls that API. + delete-branch: + if: github.event_name == 'delete' && github.event.ref_type == 'branch' + runs-on: ubuntu-24.04-arm + timeout-minutes: 10 + permissions: + contents: read + packages: write + env: + APP_FILE: ${{ inputs.app-file }} + BRANCH: ${{ github.event.ref }} + OWNER_TYPE: ${{ github.event.repository.owner.type }} + GH_TOKEN: ${{ github.token }} + steps: + # The default branch's app.yaml names the package. + - uses: actions/checkout@v7 + + - name: Install shelf + env: + SHELF_VERSION: ${{ inputs.shelf-version }} + run: | + releases=https://github.com/tweinmann/shelf/releases + if [[ "$SHELF_VERSION" == latest ]]; then + url="$releases/latest/download" + else + url="$releases/download/$SHELF_VERSION" + fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/shelf.tar.gz" "$url/shelf_linux_arm64.tar.gz" + curl -fsSL -o "$tmp/checksums.txt" "$url/checksums.txt" + (cd "$tmp" && mv shelf.tar.gz shelf_linux_arm64.tar.gz \ + && grep ' ./shelf_linux_arm64.tar.gz$' checksums.txt | sha256sum -c -) + tar -xzf "$tmp/shelf_linux_arm64.tar.gz" -C "$tmp" shelf + sudo install "$tmp/shelf" /usr/local/bin/shelf + + - name: Delete the branch's deploy artifact + run: | + plan="$(shelf build-plan "$APP_FILE" --branch "$BRANCH")" + package="$(jq -r .package <<<"$plan")" + tag="$(jq -r .tag <<<"$plan")" + owner="${GITHUB_REPOSITORY_OWNER,,}" + if [[ "$OWNER_TYPE" == Organization ]]; then + base="/orgs/$owner/packages/container/$package/versions" + else + base="/users/$owner/packages/container/$package/versions" + fi + ids="$(gh api --paginate "$base" \ + --jq ".[] | select(.metadata.container.tags | index(\"$tag\")) | .id")" + if [[ -z "$ids" ]]; then + echo "ghcr.io/$owner/$package has no version tagged $tag; nothing to delete" + exit 0 + fi + for id in $ids; do + gh api --method DELETE "$base/$id" + echo "deleted ghcr.io/$owner/$package:$tag (version $id)" + done + echo "Deleted \`ghcr.io/$owner/$package:$tag\`; its preview goes within a minute." >>"$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6771203..680d4e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} BASE: ${{ github.event.pull_request.base.sha }} run: | - all='["smoke-secrets","smoke-chart","smoke-init","smoke-apps","smoke-agent"]' + all='["smoke-secrets","smoke-chart","smoke-init","smoke-apps","smoke-agent","smoke-previews"]' has() { jq -e --arg l "$1" 'index($l) != null' <<<"$LABELS" >/dev/null; } smokes='[]' full=0 if [[ "$EVENT" == push ]]; then diff --git a/CLAUDE.md b/CLAUDE.md index eefebf8..77a4128 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -157,7 +157,7 @@ it there when it belongs to one smoke test. | `just test` | devcontainer | level-1 checks, same as CI (gofmt, vet, `go test -race`, helm lint, kubeconform, examples) | | `just golden` | devcontainer | rewrite golden files and `schema/app.schema.json` after an intended change | | `just build` | devcontainer | build `bin/shelf` (linux) | -| `just cluster-up` | devcontainer | create or start k3d cluster `shelf-dev` with its registry `shelf-registry:5000` (added to `/etc/hosts`), write kubeconfig (run after every container restart or rebuild) | +| `just cluster-up` | devcontainer | create or start k3d cluster `shelf-dev` with its registry `shelf-registry:5000` (added to `/etc/hosts`, deletes enabled), write kubeconfig (run after every container restart or rebuild) | | `just cluster-stop` | devcontainer | stop the cluster to free memory | | `just cluster-down` | devcontainer | delete the cluster | | `just cluster-reset` | devcontainer | delete and recreate the cluster | @@ -172,6 +172,7 @@ it there when it belongs to one smoke test. | `just smoke-init` | devcontainer | Phase 3 acceptance: init twice, re-push, routing and `stripPrefix` through Traefik (run `just cluster-reset` first; needs network) | | `just smoke-apps` | devcontainer | Phase 4: `shelf app add`/`rm`, rollout by polling, tampered artifact refused, restore from backup (needs network) | | `just smoke-agent` | devcontainer | Phase 9: `--json` documents, `--wait --commit` (Ready only after the commit rolled out, Failed for an image that cannot be pulled, TimedOut for a commit that never arrives; with `SMOKE_FULL=1`, the default, also Ready again for a fix after a failure, which takes about 5 more minutes because Helm times out the broken upgrade first) and `shelf app open` with the app's Host and `stripPrefix` (needs Docker Hub) | +| `just smoke-previews` | devcontainer | Phase 10: `examples/hello` with `--previews=2`; two branch tags run as isolated previews with their own database, volume and password; `open --preview`; a new commit keeps the data; the limit holds; a deleted tag removes its preview; `--no-previews` keeps the app; prints push-to-Ready times and RAM per preview (needs Docker Hub) | | `just smoke-tenant ` | devcontainer | Phase 4 acceptance with a real tenant repo and GHCR; asks for the GHCR login, waits for a push | | `just smoke-expose ` | devcontainer | Phase 8b acceptance: the app gets the Cloudflare connection `smoke`, its own tunnel, DNS record and HTTPS, then is taken off again; asks for the Cloudflare API token (set `CF_ACCOUNT_ID` if it sees several accounts); needs a cluster with a host suffix | | `just smoke-quick [app]` | devcontainer | Phase 8d acceptance: `examples/hello` as `hello-quick` with `--quick` gets a quick tunnel, answers with its own Host, gets a new name after a restart, `--private`/`--quick`, rm; keeps the cluster's host suffix; opens a public trycloudflare.com tunnel for a few minutes | diff --git a/README.md b/README.md index 0a2f1de..6da9b75 100644 --- a/README.md +++ b/README.md @@ -202,8 +202,9 @@ The second file is the same for every app: # .github/workflows/deploy.yml on: push: - branches: [main] + branches: ['**'] pull_request: + delete: permissions: contents: read packages: write @@ -213,7 +214,11 @@ jobs: ``` The workflow builds every `build` directory for `linux/arm64`, pushes the images, pins them to -their digests, and publishes the deploy artifact. Everything lives under the package name, which +their digests, and publishes the deploy artifact: as `main` for the default branch, and as +`branch-` for every other branch, which [previews](#previews-of-branches) run. Deleting a +branch deletes its artifact. Pull requests only build. Image layers are cached in the registry +(`:buildcache`), so a small change builds quickly. A workflow that keeps +`branches: [main]` keeps working, without previews. Everything lives under the package name, which is `name` in `app.yaml`: the artifact is `ghcr.io//`, an image is `ghcr.io///`. Image names never appear in `app.yaml`. Ready-made images such as `postgres:16` stay as `image:`. @@ -279,6 +284,38 @@ A coding agent working in the repository waits for its own commit with loop. `shelf app rm ` removes the app with all its data, its tunnel and its DNS record. +### Previews of branches + +Several agents, or several people, work on the same app at once, each on a branch of their own. +With previews, every branch the workflow publishes runs as an isolated copy of the app's +complete stack, database included, next to the app itself: + +```sh +shelf app credentials --previews # at most 3 branches at once; --previews=5 for more +``` + +- **Names.** A branch `feature/login` runs as the preview `--feature-login` in a namespace + of that name. The branch name is shortened to 20 characters, with a hash when it is longer. + That's why app names may not contain `--`. +- **Own state.** A preview has volumes and generated secrets of its own, never the app's, and a + quota (1 CPU, 2Gi memory and 5Gi storage requested). +- **Never exposed.** No domain, tunnel or quick tunnel, whatever the app has; it answers at + `--.shelf.internal` inside the cluster only. To look at one from this machine: + + ```sh + shelf app open --preview feature/login --browser + ``` + +- **Following a branch.** A new commit on the branch is in its preview about half a minute after + CI finished: previews are checked every 30 seconds. An agent waits for it with + `shelf app status --preview --wait --commit "$(git rev-parse HEAD)" --json`. +- **Removal.** Deleting the branch deletes its artifact, and the preview with its data goes + within a minute. `--no-previews` removes all of an app's previews; the app keeps running. +- **Limit.** When more branches are published than the limit allows, the platform picks them + alphabetically, and `shelf app status ` says the limit is reached. +- **Same slug.** Two branches whose names differ only in punctuation, `feature/login` and + `feature-login`, share one preview. + ## Writing an `app.yaml` A complete example with a web frontend, an API and a Postgres database: @@ -408,19 +445,19 @@ drift. | `shelf validate ...` | Checks one or more files without network access. Errors and warnings show file, line and field. Exits with 1 on errors. | | `shelf render ` | Validates, resolves images and prints the deploy manifest (a ConfigMap). `-o app` prints the resolved `app.yaml` instead. `--image =` supplies the image of a component with a `build` directory. Registry credentials come from `docker login`. | | `shelf schema` | Prints the JSON Schema for `app.yaml`. | -| `shelf build-plan ` | Prints the package name (`name` in `app.yaml`) and the components with a `build` directory as JSON. The workflow uses it to name the packages and to know what to build. | +| `shelf build-plan ` | Prints the package name (`name` in `app.yaml`) and the components with a `build` directory as JSON. The workflow uses it to name the packages and to know what to build. `--branch ` adds `tag`, the tag a push to that branch publishes the deploy artifact under (`branch-`). | | `shelf init cluster` | Installs Flux and the platform (Traefik, app management) into the cluster of the current kubecontext, and waits until everything is ready. `--host-suffix -dev` separates clusters that share a DNS zone: apps with a domain are then reachable at `-dev.`. Shows the target cluster and asks before changing anything (`--yes` skips the question); `--context` and `--kubeconfig` pick another cluster. Safe to run again. | | `shelf connection add registry ` | Defines a registry login for `ghcr.io` from `GHCR_USERNAME` and `GHCR_TOKEN`. It is stored in the cluster and copied only into the namespaces of the apps that chose it. | | `shelf connection add cloudflare ` | Defines a Cloudflare account from `CF_API_TOKEN` (and `CF_ACCOUNT_ID` when the token sees several). shelf checks the token and keeps it in `~/.shelf/connections/cloudflare/`, never in the cluster. | | `shelf connection list` / `rm ` | Lists the connections and the apps that use them; removes one no app uses. | | `shelf connection packages ` / `zones ` | Lists the deploy artifacts a registry connection can pull, or the zones of a Cloudflare connection's account. | -| `shelf app add ` | Deploys an app from a deploy artifact under the name `` and keeps it updated. The name is independent of the package, so one artifact can be added under several names. `--registry` picks the registry connection; `--cloudflare` with `--domain` puts the app on the internet under its own domain, `--quick` through a quick tunnel; without either it stays inside the cluster. Generates the app's secrets, stores them in the cluster and in `~/.shelf/apps//secrets.yaml`, and restores them from there after a cluster rebuild. Refuses at once if the cluster has no platform. Waits until the app is ready. Safe to run again, e.g. after adding a secret. | -| `shelf app credentials ` | Changes the app's connections or domain and deploys it again: `--registry`, `--no-registry`, `--cloudflare`, `--domain`, `--quick`, `--private`. | -| `shelf app open ` | Makes the app reachable from a browser on this machine at `http://127.0.0.1:/`, for as long as it runs: requests go through the Kubernetes API server to the cluster's Traefik with the app's own Host header, so routes and `stripPrefix` behave as through a tunnel. Works for apps that stay inside the cluster. Listens on 127.0.0.1 only; `--port` picks the port, `--browser` opens it. It is not an exposure and changes nothing about the app. | +| `shelf app add ` | Deploys an app from a deploy artifact under the name `` and keeps it updated. The name is independent of the package, so one artifact can be added under several names. `--registry` picks the registry connection; `--cloudflare` with `--domain` puts the app on the internet under its own domain, `--quick` through a quick tunnel; without either it stays inside the cluster. Generates the app's secrets, stores them in the cluster and in `~/.shelf/apps//secrets.yaml`, and restores them from there after a cluster rebuild. Refuses at once if the cluster has no platform. Waits until the app is ready. `--previews[=N]` runs a [preview](#previews-of-branches) of each published branch, at most N (default 3). Safe to run again, e.g. after adding a secret. | +| `shelf app credentials ` | Changes the app's connections, domain or previews and deploys it again: `--registry`, `--no-registry`, `--cloudflare`, `--domain`, `--quick`, `--private`, `--previews[=N]`, `--no-previews`. | +| `shelf app open ` | Makes the app reachable from a browser on this machine at `http://127.0.0.1:/`, for as long as it runs: requests go through the Kubernetes API server to the cluster's Traefik with the app's own Host header, so routes and `stripPrefix` behave as through a tunnel. Works for apps that stay inside the cluster. Listens on 127.0.0.1 only; `--port` picks the port, `--browser` opens it. It is not an exposure and changes nothing about the app. `--preview ` opens that branch's preview. | | `shelf app tags ` | Lists the tags of the app's deploy artifact, newest first. | | `shelf app redeploy ` | Fetches the app's artifact again under the tag it is registered with and rolls out what it finds. For a tag that moved, or a deploy worth another try. | | `shelf app secrets ` | Lists the names of the generated secrets; `--reveal` prints the values. | -| `shelf app status [app]` | Lists the apps with their state, or walks one app from its deploy artifact to its running pods and says which step is not ready. For one app it also shows the git commit the running artifact was built from, and names its components and where each of them answers. Same chain as the admin UI shows. `--wait` waits until the app is Ready or a step failed and prints the verdict; `--commit ` first waits until the app runs the artifact built from that commit. Exit status 0 Ready, 1 failed, 2 timed out (`--timeout`, default 10m). | +| `shelf app status [app]` | Lists the apps with their state, or walks one app from its deploy artifact to its running pods and says which step is not ready. For one app it also shows the git commit the running artifact was built from, and names its components and where each of them answers. Same chain as the admin UI shows. `--wait` waits until the app is Ready or a step failed and prints the verdict; `--commit ` first waits until the app runs the artifact built from that commit. Exit status 0 Ready, 1 failed, 2 timed out (`--timeout`, default 10m). For an app with previews it lists them; `--preview ` shows, or waits for, that preview instead. | | `shelf serve` | Runs the admin UI on port 7654 of the local network (`--listen 127.0.0.1:7654` keeps it off the network). The first start prints a setup code that claims the instance; after that a password protects it. | | `shelf app rm ` | Removes an app with its namespace, volumes, secrets, tunnel and DNS record, after asking. The secret backup stays. | | `shelf version` | Prints the version. | @@ -504,7 +541,7 @@ Useful commands: | `just build` | Build `bin/shelf` | | `just cluster-up` / `cluster-stop` / `cluster-down` / `cluster-reset` | Manage the local k3d cluster `shelf-dev` | | `just platform-push`, `just chart-push`, `just init-cluster` | Push `platform/` and the chart to the local registry; `init-cluster` pushes both and installs them with `shelf init cluster` | -| `just smoke-secrets`, `just smoke-chart`, `just smoke-init`, `just smoke-apps`, `just smoke-agent` | Smoke tests against the local cluster, also run in CI: secret expansion, the chart, `shelf init cluster` with routing through Traefik, `shelf app add`/`rm` with rollouts from the local registry, and the agent interface (`--json`, `--wait --commit`, `shelf app open`) | +| `just smoke-secrets`, `just smoke-chart`, `just smoke-init`, `just smoke-apps`, `just smoke-agent`, `just smoke-previews` | Smoke tests against the local cluster, also run in CI: secret expansion, the chart, `shelf init cluster` with routing through Traefik, `shelf app add`/`rm` with rollouts from the local registry, the agent interface (`--json`, `--wait --commit`, `shelf app open`), and branch previews | | `just smoke-registry `, `just smoke-tenant `, `just smoke-expose ` | Smoke tests that need credentials: a private image pull, a real tenant repository end to end, and an app's own Cloudflare tunnel (needs a cluster with a host suffix) | | `just smoke-quick` | A quick tunnel for `examples/hello`; puts it on the internet for a few minutes | | `hack/nuke.sh` | **Run in a terminal on your machine, not in the container.** Removes every Docker object shelf created. | diff --git a/charts/shelf-app/templates/secrets.yaml b/charts/shelf-app/templates/secrets.yaml new file mode 100644 index 0000000..4e3d006 --- /dev/null +++ b/charts/shelf-app/templates/secrets.yaml @@ -0,0 +1,26 @@ +{{- /* +A preview has no `shelf app add` that generates its secrets, so in preview mode the chart does: +every generated secret keeps the value already in shelf-secrets, or gets a new random one. The +values stay in the preview namespace, and go away with it. An app's shelf-secrets is copied by +the platform from shelf-system/app- instead; the chart never renders it then. +*/}} +{{- if .Values.platform.preview }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace "shelf-secrets" }} +{{- $old := dict }} +{{- if $existing }}{{ $old = $existing.data | default dict }}{{ end }} +apiVersion: v1 +kind: Secret +metadata: + name: shelf-secrets + namespace: {{ .Release.Namespace }} + labels: + shelf.dev/app: {{ .Values.name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +type: Opaque +data: + {{- range $name, $secret := .Values.secrets | default dict }} + {{- if and $secret $secret.generate }} + {{ $name }}: {{ index $old $name | default (randAlphaNum 26 | b64enc) }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/shelf-app/values.yaml b/charts/shelf-app/values.yaml index 64cddd3..c0c8826 100644 --- a/charts/shelf-app/values.yaml +++ b/charts/shelf-app/values.yaml @@ -22,3 +22,6 @@ platform: ingressClassName: traefik # Name of a docker-registry Secret in the app namespace used to pull images; empty for none. imagePullSecret: "" + # A preview of a branch: the chart generates the secrets in shelf-secrets itself, keeping the + # values it finds there. Set only by the platform's ResourceSet previews. + preview: false diff --git a/examples/tenant/.claude/skills/shelf/SKILL.md b/examples/tenant/.claude/skills/shelf/SKILL.md index e834262..10361b3 100644 --- a/examples/tenant/.claude/skills/shelf/SKILL.md +++ b/examples/tenant/.claude/skills/shelf/SKILL.md @@ -1,60 +1,88 @@ --- name: shelf -description: Check that a pushed commit of this app runs on shelf, and find out why it does not. Use after every git push to main, and whenever the user asks whether the app is deployed, healthy or broken. +description: Check that a pushed commit of this app runs on shelf, on main or in the preview of a branch, and find out why it does not. Use after every git push, and whenever the user asks whether the app or a branch is deployed, healthy or broken. --- # Deploying this app with shelf This repository is a shelf tenant: CI builds the images, renders `app.yaml` and pushes a deploy -artifact; shelf notices it within a minute and rolls it out. You never deploy by hand. Your job -after a push is to wait for the verdict on *your* commit and fix what it names. +artifact; shelf notices it and rolls it out. You never deploy by hand. Your job after a push is +to get the verdict on *your* commit and fix what it names, without sitting idle while CI runs. The app's name on shelf is in `CLAUDE.md` (the operator chose it; it is not `name` in `app.yaml`). Below it is written ``. +## Where your commit runs + +- **`main`** runs as the app itself. +- **Any other branch** runs as a preview of its own, if the operator switched previews on: an + isolated copy of the whole app, database included, named `--` (`feature/login` + becomes `--feature-login`). For such a branch, add `--preview ` to every command + below. +- The first push of a new branch takes about a minute longer than later ones: the preview is + created first. +- If the wait says the app has no previews, work on `main`, or ask the operator. + ## The loop -1. Commit and push to `main`. -2. Wait for the verdict on exactly that commit: +1. Commit and push. +2. Start the wait in the background, and go on with other work. Do not wait in the foreground + for CI: ```sh shelf app status --wait --commit "$(git rev-parse HEAD)" --json + shelf app status --preview --wait --commit "$(git rev-parse HEAD)" --json ``` - This blocks until the app runs that commit and is Ready, a step failed, or 10 minutes - passed (`--timeout 20m` for a slow build). It only reads the cluster; never try to make it - faster. -3. Read the exit status and the `WaitResult` document on stdout: + The first line is for `main`, the second for a branch. The wait returns when that commit runs + and is Ready, when a step failed, or after 10 minutes (`--timeout 20m` for a slow build). It + only reads the cluster; never try to make it faster. +3. If the GitHub CLI is available, watch the CI run of your push in the background as well: + `gh run watch --exit-status` on the run of your commit (`gh run list --commit `). A + failed build shows there within seconds. Read it with `gh run view --log-failed`, fix it, + and push again. Don't wait for the shelf wait to time out on an artifact that will never + come. +4. Before you report the work as done, read the wait's exit status and the `WaitResult` on + stdout: | Exit | `outcome` | What to do | |---|---|---| | 0 | `Ready` | Done. The commit runs. | | 1 | `Failed` | Read `stage.name`, `stage.reason`, `stage.message` and `stage.hint`, fix the cause in this repository, commit, push, and wait again. | - | 2 | `TimedOut` | `stage` says where it waited. "waiting for commit …" means the artifact of your commit never arrived: check the CI run of the push first (`gh run list`, `gh run view --log-failed`). | + | 2 | `TimedOut` | `stage` says where it waited. "waiting for commit …" or "waiting for the preview of branch …" means the artifact of your commit never arrived: check the CI run of the push first (`gh run list`, `gh run view --log-failed`). | After a failed commit, the next one can wait several minutes at "the app" with "waiting for the release to pick up the new values": Helm first gives up on the broken upgrade and rolls it back. That is expected; keep waiting. Any other failure prints a document of kind `Error` with a `message` (no cluster, no such - app) and exits 1 without a `WaitResult`; report it, it is not something to fix in code. + app, no previews) and exits 1 without a `WaitResult`. Report it; it is not something to fix + in code. -4. For the whole picture of one app, run `shelf app status --json`: every stage from the - deploy artifact to the pods, with the running `commit`, the components and their addresses. +5. For the whole picture, run `shelf app status --json` (or with `--preview `): + - every stage from the deploy artifact to the pods; + - the running `commit`, the components and their addresses; + - for the app, its `previews`. ## What the stages mean -`the deploy artifact` (CI did not push, or the registry refuses the login) → `the app values` -(the rendered app.yaml was rejected) → `the chart` (platform problem; report it) → `the app` -(the Helm release; often a bad `app.yaml`) → `the pods` (image pull, crash loop, out of memory, -missing secret). The first stage that is not Ready is the cause; everything after it is a -consequence. `shelf validate app.yaml` catches most app.yaml mistakes before a push. +The stages, in order: +1. `the deploy artifact`: CI did not push, or the registry refuses the login. +2. `the app values`: the rendered app.yaml was rejected. +3. `the chart`: a platform problem; report it. +4. `the app`: the Helm release; often a bad `app.yaml`. +5. `the pods`: image pull, crash loop, out of memory, missing secret, or a preview's quota. + +The first stage that is not Ready is the cause; everything after it is a consequence. +`shelf validate app.yaml` catches most app.yaml mistakes before a push. ## What you cannot do -- Make the app public. Exposure (a domain or a tunnel) is the operator's decision, made with - `shelf app add` or `shelf app credentials` outside this repository. Nothing in `app.yaml` - changes it, so do not try. -- Read secret values. `shelf app secrets` is for the operator. -- `shelf app open ` makes the app reachable in a browser on the operator's machine. It is - for the person, not for you; ask them to look if a page has to be checked by eye. +- Make the app or a preview public. Exposure (a domain or a tunnel) is the operator's decision, + made with `shelf app add` or `shelf app credentials` outside this repository. Nothing in + `app.yaml` changes it, so do not try. Previews are never exposed. +- Switch previews on or off; that is the operator's decision too. +- Read secret values. `shelf app secrets` is for the operator; a preview's secrets are its own. +- `shelf app open ` (with `--preview ` for a preview) makes the app reachable in a + browser on the operator's machine. It is for the person, not for you; ask them to look if a + page has to be checked by eye. diff --git a/examples/tenant/.github/workflows/deploy.yml b/examples/tenant/.github/workflows/deploy.yml index 37849e5..7d042d0 100644 --- a/examples/tenant/.github/workflows/deploy.yml +++ b/examples/tenant/.github/workflows/deploy.yml @@ -2,10 +2,13 @@ # ghcr.io/tweinmann/greeter-deploy. Nothing in here is specific to this app. name: deploy +# Every branch publishes, so that an app with previews runs each one on its own; deleting a branch +# deletes its artifact, and its preview with it. Pull requests only build. on: push: - branches: [main] + branches: ['**'] pull_request: + delete: permissions: contents: read diff --git a/examples/tenant/CLAUDE.md b/examples/tenant/CLAUDE.md index 1e69741..07cf3af 100644 --- a/examples/tenant/CLAUDE.md +++ b/examples/tenant/CLAUDE.md @@ -1,13 +1,15 @@ # greeter A shelf tenant: `app.yaml` describes the app, `.github/workflows/deploy.yml` builds and -publishes it, and shelf deploys every push to `main`. +publishes it, and shelf deploys every push to `main`, and every other branch as a preview of its +own if the operator switched previews on. ## Deploying The app runs on shelf as **`greeter`**. (Operator: replace this with the name you gave it in `shelf app add …`; it need not match `name` in `app.yaml`.) -After every push to `main`, use the `shelf` skill: wait with -`shelf app status greeter --wait --commit "$(git rev-parse HEAD)" --json` and fix what the -verdict names. The work is done when that command exits 0. +After every push, use the `shelf` skill: start +`shelf app status greeter --wait --commit "$(git rev-parse HEAD)" --json` in the background (on a +branch, with `--preview `), keep working, and fix what the verdict names. The work is +done when that command exits 0. diff --git a/examples/tenant/README.md b/examples/tenant/README.md index 2ad1095..5bb4168 100644 --- a/examples/tenant/README.md +++ b/examples/tenant/README.md @@ -27,6 +27,8 @@ this repository: after a push, wait for the verdict on that commit with shelf app status greeter --wait --commit "$(git rev-parse HEAD)" --json ``` -and fix what it names. Put the name you gave the app in `shelf app add` into `CLAUDE.md`. The +and fix what it names; on a branch with `--preview `, if the operator switched previews +on (`shelf app credentials greeter --previews`). Put the name you gave the app in +`shelf app add` into `CLAUDE.md`. The agent cannot make the app public: exposure is set with `shelf app add` or `shelf app credentials`, never in this repository. diff --git a/hack/cluster-up.sh b/hack/cluster-up.sh index f6a036b..a547bf6 100755 --- a/hack/cluster-up.sh +++ b/hack/cluster-up.sh @@ -27,6 +27,16 @@ else --wait fi +# registry:2 refuses to delete manifests unless told to. Previews disappear when their branch tag +# is deleted, so level 2 has to be able to delete one, as GHCR can. The config file lives in the +# registry container, which keeps it across restarts; this runs on every cluster-up. +registry_config=/etc/docker/registry/config.yml +if ! docker exec "$SHELF_REGISTRY" grep -q '^ delete:' "$registry_config"; then + docker exec "$SHELF_REGISTRY" sed -i 's/^storage:$/storage:\n delete:\n enabled: true/' "$registry_config" + docker restart "$SHELF_REGISTRY" >/dev/null + echo "registry $SHELF_REGISTRY: deleting manifests enabled" +fi + # The registry is published on this container's loopback; give it the name pods use. /etc/hosts # is recreated with the container, so this runs on every cluster-up. if ! grep -qE "^127\.0\.0\.1[[:space:]]+$SHELF_REGISTRY\$" /etc/hosts; then diff --git a/hack/smoke/previews.sh b/hack/smoke/previews.sh new file mode 100755 index 0000000..0d2ac34 --- /dev/null +++ b/hack/smoke/previews.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Checks the Phase 10 branch previews against the dev cluster, with the dev registry standing in +# for GHCR: examples/hello (web plus Postgres with a volume) gets previews; two branch tags run as +# two isolated previews, each with its own database, volume and password; `shelf app open +# --preview` reaches one with its own Host; a new commit on a branch keeps the preview's data; +# the limit holds; deleting a tag removes its preview; --no-previews removes them all and leaves +# the app running. It prints the seconds from push to Ready and the RAM of a preview. +# +# Nothing here nudges Flux: the times are what an agent waits. Installs or updates the platform +# first (idempotent). Needs network access for images. +set -euo pipefail +source "$(dirname "$0")/../lib.sh" +require_devcontainer + +repo="$(cd "$(dirname "$0")/../.." && pwd)" +work="$(mktemp -d)" +app=prev-smoke +artifact="oci://$SHELF_REGISTRY_HOST/smoke/$app-deploy" +open_pid="" +export SHELF_HOME="$work/home" + +shelf() { "$work/shelf" "$@"; } + +cleanup() { + [[ -n "$open_pid" ]] && kill "$open_pid" 2>/dev/null || true + if [[ "${KEEP:-}" == 1 ]]; then + echo "KEEP=1: leaving app $app in place; its secret backup is in $SHELF_HOME" + return + fi + shelf app rm "$app" --yes >/dev/null 2>&1 || true + rm -rf "$work" +} + +step() { echo "--- $*"; } + +commit() { head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n'; } + +# publish : pushes examples/hello as the deploy artifact under tag, built from +# commit. +publish() { + flux push artifact "$artifact:$1" --path "$work/hello" --source local --revision "$1@sha1:$2" \ + --insecure-registry >/dev/null 2>&1 || die "push of $1 failed" +} + +# unpublish : deletes a tag from the dev registry, as build.yml does on GHCR for a deleted +# branch. +unpublish() { + local path="${artifact#oci://$SHELF_REGISTRY_HOST/}" digest + digest="$(curl -fsS -I -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "http://$SHELF_REGISTRY_HOST/v2/$path/manifests/$1" | tr -d '\r' | + awk -F': ' 'tolower($1)=="docker-content-digest"{print $2}')" + [[ -n "$digest" ]] || die "no digest for $1" + curl -fsS -X DELETE "http://$SHELF_REGISTRY_HOST/v2/$path/manifests/$digest" >/dev/null \ + || die "the registry refused to delete $1; run just cluster-up" +} + +# password : the hash of the generated db-password there, never the value. +password() { + kubectl -n "$1" get secret shelf-secrets -o jsonpath='{.data.db-password}' | sha256sum | cut -c1-16 +} + +# wait_preview : waits for the preview of branch at commit and prints seconds. +wait_preview() { + local start=$SECONDS code=0 + shelf app status "$app" --preview "$1" --wait --commit "$2" --timeout 5m --json >"$work/wait.json" || code=$? + [[ "$code" == 0 ]] || die "preview of $1 at $2: exit $code, $(cat "$work/wait.json")" + echo $((SECONDS - start)) +} + +# gone : waits up to 3 minutes for a namespace to disappear. +gone() { + for _ in $(seq 90); do + kubectl get namespace "$1" >/dev/null 2>&1 || return 0 + sleep 2 + done + return 1 +} + +for leftover in "namespace/$app" "-n shelf-system resourcesetinputprovider/$app"; do + # shellcheck disable=SC2086 + kubectl get $leftover >/dev/null 2>&1 && die "$leftover already exists; remove it first (shelf app rm $app)" +done +trap cleanup EXIT + +step "build shelf, push platform and chart, init cluster" +(cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) +"$repo/hack/platform-push.sh" >/dev/null 2>&1 +"$repo/hack/chart-push.sh" >/dev/null 2>&1 +shelf init cluster --yes --host-suffix "$(cluster_host_suffix)" --insecure-registry \ + --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ + --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" >/dev/null +kubectl -n shelf-system get resourceset previews >/dev/null || die "the platform has no ResourceSet previews" + +step "add the app with previews of at most two branches" +mkdir -p "$work/hello" +(cd "$repo" && go run ./cmd/shelf render examples/hello/app.yaml) >"$work/hello/configmap.yaml" 2>/dev/null +publish main "$(commit)" +shelf app add "$app" "$artifact:main" --insecure-registry --previews=2 >/dev/null +[[ "$(kubectl -n shelf-system get resourcesetinputprovider "$app--previews" -o jsonpath='{.spec.type}')" == OCIArtifactTag ]] \ + || die "no preview provider" +shelf app status --json | jq -e --arg app "$app" '[.items[].name] | index($app + "--previews") == null' >/dev/null \ + || die "the preview provider is listed as an app" + +step "two branches run as two previews" +ca="$(commit)" +publish branch-alpha "$ca" +first="$(wait_preview alpha "$ca")" +echo "first push of a branch to Ready: $first s" +cb="$(commit)" +publish branch-beta "$cb" +wait_preview beta "$cb" >/dev/null +for ns in "$app--alpha" "$app--beta"; do + [[ "$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.shelf\.dev/app}')" == "$app" ]] \ + || die "$ns is not labelled as a preview of $app" + kubectl -n "$ns" get pvc data-db-0 >/dev/null || die "$ns has no volume of its own" + kubectl -n "$ns" get deploy cloudflared >/dev/null 2>&1 && die "$ns runs cloudflared" + kubectl -n "$ns" get resourcequota shelf-preview >/dev/null || die "$ns has no quota" +done +hashes="$(printf '%s\n' "$(password "$app")" "$(password "$app--alpha")" "$(password "$app--beta")" | sort -u | wc -l)" +[[ "$hashes" == 3 ]] || die "the app and its previews share a password" + +step "shelf app open --preview reaches the preview with its own Host" +port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')" +shelf app open "$app" --preview alpha --port "$port" >"$work/open.txt" 2>&1 & +open_pid=$! +up() { curl -fsS "http://127.0.0.1:$port/" >/dev/null 2>&1; } +for _ in $(seq 30); do up && break; sleep 1; done +curl -fsS "http://127.0.0.1:$port/" | grep -q "^Host: $app--alpha.shelf.internal" \ + || die "the preview did not get its own Host: $(cat "$work/open.txt")" +kill "$open_pid"; wait "$open_pid" 2>/dev/null || true; open_pid="" + +step "a new commit on a branch keeps the preview's data" +before="$(password "$app--alpha")" volume="$(kubectl -n "$app--alpha" get pvc data-db-0 -o jsonpath='{.metadata.uid}')" +ca2="$(commit)" +publish branch-alpha "$ca2" +later="$(wait_preview alpha "$ca2")" +echo "later commit on a branch to Ready: $later s" +[[ "$(password "$app--alpha")" == "$before" ]] || die "the new commit changed the preview's password" +[[ "$(kubectl -n "$app--alpha" get pvc data-db-0 -o jsonpath='{.metadata.uid}')" == "$volume" ]] \ + || die "the new commit replaced the preview's volume" + +step "RAM of a preview" +sleep 60 # metrics-server reports a new pod after up to a minute +kubectl top pods -n "$app--alpha" --no-headers | awk '{sub("Mi","",$3); sum+=$3} END {print "preview RAM: " sum " Mi"}' +kubectl top pods -n "$app" --no-headers | awk '{sub("Mi","",$3); sum+=$3} END {print "app RAM: " sum " Mi"}' + +step "the limit holds" +publish branch-gamma "$(commit)" +for _ in $(seq 60); do + [[ "$(kubectl get namespace -l "shelf.dev/app=$app,shelf.dev/preview" --no-headers 2>/dev/null | wc -l)" == 2 ]] \ + && kubectl get namespace "$app--gamma" >/dev/null 2>&1 && break + sleep 2 +done +shelf app status "$app" --json | jq -e '.previewLimit == 2 and .previewLimitReached == true and (.previews | length) <= 2' >/dev/null \ + || die "the limit does not hold: $(shelf app status "$app" --json | jq -c '{previews: [.previews[]?.slug], previewLimit, previewLimitReached}')" +echo "previews within the limit: $(shelf app status "$app" --json | jq -r '[.previews[].slug] | join(", ")')" + +step "deleting a branch tag removes its preview" +unpublish branch-gamma +start=$SECONDS +gone "$app--gamma" || die "the preview of a deleted tag is still there" +echo "deleted tag to preview gone: $((SECONDS - start)) s" + +step "--no-previews removes every preview and keeps the app" +shelf app credentials "$app" --no-previews >/dev/null +for ns in "$app--alpha" "$app--beta"; do gone "$ns" || die "$ns is still there after --no-previews"; done +shelf app status "$app" --json | jq -e '.phase == "Ready" and (.previews == null)' >/dev/null || die "the app did not survive --no-previews" + +step "remove the app" +shelf app rm "$app" --yes >/dev/null +echo "smoke-previews passed" diff --git a/internal/chart/chart_test.go b/internal/chart/chart_test.go index 5a8573a..6cfd01e 100644 --- a/internal/chart/chart_test.go +++ b/internal/chart/chart_test.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "testing" @@ -132,6 +133,11 @@ func TestTemplate(t *testing.T) { app: "testdata/features.app.yaml", platform: "platform:\n domain: example.com\n imagePullSecret: ghcr-pull\n", apis: []string{middlewareAPI}, + }, { + // A preview generates its secrets; the golden file holds a placeholder instead. + name: "preview", + app: "../../examples/hello/app.yaml", + platform: "platform:\n domain: shelf.internal\n imagePullSecret: shelf-registry\n preview: true\n", }, } for _, tt := range tests { @@ -151,11 +157,18 @@ func TestTemplate(t *testing.T) { if strings.Contains(out, "LoadBalancer") { t.Error("the chart must never render a LoadBalancer") } + if strings.Contains(out, "name: shelf-secrets\n namespace:") != (tt.name == "preview") { + t.Errorf("shelf-secrets rendered %v, want only in preview mode", !(tt.name == "preview")) + } + out = generatedSecret.ReplaceAllString(out, "${1}") testutil.Golden(t, filepath.Join("testdata", tt.name+".manifests.yaml"), []byte(out)) }) } } +// generatedSecret matches a value the chart generated: 26 random characters, base64-encoded. +var generatedSecret = regexp.MustCompile(`(?m)^( [a-z0-9-]+: )[A-Za-z0-9+/=]{36}$`) + // TestSecretReferences checks which secrets a container receives: only those referenced // outside an escaped $$, each once, before all other variables. func TestSecretReferences(t *testing.T) { diff --git a/internal/chart/testdata/preview.manifests.yaml b/internal/chart/testdata/preview.manifests.yaml new file mode 100644 index 0000000..70e1231 --- /dev/null +++ b/internal/chart/testdata/preview.manifests.yaml @@ -0,0 +1,288 @@ +--- +# Source: shelf-app/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: shelf-secrets + namespace: hello + labels: + shelf.dev/app: hello + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + db-password: + +--- +# Source: shelf-app/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: db + labels: + shelf.dev/app: hello + shelf.dev/component: db + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 +spec: + type: ClusterIP + selector: + shelf.dev/app: hello + shelf.dev/component: db + ports: + - name: main + port: 5432 + targetPort: main + protocol: TCP + +--- +# Source: shelf-app/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: db-headless + labels: + shelf.dev/app: hello + shelf.dev/component: db + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 +spec: + clusterIP: None + selector: + shelf.dev/app: hello + shelf.dev/component: db + ports: + - name: main + port: 5432 + targetPort: main + protocol: TCP + +--- +# Source: shelf-app/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: web + labels: + shelf.dev/app: hello + shelf.dev/component: web + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 +spec: + type: ClusterIP + selector: + shelf.dev/app: hello + shelf.dev/component: web + ports: + - name: main + port: 80 + targetPort: main + protocol: TCP + +--- +# Source: shelf-app/templates/workloads.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: check + labels: + shelf.dev/app: hello + shelf.dev/component: check + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 +spec: + replicas: 1 + revisionHistoryLimit: 3 + selector: + matchLabels: + shelf.dev/app: hello + shelf.dev/component: check + template: + metadata: + labels: + shelf.dev/app: hello + shelf.dev/component: check + spec: + # Service links would inject variables such as DB_PORT=tcp://... into every container. + enableServiceLinks: false + automountServiceAccountToken: false + imagePullSecrets: + - name: shelf-registry + containers: + - name: check + image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + command: + - "sh" + - "-c" + args: + - "while true; do psql \"$$DATABASE_URL\" -tAc 'select 1' && sleep 30 || sleep 5; done" + env: + - name: SHELF_SECRET_DB_PASSWORD + valueFrom: + secretKeyRef: + name: shelf-secrets + key: db-password + - name: DATABASE_URL + value: "postgres://app:$(SHELF_SECRET_DB_PASSWORD)@db:5432/hello" + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + memory: "64Mi" + +--- +# Source: shelf-app/templates/workloads.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + labels: + shelf.dev/app: hello + shelf.dev/component: web + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 +spec: + replicas: 2 + revisionHistoryLimit: 3 + selector: + matchLabels: + shelf.dev/app: hello + shelf.dev/component: web + template: + metadata: + labels: + shelf.dev/app: hello + shelf.dev/component: web + spec: + # Service links would inject variables such as DB_PORT=tcp://... into every container. + enableServiceLinks: false + automountServiceAccountToken: false + imagePullSecrets: + - name: shelf-registry + containers: + - name: web + image: "traefik/whoami:v1.11.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + ports: + - name: main + containerPort: 80 + protocol: TCP + readinessProbe: + httpGet: + path: "/health" + port: main + periodSeconds: 10 + failureThreshold: 3 + livenessProbe: + httpGet: + path: "/health" + port: main + periodSeconds: 10 + failureThreshold: 6 + resources: + requests: + cpu: "50m" + memory: "32Mi" + limits: + memory: "32Mi" + +--- +# Source: shelf-app/templates/workloads.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: db + labels: + shelf.dev/app: hello + shelf.dev/component: db + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 +spec: + replicas: 1 + revisionHistoryLimit: 3 + selector: + matchLabels: + shelf.dev/app: hello + shelf.dev/component: db + serviceName: db-headless + # Instances do not form a cluster, so they need not start one after the other. + podManagementPolicy: Parallel + persistentVolumeClaimRetentionPolicy: + whenScaled: Retain + whenDeleted: Delete + template: + metadata: + labels: + shelf.dev/app: hello + shelf.dev/component: db + spec: + # Service links would inject variables such as DB_PORT=tcp://... into every container. + enableServiceLinks: false + automountServiceAccountToken: false + imagePullSecrets: + - name: shelf-registry + containers: + - name: db + image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + env: + - name: SHELF_SECRET_DB_PASSWORD + valueFrom: + secretKeyRef: + name: shelf-secrets + key: db-password + - name: PGDATA + value: "/var/lib/postgresql/data/pgdata" + - name: POSTGRES_DB + value: "hello" + - name: POSTGRES_PASSWORD + value: "$(SHELF_SECRET_DB_PASSWORD)" + - name: POSTGRES_USER + value: "app" + ports: + - name: main + containerPort: 5432 + protocol: TCP + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + memory: "256Mi" + volumeMounts: + - name: data + mountPath: "/var/lib/postgresql/data" + volumeClaimTemplates: + - metadata: + name: data + labels: + shelf.dev/app: hello + shelf.dev/component: db + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: "1Gi" + +--- +# Source: shelf-app/templates/ingresses.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: web + labels: + shelf.dev/app: hello + shelf.dev/component: web + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 +spec: + ingressClassName: traefik + rules: + - host: hello.shelf.internal + http: + paths: + - path: "/" + pathType: Prefix + backend: + service: + name: web + port: + name: main diff --git a/internal/cli/app.go b/internal/cli/app.go index 424865f..2af58db 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "strconv" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -44,10 +45,15 @@ type accessFlags struct { noCloudflare bool quick bool private bool + previews int + noPreviews bool + // fs tells a --previews=0 that was given from one that was not. + fs *pflag.FlagSet } // register adds the flags; removable adds the ones that take a connection away again. func (f *accessFlags) register(fs *pflag.FlagSet, removable bool) { + f.fs = fs fs.StringVar(&f.domain, "domain", "", "a domain of the app's own: it answers at ., and without one at "+ ".shelf.internal inside the cluster; needed with --cloudflare") @@ -55,6 +61,11 @@ func (f *accessFlags) register(fs *pflag.FlagSet, removable bool) { fs.StringVar(&f.cloudflare, "cloudflare", "", "the Cloudflare connection the app is exposed through") fs.BoolVar(&f.quick, "quick", false, "expose the app through a quick tunnel at a temporary trycloudflare.com address; a tunnel and a record it had are deleted") + fs.IntVar(&f.previews, "previews", 0, fmt.Sprintf( + "run a preview of each branch the workflow publishes, at most %d at once (--previews=N for another number, up to %d); "+ + "previews are reachable from this machine only, with --preview", ops.DefaultPreviews, ops.MaxPreviews)) + fs.Lookup("previews").NoOptDefVal = strconv.Itoa(ops.DefaultPreviews) + fs.BoolVar(&f.noPreviews, "no-previews", false, "remove the app's previews, with their data") if removable { fs.BoolVar(&f.noRegistry, "no-registry", false, "pull without a login") fs.BoolVar(&f.private, "private", false, @@ -63,13 +74,29 @@ func (f *accessFlags) register(fs *pflag.FlagSet, removable bool) { } } +// exactArgs is cobra.ExactArgs, except that it names the likely mistake behind one argument too +// many: --previews takes its number after "=", so "--previews 5" leaves the 5 behind. +func exactArgs(n int) cobra.PositionalArgs { + return func(cmd *cobra.Command, args []string) error { + if len(args) == n+1 && cmd.Flags().Changed("previews") { + if v, err := strconv.Atoi(args[n]); err == nil { + return fmt.Errorf("write --previews=%d; --previews alone means %d", v, ops.DefaultPreviews) + } + } + return cobra.ExactArgs(n)(cmd, args) + } +} + // changes reports whether any flag asks for a change. func (f *accessFlags) changes() bool { return f.domain != "" || f.registry != "" || f.noRegistry || f.cloudflare != "" || f.noCloudflare || - f.quick || f.private + f.quick || f.private || f.previews != 0 || f.noPreviews } func (f *accessFlags) access() (ops.Access, error) { + if f.fs != nil && f.fs.Changed("previews") && f.previews < 1 { + return ops.Access{}, fmt.Errorf("an app may have between 1 and %d previews, not %d", ops.MaxPreviews, f.previews) + } a := ops.Access{ Domain: f.domain, Registry: f.registry, @@ -77,6 +104,8 @@ func (f *accessFlags) access() (ops.Access, error) { Cloudflare: f.cloudflare, Quick: f.quick, Private: f.private || f.noCloudflare, + Previews: f.previews, + NoPreviews: f.noPreviews, } return a, a.Check() } @@ -100,7 +129,7 @@ artifact reference, generates secrets that were added to app.yaml since, and kee values, and the app's domain and connections unless others are given. ` + accessHelp, - Args: cobra.ExactArgs(2), + Args: exactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] if err := ops.CheckAppName(name); err != nil { @@ -153,7 +182,7 @@ tunnel if the account changes; --quick moves it to a quick tunnel, and --private the internet. ` + accessHelp, - Args: cobra.ExactArgs(1), + Args: exactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] if err := ops.CheckAppName(name); err != nil { @@ -161,7 +190,7 @@ the internet. } if !access.changes() { return fmt.Errorf("nothing to change; pass --domain, --registry, --cloudflare, " + - "--no-registry, --quick or --private") + "--no-registry, --quick, --private, --previews or --no-previews") } given, err := access.access() if err != nil { diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index fc05848..37d4dbc 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -302,3 +302,15 @@ func TestAppStatusShowsTheCommit(t *testing.T) { }) } } + +func TestAppNameWithDoubleHyphen(t *testing.T) { + t.Parallel() + h := newHarness(t).public() + _, stderr, code := h.run(t, "app", "add", "hello--copy", greeterArtifact) + if code != 1 || !strings.Contains(stderr, "reserved for previews") { + t.Errorf("exit code %d, stderr %q", code, stderr) + } + if len(h.cluster.added) != 0 { + t.Error("the cluster was contacted") + } +} diff --git a/internal/cli/build.go b/internal/cli/build.go index 2a2d4f2..6d17068 100644 --- a/internal/cli/build.go +++ b/internal/cli/build.go @@ -23,6 +23,8 @@ var componentNameRE = regexp.MustCompile(schema.ComponentNamePattern) type buildPlan struct { Package string `json:"package"` Builds []buildItem `json:"builds"` + // Tag is the tag a branch's deploy artifact is published under, with --branch only. + Tag string `json:"tag,omitempty"` } // buildItem is one component that has to be built before rendering. @@ -33,7 +35,8 @@ type buildItem struct { } func newBuildPlanCmd() *cobra.Command { - return &cobra.Command{ + var branch string + cmd := &cobra.Command{ Use: "build-plan ", Short: "Print the package name and the components that have to be built, as JSON", Long: `Print {package, builds} as JSON: the package name (app.yaml's name), and every component @@ -43,7 +46,10 @@ the working directory. A CI workflow publishes the deploy artifact as ghcr.io// and the images below it: it builds each component, pushes the image and passes the result to ` + "`shelf render --image =`" + `. The app's name in shelf is chosen at -` + "`shelf app add`" + `, so one package can be deployed as several apps.`, +` + "`shelf app add`" + `, so one package can be deployed as several apps. + +With --branch, the plan also holds the tag a push to that branch publishes the deploy artifact +under, branch-. An app with previews runs it as --.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { file := args[0] @@ -52,6 +58,11 @@ it: it builds each component, pushes the image and passes the result to return err } plan := buildPlan{Package: doc.App.Name, Builds: []buildItem{}} + if branch != "" { + if plan.Tag = schema.BranchTag(branch); plan.Tag == schema.BranchTagPrefix { + return fmt.Errorf("branch %q has no letters or digits to name a preview by", branch) + } + } for _, compName := range slices.Sorted(maps.Keys(doc.App.Components)) { comp := doc.App.Components[compName] if comp == nil || !comp.IsBuilt() { @@ -71,6 +82,8 @@ it: it builds each component, pushes the image and passes the result to return err }, } + cmd.Flags().StringVar(&branch, "branch", "", "also print the tag a push to this branch publishes, branch-") + return cmd } // parseImages turns the --image flags (component=reference) into a map. diff --git a/internal/cli/build_test.go b/internal/cli/build_test.go index 7eebc11..883da66 100644 --- a/internal/cli/build_test.go +++ b/internal/cli/build_test.go @@ -94,3 +94,18 @@ func TestParseImages(t *testing.T) { t.Error("a repeated component must be rejected") } } + +func TestBuildPlanBranch(t *testing.T) { + file := writeApp(t, "apiVersion: shelf.dev/v1alpha1\nname: shop\ncomponents:\n db: { image: postgres:16 }\n") + stdout, _, code := run(t, "build-plan", file, "--branch", "feature/Login") + var plan buildPlan + if code != 0 || json.Unmarshal([]byte(stdout), &plan) != nil || plan.Tag != "branch-feature-login" { + t.Errorf("exit code %d, stdout %q", code, stdout) + } + if stdout, _, _ := run(t, "build-plan", file); strings.Contains(stdout, `"tag"`) { + t.Errorf("a plan without --branch has a tag: %s", stdout) + } + if _, stderr, code := run(t, "build-plan", file, "--branch", "///"); code == 0 || !strings.Contains(stderr, "no letters or digits") { + t.Errorf("empty slug: exit code %d, stderr %q", code, stderr) + } +} diff --git a/internal/cli/harness_test.go b/internal/cli/harness_test.go index 629c549..bd08b99 100644 --- a/internal/cli/harness_test.go +++ b/internal/cli/harness_test.go @@ -59,6 +59,9 @@ type fakeCluster struct { // rollouts are what AppRollout returns, one per call; the last one is repeated. rollouts []cluster.Rollout rolloutCalls int + // rolloutMissing is how many reads of a rollout find nothing before it appears, as a preview + // that the platform has not created yet. + rolloutMissing int // traefik is the address ForwardTraefik hands out, empty when no Traefik pod is ready. traefik string found bool @@ -81,11 +84,14 @@ func (f *fakeCluster) Install(_ context.Context, cfg *rest.Config, o cluster.Opt return nil } -func (f *fakeCluster) CheckPlatform(_ context.Context, _ *rest.Config, quick bool) error { +func (f *fakeCluster) CheckPlatform(_ context.Context, _ *rest.Config, needs cluster.PlatformNeeds) error { switch { case f.noPlatform: return errors.New("this cluster has no shelf platform; install it with `shelf init cluster`") - case f.oldPlatform && quick: + case f.oldPlatform && needs.Previews: + return errors.New("the platform in this cluster predates previews; install the platform of " + + "this shelf with `shelf init cluster` first, or leave out --previews") + case f.oldPlatform && needs.Quick: return errors.New("the platform in this cluster does not run quick tunnels yet; " + "install the platform of this shelf with `shelf init cluster` first, or add the app without --quick") } @@ -104,6 +110,7 @@ func (f *fakeCluster) AddApp(_ context.Context, _ *rest.Config, o cluster.AppOpt config := &cluster.AppConfig{ Artifact: o.Artifact, Insecure: o.Insecure, Domain: o.Domain, Registry: o.Registry, Cloudflare: o.Cloudflare, TunnelID: o.TunnelID, TunnelCredentials: o.TunnelCredentials, Quick: o.Quick, + Previews: o.Previews, } if previous := f.configs[o.Name]; previous != nil && o.TunnelCredentials == nil && o.TunnelID != "" { config.TunnelCredentials = previous.TunnelCredentials @@ -113,7 +120,7 @@ func (f *fakeCluster) AddApp(_ context.Context, _ *rest.Config, o cluster.AppOpt f.states = slices.DeleteFunc(f.states, func(s cluster.AppState) bool { return s.Name == o.Name }) f.states = append(f.states, cluster.AppState{ Name: o.Name, Artifact: o.Artifact, Domain: o.Domain, Tunnel: o.TunnelID, Quick: o.Quick, - Registry: o.Registry, Cloudflare: o.Cloudflare, + Registry: o.Registry, Cloudflare: o.Cloudflare, PreviewLimit: o.Previews, }) return nil } @@ -151,8 +158,13 @@ func (f *fakeCluster) AppDiagnosis(_ context.Context, _ *rest.Config, app string } func (f *fakeCluster) AppRollout(_ context.Context, _ *rest.Config, app string) (cluster.Rollout, error) { - if !slices.ContainsFunc(f.states, func(s cluster.AppState) bool { return s.Name == app }) { - return cluster.Rollout{}, errors.New("app " + app + " does not exist") + if f.rolloutMissing > 0 { + f.rolloutMissing-- + return cluster.Rollout{}, &cluster.NotFoundError{Name: app} + } + if !slices.ContainsFunc(f.states, func(s cluster.AppState) bool { return s.Name == app }) && + !strings.Contains(app, "--") { + return cluster.Rollout{}, &cluster.NotFoundError{Name: app} } r := f.rollouts[min(f.rolloutCalls, len(f.rollouts)-1)] f.rolloutCalls++ diff --git a/internal/cli/json.go b/internal/cli/json.go index e24885f..21fad3c 100644 --- a/internal/cli/json.go +++ b/internal/cli/json.go @@ -103,9 +103,22 @@ type stageDocument struct { Hint string `json:"hint,omitempty"` } +// previewDocument is one preview in the AppStatus of its app. The cluster knows a branch by its +// slug only, so that is what names it. +type previewDocument struct { + Slug string `json:"slug"` + Tag string `json:"tag"` + Namespace string `json:"namespace"` + Host string `json:"host"` + Phase cluster.Phase `json:"phase"` + Commit string `json:"commit,omitempty"` +} + type appStatusDocument struct { header - Name string `json:"name"` + Name string `json:"name"` + // PreviewOf is set when this is the status of a preview: the app it is a preview of. + PreviewOf string `json:"previewOf,omitempty"` Phase cluster.Phase `json:"phase"` Host string `json:"host,omitempty"` URL string `json:"url,omitempty"` @@ -118,6 +131,11 @@ type appStatusDocument struct { Deployed *time.Time `json:"deployed,omitempty"` Components []componentDocument `json:"components"` Stages []stageDocument `json:"stages"` + // Previews, PreviewLimit and PreviewLimitReached are there only for an app with previews + // switched on. + Previews []previewDocument `json:"previews,omitempty"` + PreviewLimit int `json:"previewLimit,omitempty"` + PreviewLimitReached *bool `json:"previewLimitReached,omitempty"` } type waitResultDocument struct { @@ -216,6 +234,16 @@ func appStatus(a ops.App, d cluster.Diagnosis) appStatusDocument { case a.Quick: doc.Exposure = exposure{Kind: "quick"} } + if a.PreviewLimit > 0 { + doc.PreviewLimit = a.PreviewLimit + reached := len(a.Previews) >= a.PreviewLimit + doc.PreviewLimitReached = &reached + doc.Previews = []previewDocument{} + for _, p := range a.Previews { + doc.Previews = append(doc.Previews, previewDocument{Slug: p.Slug, Tag: p.Artifact.Tag, + Namespace: p.Name, Host: ops.PreviewHost(p.Name), Phase: p.Phase, Commit: p.Commit}) + } + } for _, c := range a.Components { doc.Components = append(doc.Components, componentDocument{ Name: c.Name, Phase: c.Phase, Path: c.Path, Address: c.Address, URL: c.URL, Internal: c.Internal, diff --git a/internal/cli/open.go b/internal/cli/open.go index 6e6fd52..0d4833c 100644 --- a/internal/cli/open.go +++ b/internal/cli/open.go @@ -24,6 +24,7 @@ func newAppOpenCmd(o Options) *cobra.Command { target clusterFlags port int browser bool + preview string ) cmd := &cobra.Command{ Use: "open ", @@ -35,7 +36,10 @@ internet. This works for apps that stay inside the cluster, too. This is not an exposure: nothing about the app changes, nothing listens anywhere but on this machine's loopback interface, and the address works only for as long as this command runs. Stop -it with Ctrl-C.`, +it with Ctrl-C. + +With --preview, it opens the preview of that branch instead, at its own host +--.shelf.internal.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] @@ -50,7 +54,7 @@ it with Ctrl-C.`, return err } ctx := cmd.Context() - proxy, err := shelf.OpenApp(ctx, name) + proxy, err := shelf.OpenApp(ctx, name, preview) if err != nil { return err } @@ -62,7 +66,11 @@ it with Ctrl-C.`, local := ln.Addr().String() url := "http://" + local + "/" out := cmd.OutOrStdout() - fmt.Fprintf(out, "%s (%s) is at %s\n", name, proxy.Host, url) + label := name + if preview != "" { + label = ops.PreviewName(name, preview) + } + fmt.Fprintf(out, "%s (%s) is at %s\n", label, proxy.Host, url) fmt.Fprintln(cmd.ErrOrStderr(), "Only this machine can reach it, and only while this runs. Press Ctrl-C to stop.") srv := &http.Server{Handler: proxy.Handler(local), ReadHeaderTimeout: 30 * time.Second} @@ -89,5 +97,6 @@ it with Ctrl-C.`, target.registerTarget(cmd.Flags()) cmd.Flags().IntVar(&port, "port", 0, "the local port; a free one when not given") cmd.Flags().BoolVar(&browser, "browser", false, "open the address in the default browser") + cmd.Flags().StringVar(&preview, "preview", "", "the preview of this branch (its name or its slug) instead of the app") return cmd } diff --git a/internal/cli/previews_test.go b/internal/cli/previews_test.go new file mode 100644 index 0000000..d8f325a --- /dev/null +++ b/internal/cli/previews_test.go @@ -0,0 +1,215 @@ +package cli + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/tweinmann/shelf/internal/cluster" +) + +func TestPreviewsSwitchedOnAndOff(t *testing.T) { + t.Parallel() + h := newHarness(t).public() + h.app = appWithSecrets("greeter") + h.defineRegistry(t, "ghcr", "o", "ghcr-secret-token") + + stdout := h.mustRun(t, "app", "add", "greeter", greeterArtifact, "--registry", "ghcr", "--previews") + if last := h.cluster.added[len(h.cluster.added)-1]; last.Previews != 3 { + t.Errorf("--previews gave %d previews, want 3", last.Previews) + } + if !strings.Contains(stdout, "previews: up to 3 branches, inside the cluster only") { + t.Errorf("stdout lacks the previews line:\n%s", stdout) + } + + // Another change keeps the previews, and writes them with the new registry connection. + h.defineRegistry(t, "other", "o", "ghcr-secret-token") + h.mustRun(t, "app", "credentials", "greeter", "--registry", "other") + if last := h.cluster.added[len(h.cluster.added)-1]; last.Previews != 3 || last.Registry != "other" { + t.Errorf("after a registry change: previews %d, registry %q", last.Previews, last.Registry) + } + + h.mustRun(t, "app", "credentials", "greeter", "--previews=5") + if last := h.cluster.added[len(h.cluster.added)-1]; last.Previews != 5 { + t.Errorf("--previews=5 gave %d", last.Previews) + } + + stdout = h.mustRun(t, "app", "credentials", "greeter", "--no-previews") + if last := h.cluster.added[len(h.cluster.added)-1]; last.Previews != 0 { + t.Errorf("--no-previews left %d previews", last.Previews) + } + if !strings.Contains(stdout, "previews: none") { + t.Errorf("stdout lacks the previews line:\n%s", stdout) + } +} + +func TestPreviewsDefaultOff(t *testing.T) { + t.Parallel() + h := newHarness(t).public() + h.app = appWithSecrets("greeter") + h.mustRun(t, "app", "add", "greeter", greeterArtifact) + if last := h.cluster.added[0]; last.Previews != 0 { + t.Errorf("an app without --previews has %d previews", last.Previews) + } +} + +func TestPreviewsRefusals(t *testing.T) { + t.Parallel() + tests := map[string]struct { + args []string + old bool + want string + }{ + "zero": {args: []string{"--previews=0"}, want: "between 1 and 10 previews"}, + "too many": {args: []string{"--previews=11"}, want: "between 1 and 10 previews"}, + "number without =": {args: []string{"--previews", "5"}, want: "write --previews=5"}, + "on and off": {args: []string{"--previews", "--no-previews"}, want: "switched on and off at once"}, + "platform too old": {args: []string{"--previews"}, old: true, want: "predates previews"}, + "old platform is ok": {args: []string{}, old: true}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + h := newHarness(t).public() + h.app = appWithSecrets("greeter") + h.cluster.oldPlatform = tt.old + _, stderr, code := h.run(t, append([]string{"app", "add", "greeter", greeterArtifact}, tt.args...)...) + if tt.want == "" { + if code != 0 { + t.Errorf("exit code %d: %s", code, stderr) + } + return + } + if code != 1 || !strings.Contains(stderr, tt.want) { + t.Errorf("exit code %d, stderr %q, want %q", code, stderr, tt.want) + } + if len(h.cluster.added) != 0 { + t.Error("the app was written before the refusal") + } + }) + } +} + +// previewHarness is hello with previews of two branches, at most two, so the limit is reached. +func previewHarness(t *testing.T) *harness { + t.Helper() + h := newHarness(t).public() + hello := helloState(cluster.PhaseReady) + hello.PreviewLimit = 2 + hello.Previews = []cluster.Preview{ + {App: "hello", Slug: "feature-login", AppState: cluster.AppState{Name: "hello--feature-login", + Artifact: cluster.Artifact{URL: "oci://ghcr.io/o/hello", Tag: "branch-feature-login"}, + Revision: "branch-feature-login@sha256:def", Commit: "3f2a9c1e5b7d9f1a3c5e7b9d1f3a5c7e9b1d3f5a", + Phase: cluster.PhaseReady}}, + {App: "hello", Slug: "fix-typo", AppState: cluster.AppState{Name: "hello--fix-typo", + Artifact: cluster.Artifact{URL: "oci://ghcr.io/o/hello", Tag: "branch-fix-typo"}, + Phase: cluster.PhaseWorking, Stage: "the pods", Reason: "starting: web-0"}}, + } + h.cluster.states = []cluster.AppState{hello} + h.cluster.configs = map[string]*cluster.AppConfig{"hello": {Artifact: hello.Artifact, Previews: 2}} + h.cluster.components = []cluster.Component{{Name: "web", Ports: []cluster.Port{{Name: "main", Number: 80}}, + Path: "/", Phase: cluster.PhaseReady}} + h.cluster.stages = fiveStages(podsReady) + return h +} + +func TestStatusWithPreviews(t *testing.T) { + t.Parallel() + h := previewHarness(t) + stdout := h.mustRun(t, "app", "status", "hello") + for _, want := range []string{ + " previews 2 of at most 2, inside the cluster only\n", + " hello--feature-login Ready branch-feature-login 3f2a9c1e5b7d\n", + " hello--fix-typo Working branch-fix-typo\n", + "the preview limit of 2 is reached", + } { + if !strings.Contains(stdout, want) { + t.Errorf("stdout lacks %q:\n%s", want, stdout) + } + } + list := h.mustRun(t, "app", "status") + if want := "hello: Ready\n preview feature-login: Ready\n preview fix-typo: Working\n"; list != want { + t.Errorf("list:\n%s\nwant:\n%s", list, want) + } + goldenJSON(t, "appstatus-previews", runJSON(t, h, 0, "app", "status", "hello")) +} + +func TestStatusOfOnePreview(t *testing.T) { + t.Parallel() + h := previewHarness(t) + stdout := h.mustRun(t, "app", "status", "hello", "--preview", "feature/login") + for _, want := range []string{"hello--feature-login: Ready", "address hello--feature-login.shelf.internal", + "artifact oci://ghcr.io/o/hello:branch-feature-login", "commit 3f2a9c1e5b7d"} { + if !strings.Contains(stdout, want) { + t.Errorf("stdout lacks %q:\n%s", want, stdout) + } + } + if strings.Contains(stdout, "previews") { + t.Errorf("a preview lists previews:\n%s", stdout) + } + goldenJSON(t, "appstatus-preview", runJSON(t, h, 0, "app", "status", "hello", "--preview", "feature-login")) + + _, stderr, code := h.run(t, "app", "status", "hello", "--preview", "gone") + if code != 1 || !strings.Contains(stderr, "no preview of branch gone; it has previews of feature-login, fix-typo") { + t.Errorf("unknown branch: exit code %d, stderr %q", code, stderr) + } +} + +func TestWaitForAPreview(t *testing.T) { + t.Parallel() + h := previewHarness(t) + // The preview does not exist for the first read, as before the platform saw the branch. + h.cluster.rollouts, h.cluster.rolloutMissing = []cluster.Rollout{rolledOut(podsReady)}, 1 + out := runJSON(t, h, 0, "app", "status", "hello", "--preview", "feature/login", "--wait") + if !strings.Contains(string(out), `"app": "hello--feature-login"`) || !strings.Contains(string(out), `"outcome": "Ready"`) { + t.Errorf("verdict %s", out) + } + + h = previewHarness(t) + h.cluster.rolloutMissing = 1000 + stdout, _, code := h.run(t, "app", "status", "hello", "--preview", "new-branch", "--wait", "--timeout", "20ms") + if code != 2 || !strings.Contains(stdout, "the preview: waiting for the preview of branch new-branch") { + t.Errorf("a preview that never appears: exit code %d, stdout %q", code, stdout) + } + + h = previewHarness(t) + h.cluster.configs["hello"].Previews = 0 + if _, stderr, code := h.run(t, "app", "status", "hello", "--preview", "x", "--wait"); code != 1 || + !strings.Contains(stderr, "has no previews; `shelf app credentials hello --previews` switches them on") { + t.Errorf("previews off: exit code %d, stderr %q", code, stderr) + } +} + +func TestOpenAPreview(t *testing.T) { + t.Parallel() + h := previewHarness(t) + traefik := openHarness(t).cluster.traefik + h.cluster.traefik = traefik + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var body string + h.browser = func(url string) error { + defer cancel() + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + body = string(b) + return nil + } + stdout, stderr, code := h.runContext(t, ctx, "", "app", "open", "hello", "--preview", "feature/login", "--browser") + if code != 0 || !strings.Contains(stdout, "hello--feature-login (hello--feature-login.shelf.internal) is at http://127.0.0.1:") { + t.Fatalf("exit code %d, stdout %q, stderr %q", code, stdout, stderr) + } + if body != "host=hello--feature-login.shelf.internal path=/" { + t.Errorf("Traefik got %q", body) + } + if _, stderr, code := previewHarness(t).run(t, "app", "open", "hello", "--preview", "gone"); code != 1 || + !strings.Contains(stderr, "no preview of branch gone") { + t.Errorf("unknown branch: exit code %d, stderr %q", code, stderr) + } +} diff --git a/internal/cli/status.go b/internal/cli/status.go index a92a267..3d75f8b 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -22,6 +22,7 @@ func newAppStatusCmd(o Options) *cobra.Command { wait bool commit string timeout time.Duration + preview string ) cmd := &cobra.Command{ Use: "status [name]", @@ -35,9 +36,13 @@ the verdict is about it and not about the version before. The exit status is 0 w Ready, 1 when a step failed and 2 when the time ran out. Waiting only reads the cluster; the platform notices a new artifact within a minute by itself. +With --preview, all of this is about the preview of that branch instead of the app: its +namespace --, its artifact at the branch tag, and its stages. + This is what the admin UI shows on its pages, in text.`, - Example: ` shelf app status hello --wait --commit "$(git rev-parse HEAD)" --json`, - Args: cobra.MaximumNArgs(1), + Example: ` shelf app status hello --wait --commit "$(git rev-parse HEAD)" --json + shelf app status hello --preview feature/login --wait --commit "$(git rev-parse HEAD)" --json`, + Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if commit != "" && !wait { return errors.New("--commit needs --wait") @@ -45,6 +50,9 @@ This is what the admin UI shows on its pages, in text.`, if wait && len(args) == 0 { return errors.New("--wait needs the name of an app") } + if preview != "" && len(args) == 0 { + return errors.New("--preview needs the name of an app") + } if commit != "" { if err := ops.CheckCommit(commit); err != nil { return err @@ -63,7 +71,8 @@ This is what the admin UI shows on its pages, in text.`, return err } if wait { - result, err := shelf.Wait(cmd.Context(), name, ops.WaitOptions{Commit: commit, Timeout: timeout}) + result, err := shelf.Wait(cmd.Context(), name, + ops.WaitOptions{Commit: commit, Timeout: timeout, Preview: preview}) if err != nil { return err } @@ -77,20 +86,30 @@ This is what the admin UI shows on its pages, in text.`, } return verdictExit(result.Outcome) } - app, err := shelf.App(cmd.Context(), name) + var app ops.App + if preview != "" { + app, err = shelf.Preview(cmd.Context(), name, preview) + } else { + app, err = shelf.App(cmd.Context(), name) + } if err != nil { return err } - diagnosis, err := shelf.Diagnose(cmd.Context(), name) + diagnosis, err := shelf.Diagnose(cmd.Context(), app.Name) if err != nil { return err } if asJSON { - return printJSON(out, appStatus(app, diagnosis)) + doc := appStatus(app, diagnosis) + if preview != "" { + doc.PreviewOf = name + } + return printJSON(out, doc) } printApp(out, app) printComponents(out, app.Components, app.Public || app.Quick) printChain(out, diagnosis) + printPreviews(out, app) return nil }, } @@ -99,6 +118,7 @@ This is what the admin UI shows on its pages, in text.`, cmd.Flags().BoolVar(&wait, "wait", false, "wait until the app is Ready or a step failed, and print the verdict") cmd.Flags().StringVar(&commit, "commit", "", "with --wait: the git commit the app has to run, at least 7 hex digits") cmd.Flags().DurationVar(&timeout, "timeout", ops.DefaultWaitTimeout, "with --wait: how long to wait") + cmd.Flags().StringVar(&preview, "preview", "", "the preview of this branch (its name or its slug) instead of the app") return cmd } @@ -154,10 +174,32 @@ func listApps(cmd *cobra.Command, shelf *ops.Ops, out io.Writer, asJSON bool) er if app.Reason != "" { fmt.Fprintf(out, " %s: %s\n", app.Stage, firstLine(app.Reason)) } + for _, p := range app.Previews { + fmt.Fprintf(out, " preview %s: %s\n", p.Slug, p.Phase) + } } return nil } +// printPreviews lists an app's previews, and says when the limit leaves further branches out. +func printPreviews(out io.Writer, app ops.App) { + if app.PreviewLimit == 0 { + return + } + fmt.Fprintf(out, " previews %d of at most %d, inside the cluster only\n", len(app.Previews), app.PreviewLimit) + for _, p := range app.Previews { + line := fmt.Sprintf(" %s %s %s", p.Name, p.Phase, p.Artifact.Tag) + if p.Commit != "" { + line += " " + p.Commit[:min(len(p.Commit), 12)] + } + fmt.Fprintln(out, line) + } + if len(app.Previews) >= app.PreviewLimit { + fmt.Fprintf(out, " the preview limit of %d is reached: further branches get no preview "+ + "(`shelf app credentials %s --previews=N` raises it)\n", app.PreviewLimit, app.Name) + } +} + func printApp(out io.Writer, app ops.App) { fmt.Fprintf(out, "%s: %s\n", app.Name, app.Phase) if app.Host != "" { diff --git a/internal/cli/tenant_test.go b/internal/cli/tenant_test.go index 5b66ba8..0b9f315 100644 --- a/internal/cli/tenant_test.go +++ b/internal/cli/tenant_test.go @@ -46,7 +46,9 @@ func TestTenantAgentInstructions(t *testing.T) { } skill, _ := os.ReadFile(filepath.Join(tenant, ".claude/skills/shelf/SKILL.md")) - for _, want := range []string{`--wait --commit "$(git rev-parse HEAD)" --json`, "| 0 |", "| 1 |", "| 2 |", "Make the app public"} { + for _, want := range []string{`--wait --commit "$(git rev-parse HEAD)" --json`, "| 0 |", "| 1 |", "| 2 |", + "Make the app or a preview public", "in the background", "gh run watch --exit-status", + "--preview --wait", "takes about a minute longer"} { if !strings.Contains(string(skill), want) { t.Errorf("the skill lacks %q", want) } diff --git a/internal/cli/testdata/json/appstatus-preview.json b/internal/cli/testdata/json/appstatus-preview.json new file mode 100644 index 0000000..7128215 --- /dev/null +++ b/internal/cli/testdata/json/appstatus-preview.json @@ -0,0 +1,58 @@ +{ + "apiVersion": "shelf.dev/v1alpha1", + "kind": "AppStatus", + "name": "hello--feature-login", + "previewOf": "hello", + "phase": "Ready", + "host": "hello--feature-login.shelf.internal", + "artifact": "oci://ghcr.io/o/hello:branch-feature-login", + "exposure": { + "kind": "none" + }, + "revision": "branch-feature-login@sha256:def", + "commit": "3f2a9c1e5b7d9f1a3c5e7b9d1f3a5c7e9b1d3f5a", + "components": [ + { + "name": "web", + "phase": "Ready", + "path": "/", + "address": "hello--feature-login.shelf.internal/" + } + ], + "stages": [ + { + "name": "the deploy artifact", + "object": "OCIRepository hello/deploy", + "phase": "Ready", + "reason": "Succeeded", + "message": "stored artifact for digest 'main@sha256:abc'" + }, + { + "name": "the app values", + "object": "Kustomization hello/deploy", + "phase": "Ready", + "reason": "ReconciliationSucceeded", + "message": "Applied revision: main@sha256:abc" + }, + { + "name": "the chart", + "object": "OCIRepository hello/shelf-app", + "phase": "Ready", + "reason": "Succeeded", + "message": "stored artifact for digest '0.6.0@sha256:def'" + }, + { + "name": "the app", + "object": "HelmRelease hello/hello", + "phase": "Ready", + "reason": "UpgradeSucceeded", + "message": "Helm upgrade succeeded" + }, + { + "name": "the pods", + "object": "Pods", + "phase": "Ready", + "message": "2 running" + } + ] +} diff --git a/internal/cli/testdata/json/appstatus-previews.json b/internal/cli/testdata/json/appstatus-previews.json new file mode 100644 index 0000000..a45ff3a --- /dev/null +++ b/internal/cli/testdata/json/appstatus-previews.json @@ -0,0 +1,78 @@ +{ + "apiVersion": "shelf.dev/v1alpha1", + "kind": "AppStatus", + "name": "hello", + "phase": "Ready", + "host": "hello.shelf.internal", + "artifact": "oci://ghcr.io/o/hello:main", + "registry": "ghcr", + "exposure": { + "kind": "none" + }, + "revision": "main@sha256:abc", + "commit": "42a964eb0b4bd7248305d5f22233d6dbabcd56f3", + "deployed": "2026-10-27T13:03:00Z", + "components": [ + { + "name": "web", + "phase": "Ready", + "path": "/", + "address": "hello.shelf.internal/" + } + ], + "stages": [ + { + "name": "the deploy artifact", + "object": "OCIRepository hello/deploy", + "phase": "Ready", + "reason": "Succeeded", + "message": "stored artifact for digest 'main@sha256:abc'" + }, + { + "name": "the app values", + "object": "Kustomization hello/deploy", + "phase": "Ready", + "reason": "ReconciliationSucceeded", + "message": "Applied revision: main@sha256:abc" + }, + { + "name": "the chart", + "object": "OCIRepository hello/shelf-app", + "phase": "Ready", + "reason": "Succeeded", + "message": "stored artifact for digest '0.6.0@sha256:def'" + }, + { + "name": "the app", + "object": "HelmRelease hello/hello", + "phase": "Ready", + "reason": "UpgradeSucceeded", + "message": "Helm upgrade succeeded" + }, + { + "name": "the pods", + "object": "Pods", + "phase": "Ready", + "message": "2 running" + } + ], + "previews": [ + { + "slug": "feature-login", + "tag": "branch-feature-login", + "namespace": "hello--feature-login", + "host": "hello--feature-login.shelf.internal", + "phase": "Ready", + "commit": "3f2a9c1e5b7d9f1a3c5e7b9d1f3a5c7e9b1d3f5a" + }, + { + "slug": "fix-typo", + "tag": "branch-fix-typo", + "namespace": "hello--fix-typo", + "host": "hello--fix-typo.shelf.internal", + "phase": "Working" + } + ], + "previewLimit": 2, + "previewLimitReached": true +} diff --git a/internal/cluster/apps.go b/internal/cluster/apps.go index 1d7044b..c2c9ee7 100644 --- a/internal/cluster/apps.go +++ b/internal/cluster/apps.go @@ -66,6 +66,9 @@ type AppOptions struct { // trycloudflare.com name. Without either, the app is not exposed, and the platform stops its // cloudflared. Quick bool + // Previews is how many branches of the app may run as previews at once; 0 switches previews + // off and removes the ones that run. + Previews int // Secrets are all secret values of the app; they replace the stored ones. Secrets map[string]string Timeout time.Duration @@ -166,11 +169,21 @@ func AddApp(ctx context.Context, cfg *rest.Config, o AppOptions) error { objs = append(objs, TunnelSecret(o.Name, o.TunnelCredentials)) } objs = append(objs, AppProvider(o)) + if o.Previews > 0 { + objs = append(objs, PreviewProvider(o)) + } for _, obj := range objs { if err := c.applyReport(ctx, rep, obj); err != nil { return err } } + // Without the provider of its previews, the platform removes every preview of the app. + if o.Previews == 0 { + if err := c.deleteReport(ctx, rep, ref{gvk: providerGVK, namespace: SystemNamespace, + name: PreviewProviderName(o.Name)}); err != nil { + return err + } + } if o.TunnelID == "" { if err := c.deleteReport(ctx, rep, ref{gvk: secretGVK, namespace: SystemNamespace, name: TunnelSecretName(o.Name)}); err != nil { @@ -200,6 +213,8 @@ type AppConfig struct { TunnelCredentials []byte // Quick means the app is exposed through a quick tunnel. Quick bool + // Previews is how many previews the app may have, 0 when it has none. + Previews int } // ReadAppConfig returns how an app is registered, or nil when there is no such app. @@ -227,6 +242,12 @@ func ReadAppConfig(ctx context.Context, cfg *rest.Config, app string) (*AppConfi config.Registry, _ = values["registry"].(string) config.Cloudflare, _ = values["cloudflare"].(string) + previews, err := c.get(ctx, ref{gvk: providerGVK, namespace: SystemNamespace, name: PreviewProviderName(app)}) + if err != nil { + return nil, err + } + config.Previews = previewLimit(previews) + tunnel, err := c.get(ctx, ref{gvk: secretGVK, namespace: SystemNamespace, name: TunnelSecretName(app)}) if err != nil { return nil, err @@ -335,6 +356,11 @@ func RemoveApp(ctx context.Context, cfg *rest.Config, app string, timeout time.D rep.Report(progress.Applied(provider.String(), "deleted", "")) } + previews := ref{gvk: providerGVK, namespace: SystemNamespace, name: PreviewProviderName(app)} + if err := c.deleteReport(ctx, rep, previews); err != nil { + return false, err + } + ns := ref{gvk: namespaceGVK, name: app} err = c.step(ctx, rep, "namespace "+app+" to be deleted", func(ctx context.Context) (string, error) { return "", wait.PollUntilContextCancel(ctx, pollInterval, true, func(ctx context.Context) (bool, error) { @@ -359,6 +385,20 @@ func RemoveApp(ctx context.Context, cfg *rest.Config, app string, timeout time.D return found, err } + // The platform deletes the previews with their provider; they go before the app's secrets. + err = c.step(ctx, rep, "the previews of "+app+" to be deleted", func(ctx context.Context) (string, error) { + return "", wait.PollUntilContextCancel(ctx, pollInterval, true, func(ctx context.Context) (bool, error) { + left, err := c.list(ctx, namespaceGVK, "", AppLabel+"="+app+","+PreviewLabel) + return len(left) == 0, err + }) + }) + if err != nil { + if ctx.Err() != nil { + return found, fmt.Errorf("the previews of %s are still there; check the ResourceSet previews in the cluster", app) + } + return found, err + } + for _, name := range []string{AppSecretName(app), TunnelSecretName(app)} { secret := ref{gvk: secretGVK, namespace: SystemNamespace, name: name} deleted, err = c.delete(ctx, secret) diff --git a/internal/cluster/previews.go b/internal/cluster/previews.go new file mode 100644 index 0000000..a3253b6 --- /dev/null +++ b/internal/cluster/previews.go @@ -0,0 +1,151 @@ +package cluster + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/tweinmann/shelf/internal/schema" +) + +const ( + // PreviewsLabel marks the provider of an app's previews, with the app's name. It is not + // AppLabel: the ResourceSet apps would make an app of it. + PreviewsLabel = "shelf.dev/previews" + // PreviewLabel marks a preview's namespace, with the branch slug. + PreviewLabel = "shelf.dev/preview" + // previewInterval is how often the platform looks for new branch tags, and how often a + // preview's deploy artifact is checked: an agent waits for both. + previewInterval = "30s" +) + +// PreviewProviderName is the provider of an app's previews. "--" cannot occur in an app's name, +// so this cannot be another app's provider. +func PreviewProviderName(app string) string { return app + "--previews" } + +// PreviewNamespace is where the preview of a branch slug runs: --. +func PreviewNamespace(app, slug string) string { return app + "--" + slug } + +// PreviewProvider returns the provider that makes the platform run a preview for each branch +// tag of the app's deploy artifact, at most o.Previews of them, read with the app's registry +// login. +func PreviewProvider(o AppOptions) *unstructured.Unstructured { + spec := map[string]any{ + "type": "OCIArtifactTag", + "url": o.Artifact.URL, + "filter": map[string]any{ + "includeTag": "^" + schema.BranchTagPrefix, + "limit": int64(o.Previews), + }, + "defaultValues": map[string]any{ + "app": o.Name, + "url": o.Artifact.URL, + "insecure": o.Insecure, + "registry": o.Registry, + }, + } + if o.Insecure { + spec["insecure"] = true + } + if o.Registry != "" { + spec["secretRef"] = map[string]any{"name": RegistryConnectionSecretName(o.Registry)} + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "fluxcd.controlplane.io/v1", + "kind": "ResourceSetInputProvider", + "metadata": map[string]any{ + "name": PreviewProviderName(o.Name), + "namespace": SystemNamespace, + "labels": map[string]any{PreviewsLabel: o.Name}, + "annotations": map[string]any{"fluxcd.controlplane.io/reconcileEvery": previewInterval}, + }, + "spec": spec, + }} +} + +// previewLimit is the number of previews a provider allows, 0 for none. +func previewLimit(provider *unstructured.Unstructured) int { + if provider == nil { + return 0 + } + limit, found, _ := unstructured.NestedInt64(provider.Object, "spec", "filter", "limit") + if !found { + return 0 + } + return int(limit) +} + +// Preview is one preview of an app as it runs in the cluster. +type Preview struct { + AppState + // App is the app this is a preview of; AppState.Name is the preview's namespace. + App string `json:"app"` + // Slug is the branch slug, the part of the name after "--". + Slug string `json:"slug"` +} + +// previewLimits returns, per app, how many previews it may have; an app without previews is +// not in the map. +func (c *client) previewLimits(ctx context.Context) (map[string]int, error) { + providers, err := c.list(ctx, providerGVK, SystemNamespace, PreviewsLabel) + if err != nil { + return nil, err + } + limits := map[string]int{} + for _, p := range providers { + if app := p.GetLabels()[PreviewsLabel]; p.GetName() == PreviewProviderName(app) { + limits[app] = previewLimit(p) + } + } + return limits, nil +} + +// previewState is what a preview's objects say about it: its artifact as the source reads it, +// and its phase as the diagnosis finds it. +func previewState(name string, o *appObjects) AppState { + state := AppState{Name: name} + if o.source != nil { + state.Artifact.URL, _, _ = unstructured.NestedString(o.source.Object, "spec", "url") + state.Artifact.Tag, _, _ = unstructured.NestedString(o.source.Object, "spec", "ref", "tag") + state.Revision, _, _ = unstructured.NestedString(o.source.Object, "status", "artifact", "revision") + state.Commit = artifactCommit(o.source) + } + if o.release != nil { + state.Deployed = lastDeployed(o.release) + } + applyDiagnosis(&state, diagnose(name, o)) + return state +} + +// previewNamespaces adds the namespaces of previews to the objects of the apps, so that a +// preview is diagnosed, read and waited for like an app, by its namespace. +func (c *client) previewNamespaces(ctx context.Context, apps map[string]*appObjects) error { + namespaces, err := c.list(ctx, namespaceGVK, "", PreviewLabel) + if err != nil { + return err + } + for _, ns := range namespaces { + labels := ns.GetLabels() + app, slug := labels[AppLabel], labels[PreviewLabel] + if app == "" || ns.GetName() != PreviewNamespace(app, slug) { + continue + } + apps[ns.GetName()] = &appObjects{previewOf: app, slug: slug} + } + return nil +} + +// PlatformNeeds is what an operation needs the platform in the cluster to do. +type PlatformNeeds struct { + // Quick: run quick tunnels. + Quick bool + // Previews: run previews. + Previews bool +} + +// previewsMissing is the refusal for a platform that cannot run previews. +func previewsMissing() error { + return fmt.Errorf("the platform in this cluster predates previews; install the platform of " + + "this shelf with `shelf init cluster` first, or leave out --previews") +} diff --git a/internal/cluster/previews_test.go b/internal/cluster/previews_test.go new file mode 100644 index 0000000..2e38907 --- /dev/null +++ b/internal/cluster/previews_test.go @@ -0,0 +1,103 @@ +package cluster + +import ( + "os" + "regexp" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// TestPreviewsResourceSet pins what a preview must and must not get from the platform. +func TestPreviewsResourceSet(t *testing.T) { + t.Parallel() + data, err := os.ReadFile("../../platform/previews/resourceset.yaml") + if err != nil { + t.Fatal(err) + } + // Only what is applied counts, not what the comments say about it. + s := regexp.MustCompile(`(?m)^\s*#.*$`).ReplaceAllString(string(data), "") + for _, want := range []string{ + "key: shelf.dev/previews", "kind: ResourceQuota", "kind: LimitRange", "preview: true", + "interval: 30s", "domain: shelf.internal", "shelf.dev/preview: ", "kind: HelmRelease", + } { + if !strings.Contains(s, want) { + t.Errorf("platform/previews/resourceset.yaml lacks %q", want) + } + } + // A preview is never exposed. + for _, never := range []string{"cloudflared", "tunnel", "copyFrom: shelf-system/app-"} { + if strings.Contains(s, never) { + t.Errorf("platform/previews/resourceset.yaml mentions %q", never) + } + } + // Flux substitutes $... when it applies the platform; only ${SHELF_*} may survive that. + if m := regexp.MustCompile(`\$[A-Za-z{]`).FindAllString(s, -1); len(m) != strings.Count(s, "${SHELF_") { + t.Errorf("a $ other than ${SHELF_...}: %v", m) + } +} + +func TestPreviewProvider(t *testing.T) { + t.Parallel() + p := PreviewProvider(AppOptions{Name: "hello", Artifact: Artifact{URL: "oci://ghcr.io/acme/hello", Tag: "main"}, + Registry: "ghcr", Previews: 3}) + if p.GetName() != "hello--previews" || p.GetNamespace() != SystemNamespace { + t.Errorf("name %s/%s", p.GetNamespace(), p.GetName()) + } + if labels := p.GetLabels(); labels[PreviewsLabel] != "hello" || labels[AppLabel] != "" { + t.Errorf("labels %v: the provider of previews must not look like an app", labels) + } + if p.GetAnnotations()["fluxcd.controlplane.io/reconcileEvery"] != "30s" { + t.Errorf("annotations %v", p.GetAnnotations()) + } + spec, _, _ := unstructured.NestedMap(p.Object, "spec") + if spec["type"] != "OCIArtifactTag" || spec["url"] != "oci://ghcr.io/acme/hello" { + t.Errorf("spec %v", spec) + } + if include, _, _ := unstructured.NestedString(p.Object, "spec", "filter", "includeTag"); include != "^branch-" { + t.Errorf("includeTag %q", include) + } + if secret, _, _ := unstructured.NestedString(p.Object, "spec", "secretRef", "name"); secret != "connection-registry-ghcr" { + t.Errorf("secretRef %q", secret) + } + if previewLimit(p) != 3 || previewLimit(nil) != 0 { + t.Errorf("limit %d", previewLimit(p)) + } + if _, found, _ := unstructured.NestedFieldNoCopy(p.Object, "spec", "insecure"); found { + t.Error("insecure is set for a registry with TLS") + } + + // The dev registry: no login, no TLS. + dev := PreviewProvider(AppOptions{Name: "hello", Artifact: Artifact{URL: "oci://shelf-registry:5000/hello"}, + Insecure: true, Previews: 1}) + if insecure, _, _ := unstructured.NestedBool(dev.Object, "spec", "insecure"); !insecure { + t.Error("insecure is not set for the dev registry") + } + if _, found, _ := unstructured.NestedFieldNoCopy(dev.Object, "spec", "secretRef"); found { + t.Error("a provider without a registry connection has a secretRef") + } +} + +func TestPreviewState(t *testing.T) { + t.Parallel() + o := healthy() + o.provider, o.previewOf, o.slug = nil, "hello", "feature-login" + _ = unstructured.SetNestedMap(o.source.Object, map[string]any{"url": "oci://ghcr.io/acme/hello", + "ref": map[string]any{"tag": "branch-feature-login"}}, "spec") + _ = unstructured.SetNestedMap(o.source.Object, map[string]any{"revision": "branch-feature-login@sha256:abc", + "metadata": map[string]any{revisionAnnotation: "feature/login@sha1:42a964eb0b4bd7248305d5f22233d6dbabcd56f3"}}, + "status", "artifact") + if o.isApp() { + t.Fatal("a preview is listed as an app") + } + s := previewState("hello--feature-login", o) + if s.Name != "hello--feature-login" || s.Artifact.Tag != "branch-feature-login" || s.Phase != PhaseReady || + s.Commit != "42a964eb0b4bd7248305d5f22233d6dbabcd56f3" { + t.Errorf("state %+v", s) + } + o.pods = []*unstructured.Unstructured{pod("Pending", container(false, waiting("ImagePullBackOff", "")))} + if s := previewState("hello--feature-login", o); s.Phase != PhaseFailed || s.Stage != "the pods" { + t.Errorf("failing preview %+v", s) + } +} diff --git a/internal/cluster/rollout.go b/internal/cluster/rollout.go index 547aea5..55653ea 100644 --- a/internal/cluster/rollout.go +++ b/internal/cluster/rollout.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/hex" - "fmt" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/rest" @@ -49,7 +48,7 @@ func AppRollout(ctx context.Context, cfg *rest.Config, app string) (Rollout, err } objs, ok := apps[app] if !ok { - return Rollout{}, fmt.Errorf("app %s does not exist", app) + return Rollout{}, &NotFoundError{Name: app} } values, err := c.get(ctx, ref{gvk: configMapGVK, namespace: app, name: render.ConfigMapName}) if err != nil { diff --git a/internal/cluster/state.go b/internal/cluster/state.go index 42edeb2..6e8bcb9 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -101,10 +101,18 @@ type AppState struct { Reason string `json:"reason,omitempty"` // Stage is the step that is not ready, empty when the app is healthy. Stage string `json:"stage,omitempty"` + // PreviewLimit is how many previews the app may have, 0 when it has none switched on. + PreviewLimit int `json:"previewLimit,omitempty"` + // Previews are the app's previews that exist, ordered by name. + Previews []Preview `json:"previews,omitempty"` } // appObjects are the objects that carry one app. type appObjects struct { + // previewOf and slug are set for the namespace of a preview, which has no provider of its + // own: the provider of the app's previews stands for all of them. + previewOf string + slug string provider *unstructured.Unstructured source *unstructured.Unstructured kustomization *unstructured.Unstructured @@ -113,8 +121,17 @@ type appObjects struct { pods []*unstructured.Unstructured } -// AppStates returns every registered app with its current state. It reads the whole cluster in -// five list calls, however many apps there are, plus two for each app with a quick tunnel. +// NotFoundError means there is no app, or no preview, of that name in the cluster. +type NotFoundError struct{ Name string } + +func (e *NotFoundError) Error() string { return "app " + e.Name + " does not exist" } + +// isApp tells an app from a preview: only an app has a provider of its own. +func (o *appObjects) isApp() bool { return o.provider != nil } + +// AppStates returns every registered app with its current state and its previews. It reads the +// whole cluster in eight list calls, however many apps and previews there are, plus two for each +// app with a quick tunnel. func AppStates(ctx context.Context, cfg *rest.Config) ([]AppState, error) { c, err := newClient(cfg) if err != nil { @@ -124,9 +141,24 @@ func AppStates(ctx context.Context, cfg *rest.Config) ([]AppState, error) { if err != nil { return nil, err } + limits, err := c.previewLimits(ctx) + if err != nil { + return nil, err + } + previews := map[string][]Preview{} + for _, name := range slices.Sorted(maps.Keys(apps)) { + if o := apps[name]; !o.isApp() { + previews[o.previewOf] = append(previews[o.previewOf], + Preview{AppState: previewState(name, o), App: o.previewOf, Slug: o.slug}) + } + } states := make([]AppState, 0, len(apps)) for _, name := range slices.Sorted(maps.Keys(apps)) { + if !apps[name].isApp() { + continue + } state := appState(name, apps[name]) + state.PreviewLimit, state.Previews = limits[name], previews[name] if state.Quick { // A quick tunnel's name exists only in its cloudflared. One that cannot be asked // yet has no address, which the page says; it is not a reason to fail the list. @@ -137,7 +169,8 @@ func AppStates(ctx context.Context, cfg *rest.Config) ([]AppState, error) { return states, nil } -// AppDiagnosis returns the whole chain of one app, including its pods. +// AppDiagnosis returns the whole chain of one app, including its pods. app may also be the +// namespace of a preview. func AppDiagnosis(ctx context.Context, cfg *rest.Config, app string) (Diagnosis, error) { c, err := newClient(cfg) if err != nil { @@ -149,7 +182,7 @@ func AppDiagnosis(ctx context.Context, cfg *rest.Config, app string) (Diagnosis, } objs, ok := apps[app] if !ok { - return Diagnosis{}, fmt.Errorf("app %s does not exist", app) + return Diagnosis{}, &NotFoundError{Name: app} } return diagnose(app, objs), nil } @@ -166,6 +199,9 @@ func (c *client) appObjects(ctx context.Context) (map[string]*appObjects, error) for _, p := range providers { apps[p.GetName()] = &appObjects{provider: p} } + if err := c.previewNamespaces(ctx, apps); err != nil { + return nil, err + } sources, err := c.list(ctx, ociRepositoryGVK, "", "") if err != nil { return nil, err @@ -258,7 +294,13 @@ func appState(name string, o *appObjects) AppState { if o.release != nil { state.Deployed = lastDeployed(o.release) } - d := diagnose(name, o) + applyDiagnosis(&state, diagnose(name, o)) + return state +} + +// applyDiagnosis puts the outcome of a diagnosis into a state: the phase, and for anything but +// Ready the stage that is the cause and its reason in one line. +func applyDiagnosis(state *AppState, d Diagnosis) { state.Phase = d.Phase() if t := d.Trouble(); t != nil { state.Stage = t.Name @@ -267,7 +309,6 @@ func appState(name string, o *appObjects) AppState { state.Reason = t.Reason } } - return state } // revisionAnnotation is where `flux push artifact --revision` records the commit an artifact was diff --git a/internal/cluster/tunnel.go b/internal/cluster/tunnel.go index ec04d80..c2cf0b2 100644 --- a/internal/cluster/tunnel.go +++ b/internal/cluster/tunnel.go @@ -174,11 +174,15 @@ const appsResourceSet = "apps" // quickInput is how the platform refers to the input of a quick tunnel. const quickInput = `"quick"` +// previewsResourceSet is the ResourceSet of the platform that runs the previews of apps. +const previewsResourceSet = "previews" + // CheckPlatform fails when this cluster cannot deploy the app an operation is about to change: -// without the platform's ResourceSet nothing ever creates the app, and a platform older than -// quick tunnels never starts cloudflared for one. Either way, a wait would only end at the -// timeout, so operations call it before they write anything, here or in Cloudflare. -func CheckPlatform(ctx context.Context, cfg *rest.Config, quick bool) error { +// without the platform's ResourceSet nothing ever creates the app, a platform older than quick +// tunnels never starts cloudflared for one, and one older than previews never runs them. Either +// way, a wait would only end at the timeout, so operations call it before they write anything, +// here or in Cloudflare. +func CheckPlatform(ctx context.Context, cfg *rest.Config, needs PlatformNeeds) error { c, err := newClient(cfg) if err != nil { return err @@ -191,7 +195,17 @@ func CheckPlatform(ctx context.Context, cfg *rest.Config, quick bool) error { if err != nil { return err } - return platformError(obj, quick) + if err := platformError(obj, needs.Quick); err != nil || !needs.Previews { + return err + } + previews, err := c.get(ctx, ref{gvk: resourceSetGVK, namespace: SystemNamespace, name: previewsResourceSet}) + if err != nil { + return err + } + if previews == nil { + return previewsMissing() + } + return nil } // platformError decides CheckPlatform on the ResourceSet as read; nil means there is none. diff --git a/internal/ops/apps.go b/internal/ops/apps.go index 5b984fd..d0b0317 100644 --- a/internal/ops/apps.go +++ b/internal/ops/apps.go @@ -51,8 +51,19 @@ type Access struct { // Private takes the app off the internet: it answers inside the cluster only, and a tunnel // and a record it had are deleted. It is what a new app gets when nothing else is chosen. Private bool + // Previews gives the app previews of at most that many branches at once; 0 leaves them as + // they are. Previews are never exposed, whatever the app's own exposure. + Previews int + // NoPreviews removes the app's previews, with their namespaces and volumes. + NoPreviews bool } +// MaxPreviews is the most previews an app may have at once. +const MaxPreviews = 10 + +// DefaultPreviews is what --previews without a number gives an app. +const DefaultPreviews = 3 + // exposures counts how many ways to reach the internet an Access asks for at once. func (a Access) exposures() int { n := 0 @@ -88,6 +99,12 @@ func (a Access) Check() error { return err } } + switch { + case a.Previews != 0 && a.NoPreviews: + return errors.New("previews cannot be switched on and off at once") + case a.Previews < 0 || a.Previews > MaxPreviews: + return fmt.Errorf("an app may have between 1 and %d previews, not %d", MaxPreviews, a.Previews) + } return nil } @@ -140,9 +157,19 @@ func (o *Ops) AddApp(ctx context.Context, opts AddOptions, report progress.Repor cloudflare, quick = opts.Access.Cloudflare, false rep.Report(progress.Info("Cloudflare connection: %s", cloudflare)) } + previews := current.Previews + switch { + case opts.Access.NoPreviews: + previews = 0 + rep.Report(progress.Info("previews: none")) + case opts.Access.Previews > 0: + previews = opts.Access.Previews + rep.Report(progress.Info("previews: up to %d branches, inside the cluster only", previews)) + } // Before anything is written, here or in Cloudflare: without the platform the app would never // be deployed, and the wait for it would only end at the timeout. - if err := o.Cluster.CheckPlatform(ctx, o.config(), quick); err != nil { + needs := cluster.PlatformNeeds{Quick: quick, Previews: previews > 0} + if err := o.Cluster.CheckPlatform(ctx, o.config(), needs); err != nil { return err } domain := cmp.Or(opts.Access.Domain, current.Domain) @@ -212,6 +239,7 @@ func (o *Ops) AddApp(ctx context.Context, opts AddOptions, report progress.Repor TunnelID: exposure.tunnelID(), TunnelCredentials: exposure.credentials, Quick: quick, + Previews: previews, Secrets: values, Timeout: opts.Timeout, Report: rep, @@ -245,7 +273,7 @@ func (o *Ops) SetAccess(ctx context.Context, name string, access Access, timeout // what to do when a tag was moved, or when something failed and is worth another try. func (o *Ops) Redeploy(ctx context.Context, name string, timeout time.Duration, report progress.Reporter) error { - if err := o.Cluster.CheckPlatform(ctx, o.config(), false); err != nil { + if err := o.Cluster.CheckPlatform(ctx, o.config(), cluster.PlatformNeeds{}); err != nil { return err } return o.Cluster.Redeploy(ctx, o.config(), name, timeout, progress.OrDiscard(report)) diff --git a/internal/ops/open.go b/internal/ops/open.go index cf0c454..b75e401 100644 --- a/internal/ops/open.go +++ b/internal/ops/open.go @@ -27,8 +27,16 @@ type AppProxy struct { } // OpenApp checks that the app exists and opens the first forward, so that a cluster that -// cannot forward fails before anything listens. -func (o *Ops) OpenApp(ctx context.Context, name string) (*AppProxy, error) { +// cannot forward fails before anything listens. With a branch, it opens the app's preview of +// that branch instead, under the preview's own host. +func (o *Ops) OpenApp(ctx context.Context, name, preview string) (*AppProxy, error) { + if preview != "" { + p, err := o.Preview(ctx, name, preview) + if err != nil { + return nil, err + } + return o.openHost(ctx, p.Host) + } settings, err := o.Cluster.Settings(ctx, o.config()) if err != nil { return nil, err @@ -43,18 +51,23 @@ func (o *Ops) OpenApp(ctx context.Context, name string) (*AppProxy, error) { } // The app's own host, never a quick tunnel's random name: that is the name cloudflared // sets too, and the one the app's Ingress answers to. - p := &AppProxy{ - Host: AppHost(name, settings.HostSuffix, state.Domain), - forward: func(ctx context.Context) (*cluster.Forward, error) { return o.Cluster.ForwardTraefik(ctx, o.config()) }, - } - if _, err := p.addr(ctx); err != nil { - return nil, err - } - return p, nil + return o.openHost(ctx, AppHost(name, settings.HostSuffix, state.Domain)) } return nil, &NotFoundError{Name: name} } +// openHost opens the first forward for a proxy that sends every request under host. +func (o *Ops) openHost(ctx context.Context, host string) (*AppProxy, error) { + p := &AppProxy{ + Host: host, + forward: func(ctx context.Context) (*cluster.Forward, error) { return o.Cluster.ForwardTraefik(ctx, o.config()) }, + } + if _, err := p.addr(ctx); err != nil { + return nil, err + } + return p, nil +} + // NewAppProxy is an AppProxy over forwards made by forward, for tests. func NewAppProxy(host string, forward func(context.Context) (*cluster.Forward, error)) *AppProxy { return &AppProxy{Host: host, forward: forward} diff --git a/internal/ops/ops.go b/internal/ops/ops.go index 44b578d..571103a 100644 --- a/internal/ops/ops.go +++ b/internal/ops/ops.go @@ -77,7 +77,7 @@ type Cluster interface { Install(ctx context.Context, cfg *rest.Config, o cluster.Options) error // CheckPlatform fails when the cluster has no platform, or, with quick, one without quick // tunnels. - CheckPlatform(ctx context.Context, cfg *rest.Config, quick bool) error + CheckPlatform(ctx context.Context, cfg *rest.Config, needs cluster.PlatformNeeds) error AddApp(ctx context.Context, cfg *rest.Config, o cluster.AppOptions) error RemoveApp(ctx context.Context, cfg *rest.Config, app string, timeout time.Duration, rep progress.Reporter) (bool, error) @@ -108,8 +108,8 @@ func (liveCluster) Install(ctx context.Context, cfg *rest.Config, o cluster.Opti return cluster.Install(ctx, cfg, o) } -func (liveCluster) CheckPlatform(ctx context.Context, cfg *rest.Config, quick bool) error { - return cluster.CheckPlatform(ctx, cfg, quick) +func (liveCluster) CheckPlatform(ctx context.Context, cfg *rest.Config, needs cluster.PlatformNeeds) error { + return cluster.CheckPlatform(ctx, cfg, needs) } func (liveCluster) AddApp(ctx context.Context, cfg *rest.Config, o cluster.AppOptions) error { @@ -223,10 +223,16 @@ var ( // CheckAppName rejects a name that cannot become a namespace and a host name, or that names a // namespace of the platform. The name is chosen at // `shelf app add`; the `name` in app.yaml only names the packages. +// PreviewSeparator joins an app's name and a branch slug into the name of a preview, so no +// app name may contain it. +const PreviewSeparator = "--" + func CheckAppName(name string) error { switch { case len(name) > schema.MaxAppNameLength || !appNameRE.MatchString(name): return fmt.Errorf("app name %q must be a DNS label of at most %d characters", name, schema.MaxAppNameLength) + case strings.Contains(name, PreviewSeparator): + return fmt.Errorf("app name %q contains %q, which is reserved for previews", name, PreviewSeparator) case slices.Contains(reservedAppNames, name) || slices.ContainsFunc(reservedAppPrefixes, func(pre string) bool { return strings.HasPrefix(name, pre) }): return fmt.Errorf("app name %q is reserved for the platform", name) diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go index adb36f0..033ce9c 100644 --- a/internal/ops/ops_test.go +++ b/internal/ops/ops_test.go @@ -29,6 +29,8 @@ func TestCheckAppName(t *testing.T) { strings.Repeat("a", 40): true, strings.Repeat("a", 41): false, "a." + strings.Repeat("b", 10): false, + "hello--copy": false, + "hello-copy": true, } for name, valid := range tests { err := ops.CheckAppName(name) diff --git a/internal/ops/previews.go b/internal/ops/previews.go new file mode 100644 index 0000000..ccb03df --- /dev/null +++ b/internal/ops/previews.go @@ -0,0 +1,67 @@ +package ops + +import ( + "context" + "fmt" + "strings" + + "github.com/tweinmann/shelf/internal/cluster" + "github.com/tweinmann/shelf/internal/schema" +) + +// PreviewName is the namespace, and the name everywhere else, of the preview of a branch of an +// app. The branch may be given as its name or as its slug. +func PreviewName(app, branch string) string { + return cluster.PreviewNamespace(app, schema.BranchSlug(branch)) +} + +// PreviewNotFoundError means an app has no preview for a branch; it names the ones it has. +type PreviewNotFoundError struct { + App, Branch string + // Previews are the branch slugs of the previews the app has. + Previews []string + // Off means the app has no previews switched on at all. + Off bool +} + +func (e *PreviewNotFoundError) Error() string { + switch { + case e.Off: + return fmt.Sprintf("app %s has no previews; `shelf app credentials %s --previews` switches them on", e.App, e.App) + case len(e.Previews) == 0: + return fmt.Sprintf("app %s has no preview of branch %s, and no previews yet", e.App, e.Branch) + } + return fmt.Sprintf("app %s has no preview of branch %s; it has previews of %s", e.App, e.Branch, + strings.Join(e.Previews, ", ")) +} + +// Preview returns the preview of a branch of an app, with its address and its components, as an +// App whose Name is the preview's namespace. A preview answers inside the cluster only. +func (o *Ops) Preview(ctx context.Context, name, branch string) (App, error) { + a, err := o.App(ctx, name) + if err != nil { + return App{}, err + } + want := PreviewName(name, branch) + for _, p := range a.Previews { + if p.Name != want { + continue + } + preview := App{AppState: p.AppState, Host: PreviewHost(p.Name)} + list, err := o.Cluster.AppComponents(ctx, o.config(), p.Name) + if err != nil { + return App{}, err + } + preview.Components = components(list, preview.Host, false) + return preview, nil + } + missing := &PreviewNotFoundError{App: name, Branch: branch, Off: a.PreviewLimit == 0} + for _, p := range a.Previews { + missing.Previews = append(missing.Previews, p.Slug) + } + return App{}, missing +} + +// PreviewHost is where a preview answers: --.shelf.internal, never under the app's +// own domain, whatever that is. +func PreviewHost(namespace string) string { return namespace + "." + cluster.InternalDomain } diff --git a/internal/ops/wait.go b/internal/ops/wait.go index 11af6ec..cc14a62 100644 --- a/internal/ops/wait.go +++ b/internal/ops/wait.go @@ -38,6 +38,8 @@ type WaitOptions struct { Timeout time.Duration // Interval overrides how often the cluster is read, for tests. Interval time.Duration + // Preview waits for the preview of this branch of the app instead of the app. + Preview string } // WaitResult is the verdict of a wait. @@ -76,19 +78,40 @@ func (o *Ops) Wait(ctx context.Context, app string, opts WaitOptions) (WaitResul if timeout <= 0 { timeout = DefaultWaitTimeout } + target := app + if opts.Preview != "" { + // A preview appears only after CI and the platform's next look at the registry, so + // one that does not exist yet is on its way, as long as the app has previews at all. + config, err := o.Cluster.AppConfig(ctx, o.config(), app) + switch { + case err != nil: + return WaitResult{}, err + case config == nil: + return WaitResult{}, &NotFoundError{Name: app} + case config.Previews == 0: + return WaitResult{}, &PreviewNotFoundError{App: app, Branch: opts.Preview, Off: true} + } + target = PreviewName(app, opts.Preview) + } ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() ticker := time.NewTicker(interval) defer ticker.Stop() var last WaitResult for { - r, err := o.Cluster.AppRollout(ctx, o.config(), app) + r, err := o.Cluster.AppRollout(ctx, o.config(), target) + var missing *cluster.NotFoundError switch { case err == nil: var done bool - if last, done = Verdict(app, r, opts.Commit); done { + if last, done = Verdict(target, r, opts.Commit); done { return last, nil } + case opts.Preview != "" && errors.As(err, &missing): + last = WaitResult{Outcome: OutcomeTimedOut, App: target, Stage: &cluster.Stage{ + Name: "the preview", Object: "Namespace " + target, Phase: cluster.PhaseMissing, + Message: "waiting for the preview of branch " + opts.Preview, + }} case ctx.Err() == nil: return WaitResult{}, err } diff --git a/internal/schema/branch.go b/internal/schema/branch.go new file mode 100644 index 0000000..37d4b91 --- /dev/null +++ b/internal/schema/branch.go @@ -0,0 +1,47 @@ +package schema + +import ( + "crypto/sha256" + "encoding/hex" + "strings" +) + +const ( + // BranchTagPrefix starts the tag of every branch's deploy artifact; the platform's preview + // provider follows only tags with it. + BranchTagPrefix = "branch-" + // MaxBranchSlugLength keeps -- a DNS label: 40 + 2 + 20 is 62. + MaxBranchSlugLength = 20 +) + +// BranchSlug turns a branch name into the part of a preview's name that stands for it: lower +// case, every run of other characters one "-", no "-" at either end. A longer result keeps its +// first 13 characters and adds 6 hex digits of the branch name's SHA-256, so two long names with +// the same beginning still differ. A slug is its own slug, so a caller may pass either. +// +// The reusable workflow and the CLI both call this through shelf, so a branch maps to the same +// name wherever it is looked at; the cluster only strips the tag's prefix. +func BranchSlug(branch string) string { + var b strings.Builder + dash := false + for _, r := range strings.ToLower(branch) { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' { + b.WriteRune(r) + dash = false + continue + } + if !dash && b.Len() > 0 { + b.WriteByte('-') + dash = true + } + } + slug := strings.TrimRight(b.String(), "-") + if len(slug) <= MaxBranchSlugLength { + return slug + } + sum := sha256.Sum256([]byte(branch)) + return strings.TrimRight(slug[:13], "-") + "-" + hex.EncodeToString(sum[:])[:6] +} + +// BranchTag is the tag a branch's deploy artifact is published under. +func BranchTag(branch string) string { return BranchTagPrefix + BranchSlug(branch) } diff --git a/internal/schema/branch_test.go b/internal/schema/branch_test.go new file mode 100644 index 0000000..718dbda --- /dev/null +++ b/internal/schema/branch_test.go @@ -0,0 +1,46 @@ +package schema + +import ( + "regexp" + "strings" + "testing" +) + +func TestBranchSlug(t *testing.T) { + t.Parallel() + tests := map[string]string{ + "feature/Login": "feature-login", + "main": "main", + "fix--typo": "fix-typo", + "--lead/and/trail--": "lead-and-trail", + "feature-login": "feature-login", + "Ümlaut_ünd spaces": "mlaut-nd-spaces", + "agent/claude-1/refactor": "agent-claude-ad9bc7", + } + for branch, want := range tests { + if got := BranchSlug(branch); got != want { + t.Errorf("BranchSlug(%q) = %q, want %q", branch, got, want) + } + } + if got := BranchTag("feature/Login"); got != "branch-feature-login" { + t.Errorf("tag %q", got) + } +} + +func TestLongBranchSlugs(t *testing.T) { + t.Parallel() + label := regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + a := BranchSlug("agent/claude-1/refactor-the-billing-module") + b := BranchSlug("agent/claude-1/refactor-the-billing-api") + if a == b { + t.Errorf("two long branches share the slug %q", a) + } + for _, s := range []string{a, b, BranchSlug(strings.Repeat("x", 200)), BranchSlug("a-------------b-c-d-e-f-g")} { + if len(s) > MaxBranchSlugLength || !label.MatchString(s) { + t.Errorf("slug %q is longer than %d or not a DNS label", s, MaxBranchSlugLength) + } + } + if BranchSlug(a) != a { + t.Errorf("a slug is not its own slug: %q -> %q", a, BranchSlug(a)) + } +} diff --git a/justfile b/justfile index 7f70bbc..3946fca 100644 --- a/justfile +++ b/justfile @@ -105,6 +105,10 @@ smoke-apps: smoke-agent: hack/smoke/agent.sh +# Phase 10: two branches as isolated previews, open --preview, data kept, limit, removal, RAM and times +smoke-previews: + hack/smoke/previews.sh + # Phase 8b acceptance: expose an app through its own Cloudflare tunnel (asks for the API token) smoke-expose app domain: hack/smoke/expose.sh {{app}} {{domain}} diff --git a/openspec/changes/phase-10-branch-previews/.openspec.yaml b/openspec/changes/phase-10-branch-previews/.openspec.yaml index ad9e5be..97e3642 100644 --- a/openspec/changes/phase-10-branch-previews/.openspec.yaml +++ b/openspec/changes/phase-10-branch-previews/.openspec.yaml @@ -1,3 +1,2 @@ schema: spec-driven created: 2026-10-27 -skip_specs: true diff --git a/openspec/changes/phase-10-branch-previews/design.md b/openspec/changes/phase-10-branch-previews/design.md new file mode 100644 index 0000000..762bb31 --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/design.md @@ -0,0 +1,304 @@ +# Design + +## Context + +See proposal.md for the motivation. The facts the approach rests on: + +- The platform turns each `ResourceSetInputProvider` labelled `shelf.dev/app` in `shelf-system` + into an app (ResourceSet `apps`, `platform/apps/resourceset.yaml`). Each app's provider is of + type `Static` and carries the inputs `shelf app add` wrote. +- The embedded Flux Operator v0.60.0 offers the provider type `OCIArtifactTag`, with `includeTag` + and `excludeTag` filters, a `limit`, `insecure`, a `secretRef` holding an image pull secret, and + `defaultValues` that fill in inputs the provider does not produce. +- **Spike, 2026-10-28, dev cluster.** Three tags (`main`, `branch-feature-login`, + `branch-fix-typo`) were pushed to `shelf-registry:5000/spike/previews`, and an + `OCIArtifactTag` provider with `includeTag: ^branch-` and `defaultValues` `{app, url}` was + created next to them. + - It exported one input set per branch tag, `{id, tag, digest}` plus the defaults, and left + out `main`. The `id` is a number derived from the tag. + - A ResourceSet template rendered `<< inputs.app >>--<< trimPrefix "branch-" inputs.tag >>` + as `hello--feature-login`. + - Deleting a tag failed: the dev registry answers `DELETE` with 405, because `registry:2` + ships with deletes disabled (`/etc/docker/registry/config.yml` has no `storage.delete`). +- GHCR deletes a tag through the GitHub REST API ("delete package version"), not through the + OCI `DELETE`. That call happens in the tenant's CI, not in the cluster. +- The chart reads every secret from the Secret `shelf-secrets`, which the platform copies from + `shelf-system/app-`, where `shelf app add` stores what it generated (`rand.Text()`, 26 + URL-safe characters). Nobody runs `shelf app add` when a branch appears. +- `kubectl top` works on the dev cluster (k3s ships metrics-server), so RAM per preview can be + measured at level 2. + +## Goals / Non-Goals + +**Goals:** +- A branch tag in the registry becomes a running, isolated preview within about two minutes, + and a deleted tag removes it, with the platform reading only the registry. +- Previews behave like apps to every read: diagnosis, `--wait --commit`, `open`, JSON. +- Opting an app in is one flag for the operator. Opting out, or removing the app, leaves nothing + behind. + +**Non-Goals:** +- Exposing a preview in any way (Cloudflare Access is a later phase). +- NetworkPolicies and Pod Security (Phase 12). Quota and LimitRange are in this phase because + without them one preview can starve the app. +- Previews in the admin UI. The dashboard keeps listing apps; it lists previews in a later phase. +- Previews of pull requests from forks. Forks never push. +- Copying data from the app into a preview (seed data stays in "Later"). +- Deleting the per-branch images. Only the deploy artifact's tag is deleted; images accumulate + as they do for `main` today. + +## Decisions + +### 1. One `OCIArtifactTag` provider per app, read by a second ResourceSet + +``` + shelf app add hello oci://ghcr.io/acme/hello:main --registry ghcr --previews=3 + | + +--> shelf-system/hello Static, label shelf.dev/app --> ResourceSet apps + +--> shelf-system/hello--previews OCIArtifactTag, label shelf.dev/previews=hello + url oci://ghcr.io/acme/hello, includeTag ^branch-, limit 3, + secretRef connection-registry-ghcr, reconcileEvery 30s + defaultValues: app, url, insecure, registry + | one input set per tag: id, tag, digest + defaults + v + ResourceSet previews (platform/previews/resourceset.yaml) + per preview: Namespace, Secret shelf-registry (copy), ResourceQuota, LimitRange, + ServiceAccount/Role/RoleBinding shelf-deploy, OCIRepository deploy (tag), + Kustomization deploy, OCIRepository shelf-app, HelmRelease -- +``` + +- **Naming and label.** The provider is named `--previews`, so it cannot collide with + another app's provider, since `--` is reserved in app names. It carries + `shelf.dev/previews=` and not `shelf.dev/app`, so the ResourceSet `apps` does not pick + it up. +- **Interval.** The provider runs every 30 seconds (`fluxcd.controlplane.io/reconcileEvery: 30s`), + as does a preview's deploy artifact (decision 9). +- **Deploy chain.** Each preview's chain is the app's, with the branch tag as the OCIRepository's + `ref.tag`. The HelmRelease is named after the preview namespace, as an app's is after its own, + so the diagnosis of decision 6 needs no special case. + +*Alternatives:* +- **Flux Operator `GitHubBranch`/`GitHubPullRequest` providers.** The cluster would call the + GitHub API, against the guardrail. +- **One `Static` provider per preview, written by the host service.** A deploy path would then + depend on the host service running. +- **Previews as extra inputs of the app's own provider.** A `Static` provider exports one input + set. +- **One shared provider for all apps.** One provider has one URL. + +### 2. Previews are switched on per app, by the operator + +- **Flags.** `--previews[=N]` (1–10, default 3) on `shelf app add` and `shelf app credentials` + creates or updates `--previews` with `filter.limit: N`. `--no-previews` deletes it, and + the ResourceSet `previews` then deletes every preview of the app. +- **Where the state lives.** There is no new input on the app's provider: whether an app has + previews is whether that provider exists, and N is its limit. +- **Registry and access.** A later `shelf app credentials` that changes the app's registry + connection or `--insecure-registry` rewrites the preview provider as well, so that it reads + with the same login. +- **Same path as every operation.** The flags go through `internal/ops`, as every CLI action + does. The admin UI does not offer them in this phase, which the guardrail allows (UI ⊆ CLI). + +*Alternative:* previews for every app. A tenant's branch pushes would then use the operator's +RAM without the operator's say, the same reason exposure is the operator's decision. + +### 3. The slug is computed once, in Go, and travels in the tag + +- **Function.** `schema.BranchSlug(branch)` lower-cases the branch name, turns every run of + other characters into `-` and trims the ends. A result longer than 20 characters becomes its + first 13 characters, `-`, and 6 hex digits of the SHA-256 of the branch name. +- **Who calls it.** `shelf build-plan --branch ` prints the tag `branch-` for + `build.yml`; `--preview ` in the CLI calls the same function. A slug passes through + it unchanged, so `--preview feature-login` works as well as `--preview feature/login`. +- **Length.** 40 characters of app, `--` and 20 of slug make 62, inside the 63 of a namespace + and of a DNS label in a host name. +- **The cluster does no slugging.** It only strips the prefix: + `<< inputs.app >>--<< trimPrefix "branch-" inputs.tag >>`, shown to work in the spike. + +*Alternatives:* +- **The provider's numeric `id` (`hello-p1366820814`).** Always short and unique, but nobody + can read it; rejected by the maintainer. +- **Slugging in the ResourceSet template.** It would need truncation and hashing inside the + template, a second implementation that can drift from the CLI's. + +### 4. The chart generates a preview's secrets with `lookup` + +- **What the chart renders.** With `platform.preview: true`, set only by the ResourceSet + `previews`, `shelf-app` renders `shelf-secrets` itself: for each secret app.yaml declares with + `generate: true`, the value `lookup` finds in the existing Secret, otherwise `randAlphaNum 26`. +- **Where it ends up.** helm-controller runs `lookup` against the cluster, so upgrades keep the + values. The values end up in the Secret and in Helm's release storage in the preview + namespace. Both are in the cluster and go away with the namespace; neither reaches Git, a log, + a golden file or the host. +- **Tests.** The chart tests replace the generated values with a fixed placeholder before + comparing with the golden files. +- **Apps are unchanged.** Without `platform.preview` the chart does what it does today. + +*Alternatives:* +- **A Job in each preview namespace that creates the Secret with kubectl.** An image, RBAC and + code of our own for something Helm does. +- **Copying the app's secrets.** It would share a database password between branches and the + app, against "a preview's own state". +- **The host service generating them.** A deploy path would depend on it. + +### 5. Limits: quota, LimitRange, at most N previews + +- **Quota.** Every preview namespace gets a ResourceQuota (`requests.cpu: 1`, + `requests.memory: 2Gi`, `requests.storage: 5Gi`). +- **LimitRange.** A quota on requests makes Kubernetes refuse pods without requests, so the + LimitRange gives containers a default request (50m CPU, 64Mi memory) and a default memory + limit of 512Mi. +- **The limit of N.** It is the provider's `filter.limit`. The provider takes tags in reverse + alphabetical order, not by recency. +- **Reporting.** shelf cannot know about the branches beyond the limit without reading the + registry, so `shelf app status` reports "limit reached" whenever the number of previews + equals N. + +### 6. Reading previews: the same diagnosis, keyed by namespace + +- **Listing.** `cluster.AppStates` stays the list of apps (providers labelled `shelf.dev/app`). + A new `cluster.Previews(app)` lists the namespaces labelled `shelf.dev/app=` and + `shelf.dev/preview`. +- **Diagnosis and rollout.** They run on a namespace instead of an app name. The objects of a + preview have the same names in their namespace as an app's, so `diagnose`, `AppRollout` and + `Verdict` from Phase 9 apply unchanged. +- **CLI.** `--preview ` on `status`, `status --wait` and `open` maps the branch to its + namespace through `BranchSlug`. +- **Waiting for a preview that doesn't exist yet.** It is Working ("waiting for the preview of + branch feature/login"), because CI and the one-minute poll come first. + +### 7. Publishing and deleting branch artifacts in `build.yml` + +- **Triggers.** Tenants change their `deploy.yml` to + `on: {push: {branches: ['**']}, pull_request: {}, delete: {}}`. A tenant that keeps + `branches: [main]` keeps working and gets no previews. +- **Push to the default branch.** Unchanged. +- **Push to another branch.** Images and the deploy artifact as `sha-` and + `branch-`, from `shelf build-plan --branch "$GITHUB_REF_NAME"`. The job summary shows + `shelf app status --preview ` and `shelf app open --preview `. +- **`delete` of a branch.** The workflow computes the tag the same way, finds the package + version of `ghcr.io//` that carries it (`/users/` or `/orgs/` packages API, + depending on the owner), and deletes that version with the workflow's `GITHUB_TOKEN` + (`packages: write`). +- **Pull requests.** They still push nothing, from forks or not. + +*Alternative:* leaving branch artifacts and expiring previews by age in the cluster. That needs +state the provider does not have, and a deleted branch would keep running. + +### 8. Dev registry with deletes, and `just smoke-previews` + +- **Registry deletes.** `hack/cluster-up.sh` enables `storage.delete` in the registry + container's config and restarts it, idempotently, so that level 2 deletes tags as GHCR does. +- **What `smoke-previews` does:** + 1. Adds `examples/hello` (web plus Postgres with a volume) with `--previews=2`. + 2. Pushes `branch-a` and `branch-b`, waits for both previews with `--wait --commit`, and + checks two namespaces, two Postgres volumes and three distinct passwords. The values are + compared by hash, never printed. + 3. Checks that `open --preview` reaches a preview with its own Host, and that no preview has + cloudflared. The app stays private: only `smoke-quick` may open a quick tunnel (safety + rules), and that the template never renders cloudflared is pinned by + `TestPreviewsResourceSet`. + 4. Pushes `branch-c` and expects the limit to hold, with `previewLimitReached` set. + 5. Deletes `branch-a` and expects its namespace and volume to be gone. + 6. Runs `--no-previews` and expects all previews gone, the app untouched. + 7. Measures RAM per preview with `kubectl top` and records it in the results of this change. + 8. Removes the app. +- **CI.** It joins the full level-2 set and `select-smokes.sh`. `platform/previews/` and + `charts/**` select it. + +### 9. Turnaround: the agent should hardly notice the platform + +**Measured in Phase 9.** A push to `main` of `tweinmann/shelf-hello` was Ready on `greeter` in +127 s. That covers: +- the CI build, two images, estimated at 60–80 s; +- up to 60 s for Flux's poll; +- 10–20 s of rollout. + +A first push of a new branch adds the provider's discovery and a first install, together an +estimated 2–3.5 minutes. The maintainer asked to take the low-hanging fruit and to slow the +agent's flow as little as possible. + +- **Build cache.** `build.yml` builds with + `--cache-from type=registry,ref=:buildcache` and, in a publishing run, + `--cache-to type=registry,ref=:buildcache,mode=max`. + - The cache lives in the component's image package, not in the deploy artifact. The preview + provider (`^branch-`) and `shelf app tags` never see it. + - It needs no extra action, because the registry login is already there. + - Pull requests only read it. +- **Previews poll every 30 seconds.** The preview provider (`reconcileEvery: 30s`) and a + preview's deploy OCIRepository (`interval: 30s`). Apps stay at 1 minute: the extra registry + calls are limited to the branches agents are working on. +- **The agent does not idle.** The skill makes the wait a background job: push, start + `--wait --commit` in the background, go on, read the verdict before reporting the work done. + With the GitHub CLI it first watches the push's CI run (`gh run watch --exit-status`), so a + broken build is known in seconds rather than at the wait's timeout. That reads the forge from + the agent's machine, never from the platform. +- **Measured, not assumed.** `smoke-previews` prints the seconds from push to Ready for a first + push and for a later commit. Results go into this design. The level-3 acceptance with + `shelf-hello` adds the CI time. + +*Alternatives:* +- **A Flux Receiver webhook from CI.** It removes the poll altogether, but the cluster would + have to be reachable from the internet, which only the operator decides for an app. +- **`--reconcile` on the wait.** It was rejected in Phase 9: status would write. +- **30-second polls for apps too.** It doubles every app's registry calls for a gain only + previews need. +- **The GitHub Actions cache (`type=gha`).** `docker buildx build` in a `run` step does not see + the runtime token without an extra action. + +## Risks / Trade-offs + +- **Two branches with the same slug share a tag**, for example `feature/login` and + `feature-login`. The later push wins and both run as one preview. → Documented in the skill + and the README; it only happens without truncation, where the names are visibly alike. +- **The limit picks tags alphabetically, not by recency, and can evict.** A new branch that + sorts after the others pushes an existing preview out, and that preview's namespace and data + go with it, until a tag before it is deleted. → The status says when the limit is reached; the + operator raises N or deletes branches. A limit by recency would need a provider that sorts by + push time, which `OCIArtifactTag` does not offer. +- **Deleting through GHCR needs the token to administer the package.** Packages created by the + repository's own workflow normally allow it. → Checked at level 3 with `tweinmann/shelf-hello`; + until then a deleted branch's tag may stay, and its preview with it. +- **A preview counts against the host's RAM even when idle.** → At most N per app, a quota, and + the RAM per preview measured in `smoke-previews`. Scale-to-zero stays out of scope. +- **Generated preview secrets live in Helm's release storage.** → In the preview namespace only, + deleted with it, and never in a file of the repository. Accepted with the decision. +- **`requests.storage` counts requested sizes; local-path does not enforce them.** → As for apps + today, the README already says volume sizes are not enforced. +- **Previews read the registry every 30 seconds**: the provider per app and the deploy artifact + per preview. → Authenticated with the app's login, at most N + 1 readers per app. GHCR's limits + for authenticated reads are far above that. +- **A stale or broken build cache.** → `buildx` treats the cache as a hint; a missing or + incompatible cache only costs the time it would have saved. + +## Migration Plan + +- **Platform and chart.** The ResourceSet `previews` and the chart's preview mode ship with the + next platform and chart versions; `shelf init cluster` installs them. Apps registered before + are unchanged: they have no preview provider. +- **Tenants.** Previews need the new `deploy.yml` trigger and an operator's `--previews`. + `examples/tenant` and the README show the new trigger; `tweinmann/shelf-hello` is updated in + the acceptance. +- **Rollback.** `--no-previews` on every app, then the previous platform release. + +## Open Questions + +- Whether GHCR lets the workflow's `GITHUB_TOKEN` delete a package version of a user-owned + package without a separate PAT. This is answered at level 3. If not, deleting falls back to a + documented manual step, and the specs stay as they are. + +## Results + +`just smoke-previews` on the dev cluster (k3d in the devcontainer, 2026-10-28), times measured +from the moment the tag is in the registry, so without the CI build: + +| Measurement | Value | +|---|---| +| First push of a branch to a Ready preview (provider, namespace, quota, chain, release) | 23 s | +| Later commit on that branch to Ready | 17 s | +| Branch tag deleted to preview namespace gone | 26 s | +| RAM per preview of `examples/hello` (`kubectl top`) | 30 Mi, same as the app | + +Both previews kept their own password and volume across a new commit, the limit of two held +with three branch tags, and `--no-previews` removed the previews and kept the app. The push to +Ready of a real tenant, CI build included, is measured in the level-3 acceptance. diff --git a/openspec/changes/phase-10-branch-previews/proposal.md b/openspec/changes/phase-10-branch-previews/proposal.md index 5c62b6a..379b83a 100644 --- a/openspec/changes/phase-10-branch-previews/proposal.md +++ b/openspec/changes/phase-10-branch-previews/proposal.md @@ -1,66 +1,74 @@ # Proposal -> Draft. This change holds the roadmap text only and is marked `skip_specs: true`. -> `/opsx:propose` turns it into a full change: it writes the delta specs, `design.md` and -> `tasks.md`, revises this proposal, and removes `skip_specs` from `.openspec.yaml`. - ## Why Parallel coding agents work on several branches of the same app at once. Each branch needs its -own running, isolated copy of the app's complete stack to be tested against, on the user's own -hardware, without a shared database and without becoming reachable from the internet. +own running, isolated copy of the app's complete stack, its own database included, to be tested +against on the user's own hardware. Today a branch is built in CI and then thrown away: nothing +of it runs anywhere. Phase 9 gave agents a way to wait for a commit and read a verdict; this +phase gives them something per branch to wait for. ## What Changes -- `build.yml` pushes images and a deploy artifact also for non-default branches, tagged e.g. - `branch-` (today it builds there but pushes nothing), and deletes the tag when the branch - is deleted. Pull requests from forks never push. -- In the cluster, previews are discovered from the deploy artifact's tags. -- Each preview: its own namespace, its own secrets (generated, never copied from the app), its own - volumes, a resource quota, a limit of concurrent previews per app; deleted with its tag. -- Previews are never exposed through a tunnel in this phase; they are reached with - `shelf app open --preview `. +- **Previews are an operator decision per app.** `shelf app add` and `shelf app credentials` + take `--previews[=N]` (at most N previews, default 3) and `--no-previews`; off by default. + A tenant repository cannot give itself previews. +- **The reusable workflow publishes branches.** `build.yml` also pushes the images and the + deploy artifact for a push to a non-default branch, tagged `branch-`, and deletes that + tag when the branch is deleted. Pull requests, from forks or not, still push nothing. The + slug comes from `shelf build-plan --branch `, the one implementation the CLI shares. + Tenants change their `deploy.yml` trigger to all branches plus `delete`. +- **The cluster discovers previews from the registry.** An app with previews gets a second + provider, `-previews`, of type `OCIArtifactTag` on the app's deploy artifact, filtered to + `branch-*`. A new ResourceSet `previews` in the platform artifact turns each tag into a + preview: namespace `--`, host `--.shelf.internal`, the app's registry + login, a ResourceQuota and LimitRange, and the same deploy chain as an app. A deleted tag + removes its preview with all its volumes. No forge API is involved. +- **A preview has its own state.** The chart generates a preview's secrets itself (`lookup`, + otherwise random), never copied from the app; volumes and database are the preview's own. +- **Previews are never exposed.** No tunnel, no quick tunnel, no DNS record; they are reached + with `shelf app open --preview `. +- **The agent interface covers previews.** `shelf app status ` lists the app's previews; + `--preview ` selects one for status, `--wait --commit` and `open`, with the same JSON + documents. `shelf app rm` removes an app's previews with it. +- **Short turnaround for agents.** `build.yml` caches image layers in the registry + (`:buildcache` per component), previews poll every 30 seconds instead of 60, and the tenant + skill tells an agent to wait in the background while it keeps working, and to check the CI run + of its push first, so that a broken build shows within seconds instead of at the timeout. + `smoke-previews` measures the time from push to Ready. +- **App names may not contain `--`**, which separates an app from its previews. +- **Dev registry allows deletes**, so that level 2 can delete a branch tag. +- `just smoke-previews`: two branches side by side, each with its own database; deleting a + branch removes its preview; RAM per preview measured. ## Capabilities ### New Capabilities -- `branch-previews` (planned). +- `branch-previews`: what a preview is, how it is discovered, named, isolated, limited, reached + and removed. ### Modified Capabilities -To be decided at `/opsx:propose`; likely `app-delivery`, `app-chart` and the reusable workflow. +- `app-delivery`: app naming reserves `--`; the build plan names a branch's tag; the reusable + workflow publishes and deletes branch artifacts; status lists previews; removal removes them. +- `app-chart`: generated secrets in preview mode. +- `app-exposure`: local access to a preview. +- `agent-interface`: `--preview` on status, wait and the JSON documents; the tenant skill waits + in the background and checks CI first. +- `cluster-init`: the platform artifact also holds the ResourceSet `previews`. +- `dev-environment`: the dev registry allows deletes; `smoke-previews` in the full level-2 set. ## Impact -`.github/workflows/build.yml`, `platform/apps`, `internal/cluster`, `internal/ops`, CLI. Level 2. - -## Roadmap text (from the re-cut of 2026-10-27) - -A preview gets the app's complete stack, including its own database instance, in its own -namespace. The platform knows components, not databases; opinionated providers (e.g. -`resources: db: {type: postgres}`) may come later as an optional layer. - -**To verify at propose:** the Flux Operator's input provider for OCI artifact tags (the embedded -operator is v0.60.0), its tag filtering and polling interval. Fallback: the host service maintains -one provider per preview, as long as no deploy path depends on a forge API. - -**Open:** naming of preview namespaces and hosts within the 40-character limit and without -colliding with other app names (e.g. a reserved separator, or a label-based namespace name). - -Rejected alternatives, with their reasons: - -- Flux Operator `GitHubPullRequest` / `GitHubBranch` input providers: the cluster would talk to - the GitHub API, against the guardrail that the registry is the only interface between build and - platform. The workflow pushes an artifact per branch instead. -- A shared Postgres instance for previews: cheaper, but opinionated, and against "everything is a - component" and independent environments. -- A vcluster per preview: heavier than namespaces; only worth it for apps that bring CRDs or - cluster-scoped objects. Namespaces with quotas (and the NetworkPolicies of Phase 12) are the - default. -- A local build path into the cluster for fast iteration: the devcontainer is the inner loop; - previews verify the built artefact. - -**Acceptance (level 2):** two branches run side by side, isolated, each with its own database; -deleting a branch removes its preview; RAM per preview is measured and recorded. `examples/hello` -and `examples/tenant` serve as the reference apps (formerly Phase 13). +- `platform/previews` (new ResourceSet), `charts/shelf-app` (preview mode, generated secrets), + `.github/workflows/build.yml` (branches, delete), `.github/scripts/select-smokes.sh`. +- `internal/cluster` (preview provider, preview discovery and diagnosis), `internal/ops` + (previews on add/credentials/rm, status, wait, open), `internal/cli` (`--previews`, + `--preview`, `build-plan --branch`), a slug function shared by CLI and workflow. +- `examples/tenant` and the README (new `deploy.yml` trigger), `hack/cluster-up.sh` (registry + deletes), `hack/smoke/previews.sh`. +- Platform and chart versions change; existing apps are unaffected until previews are switched + on for them. No admin UI change in this phase. +- Level 1 for slug, chart and verdicts; level 2 for discovery, isolation and removal + (`just smoke-previews`); level 3 for branch deletion through GHCR with `tweinmann/shelf-hello`. diff --git a/openspec/changes/phase-10-branch-previews/specs/agent-interface/spec.md b/openspec/changes/phase-10-branch-previews/specs/agent-interface/spec.md new file mode 100644 index 0000000..8598984 --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/specs/agent-interface/spec.md @@ -0,0 +1,55 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Previews in the agent interface +`shelf app status --preview ` SHALL accept `--wait`, `--commit`, `--timeout` and +`--json` exactly as for the app, about that preview; the `AppStatus` of an app SHALL list its +previews under `previews`, each with `slug`, `tag`, `namespace`, `host`, `phase` and `commit`, +and `previewLimit` and `previewLimitReached` when the app has previews. + +#### Scenario: Agent waits for its branch +- **WHEN** an agent pushes commit `3f2a9c1` to the branch `feature/login` and runs `shelf app status hello --preview feature/login --wait --commit 3f2a9c1 --json` +- **THEN** it gets a `WaitResult` for `hello--feature-login` with exit status 0, 1 or 2 as for an app, and the app's own verdict plays no part + +#### Scenario: Preview appears only after the push +- **WHEN** the agent waits for a branch whose first artifact is still being built +- **THEN** the wait counts the missing preview as Working and does not fail before its timeout + +#### Scenario: Previews in the JSON of an app +- **WHEN** a user runs `shelf app status hello --json` for an app with two previews and `--previews=3` +- **THEN** `previews` has two entries, `previewLimit` is 3 and `previewLimitReached` is false + +#### Scenario: App without previews +- **WHEN** an app has no previews switched on +- **THEN** its `AppStatus` has no `previews`, `previewLimit` or `previewLimitReached` field + +## MODIFIED Requirements + +### Requirement: Agent instructions in the tenant template +`examples/tenant` SHALL hold a Claude Code skill `.claude/skills/shelf/SKILL.md` and a `CLAUDE.md` +section that tell an agent to push, run `shelf app status --wait --commit --json`, read +the verdict, and fix what its stage and hint name; the app name is a placeholder the operator +fills in. The skill SHALL tell the agent to start the wait in the background and keep working +until it needs the verdict, to check the CI run of its push first when the GitHub CLI is +available, and to wait for its branch's preview with `--preview` when it works on a branch. + +#### Scenario: Agent loop described +- **WHEN** an agent working in a copy of `examples/tenant` reads the skill +- **THEN** it finds the wait command with `--commit` and `--json`, the meaning of the three exit statuses, and that it cannot make the app public + +#### Scenario: No hook +- **WHEN** the tenant template is inspected +- **THEN** it ships no Claude Code hook that runs the wait + +#### Scenario: Waiting without idling +- **WHEN** an agent reads the skill after a push +- **THEN** it is told to run the wait in the background, to go on with other work, and to read the verdict before it reports the work as done + +#### Scenario: Broken build found early +- **WHEN** the CI run of the push fails +- **THEN** the skill tells the agent to read the failed run (`gh run watch --exit-status`, `gh run view --log-failed`) instead of waiting for the wait's timeout + +#### Scenario: Work on a branch +- **WHEN** the agent pushed to a branch other than `main` +- **THEN** the skill tells it to add `--preview ` to the wait, and that a first push of a branch takes about a minute longer than a later one diff --git a/openspec/changes/phase-10-branch-previews/specs/app-chart/spec.md b/openspec/changes/phase-10-branch-previews/specs/app-chart/spec.md new file mode 100644 index 0000000..8cee539 --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/specs/app-chart/spec.md @@ -0,0 +1,25 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Generated secrets in preview mode +With `platform.preview` set, the chart SHALL render the Secret `shelf-secrets` itself, with one key +per secret app.yaml declares as `generate: true`: the value already stored in that Secret if it +exists, otherwise a new random URL-safe value of the same length shelf generates for apps. +Without `platform.preview`, the chart MUST NOT render `shelf-secrets`. + +#### Scenario: First install of a preview +- **WHEN** a preview of an app that declares `db-password: { generate: true }` is installed +- **THEN** `shelf-secrets` in the preview namespace has a key `db-password` with a new random value + +#### Scenario: Upgrade of a preview +- **WHEN** the preview is upgraded to a new artifact +- **THEN** `db-password` keeps the value it had + +#### Scenario: An app +- **WHEN** the chart renders an app without `platform.preview` +- **THEN** it renders no `shelf-secrets`, which the platform copies from `shelf-system/app-` as before + +#### Scenario: Golden files +- **WHEN** the chart tests render preview mode +- **THEN** the golden files hold a placeholder instead of the random value, and no generated value is written to any file in the repository diff --git a/openspec/changes/phase-10-branch-previews/specs/app-delivery/spec.md b/openspec/changes/phase-10-branch-previews/specs/app-delivery/spec.md new file mode 100644 index 0000000..4f9bb30 --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/specs/app-delivery/spec.md @@ -0,0 +1,128 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: App naming +The system SHALL name an app at `shelf app add `, independent of app.yaml's `name`, and MUST +reject a name that is not a DNS label of at most 40 characters, that contains `--` (which separates +an app from its previews), or that is reserved for the platform (`default`, `flux-system`, +`traefik`, `cloudflared`, or a name starting with `kube-` or `shelf-`). + +#### Scenario: Reserved name +- **WHEN** a user runs `shelf app add shelf-web oci://ghcr.io/acme/web:main` +- **THEN** the command fails saying the name is reserved for the platform, before contacting the cluster + +#### Scenario: Invalid name +- **WHEN** the name contains upper-case letters or is longer than 40 characters +- **THEN** the command fails saying it must be a DNS label of at most 40 characters + +#### Scenario: Double hyphen +- **WHEN** a user runs `shelf app add hello--copy oci://ghcr.io/acme/hello:main` +- **THEN** the command fails saying `--` is reserved for previews, before contacting the cluster + +### Requirement: Build plan for tenant CI +`shelf build-plan ` SHALL print JSON `{package, builds}`, where `package` is app.yaml's +`name` and `builds` lists each component with a `build` directory and its context relative to +the working directory; with `--branch ` it SHALL also print `tag`, the branch tag of that +branch. + +#### Scenario: Component with a build directory +- **WHEN** app.yaml declares a component with `build: ./web` and that directory exists +- **THEN** `builds` contains that component with the context path of `./web` relative to the working directory + +#### Scenario: Missing build directory +- **WHEN** a component's build directory does not exist +- **THEN** the command fails naming the component and the directory + +#### Scenario: Branch tag +- **WHEN** the workflow runs `shelf build-plan app.yaml --branch feature/Login` +- **THEN** the JSON holds `"tag": "branch-feature-login"` + +### Requirement: Reusable tenant workflow +The reusable workflow `.github/workflows/build.yml` SHALL validate the app.yaml, build every +`build:` component for `linux/arm64` as `ghcr.io///`, render the +app.yaml, and push the deploy artifact as `ghcr.io//` tagged `sha-` +and `main` for the default branch or `branch-` for another branch, with the annotation +`dev.shelf.package=` and the source and revision of the commit; when a branch is +deleted, it SHALL delete that branch's artifact. It SHALL reuse image layers between runs +through a build cache in the registry, which only runs that publish write. + +#### Scenario: Push to the default branch +- **WHEN** the workflow runs for a push to the repository's default branch +- **THEN** it pushes the images and the deploy artifact, and outputs `oci://ghcr.io//:main` +- **AND** its job summary shows a `shelf app add` command for that artifact + +#### Scenario: Push to another branch +- **WHEN** the workflow runs for a push to the branch `feature/login` +- **THEN** it pushes the images and the deploy artifact tagged `sha-` and `branch-feature-login`, and never `main` + +#### Scenario: Pull request or other branch +- **WHEN** the workflow runs for a pull request, from a fork or not +- **THEN** it builds the images but pushes neither images nor a deploy artifact +- **AND** a push to another branch is no longer such a case: it publishes, as in "Push to another branch" + +#### Scenario: Build cache +- **WHEN** a push changes only the source of one component +- **THEN** the build of every component reads its layers from `ghcr.io///:buildcache`, a publishing run writes that cache back, and a pull request only reads it + +#### Scenario: Branch deleted +- **WHEN** the workflow runs for the deletion of the branch `feature/login` +- **THEN** it deletes the package version that carries `branch-feature-login` from `ghcr.io//`, and nothing else + +#### Scenario: Pinned shelf release +- **WHEN** the caller sets the input `shelf-version` +- **THEN** the workflow installs that release's linux/arm64 binary after checking its checksum, and otherwise the latest release + +### Requirement: App removal +`shelf app rm ` SHALL ask for confirmation unless `--yes` is given, delete the app's +provider and its preview provider, wait until its namespace and every preview namespace with all +objects and volumes are gone, and delete `app-` and `tunnel-` from `shelf-system`, +while keeping the host secret backup and saying where it is. + +#### Scenario: Remove an app +- **WHEN** a user confirms `shelf app rm hello` +- **THEN** the namespace `hello`, its volumes and the app's Secrets in `shelf-system` are deleted +- **AND** the output names the kept backup file, so a later `shelf app add hello` restores the old secret values + +#### Scenario: Remove an app with previews +- **WHEN** a user confirms `shelf app rm hello` and `hello` has the previews `hello--feature-login` and `hello--fix-typo` +- **THEN** both preview namespaces and their volumes are deleted along with the app + +#### Scenario: Namespace not created by the platform +- **WHEN** a namespace named after the app exists without the label `shelf.dev/app=` +- **THEN** removal fails saying the namespace does not belong to the app + +#### Scenario: Unknown app +- **WHEN** neither provider, namespace nor secrets exist for the name +- **THEN** the command fails saying the app does not exist + +### Requirement: App status command +`shelf app status` SHALL list every app with its state and, when not Ready, the failing stage and +reason in one line; `shelf app status ` SHALL print the app's address, artifact, registry +connection, running revision, the git commit that revision was built from, deploy time, +components, every stage with its message and, if not Ready, its hint, and the app's previews with +their branch tag, state and commit; `--preview ` SHALL print the same for one preview. + +#### Scenario: No apps +- **WHEN** the cluster has no registered apps +- **THEN** `shelf app status` prints "No apps yet." + +#### Scenario: One app in detail +- **WHEN** a user runs `shelf app status hello` for a healthy app +- **THEN** the output shows `hello: Ready`, the artifact reference, the revision, and all five stages as Ready + +#### Scenario: Commit of the running artifact +- **WHEN** the app's deploy artifact was pushed by the reusable workflow for commit `42a964eb0b4b…` +- **THEN** `shelf app status hello` shows that commit, read from the artifact's revision annotation as the cluster reports it, without calling a forge API + +#### Scenario: Artifact without a commit +- **WHEN** the deploy artifact carries no git revision +- **THEN** the commit line is left out + +#### Scenario: App with previews +- **WHEN** `hello` has previews for `feature/login` and `fix-typo` +- **THEN** `shelf app status hello` lists both with their tags, states and commits, and the list of all apps shows them under `hello`, not as apps of their own + +#### Scenario: One preview +- **WHEN** a user runs `shelf app status hello --preview feature/login` +- **THEN** the output shows the preview's namespace `hello--feature-login`, its host, artifact tag, commit, components and five stages diff --git a/openspec/changes/phase-10-branch-previews/specs/app-exposure/spec.md b/openspec/changes/phase-10-branch-previews/specs/app-exposure/spec.md new file mode 100644 index 0000000..76218e8 --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/specs/app-exposure/spec.md @@ -0,0 +1,16 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Local access to a preview +`shelf app open --preview ` SHALL forward like `shelf app open `, on the +loopback interface only, with the Host header of the preview's host `--.shelf.internal`, +so that the preview's routes and `stripPrefix` behave as the app's. + +#### Scenario: Preview in a browser +- **WHEN** a user runs `shelf app open hello --preview feature/login` +- **THEN** it prints `http://127.0.0.1:/`, and requests reach the preview `hello--feature-login`, not the app + +#### Scenario: Branch without a preview +- **WHEN** `hello` has no preview for the branch `gone` +- **THEN** the command fails saying so and naming the previews `hello` has, before it listens on any port diff --git a/openspec/changes/phase-10-branch-previews/specs/branch-previews/spec.md b/openspec/changes/phase-10-branch-previews/specs/branch-previews/spec.md new file mode 100644 index 0000000..710c8d5 --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/specs/branch-previews/spec.md @@ -0,0 +1,136 @@ +# Spec Delta + +## Purpose + +Gives each branch of an app its own isolated, running copy of the app's complete stack on the +user's own hardware, discovered from the registry, reachable only from the operator's machine, +and removed when the branch is. + +## ADDED Requirements + +### Requirement: Previews are the operator's decision +`shelf app add` and `shelf app credentials` SHALL take `--previews[=N]`, which gives the app at +most N previews (3 without a number, at most 10), and `--no-previews`, which removes them all; an app has +no previews unless the operator chose them, and nothing in app.yaml or the tenant repository can +change that. + +#### Scenario: Previews switched on +- **WHEN** a user runs `shelf app add hello oci://ghcr.io/acme/hello:main --previews` +- **THEN** the app gets previews for at most 3 branches + +#### Scenario: Default is off +- **WHEN** an app is added without `--previews` +- **THEN** no branch of it gets a preview, whatever the registry holds + +#### Scenario: Switched off again +- **WHEN** a user runs `shelf app credentials hello --no-previews` +- **THEN** every preview of `hello` is removed with its namespace and volumes, and the app itself keeps running + +#### Scenario: Platform without previews +- **WHEN** `--previews` is given for a cluster whose platform has no ResourceSet `previews` +- **THEN** the command fails at once saying the platform predates previews and that `shelf init cluster` updates it, and writes nothing + +#### Scenario: Limit out of range +- **WHEN** `--previews=0` or `--previews=11` is given +- **THEN** the command fails before contacting the cluster + +#### Scenario: Number without `=` +- **WHEN** `--previews 5` is given +- **THEN** the command fails saying to write `--previews=5`, before contacting the cluster + +### Requirement: Preview provider +An app with previews SHALL have a `ResourceSetInputProvider` `--previews` in `shelf-system`, +labelled `shelf.dev/previews=` and not `shelf.dev/app`, of type `OCIArtifactTag` on the +app's deploy artifact, filtered to tags `branch-*` and limited to N, reading the registry with +the app's registry connection, and checked every 30 seconds. + +#### Scenario: Provider of an app with a registry connection +- **WHEN** `hello` with registry connection `ghcr` and artifact `oci://ghcr.io/acme/hello:main` gets previews +- **THEN** `shelf-system/hello--previews` has type `OCIArtifactTag`, url `oci://ghcr.io/acme/hello`, `includeTag` `^branch-`, limit 3 and the image pull secret `connection-registry-ghcr` + +#### Scenario: Not an app +- **WHEN** the platform lists the apps +- **THEN** `hello--previews` is not among them + +### Requirement: Branch tag +A branch SHALL be published as the tag `branch-`, where the slug is the branch name in +lower case with every run of characters other than letters and digits replaced by one `-`, +without leading or trailing `-`, and, when longer than 20 characters, its first 13 characters +followed by `-` and the first 6 hex digits of the SHA-256 of the branch name. + +#### Scenario: Short branch name +- **WHEN** the branch is `feature/Login` +- **THEN** the tag is `branch-feature-login` + +#### Scenario: Long branch name +- **WHEN** two branches share their first 20 characters after slugging +- **THEN** their tags still differ, and neither slug is longer than 20 characters + +### Requirement: Preview naming +A preview of `` for the tag `branch-` SHALL run in the namespace `--`, +answer at `--.shelf.internal` inside the cluster, and carry the labels +`shelf.dev/app=` and `shelf.dev/preview=`. + +#### Scenario: Preview of a branch +- **WHEN** `hello` has previews and the registry holds `branch-feature-login` +- **THEN** the namespace `hello--feature-login` exists with those labels, and its web component answers at `hello--feature-login.shelf.internal` + +### Requirement: Preview contents +The platform SHALL give each preview the same deploy chain as an app (its deploy artifact at the +branch tag, checked every 30 seconds, the app values, the chart, the release), the app's registry login, a ResourceQuota +(requests of 1 CPU, 2Gi memory, 5Gi storage) and a LimitRange that gives containers without +requests a default; it MUST NOT give a preview a tunnel, cloudflared or a DNS record. + +#### Scenario: Preview of an exposed app +- **WHEN** an app exposed through a Cloudflare connection has a preview +- **THEN** the preview namespace has no cloudflared, and nothing outside the cluster reaches it + +#### Scenario: Over the quota +- **WHEN** a preview's pods would request more than 2Gi memory together +- **THEN** the pods beyond it are refused, and the preview's pods stage says so + +### Requirement: A preview's own state +A preview SHALL have its own volumes and its own secret values, generated in the cluster when the +preview is first created and kept for its lifetime; it MUST NOT share a volume, a secret value or a +database with the app or with another preview, and its secrets have no host backup. + +#### Scenario: Two branches with a database each +- **WHEN** `hello` with a Postgres component has previews for two branches +- **THEN** each preview runs its own Postgres with its own volume and its own generated password, both different from the app's + +#### Scenario: Preview updated +- **WHEN** a new commit is pushed to a branch that has a preview +- **THEN** the preview rolls out the new artifact and keeps its secret values and its data + +### Requirement: Preview removal +The platform SHALL remove a preview, with its namespace and volumes, when its tag disappears from +the registry, when the app's previews are switched off, and when the app is removed. + +#### Scenario: Branch deleted +- **WHEN** the tag `branch-feature-login` is deleted from the registry +- **THEN** within a few minutes the namespace `hello--feature-login` and its volumes are gone, and the app and its other previews are unchanged + +### Requirement: Preview limit +An app SHALL have at most N previews at a time, chosen by the provider from the branch tags, and +`shelf app status ` SHALL say when the limit is reached. + +#### Scenario: More branches than the limit +- **WHEN** `hello` has `--previews=2` and the registry holds three branch tags +- **THEN** exactly two previews exist, and `shelf app status hello` says the preview limit of 2 is reached + +### Requirement: Previews are never exposed +`--cloudflare`, `--domain` and `--quick` SHALL apply to the app only; a preview MUST be reachable +only inside the cluster and through `shelf app open --preview `. + +#### Scenario: Quick tunnel app +- **WHEN** an app with a quick tunnel has previews +- **THEN** its previews have no quick tunnel and no `*.trycloudflare.com` address + +### Requirement: Turnaround of a preview +A new commit on a branch with a preview SHALL be Ready in the preview within the CI run plus at +most 30 seconds plus its rollout, and a first push of a new branch within the CI run plus at most +60 seconds plus the first install; `smoke-previews` SHALL measure both without CI. + +#### Scenario: Measured at level 2 +- **WHEN** `just smoke-previews` pushes the first artifact of a branch and later a second one +- **THEN** it prints the seconds from each push to the preview's Ready verdict, and the results of the change record them diff --git a/openspec/changes/phase-10-branch-previews/specs/cluster-init/spec.md b/openspec/changes/phase-10-branch-previews/specs/cluster-init/spec.md new file mode 100644 index 0000000..15e24dc --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/specs/cluster-init/spec.md @@ -0,0 +1,15 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Platform contents +The platform artifact SHALL contain Traefik (Helm chart from `oci://ghcr.io/traefik/helm/traefik` +in namespace `traefik`, Service of type ClusterIP, ingress class `traefik` that is not the +default, Kubernetes Ingress and CRD providers enabled) the ResourceSet `apps` in +`shelf-system`, which generates every app from the ResourceSetInputProviders labelled +`shelf.dev/app`, and the ResourceSet `previews` in `shelf-system`, which generates every preview +from the providers labelled `shelf.dev/previews`. + +#### Scenario: Platform applied +- **WHEN** the installation completes +- **THEN** Traefik runs in `traefik` reachable at `traefik.traefik.svc.cluster.local:80` and the ResourceSets `shelf-system/apps` and `shelf-system/previews` exist diff --git a/openspec/changes/phase-10-branch-previews/specs/dev-environment/spec.md b/openspec/changes/phase-10-branch-previews/specs/dev-environment/spec.md new file mode 100644 index 0000000..efdb95b --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/specs/dev-environment/spec.md @@ -0,0 +1,52 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Dev cluster +`just cluster-up` SHALL create, or start if it exists, the k3d cluster `shelf-dev` with the k3s +image pinned in the Dockerfile, without k3s's own Traefik and without k3d's load balancer, its API +server on the container's loopback, and the plain-HTTP registry `shelf-registry:5000`, reachable +by that name from pods and from the devcontainer, which allows deleting manifests. It MUST write the kubeconfig only to +`KUBECONFIG` and never change a default kubeconfig or switch a context. + +#### Scenario: After a container restart +- **WHEN** the devcontainer was rebuilt or restarted and `just cluster-up` runs +- **THEN** the existing cluster is started, and its kubeconfig is written again + +#### Scenario: Reset +- **WHEN** `just cluster-reset` runs +- **THEN** cluster and registry are deleted and created empty, and `just init-cluster` pushes the platform and chart again before installing them + +#### Scenario: Deleting a tag +- **WHEN** a smoke test deletes a branch tag from `shelf-registry:5000` +- **THEN** the registry accepts the deletion, so a preview can be removed at level 2 as on GHCR + +### Requirement: Three test levels +Checks SHALL be split into level 1 (no cluster: `just test`, which runs gofmt, `go vet`, +`go test -race`, Helm lint, kubeconform, example validation and `openspec validate --all --strict`), +level 2 (the k3d cluster: the `just smoke-*` recipes), and level 3 (the Mac mini and real +Cloudflare), and a task that needs level 2 or 3 MUST say so. + +#### Scenario: Level 1 in CI +- **WHEN** a commit is pushed to `main` or a pull request is opened or pushed to +- **THEN** CI runs `just test` inside the devcontainer image on an arm64 runner + +#### Scenario: Level 2 in CI +- **WHEN** a pull request is opened or reopened as ready, leaves draft, or is pushed to or labelled while it carries the label `level-2` +- **THEN** CI also runs the smoke tests its changed paths need, chosen by `.github/scripts/select-smokes.sh`, each against a fresh k3d cluster of its own, side by side, with `SMOKE_FULL=0` + +#### Scenario: Documentation only +- **WHEN** a pull request changes only planning, documentation or examples +- **THEN** CI runs no smoke test, and a path the selection does not know runs all of them + +#### Scenario: Level 2 in full +- **WHEN** a commit is pushed to `main`, or a pull request carries the label `level-2-full` +- **THEN** CI runs `smoke-secrets`, `smoke-chart`, `smoke-init`, `smoke-apps`, `smoke-agent` and `smoke-previews` with `SMOKE_FULL=1` + +#### Scenario: main fails +- **WHEN** CI fails on `main` +- **THEN** it opens an issue labelled `ci-main-red`, or comments on the one that is open, with the commit and the run + +#### Scenario: Release gate +- **WHEN** a release tag is pushed for a commit whose CI run on `main` did not succeed +- **THEN** `release.yml` stops before it publishes anything diff --git a/openspec/changes/phase-10-branch-previews/tasks.md b/openspec/changes/phase-10-branch-previews/tasks.md new file mode 100644 index 0000000..af98740 --- /dev/null +++ b/openspec/changes/phase-10-branch-previews/tasks.md @@ -0,0 +1,52 @@ +# Tasks + +## 1. Names + +- [x] 1.1 Reserve `--` in `ops.CheckAppName` with the message "`--` is reserved for previews"; verify with the table test in `internal/ops` and a CLI test for `shelf app add hello--copy` (spec app-delivery "Double hyphen") +- [x] 1.2 Add `schema.BranchSlug(branch)` and `schema.BranchTag(branch)` as in design decision 3; verify with table tests: `feature/Login`, runs of punctuation, leading and trailing separators, a slug passed in again, two long names with the same 20-character prefix, every result at most 20 characters and a DNS label +- [x] 1.3 Add `--branch` to `shelf build-plan`, printing `tag`; verify with a CLI test and the README CLI reference + +## 2. Chart preview mode + +- [x] 2.1 Render `shelf-secrets` in `charts/shelf-app` when `platform.preview` is set, one key per `generate: true` secret, from `lookup` or `randAlphaNum 26`; verify that `helm lint` passes and that the chart renders no `shelf-secrets` without `platform.preview` (existing golden files unchanged) +- [x] 2.2 Add a preview-mode case to the chart tests in `internal/chart`, replacing generated values with a placeholder before comparing; verify with `just golden` and that no generated value appears in `internal/chart/testdata` + +## 3. Platform + +- [x] 3.1 Add `platform/previews/resourceset.yaml` (and the kustomization entry) as in design decision 1, with the deploy OCIRepository at `interval: 30s`: Namespace with labels, `shelf-registry` copy, ResourceQuota, LimitRange, `shelf-deploy` ServiceAccount/Role/RoleBinding, deploy OCIRepository at `<< inputs.tag >>`, Kustomization, chart OCIRepository, HelmRelease `--` with `platform.preview: true` and no cloudflared; verify with kubeconform in `just test` and the platform golden/manifest tests in `internal/cluster` +- [x] 3.2 Make `shelf init cluster` wait for the ResourceSet `previews` as for `apps`; verify with the cluster tests and `just smoke-init` (spec cluster-init "Platform applied") + +## 4. Cluster and operations + +- [x] 4.1 Add the preview provider to `internal/cluster`: create or update `--previews` (type, url, filter, limit, insecure, secretRef, defaultValues, `reconcileEvery: 30s`, label `shelf.dev/previews`), and delete it; verify with server-side-apply fixture tests that the provider matches spec "Preview provider" and is not listed by `AppStates` +- [x] 4.2 Add `cluster.Previews(app)` (namespaces labelled `shelf.dev/app` and `shelf.dev/preview`, with phase, tag and commit) and generalize `AppDiagnosis`, `AppComponents` and `AppRollout` to a namespace; verify with tests on unstructured fixtures for an app with two previews +- [x] 4.3 Wire `--previews N` / `--no-previews` through `ops.AddApp` and `ops.Credentials`, rewrite the preview provider when the registry connection or `--insecure-registry` changes, and delete it in `ops.RemoveApp`, which also waits for every preview namespace; verify with CLI tests against the fake cluster (on, off, limit out of range, registry change, removal) + +## 5. CLI and agent interface + +- [x] 5.1 Show previews in `shelf app status ` (branch tag, phase, commit, limit reached) and add `--preview ` for one preview; verify with CLI tests (spec app-delivery "App with previews", "One preview") +- [x] 5.2 Add `previews`, `previewLimit` and `previewLimitReached` to `AppStatus`, and a preview's own `AppStatus`; verify with new golden files in `internal/cli/testdata/json` and that apps without previews keep their existing golden files +- [x] 5.3 Add `--preview` to `--wait`, with a missing preview counting as Working; verify with ops and CLI tests (spec agent-interface "Agent waits for its branch", "Preview appears only after the push") +- [x] 5.4 Add `--preview` to `shelf app open`, with the preview's host and a refusal naming the existing previews; verify with CLI tests (spec app-exposure "Local access to a preview") +- [x] 5.5 Document `--previews`, `--preview` and the preview naming in the README (CLI reference, "Deploying an app") and in the tenant skill; verify with `TestTenantAgentInstructions` + +## 6. Tenant workflow + +- [x] 6.1 Publish branch artifacts in `.github/workflows/build.yml` (tags `sha-` and `branch-`, never `main`; job summary with the preview commands); verify by reading the workflow against spec "Push to another branch" and with a dry run of its tag logic in a test script +- [x] 6.2 Delete a branch's package version on `delete` in `build.yml` (`/users/` or `/orgs/` API by owner type); verify the API calls against the GitHub REST documentation; the real deletion is level 3 +- [x] 6.3 Change the trigger in `examples/tenant/.github/workflows/deploy.yml`, the header comment of `build.yml` and the README to all branches plus `delete`; verify with `just test` + +- [x] 6.4 Add the registry build cache to `build.yml` (`--cache-from` always, `--cache-to ...,mode=max` only when publishing, `:buildcache` per component image); verify by reading the build arguments for a publishing run and a pull request, and that `shelf app tags` and the preview filter never list `buildcache` +- [x] 6.5 Update the tenant skill and `CLAUDE.md` (background wait, `gh run watch --exit-status` first, `--preview ` on branches, slower first push); verify with `TestTenantAgentInstructions` (spec agent-interface "Waiting without idling", "Broken build found early", "Work on a branch") + +## 7. Level 2 and CI + +- [x] 7.1 Enable deletes in the dev registry in `hack/cluster-up.sh`, idempotently; verify that `just cluster-up` twice leaves `storage.delete.enabled: true` and that a tag can be deleted (spec dev-environment "Deleting a tag") +- [x] 7.2 Add `hack/smoke/previews.sh` and `just smoke-previews` as in design decision 8, including the seconds from push to Ready for a first push and a later commit; verify that it passes on the dev cluster and record RAM per preview and both times in design.md under "Results" +- [ ] 7.3 Add `smoke-previews` to the full set in `.github/workflows/ci.yml` and to `.github/scripts/select-smokes.sh` (`platform/*`, `charts/*`, `hack/smoke/previews.sh`, `internal/cluster/*`); verify the selection on example paths and that the PR run shows it +- [x] 7.4 Add `just smoke-previews` to the CLAUDE.md dev-environment table and the README; run `just test` and `openspec validate --all --strict`; both pass + +## Workflow follow-up + +- Level-3 acceptance by the maintainer with `tweinmann/shelf-hello`: new `deploy.yml` trigger, `--previews` on `greeter`, a pushed branch gets a preview that `shelf app open --preview` shows, deleting the branch removes it (answers the open question about GHCR deletion); push-to-Ready times with CI recorded for a first push and a later commit, with and without the build cache. +- `/opsx:archive`; update CLAUDE.md Status, README Status and the context; stop for approval before Phase 11. diff --git a/platform/kustomization.yaml b/platform/kustomization.yaml index 4b9f503..58ff135 100644 --- a/platform/kustomization.yaml +++ b/platform/kustomization.yaml @@ -5,3 +5,4 @@ kind: Kustomization resources: - traefik - apps + - previews diff --git a/platform/previews/kustomization.yaml b/platform/previews/kustomization.yaml new file mode 100644 index 0000000..ebea1e0 --- /dev/null +++ b/platform/previews/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - resourceset.yaml diff --git a/platform/previews/resourceset.yaml b/platform/previews/resourceset.yaml new file mode 100644 index 0000000..f4d4ae0 --- /dev/null +++ b/platform/previews/resourceset.yaml @@ -0,0 +1,186 @@ +# An app with previews has a second ResourceSetInputProvider in shelf-system, --previews, +# labelled shelf.dev/previews, written by `shelf app add --previews`. It is of type OCIArtifactTag +# on the app's deploy artifact and exports one input set per tag branch-: id, tag and digest, +# plus the defaults app, url, insecure and registry. For each one this ResourceSet generates a +# preview: the app's deploy chain at the branch tag, in a namespace of its own. +# +# A preview is never exposed: no tunnel, no cloudflared, no DNS record. It answers at +# --.shelf.internal inside the cluster, and `shelf app open --preview` reaches it. Its +# secrets are generated by the chart (platform.preview), never copied from the app, and its +# volumes are its own. A tag that disappears from the registry takes the preview with it. +# +# The slug was made by shelf (`shelf build-plan --branch`), so the name only strips the prefix. +# It is spelled out wherever it is needed: Flux substitutes ${...} in the platform when it applies +# it, and a template variable starting with $ would not survive that. +apiVersion: fluxcd.controlplane.io/v1 +kind: ResourceSet +metadata: + name: previews + namespace: shelf-system + annotations: + fluxcd.controlplane.io/reconcileEvery: 10m +spec: + inputsFrom: + - apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + selector: + matchExpressions: + - key: shelf.dev/previews + operator: Exists + resourcesTemplate: | + --- + apiVersion: v1 + kind: Namespace + metadata: + name: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + labels: + shelf.dev/app: << inputs.app >> + shelf.dev/preview: << trimPrefix "branch-" inputs.tag >> + --- + # The login of the app's registry connection, for the preview's images and deploy artifact. + apiVersion: v1 + kind: Secret + metadata: + name: shelf-registry + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + annotations: + fluxcd.controlplane.io/copyFrom: "shelf-system/<< if inputs.registry >>connection-registry-<< inputs.registry >><< else >>registry-anonymous<< end >>" + type: kubernetes.io/dockerconfigjson + --- + # One preview cannot take the host from the app: requests are capped, and the LimitRange gives + # containers without requests a default, which a quota on requests needs. + apiVersion: v1 + kind: ResourceQuota + metadata: + name: shelf-preview + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + spec: + hard: + requests.cpu: "1" + requests.memory: 2Gi + requests.storage: 5Gi + --- + apiVersion: v1 + kind: LimitRange + metadata: + name: shelf-preview + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + spec: + limits: + - type: Container + defaultRequest: + cpu: 50m + memory: 64Mi + default: + memory: 512Mi + --- + # As for an app: the deploy Kustomization may only manage ConfigMaps in the namespace. + apiVersion: v1 + kind: ServiceAccount + metadata: + name: shelf-deploy + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: shelf-deploy + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + rules: + - apiGroups: [""] + resources: [configmaps] + verbs: [get, list, watch, create, update, patch, delete] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: shelf-deploy + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: shelf-deploy + subjects: + - kind: ServiceAccount + name: shelf-deploy + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + --- + # Every 30 seconds, not every minute as for an app: an agent waits for this. + apiVersion: source.toolkit.fluxcd.io/v1 + kind: OCIRepository + metadata: + name: deploy + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + spec: + interval: 30s + url: << inputs.url >> + ref: + tag: << inputs.tag >> + insecure: << inputs.insecure >> + secretRef: + name: shelf-registry + --- + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: deploy + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + spec: + interval: 10m + sourceRef: + kind: OCIRepository + name: deploy + path: ./ + prune: true + wait: true + timeout: 2m + targetNamespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + serviceAccountName: shelf-deploy + commonMetadata: + labels: + shelf.dev/app: << inputs.app >> + reconcile.fluxcd.io/watch: Enabled + --- + apiVersion: source.toolkit.fluxcd.io/v1 + kind: OCIRepository + metadata: + name: shelf-app + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + spec: + interval: 1h + url: ${SHELF_CHART_URL} + ref: + tag: ${SHELF_CHART_TAG} + insecure: ${SHELF_INSECURE_REGISTRY:=false} + layerSelector: + mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip + operation: copy + --- + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + spec: + interval: 10m + chartRef: + kind: OCIRepository + name: shelf-app + valuesFrom: + - kind: ConfigMap + name: shelf-values + valuesKey: app.yaml + values: + name: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + platform: + domain: shelf.internal + hostSuffix: "" + imagePullSecret: shelf-registry + preview: true + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + remediateLastFailure: true