diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5d00e91..8e33307 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -11,6 +11,8 @@ ARG JUST_VERSION=1.58.0 ARG KUBECONFORM_VERSION=0.8.0 ARG NODE_VERSION=24.21.0 ARG OPENSPEC_VERSION=1.14.1 +ARG COSIGN_VERSION=3.1.3 +ARG CRANE_VERSION=0.20.6 # OpenSpec (spec-driven changes, see openspec/) collects usage data unless told not to. ENV OPENSPEC_TELEMETRY=0 DO_NOT_TRACK=1 NPM_CONFIG_UPDATE_NOTIFIER=false @@ -21,8 +23,8 @@ ENV SHELF_K3S_IMAGE=rancher/k3s:v1.36.4-k3s1 RUN set -eux; \ arch="$(dpkg --print-architecture)"; \ case "$arch" in \ - arm64) just_arch=aarch64; node_arch=arm64 ;; \ - amd64) just_arch=x86_64; node_arch=x64 ;; \ + arm64) just_arch=aarch64; node_arch=arm64; crane_arch=arm64 ;; \ + amd64) just_arch=x86_64; node_arch=x64; crane_arch=x86_64 ;; \ *) echo "unsupported architecture: $arch" >&2; exit 1 ;; \ esac; \ tmp="$(mktemp -d)"; \ @@ -41,10 +43,15 @@ RUN set -eux; \ | tar -xz -C /usr/local/bin kubeconform; \ curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" \ | tar -xJ -C /usr/local --strip-components=1 --exclude='*.md' --exclude=LICENSE; \ + curl -fsSL -o /usr/local/bin/cosign \ + "https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}"; \ + curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v${CRANE_VERSION}/go-containerregistry_Linux_${crane_arch}.tar.gz" \ + | tar -xz -C /usr/local/bin crane; \ npm install -g "@fission-ai/openspec@${OPENSPEC_VERSION}"; \ npm cache clean --force; \ chmod 0755 /usr/local/bin/k3d /usr/local/bin/kubectl /usr/local/bin/helm \ - /usr/local/bin/flux /usr/local/bin/just /usr/local/bin/kubeconform; \ + /usr/local/bin/flux /usr/local/bin/just /usr/local/bin/kubeconform \ + /usr/local/bin/cosign /usr/local/bin/crane; \ rm -rf "$tmp" # A named volume copies ownership and mode from the image directory it is mounted over, but only diff --git a/.github/scripts/select-smokes.sh b/.github/scripts/select-smokes.sh index f809571..0e4579c 100755 --- a/.github/scripts/select-smokes.sh +++ b/.github/scripts/select-smokes.sh @@ -7,7 +7,7 @@ # reason to skip. set -euo pipefail -all=(smoke-secrets smoke-chart smoke-init smoke-apps smoke-agent smoke-previews) +all=(smoke-secrets smoke-chart smoke-init smoke-apps smoke-agent smoke-previews smoke-verify) declare -A need=() for_path() { @@ -26,8 +26,9 @@ for_path() { hack/smoke/agent.* | internal/cli/json* | internal/cli/open* | internal/cli/wait* | internal/cli/tenant_test.go) echo smoke-agent ;; hack/smoke/previews.sh | internal/cli/previews* | internal/ops/previews*) echo smoke-previews ;; + hack/smoke/verify.sh | internal/cli/verification*) echo smoke-verify ;; internal/cli/* | internal/ops/* | internal/progress/* | internal/secrets/* | internal/hostcfg/*) - echo smoke-apps smoke-agent smoke-previews ;; + echo smoke-apps smoke-agent smoke-previews smoke-verify ;; *) echo "${all[@]}" ;; esac } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b831c5c..80e7721 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,6 +12,7 @@ # permissions: # contents: read # packages: write +# id-token: write # jobs: # shelf: # uses: tweinmann/shelf/.github/workflows/build.yml@v0 @@ -28,6 +29,15 @@ # Image layers are cached in the registry as :buildcache: every run reads the cache, and # every run that publishes writes it back. # +# A run that publishes signs every built image and the deploy artifact with keyless cosign (the +# job's GitHub OIDC token, hence `id-token: write`; the identity is this workflow, and the +# signature is recorded in the public Rekor log, repository name and commit included). It scans +# every image the rendered app.yaml names with Trivy, attaches the report to each built image as +# a `vuln` attestation, and writes the counts of CRITICAL and HIGH findings per image digest into +# the deploy artifact as the annotation dev.shelf.scan, which the artifact's signature covers. +# Findings never fail the build. The channel tag (main or branch-) moves only to a signed +# artifact. Signatures use cosign's classic format, which the Sigstore policy-controller reads. +# # A caller that keeps `branches: [main]` and no `delete` keeps working; it gets no previews. # # The package is not the app: shelf names an app when it is added (shelf app add ), @@ -47,6 +57,13 @@ on: needs a release with `shelf build-plan --branch`. type: string default: latest + platforms: + description: >- + Platforms to build the images for, comma-separated. linux/arm64 matches the hosts shelf + runs on; add linux/amd64 for a cloud target. Platforms other than the runner's are + built through QEMU, which is several times slower. + type: string + default: linux/arm64 outputs: artifact: description: The deploy artifact, oci://ghcr.io//:
; empty unless published. @@ -61,10 +78,14 @@ jobs: permissions: contents: read packages: write + id-token: write outputs: artifact: ${{ steps.publish.outputs.artifact }} env: APP_FILE: ${{ inputs.app-file }} + PLATFORMS: ${{ inputs.platforms }} + COSIGN_VERSION: v3.1.3 + TRIVY_VERSION: 0.75.0 PUBLISH: ${{ github.event_name == 'push' }} DEFAULT_BRANCH: ${{ github.ref_name == github.event.repository.default_branch }} steps: @@ -92,12 +113,42 @@ jobs: - name: Validate app.yaml run: shelf validate "$APP_FILE" + - name: Install cosign and Trivy + if: env.PUBLISH == 'true' + run: | + tmp="$(mktemp -d)" + cosign=https://github.com/sigstore/cosign/releases/download/$COSIGN_VERSION + curl -fsSL -o "$tmp/cosign-linux-arm64" "$cosign/cosign-linux-arm64" + curl -fsSL -o "$tmp/cosign_checksums.txt" "$cosign/cosign_checksums.txt" + (cd "$tmp" && grep ' cosign-linux-arm64$' cosign_checksums.txt | sha256sum -c -) + sudo install "$tmp/cosign-linux-arm64" /usr/local/bin/cosign + trivy=https://github.com/aquasecurity/trivy/releases/download/v$TRIVY_VERSION + curl -fsSL -o "$tmp/trivy_${TRIVY_VERSION}_Linux-ARM64.tar.gz" "$trivy/trivy_${TRIVY_VERSION}_Linux-ARM64.tar.gz" + curl -fsSL -o "$tmp/trivy_checksums.txt" "$trivy/trivy_${TRIVY_VERSION}_checksums.txt" + (cd "$tmp" && grep " trivy_${TRIVY_VERSION}_Linux-ARM64.tar.gz\$" trivy_checksums.txt | sha256sum -c -) + tar -xzf "$tmp/trivy_${TRIVY_VERSION}_Linux-ARM64.tar.gz" -C "$tmp" trivy + sudo install "$tmp/trivy" /usr/local/bin/trivy + cosign version + trivy --version + echo "TRIVY_DAY=$(date -u +%Y-%m-%d)" >>"$GITHUB_ENV" + + # Trivy's vulnerability database is downloaded once a day and shared between runs. + - uses: actions/cache@v4 + if: env.PUBLISH == 'true' + with: + path: ~/.cache/trivy + key: trivy-db-${{ env.TRIVY_DAY }} + restore-keys: trivy-db- + - uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ github.token }} + - uses: docker/setup-qemu-action@v4 + if: inputs.platforms != 'linux/arm64' + - uses: docker/setup-buildx-action@v4 - name: Build the images of this repository @@ -116,7 +167,7 @@ jobs: images=() while read -r component context; do image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/$package/$component" - args=(--platform linux/arm64 + args=(--platform "$PLATFORMS" --tag "$image:sha-${GITHUB_SHA::7}" --tag "$image:$channel" --cache-from "type=registry,ref=$image:buildcache" --label "org.opencontainers.image.source=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" @@ -134,9 +185,12 @@ jobs: # With the commit tag in front of the digest, the rendered app.yaml shows which # commit built the image; shelf pins it to the digest either way. images+=("--image" "$component=$image:sha-${GITHUB_SHA::7}@$digest") + echo "$digest" >>"$RUNNER_TEMP/built" + cosign sign --yes --use-signing-config=false --new-bundle-format=false "$image@$digest" fi done < <(jq -r '.builds[] | "\(.component) \(.context)"' <<<"$plan") : >"$RUNNER_TEMP/image-args" + touch "$RUNNER_TEMP/built" if (( ${#images[@]} )); then printf '%s\n' "${images[@]}" >"$RUNNER_TEMP/image-args" fi @@ -153,12 +207,40 @@ jobs: mapfile -t image_args <"$RUNNER_TEMP/image-args" mkdir -p "$RUNNER_TEMP/deploy" shelf render "$APP_FILE" "${image_args[@]}" >"$RUNNER_TEMP/deploy/configmap.yaml" + + # Scan every image the rendered app.yaml pins, built and third-party. shelf renders + # each one as @sha256:. + scanned_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + scan='{}' + while read -r ref; do + echo "::group::scan $ref" + trivy image --quiet --scanners vuln --platform "${PLATFORMS%%,*}" --format json \ + --output "$RUNNER_TEMP/report.json" "$ref" + echo "::endgroup::" + counts="$(jq -c '[.Results[]?.Vulnerabilities[]?.Severity] + | {critical: map(select(. == "CRITICAL")) | length, high: map(select(. == "HIGH")) | length}' \ + "$RUNNER_TEMP/report.json")" + scan="$(jq -c --arg d "${ref##*@}" --argjson c "$counts" '. + {($d): $c}' <<<"$scan")" + if grep -qxF "${ref##*@}" "$RUNNER_TEMP/built"; then + trivy convert --quiet --format cosign-vuln --output "$RUNNER_TEMP/vuln.json" "$RUNNER_TEMP/report.json" + cosign attest --yes --use-signing-config=false --new-bundle-format=false \ + --type vuln --predicate "$RUNNER_TEMP/vuln.json" "$ref" + fi + done < <(grep -oE '[^[:space:]]+@sha256:[0-9a-f]{64}' "$RUNNER_TEMP/deploy/configmap.yaml" | sort -u) + summary="$(jq -c --arg s "trivy $TRIVY_VERSION" --arg t "$scanned_at" \ + '{scanner: $s, scannedAt: $t, images: .}' <<<"$scan")" + repo="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/$package" flux push artifact "oci://$repo:sha-${GITHUB_SHA::7}" \ --path "$RUNNER_TEMP/deploy" \ --source "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" \ --revision "$GITHUB_REF_NAME@sha1:$GITHUB_SHA" \ - --annotations "dev.shelf.package=$package" + --annotations "dev.shelf.package=$package" \ + --annotations "dev.shelf.scan=$summary" \ + --output json >"$RUNNER_TEMP/pushed.json" + # The channel tag moves only to a signed artifact. + cosign sign --yes --use-signing-config=false --new-bundle-format=false \ + "$repo@$(jq -r .digest "$RUNNER_TEMP/pushed.json")" flux tag artifact "oci://$repo:sha-${GITHUB_SHA::7}" --tag "$channel" echo "artifact=oci://$repo:$channel" >>"$GITHUB_OUTPUT" { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 680d4e3..269a045 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} BASE: ${{ github.event.pull_request.base.sha }} run: | - all='["smoke-secrets","smoke-chart","smoke-init","smoke-apps","smoke-agent","smoke-previews"]' + all='["smoke-secrets","smoke-chart","smoke-init","smoke-apps","smoke-agent","smoke-previews","smoke-verify"]' has() { jq -e --arg l "$1" 'index($l) != null' <<<"$LABELS" >/dev/null; } smokes='[]' full=0 if [[ "$EVENT" == push ]]; then diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e64ea3..8fc79fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,8 +38,15 @@ jobs: needs: gate runs-on: ubuntu-24.04-arm timeout-minutes: 30 + # id-token: keyless cosign signs the chart and the platform artifact with this workflow's + # identity, which clusters verify before they install either. + permissions: + contents: write + packages: write + id-token: write env: VERSION: ${{ github.ref_name }} + COSIGN_VERSION: v3.1.3 steps: - uses: actions/checkout@v7 @@ -77,22 +84,41 @@ jobs: username: ${{ github.actor }} password: ${{ github.token }} - - name: Push the chart + - name: Install cosign + run: | + tmp="$(mktemp -d)" + cosign=https://github.com/sigstore/cosign/releases/download/$COSIGN_VERSION + curl -fsSL -o "$tmp/cosign-linux-arm64" "$cosign/cosign-linux-arm64" + curl -fsSL -o "$tmp/cosign_checksums.txt" "$cosign/cosign_checksums.txt" + (cd "$tmp" && grep ' cosign-linux-arm64$' cosign_checksums.txt | sha256sum -c -) + sudo install "$tmp/cosign-linux-arm64" /usr/local/bin/cosign + cosign version + + # Signatures use cosign's classic format, which both Flux and the Sigstore policy-controller + # read. + - name: Push and sign the chart run: | # Helm versions are semver without a leading v; shelf init cluster expects that. helm package charts/shelf-app --version "${VERSION#v}" --destination dist - helm push "dist/shelf-app-${VERSION#v}.tgz" "oci://ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/shelf/charts" + charts="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/shelf/charts" + helm push "dist/shelf-app-${VERSION#v}.tgz" "oci://$charts" 2>&1 | tee "$RUNNER_TEMP/helm-push.txt" + digest="$(awk '/^Digest:/ {print $2}' "$RUNNER_TEMP/helm-push.txt")" + cosign sign --yes --use-signing-config=false --new-bundle-format=false "$charts/shelf-app@$digest" - uses: fluxcd/flux2/action@v2.9.5 with: version: 2.9.5 - - name: Push the platform artifact + - name: Push and sign the platform artifact run: | - flux push artifact "oci://ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/shelf/platform:$VERSION" \ + platform="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/shelf/platform" + flux push artifact "oci://$platform:$VERSION" \ --path platform \ --source "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" \ - --revision "$VERSION@sha1:$GITHUB_SHA" + --revision "$VERSION@sha1:$GITHUB_SHA" \ + --output json >"$RUNNER_TEMP/platform.json" + cosign sign --yes --use-signing-config=false --new-bundle-format=false \ + "$platform@$(jq -r .digest "$RUNNER_TEMP/platform.json")" # Workflow and binary switch together: the major tag first, then the release becomes latest. # A tenant build that starts in the seconds between the two still gets matching versions diff --git a/CLAUDE.md b/CLAUDE.md index 811043b..040dc17 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -134,6 +134,11 @@ Next: Phase 11 (verification chain). - Exposure is an operator decision, made with `shelf app add` / `shelf app credentials` (CLI or UI, audit-logged) and never part of `app.yaml`. A tenant repository, and so an agent working in it, cannot make its app public. +- Only a verified digest goes beyond a preview: `--cloudflare` and `--quick` refuse an app that + has no tunnel yet unless its running deploy artifact and chart are signed by the cluster's + signer and the artifact's scan covers every image (`ops.AddApp`, CLI and UI alike). There is + no setting that turns verification off. A signature proves that shelf's workflow built the + artifact; which tenant built it follows from who may write to the registry path. - Agent-facing commands offer `--json` with stable fields, so an agent never parses text written for people. - The runtime output depends on no shelf-specific CRD: what runs in the cluster is Flux, Helm and @@ -166,14 +171,14 @@ it there when it belongs to one smoke test. | `just test` | devcontainer | level-1 checks, same as CI (gofmt, vet, `go test -race`, helm lint, kubeconform, examples) | | `just golden` | devcontainer | rewrite golden files and `schema/app.schema.json` after an intended change | | `just build` | devcontainer | build `bin/shelf` (linux) | -| `just cluster-up` | devcontainer | create or start k3d cluster `shelf-dev` with its registry `shelf-registry:5000` (added to `/etc/hosts`, deletes enabled), write kubeconfig (run after every container restart or rebuild) | +| `just cluster-up` | devcontainer | create or start k3d cluster `shelf-dev` with its registry `shelf-registry.local:5000` (added to `/etc/hosts`, deletes enabled; the `.local` name lets the policy-controller read it over HTTP), create the dev signer key pair under `.local/cosign/` once, write kubeconfig (run after every container restart or rebuild) | | `just cluster-stop` | devcontainer | stop the cluster to free memory | | `just cluster-down` | devcontainer | delete the cluster | | `just cluster-reset` | devcontainer | delete and recreate the cluster | -| `just platform-push` | devcontainer | push `platform/` to the dev registry as `oci://shelf-registry:5000/shelf/platform:dev` | -| `just chart-push` | devcontainer | push the chart as `oci://shelf-registry:5000/shelf/charts/shelf-app:0.0.0-dev` | +| `just platform-push` | devcontainer | push `platform/` to the dev registry as `oci://shelf-registry.local:5000/shelf/platform:dev` | +| `just chart-push` | devcontainer | push the chart as `oci://shelf-registry.local:5000/shelf/charts/shelf-app:0.0.0-dev` | | `just serve` | devcontainer | run the admin UI against the dev cluster on port 8080 (forwarded to the Mac's browser); prints the setup code on the first start | -| `just init-cluster [--yes]` | devcontainer | push the working copy's platform and chart, then `shelf init cluster` against the dev cluster with them (pass `--host-suffix -dev` if the cluster has one) | +| `just init-cluster [--yes]` | devcontainer | push the working copy's platform and chart, signed with the dev key, then `shelf init cluster --signer-key .local/cosign/cosign.pub` against the dev cluster with them (pass `--host-suffix -dev` if the cluster has one) | | `just flux-operator-update ` | devcontainer | replace the embedded Flux Operator manifest | | `just smoke-secrets` | devcontainer | check `$(VAR)` expansion from `secretKeyRef` | | `just smoke-registry ` | devcontainer | check a private GHCR pull via `imagePullSecret` | @@ -182,7 +187,8 @@ it there when it belongs to one smoke test. | `just smoke-apps` | devcontainer | Phase 4: `shelf app add`/`rm`, rollout by polling, tampered artifact refused, restore from backup (needs network) | | `just smoke-agent` | devcontainer | Phase 9: `--json` documents, `--wait --commit` (Ready only after the commit rolled out, Failed for an image that cannot be pulled, TimedOut for a commit that never arrives; with `SMOKE_FULL=1`, the default, also Ready again for a fix after a failure, which takes about 5 more minutes because Helm times out the broken upgrade first) and `shelf app open` with the app's Host and `stripPrefix` (needs Docker Hub) | | `just smoke-previews` | devcontainer | Phase 10: `examples/hello` with `--previews=2`; two branch tags run as isolated previews with their own database, volume and password; `open --preview`; a new commit keeps the data; the limit holds; a deleted tag removes its preview; `--no-previews` keeps the app; prints push-to-Ready times and RAM per preview (needs Docker Hub) | -| `just smoke-tenant ` | devcontainer | Phase 4 acceptance with a real tenant repo and GHCR; asks for the GHCR login, waits for a push | +| `just smoke-verify` | devcontainer | Phase 11: a signed and scanned artifact with a signed built image is verified and exposable; the admission policy admits it and denies an unsigned image, in the app's and a preview's namespace; a preview is verified; an artifact without a scan runs but is not verified; an unsigned artifact is not applied and the wait fails with a hint; `rm` removes the image policy; prints the policy-controller's RAM (needs Docker Hub) | +| `just smoke-tenant ` | devcontainer | Phase 4 acceptance with a real tenant repo and GHCR; installs a released platform and chart (`SHELF_RELEASE`, default the newest tag, v0.8.0 or later) with keyless verification, since the tenant's artifacts are signed keyless; asks for the GHCR login, waits for a push | | `just smoke-expose ` | devcontainer | Phase 8b acceptance: the app gets the Cloudflare connection `smoke`, its own tunnel, DNS record and HTTPS, then is taken off again; asks for the Cloudflare API token (set `CF_ACCOUNT_ID` if it sees several accounts); needs a cluster with a host suffix | | `just smoke-quick [app]` | devcontainer | Phase 8d acceptance: `examples/hello` as `hello-quick` with `--quick` gets a quick tunnel, answers with its own Host, gets a new name after a restart, `--private`/`--quick`, rm; keeps the cluster's host suffix; opens a public trycloudflare.com tunnel for a few minutes | | `openspec validate --all --strict` | devcontainer | check specs and active changes (part of `just test`) | @@ -218,6 +224,10 @@ it there when it belongs to one smoke test. record by anything but its exact name, and never one without the `-dev` suffix. - `shelf app rm` deletes an app's volumes; never run it against an app you did not create in this session. +- The dev signer key under `.local/cosign/` signs artifacts and images in the dev registry only, + without a transparency log; it never leaves the devcontainer and never signs anything pushed + to GHCR. A cluster installed with `--signer-key` accepts nothing else, so never install a + cluster that serves real apps with it. - A quick tunnel puts an app on the internet without any account. Only `smoke-quick` passes `--quick`; never give an app you did not create a quick tunnel. - `shelf init cluster` with a different `--host-suffix` moves every app with a domain to another @@ -279,6 +289,16 @@ it there when it belongs to one smoke test. structs of `ops` or `cluster`: `apiVersion` `shelf.dev/v1alpha1`, a `kind`, empty fields left out rather than null, one golden file per kind in `internal/cli/testdata/json`. Within a version fields are only added; a failing command prints an `Error` document +- Verification: the cluster settings carry the signer (`SHELF_SIGNER_*`, keyless by default with + shelf's `build.yml`/`release.yml` as subject, or a key in `shelf-system/signer-key`, copied as + `shelf-signer` into app and preview namespaces); the platform renders `verify` on every + OCIRepository from `SHELF_SIGNER_VERIFY` and a ClusterImagePolicy `shelf-` for + `//**` from `SHELF_SIGNER_AUTHORITY`, and labels app and preview namespaces + `policy.sigstore.dev/include`. Signatures use cosign's classic format + (`--new-bundle-format=false`), the only one the policy-controller reads. The scan summary is + the deploy artifact's annotation `dev.shelf.scan` (per image digest the CRITICAL and HIGH + counts), written before the artifact is signed. What "verified" means is decided in + `cluster.verification` alone - Which values a Helm release holds is decided by its `configDigest`, which `cluster.ConfigDigest` computes as helm-controller does, never by timestamps: after a failed upgrade, its failure and its rollback are both newer than the values that replaced it diff --git a/README.md b/README.md index fd7761c..25d1545 100644 --- a/README.md +++ b/README.md @@ -208,13 +208,16 @@ on: permissions: contents: read packages: write + id-token: write jobs: shelf: uses: tweinmann/shelf/.github/workflows/build.yml@v0 ``` -The workflow builds every `build` directory for `linux/arm64`, pushes the images, pins them to -their digests, and publishes the deploy artifact: as `main` for the default branch, and as +The workflow builds every `build` directory for `linux/arm64` (the input `platforms` adds +`linux/amd64`), pushes the images, pins them to their digests, signs and scans them (see +[the verification chain](#verification-chain); `id-token: write` is for the signature), and +publishes the deploy artifact: as `main` for the default branch, and as `branch-` for every other branch, which [previews](#previews-of-branches) run. Deleting a branch deletes its artifact. Pull requests only build. Image layers are cached in the registry (`:buildcache`), so a small change builds quickly. A workflow that keeps @@ -316,6 +319,52 @@ shelf app credentials --previews # at most 3 branches at once; -- - **Same slug.** Two branches whose names differ only in punctuation, `feature/login` and `feature-login`, share one preview. +### Verification chain + +A preview may run whatever a branch built. Before an app goes beyond the cluster, shelf has to +know that what runs is what the workflow built from a known commit. Every cluster verifies, and +there is no setting that turns it off: + +- **Signed.** The workflow signs every built image and the deploy artifact with keyless cosign, + using the job's GitHub identity. Every release signs its chart and platform artifact the same + way. Flux applies an artifact, and the Sigstore policy-controller admits a pod with a built + image, only with a valid signature from the cluster's signer. An unsigned artifact is not + applied, and what runs keeps running. +- **Scanned.** The workflow scans every image the app runs, built and third-party, with Trivy. It + attaches the report to each built image and writes the CRITICAL and HIGH counts into the + deploy artifact before signing it. +- **Exposure only when verified.** `--cloudflare` and `--quick` refuse an app that does not run + a signed and scanned artifact yet. A new app with such a flag is deployed privately first, and + gets its tunnel once it is verified. +- **Shown everywhere.** `shelf app status ` shows source and commit, signer, whether it is + verified, the findings per component and whether it may be exposed, and so do `--json` and the + admin UI. + +What this does **not** prove, so nobody reads more into it: + +- **Who built it.** A signature says that shelf's reusable workflow built the artifact, not which + repository ran it. That every repository calling the workflow produces a valid signature is + why the binding to a tenant rests on who may write to `ghcr.io//`. The + artifact's source, which the signature covers, is shown. +- **Third-party images.** An image such as `postgres:16` is not signed by us. Its digest is fixed + in the signed deploy artifact, and it is scanned, but no policy checks its signature. +- **Clean scans.** Findings are shown, never enforced. A scan must exist; what it found is the + operator's call. +- **Privacy.** Keyless signatures are recorded in the public Rekor log, the name of a private + tenant repository and its commit SHAs included. + +**Upgrading to a verifying platform.** Do it in this order: + +1. Each tenant adds `id-token: write` to its `deploy.yml` and pushes once with the current + workflow. +2. Then run `shelf init cluster`. + +Its confirmation lists the apps whose running artifact is not verified. Those keep running, but +get no new deploys until they are built again. + +A cluster that tests shelf verifies with a cosign key instead (`shelf init cluster --signer-key +cosign.pub`), as the dev cluster does. + ## Writing an `app.yaml` A complete example with a web frontend, an API and a Postgres database: @@ -422,6 +471,9 @@ You never have to look, but nothing is hidden. shelf keeps the substrate reachab its own tunnel, `tunnel-` with the tunnel's credentials. Registry connections are the Secrets `connection-registry-`; `registry-anonymous` is what an app without one pulls with. +- **`cosign-system`** holds the Sigstore policy-controller. Per app, the platform writes a + `ClusterImagePolicy` `shelf-` for the images built from its package, checked in the + app's namespace and its previews'. - **`flux-system`** and **`traefik`** hold Flux and Traefik. They and the `ResourceSet` belong to Flux and come from a released platform artifact. **Editing them by hand does not last**; Flux reverts the change at the next reconciliation. @@ -446,18 +498,18 @@ drift. | `shelf render ` | Validates, resolves images and prints the deploy manifest (a ConfigMap). `-o app` prints the resolved `app.yaml` instead. `--image =` supplies the image of a component with a `build` directory. Registry credentials come from `docker login`. | | `shelf schema` | Prints the JSON Schema for `app.yaml`. | | `shelf build-plan ` | Prints the package name (`name` in `app.yaml`) and the components with a `build` directory as JSON. The workflow uses it to name the packages and to know what to build. `--branch ` adds `tag`, the tag a push to that branch publishes the deploy artifact under (`branch-`). | -| `shelf init cluster` | Installs Flux and the platform (Traefik, app management) into the cluster of the current kubecontext, and waits until everything is ready. `--host-suffix -dev` separates clusters that share a DNS zone: apps with a domain are then reachable at `-dev.`. Shows the target cluster and asks before changing anything (`--yes` skips the question); `--context` and `--kubeconfig` pick another cluster. Safe to run again. | +| `shelf init cluster` | Installs Flux and the platform (Traefik, the Sigstore policy-controller, app management) into the cluster of the current kubecontext, and waits until everything is ready. The cluster verifies everything it runs: by default keyless signatures of shelf's own workflows (`--signer-issuer`, `--signer-subject`), or with `--signer-key ` a cosign public key, for clusters that test shelf. The confirmation lists apps that do not run a verified artifact. `--host-suffix -dev` separates clusters that share a DNS zone: apps with a domain are then reachable at `-dev.`. Shows the target cluster and asks before changing anything (`--yes` skips the question); `--context` and `--kubeconfig` pick another cluster. Safe to run again. | | `shelf connection add registry ` | Defines a registry login for `ghcr.io` from `GHCR_USERNAME` and `GHCR_TOKEN`. It is stored in the cluster and copied only into the namespaces of the apps that chose it. | | `shelf connection add cloudflare ` | Defines a Cloudflare account from `CF_API_TOKEN` (and `CF_ACCOUNT_ID` when the token sees several). shelf checks the token and keeps it in `~/.shelf/connections/cloudflare/`, never in the cluster. | | `shelf connection list` / `rm ` | Lists the connections and the apps that use them; removes one no app uses. | | `shelf connection packages ` / `zones ` | Lists the deploy artifacts a registry connection can pull, or the zones of a Cloudflare connection's account. | -| `shelf app add ` | Deploys an app from a deploy artifact under the name `` and keeps it updated. The name is independent of the package, so one artifact can be added under several names. `--registry` picks the registry connection; `--cloudflare` with `--domain` puts the app on the internet under its own domain, `--quick` through a quick tunnel; without either it stays inside the cluster. Generates the app's secrets, stores them in the cluster and in `~/.shelf/apps//secrets.yaml`, and restores them from there after a cluster rebuild. Refuses at once if the cluster has no platform. Waits until the app is ready. `--previews[=N]` runs a [preview](#previews-of-branches) of each published branch, at most N (default 3). Safe to run again, e.g. after adding a secret. | +| `shelf app add ` | Deploys an app from a deploy artifact under the name `` and keeps it updated. The name is independent of the package, so one artifact can be added under several names. `--registry` picks the registry connection; `--cloudflare` with `--domain` puts the app on the internet under its own domain, `--quick` through a quick tunnel, both only once it runs a [verified](#verification-chain) artifact; without either it stays inside the cluster. Generates the app's secrets, stores them in the cluster and in `~/.shelf/apps//secrets.yaml`, and restores them from there after a cluster rebuild. Refuses at once if the cluster has no platform. Waits until the app is ready. `--previews[=N]` runs a [preview](#previews-of-branches) of each published branch, at most N (default 3). Safe to run again, e.g. after adding a secret. | | `shelf app credentials ` | Changes the app's connections, domain or previews and deploys it again: `--registry`, `--no-registry`, `--cloudflare`, `--domain`, `--quick`, `--private`, `--previews[=N]`, `--no-previews`. | | `shelf app open ` | Makes the app reachable from a browser on this machine at `http://127.0.0.1:/`, for as long as it runs: requests go through the Kubernetes API server to the cluster's Traefik with the app's own Host header, so routes and `stripPrefix` behave as through a tunnel. Works for apps that stay inside the cluster. Listens on 127.0.0.1 only; `--port` picks the port, `--browser` opens it. It is not an exposure and changes nothing about the app. `--preview ` opens that branch's preview. | | `shelf app tags ` | Lists the tags of the app's deploy artifact, newest first. | | `shelf app redeploy ` | Fetches the app's artifact again under the tag it is registered with and rolls out what it finds. For a tag that moved, or a deploy worth another try. | | `shelf app secrets ` | Lists the names of the generated secrets; `--reveal` prints the values. | -| `shelf app status [app]` | Lists the apps with their state, or walks one app from its deploy artifact to its running pods and says which step is not ready. For one app it also shows the git commit the running artifact was built from, and names its components and where each of them answers. Same chain as the admin UI shows. `--wait` waits until the app is Ready or a step failed and prints the verdict; `--commit ` first waits until the app runs the artifact built from that commit. Exit status 0 Ready, 1 failed, 2 timed out (`--timeout`, default 10m). For an app with previews it lists them; `--preview ` shows, or waits for, that preview instead. | +| `shelf app status [app]` | Lists the apps with their state, or walks one app from its deploy artifact to its running pods and says which step is not ready. For one app it also shows the git commit the running artifact was built from, names its components and where each of them answers, and shows the [verification chain](#verification-chain): source, signer, verified or why not, findings per component, and whether the app may be exposed. Same chain as the admin UI shows. `--wait` waits until the app is Ready or a step failed and prints the verdict; `--commit ` first waits until the app runs the artifact built from that commit. Exit status 0 Ready, 1 failed, 2 timed out (`--timeout`, default 10m). For an app with previews it lists them; `--preview ` shows, or waits for, that preview instead. | | `shelf serve` | Runs the admin UI on port 7654 of the local network (`--listen 127.0.0.1:7654` keeps it off the network). The first start prints a setup code that claims the instance; after that a password protects it. | | `shelf app rm ` | Removes an app with its namespace, volumes, secrets, tunnel and DNS record, after asking. The secret backup stays. | | `shelf version` | Prints the version. | @@ -541,7 +593,7 @@ Useful commands: | `just build` | Build `bin/shelf` | | `just cluster-up` / `cluster-stop` / `cluster-down` / `cluster-reset` | Manage the local k3d cluster `shelf-dev` | | `just platform-push`, `just chart-push`, `just init-cluster` | Push `platform/` and the chart to the local registry; `init-cluster` pushes both and installs them with `shelf init cluster` | -| `just smoke-secrets`, `just smoke-chart`, `just smoke-init`, `just smoke-apps`, `just smoke-agent`, `just smoke-previews` | Smoke tests against the local cluster, also run in CI: secret expansion, the chart, `shelf init cluster` with routing through Traefik, `shelf app add`/`rm` with rollouts from the local registry, the agent interface (`--json`, `--wait --commit`, `shelf app open`), and branch previews | +| `just smoke-secrets`, `just smoke-chart`, `just smoke-init`, `just smoke-apps`, `just smoke-agent`, `just smoke-previews`, `just smoke-verify` | Smoke tests against the local cluster, also run in CI: secret expansion, the chart, `shelf init cluster` with routing through Traefik, `shelf app add`/`rm` with rollouts from the local registry, the agent interface (`--json`, `--wait --commit`, `shelf app open`), branch previews, and the verification chain. The dev cluster verifies with a dev key under `.local/cosign/`, which `just cluster-up` creates and which signs everything pushed to the dev registry | | `just smoke-registry `, `just smoke-tenant `, `just smoke-expose ` | Smoke tests that need credentials: a private image pull, a real tenant repository end to end, and an app's own Cloudflare tunnel (needs a cluster with a host suffix) | | `just smoke-quick` | A quick tunnel for `examples/hello`; puts it on the internet for a few minutes | | `hack/nuke.sh` | **Run in a terminal on your machine, not in the container.** Removes every Docker object shelf created. | diff --git a/examples/tenant/.claude/skills/shelf/SKILL.md b/examples/tenant/.claude/skills/shelf/SKILL.md index 10361b3..645f079 100644 --- a/examples/tenant/.claude/skills/shelf/SKILL.md +++ b/examples/tenant/.claude/skills/shelf/SKILL.md @@ -67,11 +67,13 @@ The app's name on shelf is in `CLAUDE.md` (the operator chose it; it is not `nam ## What the stages mean The stages, in order: -1. `the deploy artifact`: CI did not push, or the registry refuses the login. +1. `the deploy artifact`: CI did not push, the registry refuses the login, or the artifact is not + signed by the cluster's signer (push again; `deploy.yml` needs `id-token: write`). 2. `the app values`: the rendered app.yaml was rejected. 3. `the chart`: a platform problem; report it. 4. `the app`: the Helm release; often a bad `app.yaml`. -5. `the pods`: image pull, crash loop, out of memory, missing secret, or a preview's quota. +5. `the pods`: image pull, crash loop, out of memory, missing secret, a preview's quota, or a + built image the admission policy refused because it is not signed. The first stage that is not Ready is the cause; everything after it is a consequence. `shelf validate app.yaml` catches most app.yaml mistakes before a push. diff --git a/examples/tenant/.github/workflows/deploy.yml b/examples/tenant/.github/workflows/deploy.yml index 7d042d0..8fa659d 100644 --- a/examples/tenant/.github/workflows/deploy.yml +++ b/examples/tenant/.github/workflows/deploy.yml @@ -10,9 +10,12 @@ on: pull_request: delete: +# id-token lets the workflow sign the images and the deploy artifact with keyless cosign; the +# cluster runs nothing without that signature. permissions: contents: read packages: write + id-token: write jobs: shelf: diff --git a/hack/chart-push.sh b/hack/chart-push.sh index 3f5acb1..b0592e4 100755 --- a/hack/chart-push.sh +++ b/hack/chart-push.sh @@ -10,6 +10,7 @@ trap 'rm -rf "$work"' EXIT helm package "$repo/charts/shelf-app" --version 0.0.0-dev --destination "$work" >/dev/null helm push "$work/shelf-app-0.0.0-dev.tgz" "oci://$SHELF_REGISTRY_HOST/shelf/charts" --plain-http +sign_artifact "$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" # The dev tag never changes, so the chart sources of the running apps have to be told to look # again; in a release the chart tag changes and Flux notices by itself. for ns in $(kubectl get ocirepository -A -o json | diff --git a/hack/cluster-up.sh b/hack/cluster-up.sh index a547bf6..feed3d3 100755 --- a/hack/cluster-up.sh +++ b/hack/cluster-up.sh @@ -27,6 +27,8 @@ else --wait fi +ensure_signer_key + # registry:2 refuses to delete manifests unless told to. Previews disappear when their branch tag # is deleted, so level 2 has to be able to delete one, as GHCR can. The config file lives in the # registry container, which keeps it across restarts; this runs on every cluster-up. @@ -39,7 +41,7 @@ fi # The registry is published on this container's loopback; give it the name pods use. /etc/hosts # is recreated with the container, so this runs on every cluster-up. -if ! grep -qE "^127\.0\.0\.1[[:space:]]+$SHELF_REGISTRY\$" /etc/hosts; then +if ! grep -qE "^127\.0\.0\.1[[:space:]]+${SHELF_REGISTRY//./\\.}\$" /etc/hosts; then echo "127.0.0.1 $SHELF_REGISTRY" | sudo tee -a /etc/hosts >/dev/null fi diff --git a/hack/lib.sh b/hack/lib.sh index 707eae8..73e5147 100755 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -2,12 +2,23 @@ readonly SHELF_CLUSTER=shelf-dev readonly SHELF_KUBECONFIG=/home/vscode/.kube/shelf-dev.yaml -# The dev registry for platform, chart and deploy artifacts, created with the cluster. Pods and -# this container both reach it as shelf-registry:5000 (plain HTTP), so one reference works for -# pushing, for shelf and for Flux. -readonly SHELF_REGISTRY=shelf-registry +# The dev registry for platform, chart, deploy artifacts and built images, created with the +# cluster. Pods and this container both reach it as shelf-registry.local:5000 (plain HTTP), so one +# reference works for pushing, for shelf and for Flux. The .local name matters: the Sigstore +# policy-controller has no setting for a registry without TLS, but go-containerregistry speaks +# HTTP to a .local host. +readonly SHELF_REGISTRY=shelf-registry.local readonly SHELF_REGISTRY_PORT=5000 readonly SHELF_REGISTRY_HOST="$SHELF_REGISTRY:$SHELF_REGISTRY_PORT" +# The dev signer: a cosign key pair without a password, made once by cluster-up, which signs +# every artifact and image pushed to the dev registry. The dev cluster is installed with its +# public half (--signer-key) and so verifies like a production cluster, with a key instead of +# keyless signatures. The key is machine-local (.local is gitignored) and signs dev artifacts +# only; nothing it signs is recorded in a transparency log. +SHELF_SIGNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/.local/cosign" +readonly SHELF_SIGNER_DIR +readonly SHELF_SIGNER_KEY="$SHELF_SIGNER_DIR/cosign.key" +readonly SHELF_SIGNER_PUB="$SHELF_SIGNER_DIR/cosign.pub" die() { echo "error: $*" >&2 @@ -46,3 +57,58 @@ create_namespace() { kubectl create namespace "$1" >/dev/null kubectl -n "$1" wait --for=create serviceaccount/default --timeout=60s >/dev/null } + +# ensure_signer_key creates the dev key pair unless it exists. +ensure_signer_key() { + [[ -f "$SHELF_SIGNER_KEY" && -f "$SHELF_SIGNER_PUB" ]] && return 0 + mkdir -p "$SHELF_SIGNER_DIR" + chmod 0700 "$SHELF_SIGNER_DIR" + (cd "$SHELF_SIGNER_DIR" && COSIGN_PASSWORD='' cosign generate-key-pair >/dev/null 2>&1) \ + || die "cosign could not create the dev key pair in $SHELF_SIGNER_DIR" + echo "dev signer: key pair created in $SHELF_SIGNER_DIR" +} + +# sign_artifact : signs an artifact or image in the dev registry by digest with the +# dev key, in cosign's classic format, which both Flux and the policy-controller read, and without +# a transparency log entry. The reference has a tag or a digest and no oci:// scheme. +sign_artifact() { + [[ -f "$SHELF_SIGNER_KEY" ]] || die "no dev signer key; run just cluster-up" + local ref="${1#oci://}" repo digest + if [[ "$ref" == *@* ]]; then + repo="${ref%@*}" + else + repo="${ref%:*}" + fi + digest="$(crane digest --insecure "$ref")" || die "cannot read the digest of $ref" + COSIGN_PASSWORD='' cosign sign --key "$SHELF_SIGNER_KEY" --new-bundle-format=false \ + --use-signing-config=false --tlog-upload=false --allow-http-registry --allow-insecure-registry \ + -y "$repo@$digest" >/dev/null 2>&1 || die "cosign could not sign $ref" +} + +# dev_scan : the dev.shelf.scan summary for the deploy artifact built from dir, which +# names every image digest its ConfigMap pins. Nothing scans in the dev registry, and the +# scanner says so; the summary only lets a dev app count as verified. +dev_scan() { + grep -rhoE '@sha256:[0-9a-f]{64}' "$1" | sort -u | cut -c2- | + jq -Rnc --arg t "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{scanner: "none (dev registry)", scannedAt: $t, images: ([inputs | {(.): {critical: 0, high: 0}}] | add // {})}' +} + +# push_artifact [flux push args...]: pushes a deploy +# artifact with a scan summary and signs it, as the reusable workflow does, and prints its digest. +# Like the workflow, it pushes under another tag (staging) first and moves the tag only to the +# signed artifact, so that Flux never reads it unsigned. +push_artifact() { + local ref="$1" dir="$2" revision="$3" + shift 3 + local repo="${ref%:*}" digest + digest="$(flux push artifact "$repo:staging" --path "$dir" --source local --revision "$revision" \ + --annotations "dev.shelf.scan=$(dev_scan "$dir")" --insecure-registry --output json "$@" 2>/dev/null | + jq -r .digest)" || die "push of $ref failed" + [[ -n "$digest" && "$digest" != null ]] || die "push of $ref failed" + sign_artifact "${repo#oci://}@$digest" + # flux tag artifact has no flag for a registry without TLS. + crane tag --insecure "${repo#oci://}:staging" "${ref##*:}" >/dev/null 2>&1 \ + || die "tagging $ref failed" + echo "$digest" +} diff --git a/hack/platform-push.sh b/hack/platform-push.sh index 9c2f457..110bdb8 100755 --- a/hack/platform-push.sh +++ b/hack/platform-push.sh @@ -13,9 +13,15 @@ fi # Validate before pushing: Flux would only report a broken build in the cluster. kubectl kustomize "$repo/platform" >/dev/null -flux push artifact "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ +# Signed with the dev key before the tag moves, as release.yml signs a release, so that the +# cluster never reads an unsigned platform. +platform="oci://$SHELF_REGISTRY_HOST/shelf/platform" +flux push artifact "$platform:staging" \ --path "$repo/platform" \ --source "local" \ --revision "dev@sha1:$revision" \ --insecure-registry -echo "use: shelf init cluster --platform oci://$SHELF_REGISTRY_HOST/shelf/platform:dev --insecure-registry" +sign_artifact "${platform#oci://}:staging" +# flux tag artifact has no flag for a registry without TLS. +crane tag --insecure "${platform#oci://}:staging" dev +echo "use: shelf init cluster --platform $platform:dev --insecure-registry --signer-key $SHELF_SIGNER_PUB" diff --git a/hack/smoke/agent.sh b/hack/smoke/agent.sh index 0d5d94a..fca5d27 100755 --- a/hack/smoke/agent.sh +++ b/hack/smoke/agent.sh @@ -44,8 +44,7 @@ commit() { head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n'; } # publish : pushes the files in dir as the deploy artifact, tag main, built from # commit, and asks Flux to fetch it once the app exists. publish() { - flux push artifact "$artifact:main" --path "$1" --source local --revision "main@sha1:$2" \ - --insecure-registry >/dev/null 2>&1 || die "push failed" + push_artifact "$artifact:main" "$1" "main@sha1:$2" >/dev/null kubectl -n "$app" annotate ocirepository deploy --overwrite \ "reconcile.fluxcd.io/requestedAt=$(date +%s%N)" >/dev/null 2>&1 || true } @@ -71,7 +70,7 @@ step "build shelf, push platform and chart, init cluster" (cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) "$repo/hack/platform-push.sh" >/dev/null 2>&1 "$repo/hack/chart-push.sh" >/dev/null 2>&1 -shelf init cluster --yes --host-suffix "$(cluster_host_suffix)" --insecure-registry \ +shelf init cluster --yes --signer-key "$SHELF_SIGNER_PUB" --host-suffix "$(cluster_host_suffix)" --insecure-registry \ --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" >/dev/null diff --git a/hack/smoke/apps.sh b/hack/smoke/apps.sh index 82f6ca4..f3903f1 100755 --- a/hack/smoke/apps.sh +++ b/hack/smoke/apps.sh @@ -45,11 +45,10 @@ retry() { return 1 } -# publish : pushes the files in dir as the deploy artifact, tag main, and prints its digest. +# publish : pushes the files in dir as the deploy artifact, tag main, signed, and prints its +# digest. publish() { - flux push artifact "$artifact:main" --path "$1" --source local \ - --revision "main@sha1:$(head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n')" \ - --insecure-registry 2>&1 | sed -n 's/.*pushed to .*@\(sha256:[0-9a-f]*\).*/\1/p' + push_artifact "$artifact:main" "$1" "main@sha1:$(head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n')" } # render : writes the deploy manifest of an app.yaml into dir. @@ -72,7 +71,7 @@ step "build shelf, push platform and chart, init cluster" (cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) "$repo/hack/platform-push.sh" >/dev/null 2>&1 "$repo/hack/chart-push.sh" >/dev/null 2>&1 -shelf init cluster --yes --host-suffix "$(cluster_host_suffix)" --insecure-registry \ +shelf init cluster --yes --signer-key "$SHELF_SIGNER_PUB" --host-suffix "$(cluster_host_suffix)" --insecure-registry \ --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" >/dev/null diff --git a/hack/smoke/init.sh b/hack/smoke/init.sh index 56e1905..fb47e7b 100755 --- a/hack/smoke/init.sh +++ b/hack/smoke/init.sh @@ -48,7 +48,7 @@ trap cleanup EXIT step "build shelf" (cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) init_cluster() { - "$work/shelf" init cluster --yes --insecure-registry --host-suffix "$(cluster_host_suffix)" \ + "$work/shelf" init cluster --yes --signer-key "$SHELF_SIGNER_PUB" --insecure-registry --host-suffix "$(cluster_host_suffix)" \ --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" } diff --git a/hack/smoke/previews.sh b/hack/smoke/previews.sh index 0d2ac34..ccf8eea 100755 --- a/hack/smoke/previews.sh +++ b/hack/smoke/previews.sh @@ -38,8 +38,7 @@ commit() { head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n'; } # publish : pushes examples/hello as the deploy artifact under tag, built from # commit. publish() { - flux push artifact "$artifact:$1" --path "$work/hello" --source local --revision "$1@sha1:$2" \ - --insecure-registry >/dev/null 2>&1 || die "push of $1 failed" + push_artifact "$artifact:$1" "$work/hello" "$1@sha1:$2" >/dev/null } # unpublish : deletes a tag from the dev registry, as build.yml does on GHCR for a deleted @@ -82,11 +81,16 @@ for leftover in "namespace/$app" "-n shelf-system resourcesetinputprovider/$app" done trap cleanup EXIT +# Branch tags a run that failed left behind would be previews from the start. +for tag in $(crane ls --insecure "${artifact#oci://}" 2>/dev/null | grep '^branch-' || true); do + unpublish "$tag" +done + step "build shelf, push platform and chart, init cluster" (cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) "$repo/hack/platform-push.sh" >/dev/null 2>&1 "$repo/hack/chart-push.sh" >/dev/null 2>&1 -shelf init cluster --yes --host-suffix "$(cluster_host_suffix)" --insecure-registry \ +shelf init cluster --yes --signer-key "$SHELF_SIGNER_PUB" --host-suffix "$(cluster_host_suffix)" --insecure-registry \ --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" >/dev/null kubectl -n shelf-system get resourceset previews >/dev/null || die "the platform has no ResourceSet previews" @@ -146,7 +150,8 @@ kubectl top pods -n "$app" --no-headers | awk '{sub("Mi","",$3); sum+=$3} END {p step "the limit holds" publish branch-gamma "$(commit)" -for _ in $(seq 60); do +# Up to three minutes: the evicted preview's release and volume are removed first. +for _ in $(seq 90); do [[ "$(kubectl get namespace -l "shelf.dev/app=$app,shelf.dev/preview" --no-headers 2>/dev/null | wc -l)" == 2 ]] \ && kubectl get namespace "$app--gamma" >/dev/null 2>&1 && break sleep 2 diff --git a/hack/smoke/quick.sh b/hack/smoke/quick.sh index e4a6a3a..a693094 100755 --- a/hack/smoke/quick.sh +++ b/hack/smoke/quick.sh @@ -54,16 +54,14 @@ step "build shelf, push platform and chart, init cluster" (cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) "$repo/hack/platform-push.sh" >/dev/null 2>&1 "$repo/hack/chart-push.sh" >/dev/null 2>&1 -shelf init cluster --yes --host-suffix "$(cluster_host_suffix)" --insecure-registry \ +shelf init cluster --yes --signer-key "$SHELF_SIGNER_PUB" --host-suffix "$(cluster_host_suffix)" --insecure-registry \ --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" >/dev/null step "publish examples/hello" mkdir -p "$work/v1" (cd "$repo" && go run ./cmd/shelf render examples/hello/app.yaml) >"$work/v1/configmap.yaml" 2>/dev/null -flux push artifact "$artifact:main" --path "$work/v1" --source local \ - --revision "main@sha1:$(head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n')" \ - --insecure-registry >/dev/null 2>&1 || die "push failed" +push_artifact "$artifact:main" "$work/v1" "main@sha1:$(head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n')" >/dev/null step "shelf app add --quick gives the app a quick tunnel" start=$SECONDS diff --git a/hack/smoke/tenant.sh b/hack/smoke/tenant.sh index 8b90556..6eb9c41 100755 --- a/hack/smoke/tenant.sh +++ b/hack/smoke/tenant.sh @@ -6,6 +6,11 @@ # GHCR_USERNAME and GHCR_TOKEN (classic PAT, read:packages) are read from the environment, or # prompted for. They become the registry connection "tenant", which the app pulls with and which # is also what shelf reads the artifact with. +# +# The tenant's artifacts are signed keyless by shelf's reusable workflow, so the cluster is +# installed with a released platform and chart, which release.yml signed the same way, and the +# default signer; the dev platform is signed with the dev key, which a keyless cluster refuses. +# SHELF_RELEASE picks the release (default: the newest v* tag here); it must be v0.8.0 or later. set -euo pipefail source "$(dirname "$0")/../lib.sh" require_devcontainer @@ -43,13 +48,12 @@ cleanup() { trap cleanup EXIT step() { echo "--- $*"; } -step "build shelf, push platform and chart, init cluster" +release="${SHELF_RELEASE:-$(git -C "$repo" describe --tags --abbrev=0 --match 'v*.*.*')}" +step "build shelf, init cluster with release $release and keyless verification" (cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) -"$repo/hack/platform-push.sh" >/dev/null 2>&1 -"$repo/hack/chart-push.sh" >/dev/null 2>&1 -shelf init cluster --yes --host-suffix "$(cluster_host_suffix)" --insecure-registry \ - --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ - --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" | grep -E 'Platform' +shelf init cluster --yes --host-suffix "$(cluster_host_suffix)" \ + --platform "oci://ghcr.io/tweinmann/shelf/platform:$release" \ + --chart "oci://ghcr.io/tweinmann/shelf/charts/shelf-app:${release#v}" | grep -E 'Platform|signer' # No Docker config at all: shelf has to read the artifact with the app's own login. export DOCKER_CONFIG="$work/docker" mkdir -p "$DOCKER_CONFIG" diff --git a/hack/smoke/verify.sh b/hack/smoke/verify.sh new file mode 100755 index 0000000..a1bffea --- /dev/null +++ b/hack/smoke/verify.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Checks the Phase 11 verification chain against the dev cluster, which verifies with the dev +# key (.local/cosign) as a production cluster verifies keyless signatures: +# - an app whose signed and scanned artifact names a signed built image is verified and may be +# exposed; +# - the admission policy admits that image and denies an unsigned one, in the app's namespace +# and in a preview's; +# - a preview of a signed branch artifact is verified; +# - a signed artifact without a scan summary runs, but is not verified and may not be exposed; +# - an unsigned artifact is not applied: the wait fails at the deploy artifact with a hint, and +# what ran keeps running; +# - shelf app rm takes the app's ClusterImagePolicy with it. +# It prints the policy-controller's RAM. The exposure gate itself is tested at level 1, since +# only smoke-quick may pass --quick. Installs or updates the platform first (idempotent). Needs +# network access for images. +set -euo pipefail +source "$(dirname "$0")/../lib.sh" +require_devcontainer + +repo="$(cd "$(dirname "$0")/../.." && pwd)" +work="$(mktemp -d)" +app=verify-smoke +# As on GHCR: the deploy artifact is /, a built image /. +artifact="oci://$SHELF_REGISTRY_HOST/smoke/$app" +image="$SHELF_REGISTRY_HOST/smoke/$app/web" +export SHELF_HOME="$work/home" + +shelf() { "$work/shelf" "$@"; } + +cleanup() { + if [[ "${KEEP:-}" == 1 ]]; then + echo "KEEP=1: leaving app $app in place" + return + fi + shelf app rm "$app" --yes >/dev/null 2>&1 || true + rm -rf "$work" +} + +step() { echo "--- $*"; } + +# commit: a random 40-digit commit hash, as the reusable workflow stamps on an artifact. +commit() { head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n'; } + +# render : writes the deploy manifest of a one-component app built as image. shelf +# render looks the image up over TLS, which the dev registry has not, so it renders the Docker +# Hub image the dev ones were copied from, and the reference is swapped afterwards. +render() { + mkdir -p "$1" + local base + base="traefik/whoami:v1.11.0@$(crane digest traefik/whoami:v1.11.0)" + shelf render "$work/app.yaml" --image "web=$base" 2>/dev/null | sed "s|$base|$2|" >"$1/configmap.yaml" + grep -qF "$2" "$1/configmap.yaml" || die "the rendered manifest does not name $2" +} + +# verification [preview]: whether the app's (or a preview's) verification matches. +verification() { + local args=(app status "$app" --json) + [[ -n "${2:-}" ]] && args+=(--preview "$2") + shelf "${args[@]}" | jq -e ".verification | $1" >/dev/null +} + +# admitted : whether the admission webhook lets a pod with image start there. +# A server-side dry run goes through the webhooks without creating anything. +admitted() { + kubectl -n "$1" run "admission-check" --image="$2" --restart=Never --dry-run=server >/dev/null 2>"$work/admission.txt" +} + +for leftover in "namespace/$app" "-n shelf-system resourcesetinputprovider/$app" "clusterimagepolicy/shelf-$app"; do + # shellcheck disable=SC2086 + kubectl get $leftover >/dev/null 2>&1 && die "$leftover already exists; remove it first (shelf app rm $app)" +done +trap cleanup EXIT + +step "build shelf, push platform and chart, init cluster with the dev key" +(cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) +"$repo/hack/platform-push.sh" >/dev/null 2>&1 +"$repo/hack/chart-push.sh" >/dev/null 2>&1 +shelf init cluster --yes --signer-key "$SHELF_SIGNER_PUB" --host-suffix "$(cluster_host_suffix)" --insecure-registry \ + --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ + --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" | grep -E '^ signer' +kubectl -n cosign-system wait helmrelease/policy-controller --for=condition=Ready --timeout=300s >/dev/null \ + || die "the policy-controller is not ready" + +step "a signed and an unsigned built image of the app's package" +crane copy --insecure --platform linux/arm64 traefik/whoami:v1.11.0 "$image:base" >/dev/null 2>&1 \ + || die "cannot copy the base image" +for kind in signed unsigned; do + crane mutate --insecure -l "dev.shelf.smoke=$kind-$(commit)" "$image:base" -t "$image:$kind" >/dev/null 2>&1 \ + || die "cannot make the $kind image" +done +sign_artifact "$image:signed" +signed="$image@$(crane digest --insecure "$image:signed")" +unsigned="$image@$(crane digest --insecure "$image:unsigned")" + +cat >"$work/app.yaml" <<'EOF' +apiVersion: shelf.dev/v1alpha1 +name: verify-smoke +components: + web: + build: ./web + port: 80 + route: / + resources: { cpu: 10m, memory: 16Mi } +EOF +render "$work/signed" "$signed" + +step "a signed and scanned artifact is verified and may be exposed" +first="$(commit)" +push_artifact "$artifact:main" "$work/signed" "main@sha1:$first" >/dev/null +shelf app add "$app" "$artifact:main" --insecure-registry >/dev/null +shelf app status "$app" --wait --commit "$first" --timeout 5m --json >/dev/null || die "the signed app did not become Ready" +verification '.verified and .exposable and (.scan | length) == 1 and .scan[0].scanned' \ + || die "the signed app is not verified: $(shelf app status "$app" --json | jq -c .verification)" +[[ "$(kubectl get clusterimagepolicy "shelf-$app" -o jsonpath='{.spec.images[0].glob}')" == "$SHELF_REGISTRY_HOST/smoke/$app/**" ]] \ + || die "the app has no ClusterImagePolicy for its package" + +step "the admission policy admits the signed image and denies the unsigned one" +admitted "$app" "$signed" || die "the signed image was denied: $(cat "$work/admission.txt")" +admitted "$app" "$unsigned" && die "the unsigned image was admitted" +grep -q 'policy.sigstore.dev' "$work/admission.txt" || die "the unsigned image was not denied by the policy: $(cat "$work/admission.txt")" +admitted "$app" "postgres:16" || die "a third-party image was denied: $(cat "$work/admission.txt")" + +step "a preview of a signed branch artifact is verified, and its namespace is checked too" +shelf app credentials "$app" --previews=1 >/dev/null +branch="$(commit)" +push_artifact "$artifact:branch-check" "$work/signed" "check@sha1:$branch" >/dev/null +shelf app status "$app" --preview check --wait --commit "$branch" --timeout 5m --json >/dev/null \ + || die "the preview did not become Ready" +verification '.verified and (.exposable | not)' check || die "the preview is not verified, or counts as exposable" +admitted "$app--check" "$unsigned" && die "the unsigned image was admitted in the preview" +shelf app credentials "$app" --no-previews >/dev/null + +step "a signed artifact without a scan runs, but is not verified" +noscan="$(commit)" +flux push artifact "$artifact:staging" --path "$work/signed" --source local --revision "main@sha1:$noscan" \ + --insecure-registry >/dev/null 2>&1 || die "push failed" +sign_artifact "${artifact#oci://}:staging" +crane tag --insecure "${artifact#oci://}:staging" main >/dev/null 2>&1 || die "tag failed" +shelf app status "$app" --wait --commit "$noscan" --timeout 5m --json >/dev/null || die "the artifact without a scan did not roll out" +verification '(.verified | not) and (.exposable | not) and (.reason | test("no scan"))' \ + || die "an artifact without a scan counts as verified: $(shelf app status "$app" --json | jq -c .verification)" + +step "an unsigned artifact is not applied, and what runs keeps running" +bad="$(commit)" +flux push artifact "$artifact:main" --path "$work/signed" --source local --revision "main@sha1:$bad" \ + --annotations "dev.shelf.scan=$(dev_scan "$work/signed")" --insecure-registry >/dev/null 2>&1 || die "push failed" +code=0 +shelf app status "$app" --wait --commit "$bad" --timeout 3m --json >"$work/unsigned.json" || code=$? +[[ "$code" == 1 ]] || die "the wait for an unsigned artifact ended with $code, not 1 (Failed)" +jq -e '.stage.name == "the deploy artifact" and (.stage.hint | test("id-token: write"))' "$work/unsigned.json" >/dev/null \ + || die "the verdict does not name the signature: $(jq -c .stage "$work/unsigned.json")" +shelf app status "$app" --json | jq -e ".commit == \"$noscan\" and (.stages[-1].phase == \"Ready\")" >/dev/null \ + || die "the app no longer runs the last verified artifact" + +step "the policy-controller's RAM" +kubectl top pod -n cosign-system --no-headers 2>/dev/null | awk '{print " " $1 ": " $3}' \ + || echo " (metrics not available yet)" + +step "shelf app rm takes the image policy with it" +shelf app rm "$app" --yes >/dev/null +kubectl get clusterimagepolicy "shelf-$app" >/dev/null 2>&1 && die "the ClusterImagePolicy is still there" +echo "smoke-verify passed" diff --git a/internal/cli/harness_test.go b/internal/cli/harness_test.go index bd08b99..fe395de 100644 --- a/internal/cli/harness_test.go +++ b/internal/cli/harness_test.go @@ -62,6 +62,9 @@ type fakeCluster struct { // rolloutMissing is how many reads of a rollout find nothing before it appears, as a preview // that the platform has not created yet. rolloutMissing int + // verifications are what AppVerification returns, by app or preview namespace; without an + // entry, the app is verified and its one component scanned. + verifications map[string]cluster.Verification // traefik is the address ForwardTraefik hands out, empty when no Traefik pod is ready. traefik string found bool @@ -171,6 +174,21 @@ func (f *fakeCluster) AppRollout(_ context.Context, _ *rest.Config, app string) return r, nil } +func (f *fakeCluster) AppVerification(_ context.Context, _ *rest.Config, name string) (cluster.Verification, error) { + if v, ok := f.verifications[name]; ok { + return v, nil + } + return cluster.Verification{ + Verified: true, + Source: "https://github.com/acme/hello", + Commit: "42a964eb0b4bd7248305d5f22233d6dbabcd56f3", + Scan: &cluster.Scan{Scanner: "trivy 0.75.0", ScannedAt: "2026-10-28T12:00:00Z", Images: []cluster.ScannedImage{{ + Component: "web", Image: "traefik/whoami:v1.11.0@sha256:2006", Digest: "sha256:2006", + Scanned: true, Critical: 1, High: 26, + }}}, + }, nil +} + func (f *fakeCluster) ForwardTraefik(context.Context, *rest.Config) (*cluster.Forward, error) { if f.traefik == "" { return nil, errors.New("no Traefik pod is ready") @@ -268,7 +286,8 @@ func newHarness(t *testing.T) *harness { // exposed under a domain of their own. func (h *harness) public() *harness { h.cluster.settings = cluster.Settings{HostSuffix: "-dev", - Chart: cluster.Artifact{URL: "oci://ghcr.io/tweinmann/shelf/charts/shelf-app", Tag: "0.5.0"}} + Chart: cluster.Artifact{URL: "oci://ghcr.io/tweinmann/shelf/charts/shelf-app", Tag: "0.5.0"}, + Signer: cluster.DefaultSigner()} return h } diff --git a/internal/cli/init.go b/internal/cli/init.go index ec46013..44f0098 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "os" "strings" "time" @@ -98,6 +99,8 @@ func newInitClusterCmd(o Options) *cobra.Command { hostSuffix string insecure bool moveHosts bool + signer = cluster.DefaultSigner() + signerKey string target clusterFlags ) cmd := &cobra.Command{ @@ -111,6 +114,11 @@ A cluster has no domain: an app gets one of its own when it is added, and answer Registry logins and Cloudflare tokens are connections (` + "`shelf connection`" + `), which each app chooses for itself. +The cluster verifies every artifact and built image before it runs it: by default it accepts +keyless signatures made by shelf's own build and release workflows (--signer-issuer, +--signer-subject); --signer-key accepts a cosign public key instead, for clusters that test shelf. +There is no setting that turns verification off. + The command shows the target cluster and asks for confirmation, because it installs cluster-wide objects. It is idempotent; running it again updates what changed.`, Args: cobra.NoArgs, @@ -127,6 +135,18 @@ cluster-wide objects. It is idempotent; running it again updates what changed.`, if err := ops.CheckHostSuffix(hostSuffix); err != nil { return fmt.Errorf("--host-suffix %w", err) } + if signerKey != "" { + data, err := os.ReadFile(signerKey) + if err != nil { + return fmt.Errorf("--signer-key: %w", err) + } + if signer.Key, err = cluster.ParseSignerKey(data); err != nil { + return fmt.Errorf("--signer-key %s: %w", signerKey, err) + } + } + if err := signer.Check(); err != nil { + return fmt.Errorf("signer: %w", err) + } shelf, err := target.load(o) if err != nil { return err @@ -138,6 +158,7 @@ cluster-wide objects. It is idempotent; running it again updates what changed.`, HostSuffix: hostSuffix, Insecure: insecure, MoveHosts: moveHosts, + Signer: signer, Timeout: target.timeout, } plan := shelf.PlanCluster(opts) @@ -147,6 +168,13 @@ cluster-wide objects. It is idempotent; running it again updates what changed.`, fmt.Fprintf(out, " platform %s\n", plan.Platform) fmt.Fprintf(out, " chart %s\n", plan.Chart) fmt.Fprintf(out, " hosts %s, or .%s without a domain\n", plan.Hosts, cluster.InternalDomain) + fmt.Fprintf(out, " signer %s\n", plan.Signer) + // Best effort: a cluster without shelf has no apps, and not even their types. + if unverified, err := shelf.UnverifiedApps(cmd.Context()); err == nil && len(unverified) > 0 { + fmt.Fprintf(out, " unverified apps: %s\n", strings.Join(unverified, ", ")) + fmt.Fprintln(out, " each needs one build with the current workflow before it gets new deploys;") + fmt.Fprintln(out, " what runs now keeps running") + } if !target.yes { ok, err := confirm(cmd.InOrStdin(), out) if err != nil { @@ -169,6 +197,12 @@ cluster-wide objects. It is idempotent; running it again updates what changed.`, f.BoolVar(&insecure, "insecure-registry", false, "pull platform and chart without TLS (dev registry)") f.BoolVar(&moveHosts, "move-hosts", false, "allow a host suffix that moves the apps with a domain to different names") + f.StringVar(&signer.Issuer, "signer-issuer", cluster.DefaultSignerIssuer, + "OIDC issuer of the keyless signatures the cluster accepts") + f.StringVar(&signer.Subject, "signer-subject", cluster.DefaultSignerSubject, + "regular expression for the certificate subject of the keyless signatures the cluster accepts") + f.StringVar(&signerKey, "signer-key", "", + "cosign public key (PEM file) the cluster accepts instead of keyless signatures; for clusters that test shelf") target.register(f) return cmd } diff --git a/internal/cli/init_test.go b/internal/cli/init_test.go index 368e184..a07dd52 100644 --- a/internal/cli/init_test.go +++ b/internal/cli/init_test.go @@ -1,6 +1,8 @@ package cli import ( + "os" + "path/filepath" "strings" "testing" @@ -271,3 +273,68 @@ func TestInitClusterPinsTheFormerDomain(t *testing.T) { } } } + +func TestInitClusterSigner(t *testing.T) { + t.Parallel() + const platform = "oci://shelf-registry.local:5000/shelf/platform:dev" + const chart = "oci://shelf-registry.local:5000/shelf/charts/shelf-app:0.0.0-dev" + base := []string{"init", "cluster", "--yes", "--platform", platform, "--chart", chart} + notAKey := filepath.Join(t.TempDir(), "not-a-key.pub") + if err := os.WriteFile(notAKey, []byte("hello\n"), 0o600); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + args []string + wantMode string + wantOut string + wantErr string + }{ + {"keyless by default", nil, "keyless", "signer keyless, issuer " + cluster.DefaultSignerIssuer, ""}, + {"own subject", []string{"--signer-subject", `^https://github\.com/fork/shelf/.+$`}, "keyless", `subject ^https://github\.com/fork/shelf/.+$`, ""}, + {"key", []string{"--signer-key", filepath.Join("..", "cluster", "testdata", "signer.pub")}, "key", "signer cosign key sha256:", ""}, + {"not a key", []string{"--signer-key", notAKey}, "", "", "not a PEM public key"}, + {"missing key file", []string{"--signer-key", filepath.Join(t.TempDir(), "missing.pub")}, "", "", "--signer-key"}, + {"bad subject", []string{"--signer-subject", "("}, "", "", "signer: subject"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + h := newHarness(t) + stdout, stderr, code := h.run(t, append(append([]string{}, base...), tt.args...)...) + if tt.wantErr != "" { + if code == 0 || !strings.Contains(stderr, tt.wantErr) || len(h.cluster.installed) != 0 { + t.Fatalf("exit %d, %d installs, stderr:\n%s", code, len(h.cluster.installed), stderr) + } + return + } + if code != 0 { + t.Fatalf("exit code %d: %s", code, stderr) + } + if got := h.cluster.installed[0].Settings.Signer.Mode(); got != tt.wantMode { + t.Errorf("mode %s, want %s", got, tt.wantMode) + } + if !strings.Contains(stdout, tt.wantOut) { + t.Errorf("stdout lacks %q:\n%s", tt.wantOut, stdout) + } + }) + } +} + +func TestInitClusterListsUnverifiedApps(t *testing.T) { + t.Parallel() + h := newHarness(t).public() + h.cluster.states = []cluster.AppState{{Name: "greeter"}, {Name: "hello"}} + h.cluster.verifications = map[string]cluster.Verification{ + "greeter": {Reason: "the cluster does not verify the deploy artifact's signature"}, + } + stdout, stderr, code := h.run(t, "init", "cluster", "--yes", + "--platform", "oci://ghcr.io/tweinmann/shelf/platform:v0.8.0", "--chart", "oci://ghcr.io/tweinmann/shelf/charts/shelf-app:0.8.0") + if code != 0 { + t.Fatalf("exit code %d: %s", code, stderr) + } + if !strings.Contains(stdout, " unverified apps: greeter\n") || !strings.Contains(stdout, "one build with the current workflow") { + t.Errorf("stdout:\n%s", stdout) + } +} diff --git a/internal/cli/json.go b/internal/cli/json.go index 21fad3c..6996de0 100644 --- a/internal/cli/json.go +++ b/internal/cli/json.go @@ -136,6 +136,33 @@ type appStatusDocument struct { Previews []previewDocument `json:"previews,omitempty"` PreviewLimit int `json:"previewLimit,omitempty"` PreviewLimitReached *bool `json:"previewLimitReached,omitempty"` + // Verification is whether the app runs what the cluster's signer signed and CI scanned. + Verification *verificationDocument `json:"verification,omitempty"` +} + +type verificationDocument struct { + Verified bool `json:"verified"` + // Reason is the first thing that does not hold, and Hint what to do; both only when not + // verified. + Reason string `json:"reason,omitempty"` + Hint string `json:"hint,omitempty"` + Source string `json:"source,omitempty"` + Signer string `json:"signer"` + // Exposable is whether the app may get a tunnel; never for a preview. + Exposable bool `json:"exposable"` + // Scan is one entry per component, left out when the artifact has no scan summary. + Scan []scanDocument `json:"scan,omitempty"` +} + +type scanDocument struct { + Component string `json:"component"` + Digest string `json:"digest,omitempty"` + // Scanned is false when the summary does not name the component's image. + Scanned bool `json:"scanned"` + Critical int `json:"critical"` + High int `json:"high"` + Scanner string `json:"scanner,omitempty"` + ScannedAt string `json:"scannedAt,omitempty"` } type waitResultDocument struct { @@ -249,6 +276,17 @@ func appStatus(a ops.App, d cluster.Diagnosis) appStatusDocument { Name: c.Name, Phase: c.Phase, Path: c.Path, Address: c.Address, URL: c.URL, Internal: c.Internal, }) } + if v := a.Verification; v != nil { + doc.Verification = &verificationDocument{Verified: v.Verified, Reason: v.Reason, Hint: v.Hint, + Source: v.Source, Signer: v.Signer, Exposable: v.Exposable} + if v.Scan != nil { + for _, img := range v.Scan.Images { + doc.Verification.Scan = append(doc.Verification.Scan, scanDocument{Component: img.Component, + Digest: img.Digest, Scanned: img.Scanned, Critical: img.Critical, High: img.High, + Scanner: v.Scan.Scanner, ScannedAt: v.Scan.ScannedAt}) + } + } + } return doc } diff --git a/internal/cli/previews_test.go b/internal/cli/previews_test.go index d8f325a..88175e1 100644 --- a/internal/cli/previews_test.go +++ b/internal/cli/previews_test.go @@ -141,7 +141,8 @@ func TestStatusOfOnePreview(t *testing.T) { h := previewHarness(t) stdout := h.mustRun(t, "app", "status", "hello", "--preview", "feature/login") for _, want := range []string{"hello--feature-login: Ready", "address hello--feature-login.shelf.internal", - "artifact oci://ghcr.io/o/hello:branch-feature-login", "commit 3f2a9c1e5b7d"} { + "artifact oci://ghcr.io/o/hello:branch-feature-login", "commit 3f2a9c1e5b7d", + "verified yes", "exposure never: a preview stays inside the cluster"} { if !strings.Contains(stdout, want) { t.Errorf("stdout lacks %q:\n%s", want, stdout) } diff --git a/internal/cli/status.go b/internal/cli/status.go index 3d75f8b..491cba9 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -108,6 +108,7 @@ This is what the admin UI shows on its pages, in text.`, } printApp(out, app) printComponents(out, app.Components, app.Public || app.Quick) + printVerification(out, app.Verification, preview != "") printChain(out, diagnosis) printPreviews(out, app) return nil @@ -250,6 +251,51 @@ func printComponents(out io.Writer, comps []ops.Component, public bool) { } } +// printVerification is the chain from the commit to what runs: where the deploy artifact says it +// comes from, whether the cluster's signer signed it and CI scanned every image, and so whether +// the app may be exposed. +func printVerification(out io.Writer, v *ops.Verification, preview bool) { + if v == nil { + return + } + if v.Source != "" { + source := v.Source + if v.Commit != "" { + source += " at " + v.Commit[:min(len(v.Commit), 12)] + } + fmt.Fprintf(out, " source %s\n", source) + } + fmt.Fprintf(out, " signer %s\n", v.Signer) + if v.Verified { + fmt.Fprintln(out, " verified yes") + } else { + fmt.Fprintf(out, " verified no: %s\n", v.Reason) + fmt.Fprintf(out, " %s\n", v.Hint) + } + if v.Scan != nil { + fmt.Fprintf(out, " scan %s, %s\n", v.Scan.Scanner, v.Scan.ScannedAt) + width := 0 + for _, img := range v.Scan.Images { + width = max(width, len(img.Component)) + } + for _, img := range v.Scan.Images { + findings := "not scanned" + if img.Scanned { + findings = fmt.Sprintf("%d critical, %d high", img.Critical, img.High) + } + fmt.Fprintf(out, " %-*s %s\n", width, img.Component, findings) + } + } + switch { + case preview: + fmt.Fprintln(out, " exposure never: a preview stays inside the cluster") + case v.Exposable: + fmt.Fprintln(out, " exposure allowed") + default: + fmt.Fprintln(out, " exposure refused until it is verified") + } +} + // componentAddress is the link a browser can follow, or the address plus why it is not one. func componentAddress(c ops.Component, public bool) string { if c.URL != "" { diff --git a/internal/cli/testdata/json/appstatus-failing.json b/internal/cli/testdata/json/appstatus-failing.json index d39a2e2..699d6a9 100644 --- a/internal/cli/testdata/json/appstatus-failing.json +++ b/internal/cli/testdata/json/appstatus-failing.json @@ -63,5 +63,22 @@ "message": "Back-off pulling image", "hint": "the image cannot be pulled; check the reference and whether the cluster may read that registry" } - ] + ], + "verification": { + "verified": true, + "source": "https://github.com/acme/hello", + "signer": "keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$", + "exposable": true, + "scan": [ + { + "component": "web", + "digest": "sha256:2006", + "scanned": true, + "critical": 1, + "high": 26, + "scanner": "trivy 0.75.0", + "scannedAt": "2026-10-28T12:00:00Z" + } + ] + } } diff --git a/internal/cli/testdata/json/appstatus-preview.json b/internal/cli/testdata/json/appstatus-preview.json index 7128215..f5fc09d 100644 --- a/internal/cli/testdata/json/appstatus-preview.json +++ b/internal/cli/testdata/json/appstatus-preview.json @@ -54,5 +54,22 @@ "phase": "Ready", "message": "2 running" } - ] + ], + "verification": { + "verified": true, + "source": "https://github.com/acme/hello", + "signer": "keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$", + "exposable": false, + "scan": [ + { + "component": "web", + "digest": "sha256:2006", + "scanned": true, + "critical": 1, + "high": 26, + "scanner": "trivy 0.75.0", + "scannedAt": "2026-10-28T12:00:00Z" + } + ] + } } diff --git a/internal/cli/testdata/json/appstatus-previews.json b/internal/cli/testdata/json/appstatus-previews.json index a45ff3a..fa84b7d 100644 --- a/internal/cli/testdata/json/appstatus-previews.json +++ b/internal/cli/testdata/json/appstatus-previews.json @@ -74,5 +74,22 @@ } ], "previewLimit": 2, - "previewLimitReached": true + "previewLimitReached": true, + "verification": { + "verified": true, + "source": "https://github.com/acme/hello", + "signer": "keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$", + "exposable": true, + "scan": [ + { + "component": "web", + "digest": "sha256:2006", + "scanned": true, + "critical": 1, + "high": 26, + "scanner": "trivy 0.75.0", + "scannedAt": "2026-10-28T12:00:00Z" + } + ] + } } diff --git a/internal/cli/testdata/json/appstatus-unverified.json b/internal/cli/testdata/json/appstatus-unverified.json new file mode 100644 index 0000000..44db7f0 --- /dev/null +++ b/internal/cli/testdata/json/appstatus-unverified.json @@ -0,0 +1,22 @@ +{ + "apiVersion": "shelf.dev/v1alpha1", + "kind": "AppStatus", + "name": "hello", + "phase": "Ready", + "host": "hello.shelf.internal", + "artifact": ":", + "exposure": { + "kind": "none" + }, + "revision": "main@sha256:abc", + "components": [], + "stages": [], + "verification": { + "verified": false, + "reason": "the deploy artifact is not signed by the cluster's signer: no matching signatures were found", + "hint": "push again with the current reusable workflow, which signs and scans every build (the caller's deploy.yml needs `id-token: write`)", + "source": "https://github.com/acme/hello", + "signer": "keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$", + "exposable": false + } +} diff --git a/internal/cli/testdata/json/appstatus.json b/internal/cli/testdata/json/appstatus.json index 526a8a6..b88621e 100644 --- a/internal/cli/testdata/json/appstatus.json +++ b/internal/cli/testdata/json/appstatus.json @@ -61,5 +61,22 @@ "phase": "Ready", "message": "2 running" } - ] + ], + "verification": { + "verified": true, + "source": "https://github.com/acme/hello", + "signer": "keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$", + "exposable": true, + "scan": [ + { + "component": "web", + "digest": "sha256:2006", + "scanned": true, + "critical": 1, + "high": 26, + "scanner": "trivy 0.75.0", + "scannedAt": "2026-10-28T12:00:00Z" + } + ] + } } diff --git a/internal/cli/verification_test.go b/internal/cli/verification_test.go new file mode 100644 index 0000000..036b021 --- /dev/null +++ b/internal/cli/verification_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/tweinmann/shelf/internal/cluster" +) + +func TestAppStatusVerification(t *testing.T) { + t.Parallel() + unsigned := cluster.Verification{ + Reason: "the deploy artifact is not signed by the cluster's signer: no matching signatures were found", + Source: "https://github.com/acme/hello", Commit: "42a964eb0b4bd7248305d5f22233d6dbabcd56f3", + } + tests := []struct { + name string + v *cluster.Verification + want []string + absent []string + }{ + { + name: "verified", + want: []string{ + " source https://github.com/acme/hello at 42a964eb0b4b\n", + " signer keyless, issuer " + cluster.DefaultSignerIssuer, + " verified yes\n", + " scan trivy 0.75.0, 2026-10-28T12:00:00Z\n", + " web 1 critical, 26 high\n", + " exposure allowed\n", + }, + }, + { + name: "unsigned", + v: &unsigned, + want: []string{ + " verified no: the deploy artifact is not signed by the cluster's signer", + " push again with the current reusable workflow", + " exposure refused until it is verified\n", + }, + absent: []string{" scan "}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + h := newHarness(t).public() + h.cluster.states = []cluster.AppState{{Name: "hello", Phase: cluster.PhaseReady, Revision: "main@sha256:abc"}} + if tt.v != nil { + h.cluster.verifications = map[string]cluster.Verification{"hello": *tt.v} + } + stdout, stderr, code := h.run(t, "app", "status", "hello") + if code != 0 { + t.Fatalf("exit code %d: %s", code, stderr) + } + if tt.v != nil { + goldenJSON(t, "appstatus-unverified", runJSON(t, h, 0, "app", "status", "hello")) + } + for _, want := range tt.want { + if !strings.Contains(stdout, want) { + t.Errorf("stdout lacks %q:\n%s", want, stdout) + } + } + for _, absent := range tt.absent { + if strings.Contains(stdout, absent) { + t.Errorf("stdout has %q:\n%s", absent, stdout) + } + } + }) + } +} + +// TestExposureGate: only a verified app gets a tunnel it does not have yet, and the refusal +// comes before anything is asked of Cloudflare. The app itself is deployed either way. +func TestExposureGate(t *testing.T) { + t.Parallel() + noScan := cluster.Verification{Reason: "the deploy artifact has no scan (dev.shelf.scan)"} + unsigned := cluster.Verification{Reason: "the deploy artifact is not signed by the cluster's signer: no matching signatures were found"} + + t.Run("verified app gets its tunnel after the rollout", func(t *testing.T) { + t.Parallel() + h := cloudflareHarness(t) + stdout := h.mustRun(t, "app", "add", "greeter", greeterArtifact, "--cloudflare", "home", "--domain", "example.com") + if len(h.cluster.added) != 2 || h.cluster.added[0].Cloudflare != "" || h.cluster.added[0].TunnelID != "" { + t.Fatalf("the app was not deployed privately first: %+v", h.cluster.added) + } + if last := h.lastAdded(t); last.Cloudflare != "home" || last.TunnelID == "" || len(h.api.created) != 1 { + t.Errorf("the verified app got no tunnel: %+v, created %v", last, h.api.created) + } + if !strings.Contains(stdout, "exposure: once greeter runs a verified deploy artifact") || + !strings.Contains(stdout, "verified: signed by the cluster's signer, every image scanned") { + t.Errorf("stdout:\n%s", stdout) + } + }) + + t.Run("new app without a scan stays private", func(t *testing.T) { + t.Parallel() + h := cloudflareHarness(t) + h.cluster.verifications = map[string]cluster.Verification{"greeter": noScan} + _, stderr, code := h.run(t, "app", "add", "greeter", greeterArtifact, "--cloudflare", "home", "--domain", "example.com") + if code != 1 || !strings.Contains(stderr, "greeter is not exposed: it does not run a verified deploy artifact: the deploy artifact has no scan") || + !strings.Contains(stderr, "stays inside the cluster") { + t.Fatalf("exit code %d, stderr:\n%s", code, stderr) + } + if len(h.cluster.added) != 1 || h.lastAdded(t).Cloudflare != "" { + t.Errorf("added %+v", h.cluster.added) + } + if len(h.api.created) != 0 || len(h.api.zoneFor) != 0 || len(h.api.records) != 0 { + t.Errorf("Cloudflare was asked: created %v, zones %v, records %v", h.api.created, h.api.zoneFor, h.api.records) + } + }) + + t.Run("quick tunnel refused for an unsigned app", func(t *testing.T) { + t.Parallel() + h := cloudflareHarness(t) + h.mustRun(t, "app", "add", "greeter", greeterArtifact) + h.cluster.verifications = map[string]cluster.Verification{"greeter": unsigned} + _, stderr, code := h.run(t, "app", "credentials", "greeter", "--quick") + if code != 1 || !strings.Contains(stderr, "not signed by the cluster's signer") || !strings.Contains(stderr, "id-token: write") { + t.Fatalf("exit code %d, stderr:\n%s", code, stderr) + } + if h.lastAdded(t).Quick { + t.Error("the unsigned app got a quick tunnel") + } + }) + + t.Run("an exposed app keeps its tunnel", func(t *testing.T) { + t.Parallel() + h := cloudflareHarness(t) + h.mustRun(t, "app", "add", "greeter", greeterArtifact, "--quick") + h.cluster.verifications = map[string]cluster.Verification{"greeter": noScan} + h.mustRun(t, "app", "add", "greeter", greeterArtifact) + if !h.lastAdded(t).Quick { + t.Error("re-adding an exposed app took its quick tunnel away") + } + }) +} diff --git a/internal/cluster/apps.go b/internal/cluster/apps.go index c2c9ee7..8f02ce3 100644 --- a/internal/cluster/apps.go +++ b/internal/cluster/apps.go @@ -300,7 +300,8 @@ func (c *client) rollOut(ctx context.Context, rep progress.Reporter, name string if err != nil { return "", err } - obj, err := c.reconcileAndWait(ctx, repo, failOnAuthError(readyCondition, registryHint(name))) + obj, err := c.reconcileAndWait(ctx, repo, failOnUnverified(failOnAuthError(readyCondition, registryHint(name)), + "push again with the current reusable workflow, which signs every build (its caller needs `id-token: write`)")) if err != nil { return "", err } @@ -399,6 +400,25 @@ func RemoveApp(ctx context.Context, cfg *rest.Config, app string, timeout time.D return found, err } + // The app's image policy is cluster-wide, so it does not go with the namespace. A cluster + // without the policy-controller has none. + policy := ref{gvk: imagePolicyGVK, name: ImagePolicyName(app)} + err = c.step(ctx, rep, "the image policy of "+app+" to be deleted", func(ctx context.Context) (string, error) { + return "", wait.PollUntilContextCancel(ctx, pollInterval, true, func(ctx context.Context) (bool, error) { + obj, err := c.get(ctx, policy) + if meta.IsNoMatchError(err) { + return true, nil + } + return obj == nil, err + }) + }) + if err != nil { + if ctx.Err() != nil { + return found, fmt.Errorf("%s is still there; check the ResourceSet apps in the cluster", policy) + } + return found, err + } + for _, name := range []string{AppSecretName(app), TunnelSecretName(app)} { secret := ref{gvk: secretGVK, namespace: SystemNamespace, name: name} deleted, err = c.delete(ctx, secret) diff --git a/internal/cluster/client.go b/internal/cluster/client.go index f1b7bf7..13a8729 100644 --- a/internal/cluster/client.go +++ b/internal/cluster/client.go @@ -198,6 +198,21 @@ func failOnAuthError(ready readyFunc, hint string) readyFunc { } } +// failOnUnverified turns a failed signature check into an error: Flux retries it forever, but +// the artifact will not get another signature by waiting. +func failOnUnverified(ready readyFunc, hint string) readyFunc { + return func(obj *unstructured.Unstructured) (bool, string, error) { + ok, msg, err := ready(obj) + if err == nil && !ok { + if cond := condition(obj, sourceVerified); cond != nil && cond["status"] == "False" { + message, _ := cond["message"].(string) + return false, msg, fmt.Errorf("%s is not signed by the cluster's signer: %s\n%s", describe(obj), message, hint) + } + } + return ok, msg, err + } +} + // waitFor polls until ready reports true or the context ends. A missing object counts as not // ready yet, because controllers create some of the objects shelf waits for. func (c *client) waitFor(ctx context.Context, r ref, ready readyFunc) error { diff --git a/internal/cluster/cluster_test.go b/internal/cluster/cluster_test.go index cf82fd9..dd51eb9 100644 --- a/internal/cluster/cluster_test.go +++ b/internal/cluster/cluster_test.go @@ -124,19 +124,23 @@ func TestParseArtifact(t *testing.T) { } } +// TestFluxInstance covers a release cluster, which verifies keyless signatures, and the dev +// cluster, which reads the platform without TLS and verifies with a key. func TestFluxInstance(t *testing.T) { for _, insecure := range []bool{false, true} { name := "fluxinstance" ref := "oci://ghcr.io/tweinmann/shelf/platform:v0.3.0" + signer := DefaultSigner() if insecure { - name, ref = "fluxinstance-insecure", "oci://shelf-registry:5000/shelf/platform:dev" + name, ref = "fluxinstance-insecure", "oci://shelf-registry.local:5000/shelf/platform:dev" + signer = Signer{Key: testSignerKey(t)} } t.Run(name, func(t *testing.T) { p, err := ParseArtifact(ref) if err != nil { t.Fatal(err) } - out, err := yaml.Marshal(FluxInstance(p, insecure).Object) + out, err := yaml.Marshal(FluxInstance(p, insecure, signer).Object) if err != nil { t.Fatal(err) } @@ -206,7 +210,7 @@ func TestPlatformObjects(t *testing.T) { if err != nil { t.Fatal(err) } - objs := ConfigObjects(Settings{Chart: chart, InsecureRegistry: true}) + objs := ConfigObjects(Settings{Chart: chart, InsecureRegistry: true, Signer: DefaultSigner()}) withLogin, err := RegistryConnectionSecret("ghcr", RegistryAuth{Username: "tobi", Token: "not-a-real-token"}) if err != nil { t.Fatal(err) @@ -283,21 +287,22 @@ func TestSettingsRoundTrip(t *testing.T) { if err != nil { t.Fatal(err) } - want := Settings{ - HostSuffix: "-dev", - Chart: chart, - InsecureRegistry: true, - } - var data map[string]string - for _, obj := range ConfigObjects(want) { - if obj.GetKind() != "ConfigMap" { - continue + settingsData := func(s Settings) map[string]string { + for _, obj := range ConfigObjects(s) { + if obj.GetKind() == "ConfigMap" { + data, _, _ := unstructured.NestedStringMap(obj.Object, "data") + return data + } } - data, _, _ = unstructured.NestedStringMap(obj.Object, "data") + return nil } - if got := ParseSettings(data); got != want { - t.Errorf("got %+v, want %+v", got, want) + for _, signer := range []Signer{DefaultSigner(), {Key: testSignerKey(t)}} { + want := Settings{HostSuffix: "-dev", Chart: chart, InsecureRegistry: true, Signer: signer} + if got := ParseSettings(settingsData(want)); got != want { + t.Errorf("got %+v, want %+v", got, want) + } } + data := settingsData(Settings{HostSuffix: "-dev", Chart: chart, InsecureRegistry: true, Signer: DefaultSigner()}) if got := ParseSettings(nil); got != (Settings{}) || got.Installed() { t.Errorf("a cluster without settings must read as zero, got %+v", got) } diff --git a/internal/cluster/config.go b/internal/cluster/config.go index 334301d..97be05a 100644 --- a/internal/cluster/config.go +++ b/internal/cluster/config.go @@ -46,6 +46,9 @@ type Settings struct { // LegacyDomain is the domain a cluster had before clusters had none, read back and never // written. `shelf init cluster` gives it to the apps that answered under it. LegacyDomain string + // Signer is who must have signed what the cluster runs. Zero only in a cluster installed + // before Phase 11. + Signer Signer } // Installed reports whether `shelf init cluster` has run against the cluster. @@ -57,7 +60,10 @@ type RegistryAuth struct { Token string } -// ConfigObjects returns the system namespace and the ConfigMap with the cluster settings. +// ConfigObjects returns the system namespace, the ConfigMap with the cluster settings and the +// signer's key in shelf-system, for the app namespaces, and in flux-system, for the platform. +// Besides the signer's own fields, the settings carry what the platform needs of it as JSON: +// the verify block of an OCIRepository and the authority of a ClusterImagePolicy. func ConfigObjects(s Settings) []*unstructured.Unstructured { return []*unstructured.Unstructured{ {Object: map[string]any{ @@ -79,8 +85,16 @@ func ConfigObjects(s Settings) []*unstructured.Unstructured { "SHELF_CHART_URL": s.Chart.URL, "SHELF_CHART_TAG": s.Chart.Tag, "SHELF_INSECURE_REGISTRY": strconv.FormatBool(s.InsecureRegistry), + "SHELF_SIGNER_MODE": s.Signer.Mode(), + "SHELF_SIGNER_ISSUER": s.Signer.Issuer, + "SHELF_SIGNER_SUBJECT": s.Signer.Subject, + "SHELF_SIGNER_KEY": s.Signer.Key, + "SHELF_SIGNER_VERIFY": compactJSON(s.Signer.verify(SignerSecretName)), + "SHELF_SIGNER_AUTHORITY": compactJSON(s.Signer.authority()), }, }}, + signerSecret(SystemNamespace, signerKeySecret, s.Signer), + signerSecret(FluxNamespace, SignerSecretName, s.Signer), } } @@ -103,6 +117,11 @@ func ParseSettings(data map[string]string) Settings { Chart: Artifact{URL: data["SHELF_CHART_URL"], Tag: data["SHELF_CHART_TAG"]}, InsecureRegistry: insecure, LegacyDomain: data[legacyDomainKey], + Signer: Signer{ + Issuer: data["SHELF_SIGNER_ISSUER"], + Subject: data["SHELF_SIGNER_SUBJECT"], + Key: data["SHELF_SIGNER_KEY"], + }, } } diff --git a/internal/cluster/fluxinstance.go b/internal/cluster/fluxinstance.go index 7d5a3f4..5639888 100644 --- a/internal/cluster/fluxinstance.go +++ b/internal/cluster/fluxinstance.go @@ -63,9 +63,15 @@ const syncKustomizationPatch = `- op: add ` // FluxInstance returns the FluxInstance that installs Flux and syncs the platform. insecure -// allows a platform registry without TLS. -func FluxInstance(platform Artifact, insecure bool) *unstructured.Unstructured { - patches := []any{jsonPatch("Kustomization", PlatformSyncName, syncKustomizationPatch)} +// allows a platform registry without TLS. The platform artifact is verified against signer +// before Flux applies it. +func FluxInstance(platform Artifact, insecure bool, signer Signer) *unstructured.Unstructured { + patches := []any{ + jsonPatch("Kustomization", PlatformSyncName, syncKustomizationPatch), + // spec.sync has no field for verification either. + jsonPatch("OCIRepository", PlatformSyncName, + "- op: add\n path: /spec/verify\n value: "+compactJSON(signer.verify(SignerSecretName))+"\n"), + } if insecure { // spec.sync has no field for registries without TLS. patches = append(patches, diff --git a/internal/cluster/install.go b/internal/cluster/install.go index ebd5625..9cafa76 100644 --- a/internal/cluster/install.go +++ b/internal/cluster/install.go @@ -101,7 +101,7 @@ func Install(ctx context.Context, cfg *rest.Config, opts Options) error { return err } - instance := FluxInstance(opts.Platform, opts.Settings.InsecureRegistry) + instance := FluxInstance(opts.Platform, opts.Settings.InsecureRegistry, opts.Settings.Signer) action, err := c.apply(ctx, instance) if err != nil { return err @@ -376,8 +376,9 @@ func (c *client) waitForPlatform(ctx context.Context, p Artifact) (string, error if err != nil { return "", err } - obj, err := c.reconcileAndWait(ctx, repo, failOnAuthError(readyCondition, - "the platform artifact is read without a login, so it has to be public")) + obj, err := c.reconcileAndWait(ctx, repo, failOnUnverified(failOnAuthError(readyCondition, + "the platform artifact is read without a login, so it has to be public"), + "install a released platform, which release.yml signs, or one signed with the cluster's --signer-key")) if err != nil { return "", err } diff --git a/internal/cluster/platform_test.go b/internal/cluster/platform_test.go new file mode 100644 index 0000000..4060c55 --- /dev/null +++ b/internal/cluster/platform_test.go @@ -0,0 +1,171 @@ +package cluster + +import ( + "bytes" + "os" + "regexp" + "strings" + "testing" + "text/template" + + "go.yaml.in/yaml/v3" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + sigsyaml "sigs.k8s.io/yaml" +) + +// platformObjects renders a ResourceSet of the platform the way the cluster does: Flux first +// substitutes ${SHELF_...} from the cluster settings into the YAML of the whole object, then the +// Flux Operator renders << >> per input set, both the resources and the resourcesTemplate. +func platformObjects(t *testing.T, file string, settings Settings, inputs map[string]any) []*unstructured.Unstructured { + t.Helper() + raw, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + var obj map[string]any + if err := yaml.Unmarshal(raw, &obj); err != nil { + t.Fatal(err) + } + // kustomize-controller marshals the object again before it substitutes, so the + // substitution sees the YAML as Flux writes it, not as the file spells it. + encoded, err := yaml.Marshal(obj) + if err != nil { + t.Fatal(err) + } + var data map[string]string + for _, o := range ConfigObjects(settings) { + if o.GetKind() == "ConfigMap" { + data, _, _ = unstructured.NestedStringMap(o.Object, "data") + } + } + substituted := regexp.MustCompile(`\$\{([A-Z_]+)(?::=([^}]*))?\}`).ReplaceAllStringFunc(string(encoded), func(m string) string { + parts := regexp.MustCompile(`\$\{([A-Z_]+)(?::=([^}]*))?\}`).FindStringSubmatch(m) + if v, ok := data[parts[1]]; ok && v != "" { + return v + } + return parts[2] + }) + var rs map[string]any + if err := yaml.Unmarshal([]byte(substituted), &rs); err != nil { + t.Fatalf("%s after substitution: %v", file, err) + } + spec := rs["spec"].(map[string]any) + + render := func(text string) []byte { + tmpl, err := template.New(file).Delims("<<", ">>").Funcs(template.FuncMap{ + "trimPrefix": func(prefix, s string) string { return strings.TrimPrefix(s, prefix) }, + "inputs": func() map[string]any { return inputs }, + }).Option("missingkey=zero").Parse(text) + if err != nil { + t.Fatalf("%s: %v", file, err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, nil); err != nil { + t.Fatalf("%s: %v", file, err) + } + return out.Bytes() + } + var objs []*unstructured.Unstructured + add := func(doc []byte) { + if len(bytes.TrimSpace(doc)) == 0 { + return + } + u := &unstructured.Unstructured{} + if err := sigsyaml.Unmarshal(doc, &u.Object); err != nil { + t.Fatalf("%s: rendered object does not parse: %v\n%s", file, err, doc) + } + if len(u.Object) > 0 { + objs = append(objs, u) + } + } + if resources, ok := spec["resources"].([]any); ok { + for _, r := range resources { + doc, err := yaml.Marshal(r) + if err != nil { + t.Fatal(err) + } + add(render(string(doc))) + } + } + if tmpl, ok := spec["resourcesTemplate"].(string); ok { + for _, doc := range strings.Split(string(render(tmpl)), "\n---") { + add([]byte(doc)) + } + } + return objs +} + +func find(objs []*unstructured.Unstructured, kind, name string) *unstructured.Unstructured { + for _, o := range objs { + if o.GetKind() == kind && o.GetName() == name { + return o + } + } + return nil +} + +// TestPlatformVerifies renders the ResourceSets apps and previews with both kinds of signer and +// checks that everything they fetch is verified, that their namespaces are checked by the +// policy-controller, and that each app gets the policy for its built images. +func TestPlatformVerifies(t *testing.T) { + t.Parallel() + chart := Artifact{URL: "oci://ghcr.io/tweinmann/shelf/charts/shelf-app", Tag: "0.8.0"} + signers := map[string]Signer{"keyless": DefaultSigner(), "key": {Key: testSignerKey(t)}} + for mode, signer := range signers { + t.Run(mode, func(t *testing.T) { + t.Parallel() + settings := Settings{Chart: chart, Signer: signer} + app := platformObjects(t, "../../platform/apps/resourceset.yaml", settings, map[string]any{ + "name": "hello", "url": "oci://ghcr.io/acme/hello", "tag": "main", "insecure": false, + "domain": "", "tunnel": "", "quick": false, "registry": "ghcr", "cloudflare": "", + }) + preview := platformObjects(t, "../../platform/previews/resourceset.yaml", settings, map[string]any{ + "app": "hello", "url": "oci://ghcr.io/acme/hello", "tag": "branch-feature-login", + "insecure": false, "registry": "ghcr", + }) + for name, objs := range map[string][]*unstructured.Unstructured{"app": app, "preview": preview} { + for _, repo := range []string{"deploy", "shelf-app"} { + o := find(objs, "OCIRepository", repo) + if o == nil { + t.Fatalf("%s: no OCIRepository %s", name, repo) + } + verify, _, _ := unstructured.NestedMap(o.Object, "spec", "verify") + if verify["provider"] != "cosign" { + t.Errorf("%s: OCIRepository %s verify %v", name, repo, verify) + } + if (verify["secretRef"] != nil) != (mode == "key") || (verify["matchOIDCIdentity"] != nil) != (mode == "keyless") { + t.Errorf("%s: OCIRepository %s verify %v for a %s signer", name, repo, verify, mode) + } + } + var ns *unstructured.Unstructured + for _, o := range objs { + if o.GetKind() == "Namespace" { + ns = o + } + } + if ns == nil || ns.GetLabels()["policy.sigstore.dev/include"] != "true" { + t.Errorf("%s: namespace %v is not checked by the policy-controller", name, ns) + } + secret := find(objs, "Secret", SignerSecretName) + if secret == nil || secret.GetAnnotations()["fluxcd.controlplane.io/copyFrom"] != SystemNamespace+"/"+signerKeySecret { + t.Errorf("%s: no copy of the signer key: %v", name, secret) + } + } + policy := find(app, "ClusterImagePolicy", "shelf-hello") + if policy == nil { + t.Fatal("no ClusterImagePolicy shelf-hello") + } + images, _, _ := unstructured.NestedSlice(policy.Object, "spec", "images") + authorities, _, _ := unstructured.NestedSlice(policy.Object, "spec", "authorities") + if len(images) != 1 || images[0].(map[string]any)["glob"] != "ghcr.io/acme/hello/**" { + t.Errorf("images %v", images) + } + if len(authorities) != 1 || authorities[0].(map[string]any)[map[string]string{"key": "key", "keyless": "keyless"}[mode]] == nil { + t.Errorf("authorities %v for a %s signer", authorities, mode) + } + if find(preview, "ClusterImagePolicy", "shelf-hello") != nil { + t.Error("a preview has a policy of its own; the app's covers its images") + } + }) + } +} diff --git a/internal/cluster/signer.go b/internal/cluster/signer.go new file mode 100644 index 0000000..b439a9e --- /dev/null +++ b/internal/cluster/signer.go @@ -0,0 +1,190 @@ +package cluster + +import ( + "bytes" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "regexp" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// The identity a keyless signature has when shelf's own workflows made it. With a reusable +// workflow, the certificate's subject is the reusable workflow (build.yml of tweinmann/shelf at +// the ref the tenant uses), not the tenant's repository; which tenant it belongs to follows from +// who may write to the registry path. +const ( + DefaultSignerIssuer = "https://token.actions.githubusercontent.com" + DefaultSignerSubject = `^https://github\.com/tweinmann/shelf/\.github/workflows/(build|release)\.yml@refs/(tags|heads)/.+$` +) + +// The public instance of Sigstore that keyless signatures are checked against. +const ( + fulcioURL = "https://fulcio.sigstore.dev" + rekorURL = "https://rekor.sigstore.dev" +) + +// Where a signer key lives: shelf-system/signer-key, copied by the platform into every app and +// preview namespace as SignerSecretName, and written into flux-system for the platform itself. +// In keyless mode the Secret exists too, without a key, so the copy always has a source. +const ( + SignerSecretName = "shelf-signer" + signerKeySecret = "signer-key" + signerKeyFile = "cosign.pub" +) + +// imagePolicyGVK is the Sigstore policy-controller's ClusterImagePolicy. The platform gives every +// app one, ImagePolicyName, for the images built from its package. +var imagePolicyGVK = schema.GroupVersionKind{Group: "policy.sigstore.dev", Version: "v1beta1", Kind: "ClusterImagePolicy"} + +// ImagePolicyName is the ClusterImagePolicy of an app. +func ImagePolicyName(app string) string { return "shelf-" + app } + +// Signer is who must have signed every artifact and built image the cluster runs: an identity +// for keyless signatures, or a cosign public key. There is no signer that accepts everything. +type Signer struct { + // Issuer and Subject are the OIDC issuer and a regular expression for the certificate's + // subject of a keyless signature. They are ignored when Key is set. + Issuer string + Subject string + // Key is a PEM-encoded public key. A cluster with a key accepts only signatures made with + // it, without a transparency log; it is meant for clusters that test shelf. + Key string +} + +// DefaultSigner accepts keyless signatures made by shelf's build and release workflows. +func DefaultSigner() Signer { + return Signer{Issuer: DefaultSignerIssuer, Subject: DefaultSignerSubject} +} + +// Mode is "key" or "keyless". +func (s Signer) Mode() string { + if s.Key != "" { + return "key" + } + return "keyless" +} + +// IsZero reports whether no signer is set, as in a cluster installed before Phase 11. +func (s Signer) IsZero() bool { return s.Key == "" && s.Issuer == "" && s.Subject == "" } + +func (s Signer) String() string { + if s.Key != "" { + return "cosign key " + keyFingerprint(s.Key) + } + return fmt.Sprintf("keyless, issuer %s, subject %s", s.Issuer, s.Subject) +} + +// Check reports a signer that cannot verify anything. +func (s Signer) Check() error { + if s.Key != "" { + _, err := ParseSignerKey([]byte(s.Key)) + return err + } + if s.Issuer == "" || s.Subject == "" { + return errors.New("a keyless signer needs an issuer and a subject") + } + if _, err := regexp.Compile(s.Subject); err != nil { + return fmt.Errorf("subject: %w", err) + } + return nil +} + +// ParseSignerKey checks that data is one PEM-encoded public key and returns it normalised. +func ParseSignerKey(data []byte) (string, error) { + block, rest := pem.Decode(data) + if block == nil || block.Type != "PUBLIC KEY" { + return "", errors.New("not a PEM public key (-----BEGIN PUBLIC KEY-----)") + } + if len(bytes.TrimSpace(rest)) > 0 { + return "", errors.New("more than one PEM block") + } + if _, err := x509.ParsePKIXPublicKey(block.Bytes); err != nil { + return "", fmt.Errorf("not a public key: %w", err) + } + return string(pem.EncodeToMemory(block)), nil +} + +// keyFingerprint names a key in output: the first bytes of the SHA-256 of its DER encoding. +func keyFingerprint(key string) string { + block, _ := pem.Decode([]byte(key)) + if block == nil { + return "(unreadable)" + } + sum := sha256.Sum256(block.Bytes) + return "sha256:" + hex.EncodeToString(sum[:8]) +} + +// verify is the spec.verify of a Flux OCIRepository that accepts what this signer signed. +// Flux matches issuer and subject as regular expressions. +func (s Signer) verify(secretName string) map[string]any { + if s.Key != "" { + return map[string]any{ + "provider": "cosign", + "secretRef": map[string]any{"name": secretName}, + } + } + return map[string]any{ + "provider": "cosign", + "matchOIDCIdentity": []any{map[string]any{ + "issuer": "^" + regexp.QuoteMeta(s.Issuer) + "$", + "subject": s.Subject, + }}, + } +} + +// authority is the authority of a Sigstore ClusterImagePolicy that admits what this signer +// signed. A key authority without a ctlog looks up no transparency log, as dev signatures have +// no entry there. +func (s Signer) authority() map[string]any { + if s.Key != "" { + return map[string]any{"key": map[string]any{"data": s.Key}} + } + return map[string]any{ + "keyless": map[string]any{ + "url": fulcioURL, + "identities": []any{map[string]any{ + "issuer": s.Issuer, + "subjectRegExp": s.Subject, + }}, + }, + "ctlog": map[string]any{"url": rekorURL}, + } +} + +// signerSecret holds the signer's key, empty for a keyless signer. It is written as data, not +// stringData, which the API server never returns, so that applying it again changes nothing. +func signerSecret(namespace, name string, s Signer) *unstructured.Unstructured { + data := map[string]any{} + if s.Key != "" { + data[signerKeyFile] = base64.StdEncoding.EncodeToString([]byte(s.Key)) + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]any{ + "name": name, + "namespace": namespace, + "labels": map[string]any{WatchLabel: "Enabled"}, + }, + "type": "Opaque", + "data": data, + }} +} + +// compactJSON is what the cluster settings carry for the platform: Flux substitutes the value +// into the ResourceSets, where a JSON object is read as the YAML mapping it is. +func compactJSON(v any) string { + data, err := json.Marshal(v) + if err != nil { + panic(err) + } + return string(data) +} diff --git a/internal/cluster/signer_test.go b/internal/cluster/signer_test.go new file mode 100644 index 0000000..1a00830 --- /dev/null +++ b/internal/cluster/signer_test.go @@ -0,0 +1,117 @@ +package cluster + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// testSignerKey is a public key made for these tests; its private half was never kept. +func testSignerKey(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "signer.pub")) + if err != nil { + t.Fatal(err) + } + key, err := ParseSignerKey(data) + if err != nil { + t.Fatal(err) + } + return key +} + +func TestParseSignerKey(t *testing.T) { + key := testSignerKey(t) + tests := []struct { + name, data, wantErr string + }{ + {"public key", key, ""}, + {"with surrounding blank lines", "\n" + key + "\n\n", ""}, + {"empty", "", "not a PEM public key"}, + {"private key", strings.ReplaceAll(key, "PUBLIC KEY", "PRIVATE KEY"), "not a PEM public key"}, + {"two keys", key + key, "more than one PEM block"}, + {"garbage in the block", "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n", "not a public key"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseSignerKey([]byte(tt.data)) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error %v, want %q", err, tt.wantErr) + } + return + } + if err != nil || got != key { + t.Fatalf("got %q, %v", got, err) + } + }) + } +} + +func TestSignerCheck(t *testing.T) { + tests := []struct { + name string + signer Signer + wantErr bool + }{ + {"default", DefaultSigner(), false}, + {"key", Signer{Key: testSignerKey(t)}, false}, + {"nothing", Signer{}, true}, + {"no subject", Signer{Issuer: DefaultSignerIssuer}, true}, + {"bad subject", Signer{Issuer: DefaultSignerIssuer, Subject: "("}, true}, + {"bad key", Signer{Key: "not a key"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.signer.Check(); (err != nil) != tt.wantErr { + t.Fatalf("error %v, want error %v", err, tt.wantErr) + } + }) + } +} + +// TestDefaultSubject pins which certificates the default accepts: shelf's build and release +// workflows at a tag or branch of tweinmann/shelf, and nothing of a fork or another workflow. +func TestDefaultSubject(t *testing.T) { + re := regexp.MustCompile(DefaultSignerSubject) + for subject, want := range map[string]bool{ + "https://github.com/tweinmann/shelf/.github/workflows/build.yml@refs/tags/v0": true, + "https://github.com/tweinmann/shelf/.github/workflows/build.yml@refs/heads/main": true, + "https://github.com/tweinmann/shelf/.github/workflows/release.yml@refs/tags/v0.8.0": true, + "https://github.com/tweinmann/shelf/.github/workflows/ci.yml@refs/heads/main": false, + "https://github.com/someone/shelf/.github/workflows/build.yml@refs/tags/v0": false, + "https://github.com/tweinmann/shelf-hello/.github/workflows/deploy.yml@refs/heads/main": false, + "https://github.com/tweinmann/shelfXgithub/workflows/build.yml@refs/tags/v0": false, + } { + if got := re.MatchString(subject); got != want { + t.Errorf("%s: match %v, want %v", subject, got, want) + } + } +} + +func TestSignerForms(t *testing.T) { + keyless := DefaultSigner() + if v := keyless.verify("s"); v["secretRef"] != nil || v["matchOIDCIdentity"] == nil { + t.Errorf("keyless verify %v", v) + } + issuer := keyless.verify("s")["matchOIDCIdentity"].([]any)[0].(map[string]any)["issuer"].(string) + if !regexp.MustCompile(issuer).MatchString(DefaultSignerIssuer) || regexp.MustCompile(issuer).MatchString(DefaultSignerIssuer+".evil") { + t.Errorf("issuer regex %q", issuer) + } + if a := keyless.authority(); a["keyless"] == nil || a["ctlog"] == nil { + t.Errorf("keyless authority %v", a) + } + key := Signer{Key: testSignerKey(t)} + if v := key.verify("s"); v["secretRef"] == nil || v["matchOIDCIdentity"] != nil { + t.Errorf("key verify %v", v) + } + // A dev signature has no transparency log entry, so a key authority must not ask for one. + if a := key.authority(); a["key"] == nil || a["ctlog"] != nil { + t.Errorf("key authority %v", a) + } + if !strings.HasPrefix(key.String(), "cosign key sha256:") || !strings.HasPrefix(keyless.String(), "keyless, issuer ") { + t.Errorf("String() = %q, %q", key.String(), keyless.String()) + } +} diff --git a/internal/cluster/state.go b/internal/cluster/state.go index 6e8bcb9..af70957 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -15,7 +15,10 @@ import ( "k8s.io/client-go/rest" ) -var podGVK = schema.GroupVersionKind{Version: "v1", Kind: "Pod"} +var ( + podGVK = schema.GroupVersionKind{Version: "v1", Kind: "Pod"} + eventGVK = schema.GroupVersionKind{Version: "v1", Kind: "Event"} +) // Phase is how far an app, or one step towards it, has got. type Phase string @@ -119,6 +122,9 @@ type appObjects struct { chart *unstructured.Unstructured release *unstructured.Unstructured pods []*unstructured.Unstructured + // refusals are the FailedCreate events in the namespace: a pod the API server did not + // create, because the admission webhook or a quota refused it, never shows up as a pod. + refusals []*unstructured.Unstructured } // NotFoundError means there is no app, or no preview, of that name in the cluster. @@ -130,7 +136,7 @@ func (e *NotFoundError) Error() string { return "app " + e.Name + " does not exi func (o *appObjects) isApp() bool { return o.provider != nil } // AppStates returns every registered app with its current state and its previews. It reads the -// whole cluster in eight list calls, however many apps and previews there are, plus two for each +// whole cluster in nine list calls, however many apps and previews there are, plus two for each // app with a quick tunnel. func AppStates(ctx context.Context, cfg *rest.Config) ([]AppState, error) { c, err := newClient(cfg) @@ -245,18 +251,38 @@ func (c *client) appObjects(ctx context.Context) (map[string]*appObjects, error) app.pods = append(app.pods, p) } } + refusals, err := c.listFields(ctx, eventGVK, "reason=FailedCreate") + if err != nil { + return nil, err + } + for _, e := range refusals { + if app, ok := apps[e.GetNamespace()]; ok { + app.refusals = append(app.refusals, e) + } + } return apps, nil } // list returns the objects of a kind. An empty namespace means every namespace, an empty // selector every object. func (c *client) list(ctx context.Context, gvk schema.GroupVersionKind, namespace, selector string) ( + []*unstructured.Unstructured, error) { + return c.listWith(ctx, gvk, namespace, metav1.ListOptions{LabelSelector: selector}) +} + +// listFields returns the objects of a kind in every namespace that match a field selector. +func (c *client) listFields(ctx context.Context, gvk schema.GroupVersionKind, selector string) ( + []*unstructured.Unstructured, error) { + return c.listWith(ctx, gvk, "", metav1.ListOptions{FieldSelector: selector}) +} + +func (c *client) listWith(ctx context.Context, gvk schema.GroupVersionKind, namespace string, opts metav1.ListOptions) ( []*unstructured.Unstructured, error) { ri, err := c.resource(gvk, namespace) if err != nil { return nil, err } - list, err := ri.List(ctx, metav1.ListOptions{LabelSelector: selector}) + list, err := ri.List(ctx, opts) if err != nil { return nil, fmt.Errorf("listing %s: %w", gvk.Kind, err) } @@ -352,7 +378,7 @@ func diagnose(name string, o *appObjects) Diagnosis { "the platform has not created the app yet", "the app.yaml was rendered but the release did not succeed"), ) - d.Stages = append(d.Stages, podStage(o.pods)) + d.Stages = append(d.Stages, podStage(o.pods, o.refusals)) return d } @@ -394,6 +420,15 @@ func fluxStage(name, object string, obj *unstructured.Unstructured, missing, fai s.Hint = "the registry refused to hand out the deploy artifact; " + registryHint(app) } } + // Flux fetched the artifact but its signature is not the cluster's signer's. + if s.Reason == "VerificationError" { + if strings.HasSuffix(s.Object, "/"+chartName) { + s.Hint = "the chart is not signed by the cluster's signer; install a released platform and chart" + } else { + s.Hint = "the deploy artifact is not signed by the cluster's signer; push again with the " + + "current workflow, which signs it (its caller needs `id-token: write`)" + } + } } return s } @@ -405,9 +440,20 @@ func working(reason string) bool { } // podStage looks at the pods the chart created, because a release can succeed while the -// container it starts does not run at all. -func podStage(pods []*unstructured.Unstructured) Stage { +// container it starts does not run at all, or never gets a pod: a refusal newer than every pod +// means a workload is still being refused. An older one was about a pod that exists by now. +func podStage(pods, refusals []*unstructured.Unstructured) Stage { s := Stage{Name: "the pods", Object: "Pods"} + if refused := currentRefusal(pods, refusals); refused != nil { + involved, _, _ := unstructured.NestedMap(refused.Object, "involvedObject") + kind, _ := involved["kind"].(string) + name, _ := involved["name"].(string) + message, _, _ := unstructured.NestedString(refused.Object, "message") + return Stage{ + Name: "the pods", Object: kind + " " + refused.GetNamespace() + "/" + name, + Phase: PhaseFailed, Reason: "FailedCreate", Message: message, Hint: podHint("FailedCreate", message), + } + } if len(pods) == 0 { s.Phase, s.Message = PhaseMissing, "no pods yet" return s @@ -484,6 +530,15 @@ var missingSecret = regexp.MustCompile(`couldn't find key (\S+) in Secret \S+/` func podHint(reason, message string) string { switch reason { + case "FailedCreate": + switch { + case strings.Contains(message, "policy.sigstore.dev"): + return "a built image is not signed by the cluster's signer; push again, so that the " + + "reusable workflow builds and signs it" + case strings.Contains(message, "exceeded quota"): + return "the preview's quota is used up; give the components fewer resources, or fewer instances" + } + return "the workload cannot create its pods; the message says why" case "ImagePullBackOff", "ErrImagePull": return "the image cannot be pulled; check the reference and whether the cluster may read that registry" case "CrashLoopBackOff": @@ -500,6 +555,28 @@ func podHint(reason, message string) string { return "" } +// currentRefusal returns the newest refusal that no pod came after, or nil. +func currentRefusal(pods, refusals []*unstructured.Unstructured) *unstructured.Unstructured { + var newestPod string + for _, p := range pods { + if created := p.GetCreationTimestamp().UTC().Format(time.RFC3339); created > newestPod { + newestPod = created + } + } + var found *unstructured.Unstructured + var foundAt string + for _, e := range refusals { + at, _, _ := unstructured.NestedString(e.Object, "lastTimestamp") + if at == "" { + at, _, _ = unstructured.NestedString(e.Object, "eventTime") + } + if at > newestPod && at > foundAt { + found, foundAt = e, at + } + } + return found +} + // nested returns a nested map, e.g. state.waiting. func nested(m map[string]any, keys ...string) (map[string]any, bool) { out, found, err := unstructured.NestedMap(m, keys...) diff --git a/internal/cluster/testdata/fluxinstance-insecure.yaml b/internal/cluster/testdata/fluxinstance-insecure.yaml index 7183eb4..ecf1986 100644 --- a/internal/cluster/testdata/fluxinstance-insecure.yaml +++ b/internal/cluster/testdata/fluxinstance-insecure.yaml @@ -30,6 +30,13 @@ spec: target: kind: Kustomization name: flux-system + - patch: | + - op: add + path: /spec/verify + value: {"provider":"cosign","secretRef":{"name":"shelf-signer"}} + target: + kind: OCIRepository + name: flux-system - patch: | - op: add path: /spec/insecure @@ -42,4 +49,4 @@ spec: kind: OCIRepository path: ./ ref: dev - url: oci://shelf-registry:5000/shelf/platform + url: oci://shelf-registry.local:5000/shelf/platform diff --git a/internal/cluster/testdata/fluxinstance.yaml b/internal/cluster/testdata/fluxinstance.yaml index 4a4d5cb..8c0dd62 100644 --- a/internal/cluster/testdata/fluxinstance.yaml +++ b/internal/cluster/testdata/fluxinstance.yaml @@ -30,6 +30,13 @@ spec: target: kind: Kustomization name: flux-system + - patch: | + - op: add + path: /spec/verify + value: {"matchOIDCIdentity":[{"issuer":"^https://token\\.actions\\.githubusercontent\\.com$","subject":"^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$"}],"provider":"cosign"} + target: + kind: OCIRepository + name: flux-system sync: interval: 1m kind: OCIRepository diff --git a/internal/cluster/testdata/objects.yaml b/internal/cluster/testdata/objects.yaml index 663716a..e1f4d32 100644 --- a/internal/cluster/testdata/objects.yaml +++ b/internal/cluster/testdata/objects.yaml @@ -12,12 +12,38 @@ data: SHELF_CHART_URL: oci://shelf-registry:5000/shelf/charts/shelf-app SHELF_HOST_SUFFIX: "" SHELF_INSECURE_REGISTRY: "true" + SHELF_SIGNER_AUTHORITY: '{"ctlog":{"url":"https://rekor.sigstore.dev"},"keyless":{"identities":[{"issuer":"https://token.actions.githubusercontent.com","subjectRegExp":"^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$"}],"url":"https://fulcio.sigstore.dev"}}' + SHELF_SIGNER_ISSUER: https://token.actions.githubusercontent.com + SHELF_SIGNER_KEY: "" + SHELF_SIGNER_MODE: keyless + SHELF_SIGNER_SUBJECT: ^https://github\.com/tweinmann/shelf/\.github/workflows/(build|release)\.yml@refs/(tags|heads)/.+$ + SHELF_SIGNER_VERIFY: '{"matchOIDCIdentity":[{"issuer":"^https://token\\.actions\\.githubusercontent\\.com$","subject":"^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$"}],"provider":"cosign"}' kind: ConfigMap metadata: name: shelf-config namespace: flux-system --- apiVersion: v1 +data: {} +kind: Secret +metadata: + labels: + reconcile.fluxcd.io/watch: Enabled + name: signer-key + namespace: shelf-system +type: Opaque +--- +apiVersion: v1 +data: {} +kind: Secret +metadata: + labels: + reconcile.fluxcd.io/watch: Enabled + name: shelf-signer + namespace: flux-system +type: Opaque +--- +apiVersion: v1 data: .dockerconfigjson: eyJhdXRocyI6eyJnaGNyLmlvIjp7ImF1dGgiOiJkRzlpYVRwdWIzUXRZUzF5WldGc0xYUnZhMlZ1IiwicGFzc3dvcmQiOiJub3QtYS1yZWFsLXRva2VuIiwidXNlcm5hbWUiOiJ0b2JpIn19fQ== kind: Secret diff --git a/internal/cluster/testdata/signer.pub b/internal/cluster/testdata/signer.pub new file mode 100644 index 0000000..3932355 --- /dev/null +++ b/internal/cluster/testdata/signer.pub @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE6dD7Pyv/zGh3xxdaiRgXux3x05Fg +KM2BH7VyRC/m4L7w2ga7TlXK2UecwQhwoxrx7VH64eM6xW+iAgXzzzNUNg== +-----END PUBLIC KEY----- diff --git a/internal/cluster/verification.go b/internal/cluster/verification.go new file mode 100644 index 0000000..b0268e1 --- /dev/null +++ b/internal/cluster/verification.go @@ -0,0 +1,195 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/rest" + "sigs.k8s.io/yaml" + + "github.com/tweinmann/shelf/internal/render" +) + +// Annotations of the deploy artifact that verification reads, as Flux copies them into the +// OCIRepository's status.artifact.metadata. The scan summary is written by the reusable workflow +// before it signs the artifact, so the signature covers it. +const ( + ScanAnnotation = "dev.shelf.scan" + sourceAnnotation = "org.opencontainers.image.source" +) + +// sourceVerified is the condition Flux sets on an OCIRepository with verify. +const sourceVerified = "SourceVerified" + +// Verification says whether what an app or preview runs is what the cluster's signer signed and +// CI scanned. Only a verified app may be exposed. +type Verification struct { + Verified bool + // Reason is the first thing that does not hold, when not verified. + Reason string + // Source is the repository the deploy artifact says it was built from, and Commit the + // commit; both come from annotations the signature covers. + Source string + Commit string + // Scan is the scan summary of the running deploy artifact, nil when it has none. + Scan *Scan +} + +// Scan is what CI found in the images of one deploy artifact. +type Scan struct { + Scanner string + ScannedAt string + // Images are the app's components with their image digest and findings, by component name. + Images []ScannedImage +} + +// ScannedImage is one component's image and what the scan found in it. Scanned is false when +// the summary does not name the image's digest. +type ScannedImage struct { + Component string + Image string + Digest string + Scanned bool + Critical int + High int +} + +// scanSummary is the JSON of the annotation dev.shelf.scan. +type scanSummary struct { + Scanner string `json:"scanner"` + ScannedAt string `json:"scannedAt"` + Images map[string]struct { + Critical int `json:"critical"` + High int `json:"high"` + } `json:"images"` +} + +// AppVerification reads whether an app or a preview (by its namespace) runs a verified deploy +// artifact, without changing anything. +func AppVerification(ctx context.Context, cfg *rest.Config, name string) (Verification, error) { + c, err := newClient(cfg) + if err != nil { + return Verification{}, err + } + apps, err := c.appObjects(ctx) + if err != nil { + return Verification{}, err + } + objs, ok := apps[name] + if !ok { + return Verification{}, &NotFoundError{Name: name} + } + values, err := c.get(ctx, ref{gvk: configMapGVK, namespace: name, name: render.ConfigMapName}) + if err != nil { + return Verification{}, err + } + return verification(name, objs, values), nil +} + +// verification checks, in the order things happen: the deploy artifact's signature, that the +// release runs that artifact's values, the chart's signature, and that the scan summary covers +// every image the values name. +func verification(name string, o *appObjects, values *unstructured.Unstructured) Verification { + v := Verification{} + if o.source != nil { + v.Source, _, _ = unstructured.NestedString(o.source.Object, "status", "artifact", "metadata", sourceAnnotation) + v.Commit = artifactCommit(o.source) + } + if reason := signatureReason("deploy artifact", o.source); reason != "" { + v.Reason = reason + return v + } + if !rollout(name, o, values).Running() { + v.Reason = "the release does not run the verified deploy artifact yet" + return v + } + if reason := signatureReason("chart", o.chart); reason != "" { + v.Reason = reason + return v + } + images, err := valueImages(values) + if err != nil { + v.Reason = err.Error() + return v + } + raw, _, _ := unstructured.NestedString(o.source.Object, "status", "artifact", "metadata", ScanAnnotation) + if raw == "" { + v.Reason = "the deploy artifact has no scan (" + ScanAnnotation + ")" + return v + } + var summary scanSummary + if err := json.Unmarshal([]byte(raw), &summary); err != nil { + v.Reason = "the deploy artifact's scan (" + ScanAnnotation + ") cannot be read: " + err.Error() + return v + } + v.Scan = &Scan{Scanner: summary.Scanner, ScannedAt: summary.ScannedAt} + var unscanned []string + for _, img := range images { + found, ok := summary.Images[img.Digest] + img.Scanned = ok + img.Critical, img.High = found.Critical, found.High + if !ok { + unscanned = append(unscanned, img.Component) + } + v.Scan.Images = append(v.Scan.Images, img) + } + if len(unscanned) > 0 { + v.Reason = "the scan does not cover the image of " + strings.Join(unscanned, ", ") + return v + } + v.Verified = true + return v +} + +// signatureReason is why a source's artifact does not count as signed, or "" if it does. A +// source without the condition is one the cluster does not verify, as on a platform from before +// verification. +func signatureReason(what string, source *unstructured.Unstructured) string { + if source == nil { + return "the " + what + " is not there yet" + } + cond := condition(source, sourceVerified) + if cond == nil { + return "the cluster does not verify the " + what + "'s signature" + } + if cond["status"] != "True" { + msg, _ := cond["message"].(string) + return "the " + what + " is not signed by the cluster's signer: " + msg + } + return "" +} + +// valueImages returns the image of every component in the app values, sorted by component. +// shelf render pins each one as @sha256:. +func valueImages(values *unstructured.Unstructured) ([]ScannedImage, error) { + if values == nil { + return nil, fmt.Errorf("the app values (%s) are not there yet", render.ConfigMapName) + } + data, _, _ := unstructured.NestedString(values.Object, "data", render.ConfigMapKey) + var app struct { + Components map[string]struct { + Image string `json:"image"` + } `json:"components"` + } + if err := yaml.Unmarshal([]byte(data), &app); err != nil { + return nil, fmt.Errorf("the app values cannot be read: %w", err) + } + var out []ScannedImage + for name, comp := range app.Components { + _, digest, _ := strings.Cut(comp.Image, "@") + out = append(out, ScannedImage{Component: name, Image: comp.Image, Digest: digest}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Component < out[j].Component }) + return out, nil +} + +// Running reports whether the release runs the values of the deploy artifact the source holds: +// the artifact is applied and the release's latest revision has its values. +func (r Rollout) Running() bool { + return r.Revision != "" && r.AppliedRevision == r.Revision && + r.ValuesDigest != "" && r.ReleasedDigest == r.ValuesDigest +} diff --git a/internal/cluster/verification_test.go b/internal/cluster/verification_test.go new file mode 100644 index 0000000..def923f --- /dev/null +++ b/internal/cluster/verification_test.go @@ -0,0 +1,228 @@ +package cluster + +import ( + "strings" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const ( + webDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + dbDigest = "sha256:2222222222222222222222222222222222222222222222222222222222222222" +) + +const verifiedValues = `name: shop-pkg +components: + web: + image: ghcr.io/acme/shop/web:sha-42a964e@` + webDigest + ` + db: + image: postgres:16@` + dbDigest + ` +` + +func setCondition(t *testing.T, obj *unstructured.Unstructured, kind, status, message string) { + t.Helper() + conditions, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + kept := []any{} + for _, c := range conditions { + if c.(map[string]any)["type"] != kind { + kept = append(kept, c) + } + } + kept = append(kept, map[string]any{"type": kind, "status": status, "message": message}) + if err := unstructured.SetNestedSlice(obj.Object, kept, "status", "conditions"); err != nil { + t.Fatal(err) + } +} + +// verifiedApp is an app whose signed and scanned deploy artifact has rolled out. +func verifiedApp(t *testing.T, scan string) (*appObjects, *unstructured.Unstructured) { + t.Helper() + o := healthy() + metadata := map[string]any{ + revisionAnnotation: "main@sha1:42a964eb0b4bd7248305d5f22233d6dbabcd56f3", + sourceAnnotation: "https://github.com/acme/shop", + } + if scan != "" { + metadata[ScanAnnotation] = scan + } + _ = unstructured.SetNestedMap(o.source.Object, map[string]any{"revision": "main@sha256:new", "metadata": metadata}, "status", "artifact") + setCondition(t, o.source, sourceVerified, "True", "verified signature of revision main@sha256:new") + setCondition(t, o.chart, sourceVerified, "True", "verified signature of revision 0.8.0@sha256:c") + _ = unstructured.SetNestedField(o.kustomization.Object, "main@sha256:new", "status", "lastAppliedRevision") + inline := map[string]any{"name": "shop"} + _ = unstructured.SetNestedMap(o.release.Object, inline, "spec", "values") + digest, err := ConfigDigest(verifiedValues, inline) + if err != nil { + t.Fatal(err) + } + _ = unstructured.SetNestedSlice(o.release.Object, []any{map[string]any{"version": int64(2), "configDigest": digest}}, "status", "history") + values := &unstructured.Unstructured{Object: map[string]any{"data": map[string]any{"app.yaml": verifiedValues}}} + return o, values +} + +const fullScan = `{"scanner":"trivy 0.75.0","scannedAt":"2026-10-28T12:00:00Z","images":{"` + + webDigest + `":{"critical":0,"high":2},"` + dbDigest + `":{"critical":2,"high":86}}}` + +func TestVerification(t *testing.T) { + t.Parallel() + tests := []struct { + name string + scan string + change func(t *testing.T, o *appObjects, values *unstructured.Unstructured) + wantReason string + }{ + {name: "verified", scan: fullScan}, + {name: "findings do not matter", scan: strings.ReplaceAll(fullScan, `"critical":0`, `"critical":9`)}, + { + name: "unsigned", scan: fullScan, + change: func(t *testing.T, o *appObjects, _ *unstructured.Unstructured) { + setCondition(t, o.source, sourceVerified, "False", "no matching signatures were found") + }, + wantReason: "the deploy artifact is not signed by the cluster's signer: no matching signatures were found", + }, + { + name: "platform that does not verify", scan: fullScan, + change: func(t *testing.T, o *appObjects, _ *unstructured.Unstructured) { + _ = unstructured.SetNestedSlice(o.source.Object, []any{}, "status", "conditions") + }, + wantReason: "the cluster does not verify the deploy artifact's signature", + }, + { + name: "rollout pending", scan: fullScan, + change: func(t *testing.T, o *appObjects, _ *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.kustomization.Object, "main@sha256:old", "status", "lastAppliedRevision") + }, + wantReason: "the release does not run the verified deploy artifact yet", + }, + { + name: "release on older values", scan: fullScan, + change: func(t *testing.T, o *appObjects, _ *unstructured.Unstructured) { + _ = unstructured.SetNestedSlice(o.release.Object, []any{map[string]any{"configDigest": "sha256:older"}}, "status", "history") + }, + wantReason: "the release does not run the verified deploy artifact yet", + }, + { + name: "chart unsigned", scan: fullScan, + change: func(t *testing.T, o *appObjects, _ *unstructured.Unstructured) { + setCondition(t, o.chart, sourceVerified, "False", "no matching signatures were found") + }, + wantReason: "the chart is not signed by the cluster's signer", + }, + {name: "no scan", wantReason: "the deploy artifact has no scan (dev.shelf.scan)"}, + {name: "unreadable scan", scan: "{", wantReason: "cannot be read"}, + { + name: "scan misses an image", + scan: `{"scanner":"trivy 0.75.0","images":{"` + webDigest + `":{"critical":0,"high":0}}}`, + wantReason: "the scan does not cover the image of db", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + o, values := verifiedApp(t, tt.scan) + if tt.change != nil { + tt.change(t, o, values) + } + v := verification("shop", o, values) + if v.Source != "https://github.com/acme/shop" || v.Commit != "42a964eb0b4bd7248305d5f22233d6dbabcd56f3" { + t.Errorf("source %q, commit %q", v.Source, v.Commit) + } + if tt.wantReason == "" { + if !v.Verified || v.Reason != "" { + t.Fatalf("not verified: %s", v.Reason) + } + if v.Scan == nil || len(v.Scan.Images) != 2 || v.Scan.Images[0].Component != "db" || v.Scan.Images[0].High != 86 || + v.Scan.Images[1].Digest != webDigest || v.Scan.Scanner != "trivy 0.75.0" { + t.Errorf("scan %+v", v.Scan) + } + return + } + if v.Verified || !strings.Contains(v.Reason, tt.wantReason) { + t.Fatalf("verified %v, reason %q, want %q", v.Verified, v.Reason, tt.wantReason) + } + }) + } +} + +func refusal(at, kind, name, message string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", "kind": "Event", + "metadata": map[string]any{"name": "e", "namespace": "shop"}, + "involvedObject": map[string]any{"kind": kind, "name": name}, + "reason": "FailedCreate", "message": message, "lastTimestamp": at, + }} +} + +func podCreated(at string) *unstructured.Unstructured { + p := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", "kind": "Pod", + "metadata": map[string]any{"name": "web-1", "namespace": "shop", "creationTimestamp": at}, + "status": map[string]any{"phase": "Running", "containerStatuses": []any{map[string]any{"name": "web", "ready": true}}}, + }} + return p +} + +// TestRefusedPods covers pods the API server never created: the pods stage names the workload, +// the refusal and what to do, and forgets a refusal once a pod came after it. +func TestRefusedPods(t *testing.T) { + t.Parallel() + denied := `admission webhook "policy.sigstore.dev" denied the request: validation failed: no signatures found` + quota := `pods "web-1" is forbidden: exceeded quota: shelf-preview, requested: requests.memory=1Gi` + tests := []struct { + name string + pods []*unstructured.Unstructured + refusals []*unstructured.Unstructured + wantPhase Phase + wantHint string + }{ + {"unsigned image, no pod", nil, []*unstructured.Unstructured{refusal("2026-10-28T12:00:00Z", "ReplicaSet", "web-6d7b", denied)}, + PhaseFailed, "not signed by the cluster's signer"}, + {"quota of a preview", nil, []*unstructured.Unstructured{refusal("2026-10-28T12:00:00Z", "StatefulSet", "db", quota)}, + PhaseFailed, "quota is used up"}, + {"refusal after the last pod", []*unstructured.Unstructured{podCreated("2026-10-28T11:00:00Z")}, + []*unstructured.Unstructured{refusal("2026-10-28T12:00:00Z", "ReplicaSet", "web-6d7b", denied)}, PhaseFailed, "not signed"}, + {"pod after the refusal", []*unstructured.Unstructured{podCreated("2026-10-28T12:05:00Z")}, + []*unstructured.Unstructured{refusal("2026-10-28T12:00:00Z", "ReplicaSet", "web-6d7b", denied)}, PhaseReady, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := podStage(tt.pods, tt.refusals) + if s.Phase != tt.wantPhase || !strings.Contains(s.Hint, tt.wantHint) { + t.Fatalf("stage %+v", s) + } + if s.Phase == PhaseFailed && (s.Reason != "FailedCreate" || !strings.HasPrefix(s.Object, tt.refusals[0].Object["involvedObject"].(map[string]any)["kind"].(string)+" shop/")) { + t.Errorf("stage %+v", s) + } + }) + } +} + +func TestVerificationErrorHint(t *testing.T) { + t.Parallel() + o := healthy() + _ = unstructured.SetNestedSlice(o.source.Object, []any{map[string]any{ + "type": "Ready", "status": "False", "reason": "VerificationError", + "message": "failed to verify the signature using provider 'cosign': no matching signatures were found", + }}, "status", "conditions") + d := diagnose("shop", o) + if s := d.Stages[0]; s.Phase != PhaseFailed || !strings.Contains(s.Hint, "id-token: write") { + t.Errorf("deploy artifact stage %+v", s) + } +} + +func TestFailOnUnverified(t *testing.T) { + t.Parallel() + o := healthy() + ready := failOnUnverified(readyCondition, "push again") + setCondition(t, o.source, "Ready", "False", "failed to verify the signature") + setCondition(t, o.source, sourceVerified, "False", "no matching signatures were found") + if _, _, err := ready(o.source); err == nil || !strings.Contains(err.Error(), "no matching signatures were found\npush again") { + t.Errorf("an unsigned artifact: %v", err) + } + setCondition(t, o.source, sourceVerified, "True", "verified signature") + if _, _, err := ready(o.source); err != nil { + t.Errorf("a verified artifact that is not ready yet: %v", err) + } +} diff --git a/internal/ops/apps.go b/internal/ops/apps.go index d0b0317..a103b38 100644 --- a/internal/ops/apps.go +++ b/internal/ops/apps.go @@ -176,6 +176,9 @@ func (o *Ops) AddApp(ctx context.Context, opts AddOptions, report progress.Repor if cloudflare != "" && domain == "" { return noDomain(opts.Name, cloudflare) } + if cloudflare != "" && !PublicDomain(domain) { + return notPublic(opts.Name, domain) + } if opts.Access.Cloudflare != "" && opts.Access.Cloudflare != current.Cloudflare { conn, err := o.Env.Connections.Cloudflare(opts.Access.Cloudflare) if err != nil { @@ -225,30 +228,58 @@ func (o *Ops) AddApp(ctx context.Context, opts AddOptions, report progress.Repor if err != nil { return err } + deployment := cluster.AppOptions{ + Name: opts.Name, + Artifact: opts.Artifact, + Insecure: opts.Insecure, + Domain: domain, + Registry: registry, + Previews: previews, + Secrets: values, + Timeout: opts.Timeout, + Report: rep, + } + // Only a verified app goes beyond a preview. An app that is not exposed yet is deployed + // privately first, and gets its tunnel only once it runs a verified artifact; until then + // nothing is asked of Cloudflare. + if (cloudflare != "" || quick) && current.TunnelID == "" && !current.Quick { + rep.Report(progress.Info("exposure: once %s runs a verified deploy artifact", opts.Name)) + if err := o.Cluster.AddApp(ctx, o.config(), deployment); err != nil { + return err + } + v, err := o.verify(ctx, opts.Name, settings, false) + if err != nil { + return err + } + if !v.Exposable { + return &NotVerifiedError{App: opts.Name, Reason: v.Reason, Hint: v.Hint} + } + rep.Report(progress.Info("verified: signed by the cluster's signer, every image scanned")) + } exposure, err := o.planExposure(ctx, opts.Name, settings, current, domain, cloudflare, rep) if err != nil { return err } - if err := o.Cluster.AddApp(ctx, o.config(), cluster.AppOptions{ - Name: opts.Name, - Artifact: opts.Artifact, - Insecure: opts.Insecure, - Domain: domain, - Registry: registry, - Cloudflare: cloudflare, - TunnelID: exposure.tunnelID(), - TunnelCredentials: exposure.credentials, - Quick: quick, - Previews: previews, - Secrets: values, - Timeout: opts.Timeout, - Report: rep, - }); err != nil { + deployment.Cloudflare, deployment.Quick = cloudflare, quick + deployment.TunnelID, deployment.TunnelCredentials = exposure.tunnelID(), exposure.credentials + if err := o.Cluster.AddApp(ctx, o.config(), deployment); err != nil { return err } return o.finish(ctx, exposure, rep) } +// NotVerifiedError means an app was not exposed because it does not run a verified deploy +// artifact. The app is deployed and stays inside the cluster. +type NotVerifiedError struct { + App, Reason, Hint string +} + +func (e *NotVerifiedError) Error() string { + return fmt.Sprintf("%s is not exposed: it does not run a verified deploy artifact: %s\n%s\n"+ + "%s is deployed and stays inside the cluster; `shelf app status %s` shows the chain.", + e.App, e.Reason, e.Hint, e.App, e.App) +} + // SetAccess changes how an app reaches its registry and the internet, and nothing else: it // deploys the artifact the app is registered with, the same way AddApp does. func (o *Ops) SetAccess(ctx context.Context, name string, access Access, timeout time.Duration, diff --git a/internal/ops/init.go b/internal/ops/init.go index 2e165af..e0331b0 100644 --- a/internal/ops/init.go +++ b/internal/ops/init.go @@ -24,7 +24,9 @@ type InitOptions struct { // name. Without it such a change is refused, because it silently takes those apps off the // name they answer under. MoveHosts bool - Timeout time.Duration + // Signer is who must have signed every artifact and built image the cluster runs. + Signer cluster.Signer + Timeout time.Duration } // HostsMoveError is returned when an installation would rename every app and was not asked to. @@ -63,6 +65,8 @@ type InitPlan struct { Chart cluster.Artifact // Hosts is the pattern the apps with a domain answer under, such as -dev.. Hosts string + // Signer is who the cluster will expect to have signed what it runs. + Signer cluster.Signer } // CheckDomain rejects a domain that cannot carry host names. The message has no leading field @@ -117,9 +121,31 @@ func (o *Ops) PlanCluster(opts InitOptions) InitPlan { Platform: opts.Platform, Chart: opts.Chart, Hosts: Hosts(opts.HostSuffix), + Signer: opts.Signer, } } +// UnverifiedApps returns the apps whose running deploy artifact is not verified, so that an +// upgrade to a platform that verifies can say which of them get no new deploys until they were +// built once with the current workflow. A cluster without shelf has none. +func (o *Ops) UnverifiedApps(ctx context.Context) ([]string, error) { + states, err := o.Cluster.AppStates(ctx, o.config()) + if err != nil { + return nil, err + } + var out []string + for _, s := range states { + v, err := o.Cluster.AppVerification(ctx, o.config(), s.Name) + if err != nil { + return nil, err + } + if !v.Verified { + out = append(out, s.Name) + } + } + return out, nil +} + // InitCluster installs Flux and the platform and waits until everything is ready. A cluster // from when clusters had a domain gives it to the apps that answered under it, as a domain of // their own, if it is a public one: they keep their names and their DNS records. @@ -132,6 +158,9 @@ func (o *Ops) InitCluster(ctx context.Context, opts InitOptions, rep progress.Re if PublicDomain(before.LegacyDomain) { pin = before.LegacyDomain } + if err := opts.Signer.Check(); err != nil { + return fmt.Errorf("signer: %w", err) + } if err := o.checkHostsStay(ctx, opts, before, pin); err != nil { return err } @@ -142,6 +171,7 @@ func (o *Ops) InitCluster(ctx context.Context, opts InitOptions, rep progress.Re HostSuffix: opts.HostSuffix, Chart: opts.Chart, InsecureRegistry: opts.Insecure, + Signer: opts.Signer, }, Timeout: opts.Timeout, Report: rep, diff --git a/internal/ops/ops.go b/internal/ops/ops.go index 571103a..567eaaf 100644 --- a/internal/ops/ops.go +++ b/internal/ops/ops.go @@ -89,6 +89,9 @@ type Cluster interface { AppComponents(ctx context.Context, cfg *rest.Config, app string) ([]cluster.Component, error) AppDiagnosis(ctx context.Context, cfg *rest.Config, app string) (cluster.Diagnosis, error) AppRollout(ctx context.Context, cfg *rest.Config, app string) (cluster.Rollout, error) + // AppVerification reads whether an app or a preview, by its namespace, runs a verified + // deploy artifact. + AppVerification(ctx context.Context, cfg *rest.Config, name string) (cluster.Verification, error) ForwardTraefik(ctx context.Context, cfg *rest.Config) (*cluster.Forward, error) Settings(ctx context.Context, cfg *rest.Config) (cluster.Settings, error) // AppConfig returns how an app is registered, or nil when there is no such app. @@ -130,6 +133,10 @@ func (liveCluster) AppSecrets(ctx context.Context, cfg *rest.Config, app string) return cluster.AppSecrets(ctx, cfg, app) } +func (liveCluster) AppVerification(ctx context.Context, cfg *rest.Config, name string) (cluster.Verification, error) { + return cluster.AppVerification(ctx, cfg, name) +} + func (liveCluster) AppNames(ctx context.Context, cfg *rest.Config) ([]string, error) { return cluster.AppNames(ctx, cfg) } diff --git a/internal/ops/previews.go b/internal/ops/previews.go index ccb03df..a8a4e3b 100644 --- a/internal/ops/previews.go +++ b/internal/ops/previews.go @@ -53,6 +53,13 @@ func (o *Ops) Preview(ctx context.Context, name, branch string) (App, error) { return App{}, err } preview.Components = components(list, preview.Host, false) + settings, err := o.Cluster.Settings(ctx, o.config()) + if err != nil { + return App{}, err + } + if preview.Verification, err = o.verify(ctx, p.Name, settings, true); err != nil { + return App{}, err + } return preview, nil } missing := &PreviewNotFoundError{App: name, Branch: branch, Off: a.PreviewLimit == 0} diff --git a/internal/ops/status.go b/internal/ops/status.go index 4ac5736..ed6e2a2 100644 --- a/internal/ops/status.go +++ b/internal/ops/status.go @@ -30,6 +30,51 @@ type App struct { // Components is what the app is made of, filled by App but not by Apps: a list of apps // does not need it, and it costs a read per app. Components []Component `json:"components,omitempty"` + // Verification says whether the app runs what the cluster's signer signed and CI scanned, + // filled by App and Preview but not by Apps. + Verification *Verification `json:"verification,omitempty"` +} + +// Verification is the chain of an app or a preview from the commit to what runs, and whether +// it may be exposed. +type Verification struct { + cluster.Verification + // Signer is who the cluster expects to have signed what it runs. + Signer string + // Exposable is whether the app may get a tunnel. A preview never may. + Exposable bool + // Hint is what to do when it is not verified. + Hint string +} + +// verify reads the verification of an app or a preview, by its namespace. +func (o *Ops) verify(ctx context.Context, name string, settings cluster.Settings, preview bool) (*Verification, error) { + v, err := o.Cluster.AppVerification(ctx, o.config(), name) + if err != nil { + return nil, err + } + out := &Verification{Verification: v, Signer: settings.Signer.String(), Exposable: v.Verified && !preview} + if settings.Signer.IsZero() { + out.Signer = "none: the cluster was installed before shelf verified anything" + } + if !v.Verified { + out.Hint = verificationHint(v.Reason) + } + return out, nil +} + +// verificationHint is what to do about the first thing verification found missing. +func verificationHint(reason string) string { + switch { + case strings.Contains(reason, "does not run the verified deploy artifact yet"): + return "wait for the rollout to finish" + case strings.Contains(reason, "cluster does not verify"): + return "install the current platform with `shelf init cluster`" + case strings.Contains(reason, "chart"): + return "install a released platform and chart with `shelf init cluster`" + } + return "push again with the current reusable workflow, which signs and scans every build " + + "(the caller's deploy.yml needs `id-token: write`)" } // Component is one part of an app together with the address it answers at. @@ -95,8 +140,12 @@ func (o *Ops) Apps(ctx context.Context) ([]App, error) { return apps, nil } -// App returns one app with its state and its address. +// App returns one app with its state, its address, its components and its verification. func (o *Ops) App(ctx context.Context, name string) (App, error) { + settings, err := o.Cluster.Settings(ctx, o.config()) + if err != nil { + return App{}, err + } apps, err := o.Apps(ctx) if err != nil { return App{}, err @@ -118,6 +167,9 @@ func (o *Ops) App(ctx context.Context, name string) (App, error) { } a.CloudflareMissing = a.Cloudflare != "" && conn == nil a.Components = components(list, a.Host, a.URL != "") + if a.Verification, err = o.verify(ctx, name, settings, false); err != nil { + return App{}, err + } return a, nil } return App{}, &NotFoundError{Name: name} diff --git a/internal/ops/tunnel.go b/internal/ops/tunnel.go index b0accac..46afaf7 100644 --- a/internal/ops/tunnel.go +++ b/internal/ops/tunnel.go @@ -120,8 +120,7 @@ func (o *Ops) planExposure(ctx context.Context, name string, settings cluster.Se return nil, noDomain(name, connection) } if !PublicDomain(domain) { - return nil, fmt.Errorf("%s cannot be exposed under %s, which is not a public domain; give it a public domain of its own", - name, domain) + return nil, notPublic(name, domain) } e.api = o.NewCloudflare(e.access.Token) zone, err := e.api.ZoneFor(ctx, domain) @@ -156,6 +155,11 @@ func (o *Ops) planExposure(ctx context.Context, name string, settings cluster.Se return e, nil } +func notPublic(app, domain string) error { + return fmt.Errorf("%s cannot be exposed under %s, which is not a public domain; give it a public domain of its own", + app, domain) +} + // finish points the app's name at its tunnel, and removes the record and the tunnel the cluster // no longer uses. It runs once cloudflared has moved, because Cloudflare refuses to delete a // tunnel that still has connections. diff --git a/internal/server/server_test.go b/internal/server/server_test.go index ff703db..b7800a2 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -208,6 +208,17 @@ func runningPlatform() *fakePlatform { Public: true, URL: "https://greeter-dev.example.com/", CloudflareAccount: "acc-1", + Verification: &ops.Verification{ + Verification: cluster.Verification{ + Verified: true, Source: "https://github.com/tweinmann/shelf-hello", + Commit: "42a964eb0b4bd7248305d5f22233d6dbabcd56f3", + Scan: &cluster.Scan{Scanner: "trivy 0.75.0", ScannedAt: "2026-10-28T12:00:00Z", Images: []cluster.ScannedImage{ + {Component: "db", Scanned: true, Critical: 2, High: 86}, + {Component: "web", Scanned: true, High: 3}, + }}, + }, + Signer: "keyless, issuer https://token.actions.githubusercontent.com", Exposable: true, + }, Components: []ops.Component{ { Component: cluster.Component{Name: "check", Phase: cluster.PhaseWorking}, @@ -242,7 +253,12 @@ func runningPlatform() *fakePlatform { Host: "shop-dev.shop.ch", Public: true, CloudflareMissing: true, - URL: "https://shop-dev.shop.ch/", + Verification: &ops.Verification{ + Verification: cluster.Verification{Reason: "the deploy artifact has no scan (dev.shelf.scan)"}, + Signer: "keyless, issuer https://token.actions.githubusercontent.com", + Hint: "push again with the current reusable workflow, which signs and scans every build", + }, + URL: "https://shop-dev.shop.ch/", Components: []ops.Component{ { Component: cluster.Component{Name: "web", Phase: cluster.PhaseReady, diff --git a/internal/server/testdata/app-failed.html b/internal/server/testdata/app-failed.html index c9b2040..a661c4e 100644 --- a/internal/server/testdata/app-failed.html +++ b/internal/server/testdata/app-failed.html @@ -69,6 +69,23 @@

Components

+
+

Verification

+
+ +
Signer
keyless, issuer https://token.actions.githubusercontent.com
+
Verified
no: the deploy artifact has no scan (dev.shelf.scan)
+ +
Exposure
refused until it is verified
+
+

push again with the current reusable workflow, which signs and scans every build

+ +

The cluster runs only what this signer signed. A scan must exist before the app can +be exposed; how many findings it has is shown, never enforced.

+
+ + +

Deploy

diff --git a/internal/server/testdata/app-no-domain.html b/internal/server/testdata/app-no-domain.html index b896419..d1738f1 100644 --- a/internal/server/testdata/app-no-domain.html +++ b/internal/server/testdata/app-no-domain.html @@ -63,6 +63,8 @@

Components

+ +

Deploy

diff --git a/internal/server/testdata/app-quick.html b/internal/server/testdata/app-quick.html index 0f46faa..095c755 100644 --- a/internal/server/testdata/app-quick.html +++ b/internal/server/testdata/app-quick.html @@ -64,6 +64,8 @@

Components

+ +

Deploy

diff --git a/internal/server/testdata/app-ready.html b/internal/server/testdata/app-ready.html index 07ed431..a1188bc 100644 --- a/internal/server/testdata/app-ready.html +++ b/internal/server/testdata/app-ready.html @@ -72,6 +72,33 @@

Components

+
+

Verification

+
+
Source
https://github.com/tweinmann/shelf-hello at 42a964eb0b4b
+
Signer
keyless, issuer https://token.actions.githubusercontent.com
+
Verified
yes
+
Scan
trivy 0.75.0, 2026-10-28T12:00:00Z
+
Exposure
allowed
+
+ + +
    + +
  • db +2 critical, 86 high
  • + +
  • web +0 critical, 3 high
  • + +
+ +

The cluster runs only what this signer signed. A scan must exist before the app can +be exposed; how many findings it has is shown, never enforced.

+
+ + +

Deploy

diff --git a/internal/server/ui/app.html b/internal/server/ui/app.html index c859c2a..62a544b 100644 --- a/internal/server/ui/app.html +++ b/internal/server/ui/app.html @@ -52,6 +52,30 @@

Components

{{end}}
+{{with .App.Verification}} +
+

Verification

+
+{{if .Source}}
Source
{{.Source}}{{with .Commit}} at {{printf "%.12s" .}}{{end}}
{{end}} +
Signer
{{.Signer}}
+
Verified
{{if .Verified}}yes{{else}}no: {{.Reason}}{{end}}
+{{with .Scan}}
Scan
{{.Scanner}}, {{.ScannedAt}}
{{end}} +
Exposure
{{if .Exposable}}allowed{{else}}refused until it is verified{{end}}
+
+{{if not .Verified}}

{{.Hint}}

{{end}} +{{with .Scan}} +
    +{{range .Images}} +
  • {{.Component}} +{{if .Scanned}}{{.Critical}} critical, {{.High}} high{{else}}not scanned{{end}}
  • +{{end}} +
+{{end}} +

The cluster runs only what this signer signed. A scan must exist before the app can +be exposed; how many findings it has is shown, never enforced.

+
+{{end}} + {{if .Running}}

A change is running: {{.Running.Title}}. The buttons below wait until it is done.

diff --git a/justfile b/justfile index 3946fca..66cf6d7 100644 --- a/justfile +++ b/justfile @@ -76,9 +76,9 @@ chart-push: # Install Flux and the platform into the dev cluster from the dev registry. The working copy's # platform and chart are pushed first: a reset cluster comes with an empty registry. init-cluster *flags: platform-push chart-push - go run ./cmd/shelf init cluster --insecure-registry \ - --platform oci://shelf-registry:5000/shelf/platform:dev \ - --chart oci://shelf-registry:5000/shelf/charts/shelf-app:0.0.0-dev {{flags}} + go run ./cmd/shelf init cluster --insecure-registry --signer-key .local/cosign/cosign.pub \ + --platform oci://shelf-registry.local:5000/shelf/platform:dev \ + --chart oci://shelf-registry.local:5000/shelf/charts/shelf-app:0.0.0-dev {{flags}} # Run the admin UI against the dev cluster. The devcontainer forwards 8080, so the browser on # the Mac reaches it at http://localhost:8080; the setup code is printed on the first start. @@ -109,6 +109,10 @@ smoke-agent: smoke-previews: hack/smoke/previews.sh +# Phase 11: signed and scanned is verified, admission of built images, previews, no scan, unsigned refused +smoke-verify: + hack/smoke/verify.sh + # Phase 8b acceptance: expose an app through its own Cloudflare tunnel (asks for the API token) smoke-expose app domain: hack/smoke/expose.sh {{app}} {{domain}} diff --git a/openspec/changes/phase-11-verification-chain/.openspec.yaml b/openspec/changes/phase-11-verification-chain/.openspec.yaml index ad9e5be..97e3642 100644 --- a/openspec/changes/phase-11-verification-chain/.openspec.yaml +++ b/openspec/changes/phase-11-verification-chain/.openspec.yaml @@ -1,3 +1,2 @@ schema: spec-driven created: 2026-10-27 -skip_specs: true diff --git a/openspec/changes/phase-11-verification-chain/design.md b/openspec/changes/phase-11-verification-chain/design.md new file mode 100644 index 0000000..5e27c57 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/design.md @@ -0,0 +1,468 @@ +# Design + +## Context + +What runs today, and what it trusts: + +- **Workflow.** The reusable workflow `build.yml` builds the `build:` components, renders the + app.yaml with `shelf render` and pushes the deploy artifact with `flux push artifact`. `render` + pins every image by digest, built and third-party (for example `postgres:16` of + `examples/hello`), so the artifact names exactly what runs. Nothing is signed, and nothing is + scanned. +- **Cluster.** Per app and per preview, Flux has an OCIRepository `deploy` (the artifact) and + one `shelf-app` (the chart). The platform itself comes through the FluxInstance's sync + OCIRepository. All three trust whoever can write to the registry. +- **What shelf reads.** shelf learns about a running app only through the API server. The + OCIRepository's `status.artifact.metadata` already carries the artifact's annotations; that + is how `shelf app status` shows the commit without a forge API. +- **Tenants are private.** `tweinmann/shelf-hello` is a private repository on a free plan. As far + as we know, GitHub's own artifact attestations are only available for private repositories on + GitHub Enterprise Cloud, so they are out; cosign is what is left. +- **Hardware.** The target host is small (a Mac mini or a small Linux box), so every controller + added to the platform counts in RAM. + +## Goals / Non-Goals + +**Goals:** + +- **Signed.** Every deploy artifact, built image, chart and platform artifact the cluster takes + from a release or a tenant build is signed by the expected identity, and the cluster refuses + anything that is not. +- **Scanned.** Every image an app runs is scanned in CI. The result reaches shelf through the + signed deploy artifact. +- **Gate.** Only a verified app can be exposed (`--cloudflare`, `--quick`), in the CLI and the + UI alike. +- **Queryable.** The chain can be read per running version, for people and as JSON for agents. +- **Dev mirrors production.** The dev cluster verifies like production, with a key pair instead + of keyless signing, so level 2 tests the real checks. + +**Non-Goals:** + +- **Blocking on findings.** A scan must exist; its findings are shown, never enforced. +- **Signing the CLI binaries** or their checksums. +- **SBOMs and SLSA provenance** beyond what cosign and Trivy produce. +- **Admission for third-party images** beyond the digest pinned in the signed artifact. +- **A switch that turns verification off.** A cluster always verifies, with keyless signing or + a key. +- **Re-checking already running pods** after the cluster is upgraded. A pod that runs stays + running. + +## Decisions + +### 1. Admission with the Sigstore policy-controller + +The platform installs the Sigstore policy-controller (Helm chart, pinned, namespace +`cosign-system`): +- **Policy per app.** The ResourceSet `apps` generates a ClusterImagePolicy `shelf-` + covering the app's built images, `ghcr.io///**`. Its identity is the cluster's + signer. +- **Namespaces opt in.** App and preview namespaces carry `policy.sigstore.dev/include: "true"`, + and only those are checked. +- **Unmatched images pass.** Images no policy matches, the third-party ones, are allowed + (`no-match-policy: allow`). +- **Webhook failure.** If the webhook does not answer, pod creation in those namespaces fails + (`failurePolicy: Fail`). + +*Alternatives:* +- **Kyverno** is a general policy engine and could serve Phase 12 too, but a default install is + several controllers with several hundred Mi of RAM. +- **No admission controller** is the leanest option. But then an unsigned image could start as + long as someone writes a values ConfigMap by hand, and the acceptance "an unsigned artefact + does not reach the cluster" would hold for artifacts only. + +The maintainer chose the policy-controller. ClusterImagePolicy is a Sigstore CRD, not a +shelf-specific one; an ejected app can keep it or drop it. + +### 2. Keyless cosign with the public Sigstore + +CI signs with `cosign sign` and `cosign attest` using the job's GitHub OIDC token, so the +callers' `deploy.yml` needs `permissions: id-token: write`. There is no key to create, store or +rotate. + +The cost is privacy. Every signature is recorded in the public Rekor log, and with it the name of +the private tenant repository, the workflow and the commit SHA, permanently. + +*Alternatives:* +- **A cosign key pair per tenant:** nothing public, but keys to create, keep secret in the + tenant repo and rotate. +- **Keyless with a key pair option per app:** more code and tests. + +The maintainer chose keyless signing, accepting the entries in the public log. + +### 3. The expected identity is shelf's workflow + +With a reusable workflow, the Fulcio certificate's subject is the reusable workflow, not the +caller: +`https://github.com/tweinmann/shelf/.github/workflows/build.yml@refs/tags/v0`. + +The caller's repository is only in a certificate extension. Neither Flux's `matchOIDCIdentity` nor +the policy-controller's identities can match extensions. So the expected identity is: + +- issuer `https://token.actions.githubusercontent.com`; +- subject regex `^https://github\.com/tweinmann/shelf/\.github/workflows/(build|release)\.yml@refs/(tags|heads)/.+$`. + +These are the defaults of the cluster settings `SHELF_SIGNER_ISSUER` and +`SHELF_SIGNER_SUBJECT`. A fork of shelf sets its own with `shelf init cluster --signer-subject`. + +What this does and does not prove: + +- **What it proves:** shelf's workflow built the artifact. It does not prove which tenant + repository ran the workflow. +- **Repository:** the binding to the tenant comes from the registry. Only someone with write + access to `ghcr.io//` can push an artifact or a signature there. +- **Source annotation:** shelf also shows the artifact's `org.opencontainers.image.source`. That + annotation is part of the signed manifest. + +*Alternatives:* +- **Signing in the tenant's own workflow** would put the caller into the subject. But it moves + the signing steps into every tenant's boilerplate, against Phase 4b's one-line workflow. +- **Checking the extension with cosign on the host** would need the Sigstore Go libraries in + shelf and a registry call from `app status`. A later phase can add it. + +### 4. Signatures everywhere, the gate only for exposure + +Branch builds are signed by the same workflow. The verification of the deploy chain and the +admission policy therefore apply to previews exactly as to apps: an unsigned artifact reaches no +namespace. The exposure gate (Decision 8) concerns only the app, since a preview is never +exposed. + +*Alternative:* previews unchecked, which means fewer refusals while working. But the acceptance +would then hold for `main` only. The maintainer chose signatures everywhere. + +### 5. Third-party images: pinned by the signed artifact + +An image such as `postgres:16@sha256:…` is not ours to sign: the signature would have to be +pushed into Docker Hub's repository. + +- **Integrity:** the digest is fixed in the deploy artifact, which is signed and verified. A + values ConfigMap cannot change without a new signed artifact, so the digest that runs is the + one CI resolved. +- **Admission:** the policy does not match third-party images (Decision 1). The scan covers them + (Decision 6). + +*Alternatives:* +- **Copy third-party images** into `ghcr.io///` and sign the copies. That means + more registry storage and a slower build, and it changes the image references in app.yaml. +- **Deny unmatched images.** That would break every app with a database. + +### 6. Scan in CI, summary in the signed artifact + +**What CI does:** +1. After `shelf render`, CI scans every image digest the rendered values name with Trivy (pinned, + its DB cached between runs). +2. For each built image, it attaches the full report as a cosign attestation (`--type vuln`). +3. It writes a summary into the deploy artifact as the annotation `dev.shelf.scan`: per image + digest the CRITICAL and HIGH counts, plus scanner version and scan time, as compact JSON. It + uses `flux push artifact --annotations`. +4. Then it signs the artifact. + +The signature covers the manifest, and with it the summary. + +**How shelf reads it:** Flux copies manifest annotations into the OCIRepository's +`status.artifact.metadata`. shelf reads the summary from the cluster: no Sigstore library in +shelf, and no registry call from `app status`. + +**What it blocks:** a scan must exist for exposure; how many findings it reports blocks nothing +(Decision 8). + +*Alternatives:* +- **Read and verify the attestations from shelf:** needs the Sigstore libraries and registry + access from the host. +- **Require the attestation at admission:** blocks previews and pods for a missing scan, against + Decision 4. +- **Block on findings:** a new CVE could make an app impossible to expose overnight. The + maintainer chose to show findings only. + +### 7. What "verified" means for a running app + +An app is **verified** when all of these hold: + +- **Signature:** its `deploy` OCIRepository has `SourceVerified=True` for the revision it holds. +- **Release:** the release runs that revision's values (the Phase 9 configDigest comparison). +- **Chart:** its `shelf-app` OCIRepository has `SourceVerified=True`. +- **Scan:** `dev.shelf.scan` names every image digest in the running values. + +A running pod's built images are admitted by the policy-controller, so they need no extra check. +The definition is computed in `internal/cluster` from what the API server reports, and reported +as a `Verification`: + +- `Verified`; +- `Reason` (the first thing missing, in the order above); +- `Source` (the artifact's source annotation); +- `Signer` (the cluster's identity); +- `Scan` (per component: digest, critical, high, scanner, time). + +### 8. The exposure gate, and add before expose + +- **Where it runs:** `internal/ops` checks the gate whenever a call would give an app a named or + quick tunnel it does not have yet: `shelf app add` or `shelf app credentials` with + `--cloudflare` or `--quick`, and the UI's access form. The CLI and the UI share the code. +- **Refusal:** the gate refuses with the reason from Decision 7 and touches no Cloudflare API and + no tunnel. +- **Order for a new app:** an app that does not run yet cannot be verified. `shelf app add` with + an exposure flag first adds the app privately and waits for its rollout as today. Then it + checks verification, and only then creates the tunnel. +- **If the check fails:** the app stays added and private, and the command exits 1 with the + reason and a hint: push again with the current workflow, then run `shelf app credentials`. +- **Already exposed apps:** they keep their exposure. Their new artifacts are verified by Flux + anyway, so an unsigned one never runs. A missing scan on a new artifact is shown in status and + does not take the app off the internet. + +*Alternative:* verify before adding, on the host. That needs cosign on the operator's machine; +rejected for the same reason as in Decision 6. + +### 9. The signer is a cluster setting, keyless or a key + +`shelf init cluster` writes the signer into the cluster settings. + +**Keyless (the default):** +- `SHELF_SIGNER_ISSUER` and `SHELF_SIGNER_SUBJECT`, from `--signer-issuer` and + `--signer-subject`, with the defaults of Decision 3. + +**A key (test clusters):** +- `--signer-key ` with a cosign public key writes `shelf-system/signer-key`. +- shelf copies that Secret into every app and preview namespace as `shelf-signer`, as it does + with the registry login. +- The platform's `verify` uses `secretRef: shelf-signer`, and the ClusterImagePolicy uses the + key. + +**Templating:** the ResourceSets choose the form at render time. The sync Kustomization +substitutes `${SHELF_SIGNER_MODE}` into the template's `<< if >>` before the operator renders it. + +**No off switch:** `--signer-key` is for clusters that test shelf. A cluster without verification +would make the gate meaningless. + +*Alternative:* an `--verify=off` mode for test clusters. But then level 2 would not test what +production runs. + +### 10. The dev cluster signs with a dev key + +- **Key:** `just cluster-up` creates a cosign key pair once, with an empty password, under + `.local/cosign/`. That directory is machine-local and gitignored, and the key never leaves the + devcontainer. +- **Install:** `just init-cluster` installs with `--signer-key .local/cosign/cosign.pub`. +- **Signing:** a helper in `hack/lib.sh` (`sign_artifact`) signs every artifact a recipe or smoke + test pushes: platform, chart, deploy artifacts and built images. It signs with + `--tlog-upload=false`, so nothing from the dev cluster reaches Rekor. +- **CI:** each CI job creates its own key pair. + +### 11. Chart and platform signed by the release + +- **Signing:** `release.yml` signs the chart and the platform artifact keyless. Its subject is + `release.yml@refs/tags/v…`, which the default subject regex covers. +- **Platform:** the FluxInstance's sync OCIRepository gets `verify` through the existing + kustomize patch, so `shelf init cluster` installs only a signed platform. +- **Chart:** the `shelf-app` OCIRepository of every app and preview gets `verify` too. + +### 12. Platforms + +`build.yml` gets an input `platforms` (default `linux/arm64`), passed to `docker buildx build +--platform`. More than one platform builds a multi-arch index, which is what gets signed and +pinned. Trivy scans the index's first platform. Building amd64 on the arm64 runner goes through +QEMU and is slow; the input's description says so. + +### 13. Migration + +An app's running artifact was built before this phase and is unsigned, so the upgraded platform +no longer applies a new artifact of it. The pods that run keep running. A pod that is recreated +with a built image is denied, because the old image has no signature. + +The order is therefore: +1. Release. +2. Every tenant pushes once with the new workflow. On `@v0` that is automatic, apart from adding + `id-token: write`. +3. The operator upgrades the cluster. + +Two places say so: +- **`shelf init cluster`:** on an upgrade, its confirmation lists the apps and says that each + needs one build with the new workflow. +- **`shelf app status`:** shows an app whose artifact is unverified with the reason and the hint. + +### 14. Status, JSON and UI + +- **`shelf app status `:** a block between the components and the stages: + - `source`: repository and commit; + - `signed`: `verified, signer `, or the reason; + - `scan`: per component ` critical, high`, with scanner and time; + - `exposure`: `allowed`, or `refused: `. +- **`--json`:** `AppStatus.verification` holds `verified`, `reason`, `source`, `signer`, + `exposable` and `scan[]` with `component`, `digest`, `critical`, `high`, `scanner` and + `scannedAt`. Fields are only added (Phase 9 rule). +- **Previews:** they show `signed` and `scan` the same way. +- **Admin UI:** the app page shows the same block. The access form shows the gate's refusal as + the job log's last line. +- **Diagnosis:** + - A denied pod shows in the pods stage with the webhook's message and a hint about the + signature. + - A failed verification shows in the deploy artifact stage with Flux's message and a hint. + +### 15. Turnaround + +Signing adds a few seconds per image. Trivy's DB download is cached with `actions/cache`, so a +scan should take seconds per image. The agent wait (`--wait --commit`) is unchanged: it waits +for the rollout, which now includes verification by Flux. `smoke-verify` and the level-3 run +measure the CI and push-to-Ready times again, against Phase 10's 93 s and 11 s. + +### 16. Tests + +**Level 1:** +- `Verification` from fake cluster objects; +- the gate and the order of add then expose in `internal/ops` (fake cluster and Cloudflare); +- the status text and the JSON golden file; +- the UI page golden file; +- the ResourceSets rendered in both signer modes; +- the FluxInstance patch. + +**Level 2, `just smoke-verify`:** +- **Signed:** with the dev key, a signed artifact with a signed built image becomes Ready and + verified. +- **Unsigned artifact:** it is not applied, and the stage says why. +- **Unsigned image:** an artifact naming an unsigned built image is applied, but its pod is + denied. +- **No scan:** an artifact without a scan summary runs, but is not exposable. +- **Previews:** a preview of a signed branch artifact is verified. +- **RAM:** prints the policy-controller's RAM. + +The refusal itself is tested at level 1, because only `smoke-quick` may pass `--quick`. + +**Level 3:** +- keyless signing in `shelf-hello`; +- the migration of `gretter`; +- the gate against the real Cloudflare connection. + +### 17. Signatures in cosign's classic format (from the spike) + +cosign v3 stores a signature by default as a Sigstore bundle attached as an OCI referrer. Flux +reads it, but the policy-controller (chart 0.10.8, controller 0.13.1) does not: it reports "no +signatures found". Every signature and attestation is therefore made with +`--new-bundle-format=false`, which writes the classic `sha256-.sig` tag that both read. +Signing without a transparency log (the dev key) also needs `--use-signing-config=false +--tlog-upload=false`. A ClusterImagePolicy with a key and no `ctlog` does not look up a log. + +*Alternative:* wait for a policy-controller release that reads bundles. Rejected for now, but +switching back is one flag once it does. + +### 18. The dev registry is `shelf-registry.local:5000` (from the spike) + +The policy-controller has no setting for a registry without TLS, but go-containerregistry +speaks HTTP to a registry whose name ends in `.local`. With the dev registry named +`shelf-registry.local`, it verifies signatures there as on GHCR. +- **What changes:** `just cluster-up` creates the registry under that name, so k3d writes it into + CoreDNS and the containerd mirror configuration, and `/etc/hosts` in the devcontainer gets the + new name. +- **Moving over:** one `just cluster-reset`. + +*Alternatives:* +- **TLS with a CA of our own,** trusted by containerd, Flux, the policy-controller + (`registryCaBundle`) and the devcontainer. Closest to production, but the most work. +- **Admission tested at level 3 only.** + +The maintainer chose the rename. + +## Risks / Trade-offs + +- **Public log** → the names of private tenant repos and their commit SHAs become public in + Rekor. Accepted by the maintainer (Decision 2). A key pair option can be added later without + changing the cluster side (Decision 9 already supports keys). +- **Shared identity** → any repository calling shelf's workflow produces a valid signature. + Mitigated by registry write access and by showing the source annotation (Decision 3). It is + stated in the README, so nobody reads more into the signature than it proves. +- **Sigstore down** → no new artifact or pod with a built image can be verified, so new deploys + wait while running pods keep running. A recreated pod with a built image fails while the + webhook cannot verify. Accepted for now; Phase 15 (operations) covers outages. +- **Webhook down** → with `failurePolicy: Fail`, no pod in an app or preview namespace can be + created. The policy-controller runs two replicas if RAM allows; otherwise one, measured in + `smoke-verify`. +- **Plain-HTTP dev registry** → the spike showed the policy-controller only reaches it under a + `.local` name (Decision 18). +- **Signature format** → the policy-controller reads only cosign's classic format (Decision 17). + A later cosign that drops `--new-bundle-format=false` before the policy-controller reads + bundles would block an upgrade of cosign; it is pinned. +- **Migration** → an operator who upgrades the cluster before the tenants pushed sees new + deploys stop. The confirmation and the status say so (Decision 13). Nothing is deleted. +- **Turnaround** → the scan and signing add to every push, also of branches. Measured and + recorded (Decision 15). If the scan dominates, it moves to a job of its own that runs parallel + to the build of other components. +- **More of a "security" claim than exists** → the README says exactly what is verified and what + is not (third-party images only by digest, the shared identity, findings not enforced). + +## Migration Plan + +1. **Release** v0.8.0: workflow, chart and platform, signed. +2. **Tenants:** add `id-token: write` to `deploy.yml`. `examples/tenant` shows it, and + `shelf-hello` is updated in the acceptance. Push once. +3. **Clusters:** `shelf init cluster` with the new platform. The confirmation lists apps without + a verified artifact. +4. **Rollback:** the previous platform release removes verification and the policy-controller. + The signed artifacts keep working there. + +## Open Questions + +- Whether two replicas of the policy-controller fit; one uses 34 Mi (spike), measured again in + `smoke-verify`. + +## Results + +Spike on the dev cluster (2026-10-28): +- **Versions:** cosign v3.1.3, policy-controller chart 0.10.8 (controller 0.13.1), Flux 2.9.5. + +| Check | Result | +|---|---| +| Flux `verify` with a key, no tlog, plain HTTP | verifies a signed artifact (`SourceVerified=True`) and refuses an unsigned one, in both signature formats | +| policy-controller and `shelf-registry:5000` | fails: it only tries HTTPS | +| policy-controller and `shelf-registry.local:5000` | reaches the registry over HTTP | +| policy-controller and a cosign v3 bundle | "no signatures found" | +| policy-controller, classic `.sig`, key, no tlog | admits the signed image, denies the unsigned one | +| policy-controller RAM, one replica, idle | 34 Mi | + +Building images: the devcontainer's Docker has no buildx and does not push to a registry +without TLS. Smoke tests push images with `crane` (go-containerregistry), which is pinned with +the other tools. + +`just smoke-verify` on the dev cluster (2026-10-28), with the dev registry renamed to +`shelf-registry.local:5000`, passed: +- a signed and scanned artifact naming a signed built image is verified and exposable; +- the policy admits that image and `postgres:16`, and denies the unsigned image, in the app's + namespace and in a preview's; +- a preview of a signed branch artifact is verified, and never exposable; +- a signed artifact without `dev.shelf.scan` rolls out but is not verified; +- an unsigned artifact is not applied: the wait ends Failed at the deploy artifact with the + `id-token: write` hint, and the app keeps running the last verified commit; +- `shelf app rm` removes `shelf-`. + +The policy-controller used 34 Mi and 40 Mi for its two replicas. + +`smoke-apps`, `smoke-agent` (`SMOKE_FULL=0`) and `smoke-previews` passed on the same cluster, +every artifact signed with the dev key. Their times are within a few seconds of Phase 10's, so +verification adds no noticeable wait in the cluster: + +| Measurement | Phase 10 | Phase 11 | +|---|---|---| +| First push of a branch to Ready | 23 s | 29 s | +| Later commit on that branch | 17 s | 18 s | +| Deleted tag to preview gone | 26 s | 31 s | + +Two fixes came out of these runs: +- **The wait for an evicted preview.** One run of `smoke-previews` saw the evicted preview + outlast the 120 s wait for it; the wait is now 180 s. +- **Leftover branch tags.** The script now deletes the branch tags a failed run left in the + registry before it starts. + +Deviations found while implementing: +- **No pod to show.** A pod the admission webhook (or a preview's quota) refuses never exists, so + the pods stage reads the namespace's `FailedCreate` events, one list call for all namespaces. + A refusal counts only while no pod is newer than it. +- **Waits fail fast.** `shelf app add` and `shelf init cluster` fail at once when Flux reports + `SourceVerified=False`, instead of waiting for the timeout: Flux retries forever, but a + signature does not appear by waiting. +- **Public domain checked first.** The check that the app's domain is public moved before the + private deploy, so a name that cannot exist is still refused before anything is written. +- **Tools in the dev scripts.** + - `flux tag artifact` has no option for a registry without TLS, so the dev scripts tag with + `crane tag --insecure`. + - `shelf render` looks built images up over TLS, so `smoke-verify` renders the Docker Hub + image its dev images were copied from and swaps the reference. +- **Dev scan summary.** Dev artifacts carry a scan summary whose scanner is `none (dev + registry)`: nothing scans in level 2, and the summary only lets a dev app count as verified. +- **`smoke-tenant`.** It installs a released platform with keyless verification, because the + tenant's artifacts are signed keyless and a cluster accepts one signer. diff --git a/openspec/changes/phase-11-verification-chain/proposal.md b/openspec/changes/phase-11-verification-chain/proposal.md index 18735ed..fafdd2e 100644 --- a/openspec/changes/phase-11-verification-chain/proposal.md +++ b/openspec/changes/phase-11-verification-chain/proposal.md @@ -1,54 +1,90 @@ # Proposal -> Draft. This change holds the roadmap text only and is marked `skip_specs: true`. -> `/opsx:propose` turns it into a full change: it writes the delta specs, `design.md` and -> `tasks.md`, revises this proposal, and removes `skip_specs` from `.openspec.yaml`. - ## Why -A preview may run anything a branch built. Before an app goes beyond a preview, shelf has to know -that the running digest is the one CI built from a given commit, signed by the expected identity -and scanned. +Since Phase 10, every branch a coding agent pushes runs as a preview. A preview may run anything +the branch built, and so far shelf has no way of telling a running digest that CI built from a +known commit from one someone pushed by hand. Before an app goes beyond a preview, shelf has to +know that the running deploy artifact and its images are the ones the reusable workflow built, +signed by the expected identity, and scanned. Only then can the exposure decision rest on more +than trust in whoever can write to the registry. ## What Changes -- `build.yml`: multi-arch build option (arm64 default for shelf hosts, amd64 for cloud targets), - image scan (e.g. Trivy), signature and build provenance for images and the deploy artifact - (cosign keyless / GitHub artifact attestations), the scan result attached as an attestation. -- Cluster: Flux verifies the deploy artifact's signature (OCIRepository `verify`); an admission - policy accepts only signed image digests from the expected identity. -- `shelf app status` shows the chain: commit → digest → signature → scan → deployed. -- Exposure gate: `--cloudflare` and `--quick` refuse an app whose running digest is not verified. +- **Reusable workflow** (`build.yml`): + - signs every built image and the deploy artifact with keyless cosign (GitHub OIDC, the public + Sigstore); + - scans every image the app runs, built and third-party, with Trivy; + - attaches the full report to each built image as a signed attestation; + - writes a summary per image digest into the deploy artifact as the annotation + `dev.shelf.scan`, covered by the artifact's signature; + - gets an input `platforms` (default `linux/arm64`, for example `linux/arm64,linux/amd64` for + cloud targets). +- **Release** (`release.yml`): signs the `shelf-app` chart and the platform artifact the same + way. +- **Cluster: Flux verifies.** The deploy artifact, the chart and the platform artifact are + verified (OCIRepository `verify`), for apps and previews alike. An artifact without a valid + signature is not applied. +- **Cluster: admission.** The platform installs the Sigstore policy-controller. A + ClusterImagePolicy per app admits a built image of the app only with a valid signature from + the expected identity, in the app's namespace and in its previews' namespaces. Third-party + images are pinned by digest in the signed deploy artifact. +- **Signer.** `shelf init cluster` takes the signer as a cluster setting: keyless with the + identity of shelf's workflows by default, or `--signer-key ` with a cosign public key for + test clusters. The dev cluster uses a dev key pair, so level 2 verifies like production. +- **Exposure gate.** `--cloudflare` and `--quick` (CLI and admin UI, one function in + `internal/ops`) refuse an app whose running deploy artifact is not verified or not scanned. A + new app with an exposure flag is added privately, waited for, checked, and only then exposed. +- **Status.** `shelf app status` shows the chain: commit → deploy artifact → signature → images → + scan → deployed. `--json` gets a `verification` object, and the admin UI's app page shows the + same. +- **Findings are shown, never block.** A scan must exist for exposure; how many critical and high + findings it has is shown and left to the operator. +- **CLAUDE.md** gets the guardrail "only a verified digest goes beyond a preview". ## Capabilities ### New Capabilities -- `verification` (planned). +- `verification`: + - signing and scanning in the reusable workflow and the release; + - the signer setting; + - verification of artifacts by Flux; + - the admission policy for built images; + - what "verified" means for a running app; + - how the chain is reported. ### Modified Capabilities -To be decided at `/opsx:propose`; likely `app-delivery`, `app-exposure`, `cluster-init` and the -reusable workflow. +- `app-delivery`: the reusable workflow builds for the input `platforms`; the release signs chart + and platform; `shelf app status` prints the verification chain. +- `app-exposure`: `--cloudflare` and `--quick` refuse an unverified app (new requirement), and a + new app with an exposure flag is exposed only after it is verified. +- `cluster-init`: + - the platform contains the policy-controller; + - the cluster settings hold the signer; + - the FluxInstance verifies the platform artifact. +- `branch-previews`: a preview's deploy chain verifies like an app's, and its built images are + admitted by the app's policy. +- `admin-ui`: the app page shows the verification chain and why an app cannot be exposed. +- `agent-interface`: `AppStatus` carries a `verification` object. +- `dev-environment`: + - cosign is pinned in the devcontainer; + - the dev cluster verifies with a dev key pair, and every dev artifact is signed; + - `smoke-verify` joins level 2. ## Impact -`.github/workflows/build.yml`, `platform/`, `internal/cluster`, `internal/ops`, CLI and admin UI. -Level 2. - -## Roadmap text (from the re-cut of 2026-10-27) - -Three separate states per app and preview: **build** (built, scanned, signed), **deployment** -(which digest runs where), **exposure** (cluster only, port-forward, tunnel). Only a verified -artefact goes beyond a preview. - -**Open:** Kyverno vs. Sigstore policy-controller for the admission policy; the RAM footprint on a -small host matters. - -This change adds the guardrail "only a verified digest can be exposed beyond a preview" to -CLAUDE.md. It was deliberately not adopted with the re-cut, because it describes behaviour that -does not exist before this phase. shelf is not marketed as more secure than alternatives before -this chain exists. - -**Acceptance (level 2):** an unsigned or unscanned artefact does not reach the cluster; exposing -an unverified app is refused; the chain is queryable per running version. +- **Workflows:** `.github/workflows/build.yml` (cosign, Trivy, `id-token: write`, + `attestations`), `.github/workflows/release.yml`. +- **Platform and cluster code:** `platform/` (policy-controller, `verify` in `apps` and + `previews`, ClusterImagePolicy per app), `internal/cluster` (signer settings, FluxInstance + patch, verification state), `internal/ops` (exposure gate, order of add and expose). +- **Interfaces:** `internal/cli` (`--signer-key`, status, JSON), `internal/server` (app page). +- **Dev environment:** `.devcontainer/Dockerfile` (cosign), `hack/` (dev key, signing every + pushed artifact, `smoke-verify`), CI selection. +- **Tenants:** need `id-token: write` in their `deploy.yml` (in `examples/tenant` and + `shelf-hello`). Every app needs one push with the new workflow before its cluster is upgraded: + an unsigned artifact is no longer applied. +- **Levels:** level 2 for the cluster and gate; level 3 with `shelf-hello` and real Cloudflare + for keyless signing and the exposure gate. diff --git a/openspec/changes/phase-11-verification-chain/specs/admin-ui/spec.md b/openspec/changes/phase-11-verification-chain/specs/admin-ui/spec.md new file mode 100644 index 0000000..ed6e3b4 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/specs/admin-ui/spec.md @@ -0,0 +1,24 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: App page +The system SHALL show at `/apps/` the app's phase, the first stage that is not ready with +its message and hint, its address, artifact, revision and deploy time, its components each with +its phase and address (routed ones as links, the others marked `inside the cluster only` or `not +exposed`), its access settings, the names of its generated secrets, its verification as `shelf +app status` prints it, and the diagnosis chain under `How it gets here`. + +#### Scenario: Routed and internal components +- **WHEN** an app has a routed `web` component and an unrouted `db` component +- **THEN** `web` is shown with a link to its URL and `db` with its address and `inside the + cluster only` + +#### Scenario: Unknown app +- **WHEN** `/apps/` is requested for an app that does not exist or a name that is not a + valid app name +- **THEN** an error page with status 404 is shown + +#### Scenario: Unverified app +- **WHEN** the running artifact of an app is unsigned +- **THEN** the page says it is not verified and why, and that it cannot be exposed diff --git a/openspec/changes/phase-11-verification-chain/specs/agent-interface/spec.md b/openspec/changes/phase-11-verification-chain/specs/agent-interface/spec.md new file mode 100644 index 0000000..8f3e3d1 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/specs/agent-interface/spec.md @@ -0,0 +1,17 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Verification in the agent interface +The `AppStatus` of an app or preview SHALL hold `verification` with `verified`, `reason` and +`hint` when not verified, `source`, `signer`, `exposable`, and `scan`, one entry per component +with `component`, `digest`, `scanned`, `critical`, `high`, `scanner` and `scannedAt`; a +`WaitResult` with outcome `Ready` says nothing about verification. + +#### Scenario: Verified app as JSON +- **WHEN** a user runs `shelf app status hello --json` for a verified app with `web` and `db` +- **THEN** `verification.verified` and `verification.exposable` are true and `scan` has two entries + +#### Scenario: Agent reads why it cannot be exposed +- **WHEN** the running artifact has no scan summary +- **THEN** `verification.verified` is false and `reason` names the missing scan diff --git a/openspec/changes/phase-11-verification-chain/specs/app-delivery/spec.md b/openspec/changes/phase-11-verification-chain/specs/app-delivery/spec.md new file mode 100644 index 0000000..7e73c30 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/specs/app-delivery/spec.md @@ -0,0 +1,82 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Reusable tenant workflow +The reusable workflow `.github/workflows/build.yml` SHALL validate the app.yaml, build every +`build:` component for the platforms of its input `platforms` (default `linux/arm64`) as +`ghcr.io///`, render the app.yaml, and push the deploy artifact +`ghcr.io//` tagged `sha-` and `main` or `branch-`, annotated with +`dev.shelf.package=` and the commit's source and revision. When a branch is deleted, it +SHALL delete that branch's artifact. + +#### Scenario: Push to the default branch +- **WHEN** the workflow runs for a push to the repository's default branch +- **THEN** it pushes the images and the deploy artifact, and outputs `oci://ghcr.io//:main` +- **AND** its job summary shows a `shelf app add` command for that artifact + +#### Scenario: Push to another branch +- **WHEN** the workflow runs for a push to the branch `feature/login` +- **THEN** it pushes the images and the deploy artifact tagged `sha-` and `branch-feature-login`, and never `main` + +#### Scenario: Pull request or other branch +- **WHEN** the workflow runs for a pull request, from a fork or not +- **THEN** it builds the images but pushes neither images nor a deploy artifact +- **AND** a push to another branch is no longer such a case: it publishes, as in "Push to another branch" + +#### Scenario: Branch deleted +- **WHEN** the workflow runs for the deletion of the branch `feature/login` +- **THEN** it deletes the package version that carries `branch-feature-login` from `ghcr.io//`, and nothing else + +#### Scenario: Pinned shelf release +- **WHEN** the caller sets the input `shelf-version` +- **THEN** the workflow installs that release's linux/arm64 binary after checking its checksum, and otherwise the latest release + +#### Scenario: Two platforms +- **WHEN** the caller sets `platforms: linux/arm64,linux/amd64` +- **THEN** every built image is a multi-arch index of both, and the rendered values pin the index digest + +### Requirement: Release publication +Pushing a `v*` tag MUST publish a shelf release: CLI binaries for linux and darwin (amd64 and +arm64) with checksums, the `shelf-app` chart and the platform artifact, both signed with keyless +cosign by `release.yml`, all at the release version, and then move the major tag (for example +`v0`) before marking the release as latest. + +#### Scenario: Tenant on the major tag +- **WHEN** a tenant uses `tweinmann/shelf/.github/workflows/build.yml@v0` and a new v0 release is published +- **THEN** the tenant's next build uses the new workflow together with the matching latest binary + +#### Scenario: Signed chart and platform +- **WHEN** release `v0.8.0` is published +- **THEN** `ghcr.io/tweinmann/shelf/charts/shelf-app:0.8.0` and the platform artifact `v0.8.0` carry a signature whose subject is `release.yml` of `tweinmann/shelf` at that tag + +### Requirement: App status command +`shelf app status` SHALL list every app with its state and, when not Ready, the failing stage and +reason in one line; `shelf app status ` SHALL print the app's address, artifact, registry +connection, running revision and commit, deploy time, components, its verification, every stage +with its message and, if not Ready, its hint, and its previews with tag, state and commit; +`--preview ` SHALL print the same for one preview. + +#### Scenario: No apps +- **WHEN** the cluster has no registered apps +- **THEN** `shelf app status` prints "No apps yet." + +#### Scenario: One app in detail +- **WHEN** a user runs `shelf app status hello` for a healthy app +- **THEN** the output shows `hello: Ready`, the artifact reference, the revision, and all five stages as Ready + +#### Scenario: Commit of the running artifact +- **WHEN** the app's deploy artifact was pushed by the reusable workflow for commit `42a964eb0b4b…` +- **THEN** `shelf app status hello` shows that commit, read from the artifact's revision annotation as the cluster reports it, without calling a forge API + +#### Scenario: Artifact without a commit +- **WHEN** the deploy artifact carries no git revision +- **THEN** the commit line is left out + +#### Scenario: App with previews +- **WHEN** `hello` has previews for `feature/login` and `fix-typo` +- **THEN** `shelf app status hello` lists both with their tags, states and commits, and the list of all apps shows them under `hello`, not as apps of their own + +#### Scenario: One preview +- **WHEN** a user runs `shelf app status hello --preview feature/login` +- **THEN** the output shows the preview's namespace `hello--feature-login`, its host, artifact tag, commit, components and five stages diff --git a/openspec/changes/phase-11-verification-chain/specs/app-exposure/spec.md b/openspec/changes/phase-11-verification-chain/specs/app-exposure/spec.md new file mode 100644 index 0000000..c17fdb7 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/specs/app-exposure/spec.md @@ -0,0 +1,29 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Only a verified app is exposed +`shelf app add` and `shelf app credentials` SHALL refuse `--cloudflare` and `--quick` for an app +that has no tunnel yet unless the app is verified, with the reason, before any Cloudflare call or +tunnel; a new app with such a flag SHALL be added privately and waited for first, and stays added +and private when the check fails. An app that is already exposed keeps its exposure. + +#### Scenario: Unverified app refused +- **WHEN** a user runs `shelf app credentials hello --quick` and the running artifact of `hello` is unsigned +- **THEN** the command fails with the reason and a hint to push again with the current workflow, and no cloudflared and no tunnel are created + +#### Scenario: New app exposed after its rollout +- **WHEN** a user runs `shelf app add hello oci://ghcr.io/acme/hello:main --cloudflare acme --domain hello.example.com` for a signed and scanned artifact +- **THEN** the app is added, rolls out privately, is found verified, and only then gets its tunnel and DNS record + +#### Scenario: New app that is not verified +- **WHEN** the same command runs for an artifact without a scan summary +- **THEN** the app is added and private, and the command exits 1 naming the missing scan + +#### Scenario: Same gate in the admin UI +- **WHEN** the access form of the app page asks for a quick tunnel for an unverified app +- **THEN** the job fails with the same reason, as it calls the same operation + +#### Scenario: Exposed app keeps its tunnel +- **WHEN** an exposed app gets a new artifact without a scan summary +- **THEN** it keeps its tunnel and its status shows it is not verified diff --git a/openspec/changes/phase-11-verification-chain/specs/branch-previews/spec.md b/openspec/changes/phase-11-verification-chain/specs/branch-previews/spec.md new file mode 100644 index 0000000..b7aa705 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/specs/branch-previews/spec.md @@ -0,0 +1,26 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Preview contents +The platform SHALL give each preview the same deploy chain as an app (its deploy artifact at the +branch tag, checked every 30 seconds, the app values, the chart, the release), verified against +the cluster's signer, the app's registry login, a ResourceQuota (requests of 1 CPU, 2Gi memory, +5Gi storage) and a LimitRange that gives containers without requests a default; it MUST NOT give +a preview a tunnel, cloudflared or a DNS record. + +#### Scenario: Preview of an exposed app +- **WHEN** an app exposed through a Cloudflare connection has a preview +- **THEN** the preview namespace has no cloudflared, and nothing outside the cluster reaches it + +#### Scenario: Over the quota +- **WHEN** a preview's pods would request more than 2Gi memory together +- **THEN** the pods beyond it are refused, and the preview's pods stage says so + +#### Scenario: Unsigned branch artifact +- **WHEN** an artifact without a signature is pushed as `branch-feature-login` +- **THEN** the preview `hello--feature-login` does not apply it, and its deploy artifact stage says the verification failed + +#### Scenario: Built image of a preview +- **WHEN** a preview's values name an unsigned built image of the app's package +- **THEN** its pod is denied by the app's admission policy, as in the app's namespace diff --git a/openspec/changes/phase-11-verification-chain/specs/cluster-init/spec.md b/openspec/changes/phase-11-verification-chain/specs/cluster-init/spec.md new file mode 100644 index 0000000..3559339 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/specs/cluster-init/spec.md @@ -0,0 +1,70 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Confirmation before changing the cluster +The system SHALL show the target context, its API server, the platform artifact, the chart and +the host name pattern, and SHALL ask `Proceed? [y/N]` before applying anything; only `y` or +`yes` (case-insensitive) proceeds, and `--yes`/`-y` skips the question. + +#### Scenario: Confirmed +- **WHEN** the user answers `y` +- **THEN** the installation runs + +#### Scenario: Declined or no answer +- **WHEN** the user answers anything other than `y`/`yes`, or input ends without an answer +- **THEN** the command exits non-zero with "aborted; nothing was changed" and the cluster is untouched + +#### Scenario: Non-interactive +- **WHEN** `--yes` is passed +- **THEN** no question is asked and the installation runs + +#### Scenario: Apps without a verified artifact +- **WHEN** the platform is upgraded to one that verifies, and apps run artifacts that are not verified +- **THEN** the confirmation lists those apps and says that each needs one build with the current workflow before it gets new deploys, and that their running pods keep running + +### Requirement: FluxInstance syncing the platform +The system SHALL apply a FluxInstance `flux` in `flux-system` with the pinned Flux distribution +from `ghcr.io/fluxcd`, the components source-, kustomize-, helm- and notification-controller, +network policies enabled, and a sync of the platform artifact as an OCIRepository at path `./` +every `1m`, verified against the cluster's signer; the sync Kustomization MUST wait for +everything it applied and substitute variables from `flux-system/shelf-config`. + +#### Scenario: Insecure dev registry +- **WHEN** `--insecure-registry` is passed +- **THEN** the platform OCIRepository is patched to `insecure: true` and `SHELF_INSECURE_REGISTRY` is `true` + +#### Scenario: Flux ready +- **WHEN** the FluxInstance has been applied +- **THEN** the command waits until it reports Ready for its current generation + +#### Scenario: Unsigned platform +- **WHEN** `shelf init cluster --platform` names a platform artifact without a valid signature +- **THEN** the platform OCIRepository reports a failed verification, and the command fails while waiting for the platform with that message + +### Requirement: Platform contents +The platform artifact SHALL contain Traefik (Helm chart from `oci://ghcr.io/traefik/helm/traefik` +in namespace `traefik`, ClusterIP Service, ingress class `traefik` that is not the default, +Ingress and CRD providers enabled), the Sigstore policy-controller in `cosign-system`, and, in +`shelf-system`, the ResourceSet `apps`, which generates every app from the providers labelled +`shelf.dev/app`, and the ResourceSet `previews`, which generates every preview from those +labelled `shelf.dev/previews`. + +#### Scenario: Platform applied +- **WHEN** the installation completes +- **THEN** Traefik runs in `traefik` reachable at `traefik.traefik.svc.cluster.local:80` the ResourceSets `shelf-system/apps` and `shelf-system/previews` exist, and the policy-controller's webhook is ready + +### Requirement: System namespace and cluster settings +The system SHALL apply the namespace `shelf-system` (labelled +`app.kubernetes.io/managed-by: shelf`) and the ConfigMap `flux-system/shelf-config` with the keys +`SHELF_HOST_SUFFIX`, `SHELF_CHART_URL`, `SHELF_CHART_TAG`, `SHELF_INSECURE_REGISTRY` and the +signer keys `SHELF_SIGNER_MODE`, `SHELF_SIGNER_ISSUER` and `SHELF_SIGNER_SUBJECT`; the cluster has +no domain setting. + +#### Scenario: Settings written +- **WHEN** `shelf init cluster --host-suffix -dev` completes +- **THEN** `shelf-config` holds `SHELF_HOST_SUFFIX=-dev` and the chart URL and tag that were chosen + +#### Scenario: No cluster domain flag +- **WHEN** `--domain` is passed +- **THEN** the command fails with "unknown flag: --domain" diff --git a/openspec/changes/phase-11-verification-chain/specs/dev-environment/spec.md b/openspec/changes/phase-11-verification-chain/specs/dev-environment/spec.md new file mode 100644 index 0000000..8619793 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/specs/dev-environment/spec.md @@ -0,0 +1,56 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Dev cluster +`just cluster-up` SHALL create, or start if it exists, the k3d cluster `shelf-dev` with the k3s +image pinned in the Dockerfile, without k3s's own Traefik and without k3d's load balancer, its API +server on the container's loopback, the plain-HTTP registry `shelf-registry.local:5000` that allows +deleting manifests, and a dev cosign key pair under `.local/cosign/`. It MUST write the +kubeconfig only to `KUBECONFIG` and never change a default kubeconfig or switch a context. + +#### Scenario: After a container restart +- **WHEN** the devcontainer was rebuilt or restarted and `just cluster-up` runs +- **THEN** the existing cluster is started, and its kubeconfig is written again + +#### Scenario: Reset +- **WHEN** `just cluster-reset` runs +- **THEN** cluster and registry are deleted and created empty, and `just init-cluster` pushes the platform and chart again before installing them + +#### Scenario: Deleting a tag +- **WHEN** a smoke test deletes a branch tag from `shelf-registry.local:5000` +- **THEN** the registry accepts the deletion, so a preview can be removed at level 2 as on GHCR + +#### Scenario: Dev artifacts are signed +- **WHEN** `just init-cluster`, `just platform-push`, `just chart-push` or a smoke test pushes an artifact or a built image to `shelf-registry.local:5000` +- **THEN** it is signed with the dev key without a transparency log entry, and the cluster, installed with `--signer-key .local/cosign/cosign.pub`, verifies it + +### Requirement: Three test levels +Checks SHALL be split into level 1 (no cluster: `just test`, which runs gofmt, `go vet`, +`go test -race`, Helm lint, kubeconform, example validation and `openspec validate --all --strict`), +level 2 (the k3d cluster: the `just smoke-*` recipes), and level 3 (the Mac mini and real +Cloudflare), and a task that needs level 2 or 3 MUST say so. + +#### Scenario: Level 1 in CI +- **WHEN** a commit is pushed to `main` or a pull request is opened or pushed to +- **THEN** CI runs `just test` inside the devcontainer image on an arm64 runner + +#### Scenario: Level 2 in CI +- **WHEN** a pull request is opened or reopened as ready, leaves draft, or is pushed to or labelled while it carries the label `level-2` +- **THEN** CI also runs the smoke tests its changed paths need, chosen by `.github/scripts/select-smokes.sh`, each against a fresh k3d cluster of its own, side by side, with `SMOKE_FULL=0` + +#### Scenario: Documentation only +- **WHEN** a pull request changes only planning, documentation or examples +- **THEN** CI runs no smoke test, and a path the selection does not know runs all of them + +#### Scenario: Level 2 in full +- **WHEN** a commit is pushed to `main`, or a pull request carries the label `level-2-full` +- **THEN** CI runs `smoke-secrets`, `smoke-chart`, `smoke-init`, `smoke-apps`, `smoke-agent`, `smoke-previews` and `smoke-verify` with `SMOKE_FULL=1` + +#### Scenario: main fails +- **WHEN** CI fails on `main` +- **THEN** it opens an issue labelled `ci-main-red`, or comments on the one that is open, with the commit and the run + +#### Scenario: Release gate +- **WHEN** a release tag is pushed for a commit whose CI run on `main` did not succeed +- **THEN** `release.yml` stops before it publishes anything diff --git a/openspec/changes/phase-11-verification-chain/specs/verification/spec.md b/openspec/changes/phase-11-verification-chain/specs/verification/spec.md new file mode 100644 index 0000000..cf1c14f --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/specs/verification/spec.md @@ -0,0 +1,123 @@ +# Spec Delta + +## Purpose + +Lets shelf and its operator know that what runs is what the reusable workflow built from a +known commit: signed by the expected identity, scanned, and verified by the cluster before it +runs, so that only a verified app goes beyond a preview. + +## ADDED Requirements + +### Requirement: Signing in the tenant workflow +When it publishes, the reusable workflow SHALL sign every built image and the deploy artifact by +digest with keyless cosign, using the job's GitHub OIDC token, so the caller's workflow MUST grant +`id-token: write`; a run that cannot sign MUST fail without moving the `main` or `branch-` +tag. + +#### Scenario: Published build +- **WHEN** the workflow publishes for a push to `main` +- **THEN** every built image and the deploy artifact have a cosign signature whose certificate names shelf's workflow as subject and `https://token.actions.githubusercontent.com` as issuer + +#### Scenario: Missing permission +- **WHEN** the caller's `deploy.yml` does not grant `id-token: write` +- **THEN** the run fails at signing, and the channel tag still points at the previous artifact + +#### Scenario: Pull request +- **WHEN** the workflow runs for a pull request +- **THEN** it signs nothing, as it publishes nothing + +### Requirement: Scan in the tenant workflow +When it publishes, the reusable workflow SHALL scan every image digest the rendered values name, +built and third-party, with a pinned Trivy, attach the full report to each built image as a +signed cosign attestation of type `vuln`, and write the annotation `dev.shelf.scan` to the deploy +artifact before it is signed: per image digest the CRITICAL and HIGH counts, the scanner version +and the scan time. + +#### Scenario: App with a third-party database +- **WHEN** an app has a built `web` and a `db` with `image: postgres:16` +- **THEN** `dev.shelf.scan` lists both digests with their counts, and only the `web` image has a `vuln` attestation + +#### Scenario: Findings do not fail the build +- **WHEN** an image has CRITICAL findings +- **THEN** the workflow still publishes, and the counts are in `dev.shelf.scan` + +### Requirement: Signer of a cluster +The cluster settings SHALL name the signer every verification expects: keyless by default with +`SHELF_SIGNER_ISSUER` and the subject regex `SHELF_SIGNER_SUBJECT` covering shelf's `build.yml` +and `release.yml` (`--signer-issuer`, `--signer-subject`), or a cosign public key given with +`--signer-key ` and kept in `shelf-system/signer-key`; there MUST be no setting that turns +verification off. + +#### Scenario: Default signer +- **WHEN** `shelf init cluster` runs without signer flags +- **THEN** `shelf-config` holds the GitHub Actions issuer and a subject regex matching `https://github.com/tweinmann/shelf/.github/workflows/build.yml@refs/tags/v0` + +#### Scenario: Key for a test cluster +- **WHEN** `shelf init cluster --signer-key cosign.pub` runs +- **THEN** `shelf-system/signer-key` holds the key, and every app and preview namespace gets it as `shelf-signer` + +#### Scenario: Unreadable key +- **WHEN** the file given to `--signer-key` is not a PEM public key +- **THEN** the command fails before it changes the cluster + +### Requirement: Artifacts verified by Flux +Every OCIRepository the platform creates for an app or a preview, the deploy artifact and the +`shelf-app` chart, SHALL verify the cosign signature against the cluster's signer, so that an +artifact without a valid signature is not applied and what already runs keeps running. + +#### Scenario: Unsigned deploy artifact +- **WHEN** an artifact without a signature is pushed under an app's tag +- **THEN** the app's `deploy` OCIRepository reports a failed verification, the release keeps the previous values, and the pods keep running + +#### Scenario: Signed by someone else +- **WHEN** an artifact is signed by an identity the subject regex does not match +- **THEN** it is not applied, as if unsigned + +### Requirement: Admission of built images +The platform SHALL run the Sigstore policy-controller and give every app a ClusterImagePolicy +`shelf-` that admits an image under `ghcr.io///` only with a valid signature +from the cluster's signer, in the namespaces of the app and its previews; images no policy +matches SHALL be admitted, as their digests are fixed in the signed deploy artifact. + +#### Scenario: Unsigned built image +- **WHEN** the values of an app name a built image of its package that has no signature +- **THEN** the pod is denied by the admission webhook, and the pods stage shows the denial with a hint about the signature + +#### Scenario: Third-party image +- **WHEN** an app runs `postgres:16@sha256:…` from Docker Hub +- **THEN** the pod is admitted + +#### Scenario: Removed with the app +- **WHEN** an app is removed with `shelf app rm` +- **THEN** its ClusterImagePolicy is gone + +### Requirement: Verified app +An app or preview SHALL count as verified when its deploy artifact and chart sources report a +verified signature for the revisions they hold, its release runs the values of that deploy +artifact revision, and the artifact's `dev.shelf.scan` names every image digest in those values; +otherwise the reason SHALL be the first of these that does not hold. + +#### Scenario: Verified +- **WHEN** a signed and scanned artifact has rolled out +- **THEN** the app is verified, and its scan summary per component is known + +#### Scenario: No scan summary +- **WHEN** a signed artifact without `dev.shelf.scan` runs +- **THEN** the app is not verified, with the reason that the artifact has no scan + +#### Scenario: Rollout still pending +- **WHEN** a new signed artifact is fetched but its values are not released yet +- **THEN** the app is not verified until the release runs them + +### Requirement: Verification in the status +`shelf app status ` and `--preview` SHALL print the artifact's source and commit, whether the +signature is verified and by which signer or why not, the CRITICAL and HIGH counts per component +with scanner and scan time, and whether the app may be exposed or why not. + +#### Scenario: Verified app +- **WHEN** a user runs `shelf app status hello` for a verified app +- **THEN** the output shows the source and commit, the signer, `verified yes`, a scan line per component, and `exposure allowed` + +#### Scenario: Unverified app +- **WHEN** the running artifact of `hello` is unsigned +- **THEN** the output shows `verified no` with the reason and a hint to push again with the current workflow, and `exposure refused` diff --git a/openspec/changes/phase-11-verification-chain/tasks.md b/openspec/changes/phase-11-verification-chain/tasks.md new file mode 100644 index 0000000..9c3e6e8 --- /dev/null +++ b/openspec/changes/phase-11-verification-chain/tasks.md @@ -0,0 +1,185 @@ +# Tasks + +## 1. Spike and toolchain + +- [x] 1.1 Spike on the dev cluster, by hand and not committed: + - install the policy-controller chart; + - sign an image and a `flux push artifact` in `shelf-registry:5000` with a key and + `--tlog-upload=false`; + - check that Flux's OCIRepository `verify` (key and keyless forms) and a ClusterImagePolicy + with that key accept them, and refuse an unsigned image; + - measure the policy-controller's RAM. + + Record the results and any deviation in design.md before task group 4 (design risks + "Plain-HTTP dev registry" and "Key verification and the transparency log"). +- [x] 1.2 Pin cosign (v3.1.3) and crane (go-containerregistry v0.20.6) in + `.devcontainer/Dockerfile` (an `ARG` each, like the other tools), and check `cosign version` + and `crane version` in the container build. +- [x] 1.3 Rename the dev registry to `shelf-registry.local:5000` (design decision 18): + - `hack/lib.sh`, `hack/cluster-up.sh` (k3d `--registry-create`, `/etc/hosts`), every hack + script and justfile recipe that names it; + - the CLAUDE.md dev table and the README; + - the dev-environment delta spec. + + Then `just cluster-reset`, `just init-cluster` and `just smoke-init` (level 2). Afterwards, re-add + `gretter` with the maintainer: the reset removes it. + +## 2. Workflows + +- [x] 2.1 `build.yml`: + - input `platforms` (default `linux/arm64`), passed to `docker buildx build --platform`; + - `permissions: id-token: write` documented in the header; + - after the push of each built image, `cosign sign` its digest (spec verification "Signing in + the tenant workflow"). +- [x] 2.2 `build.yml` scan step (spec verification "Scan in the tenant workflow"): + - after `shelf render`, scan every image digest of the rendered values with a pinned Trivy, its + DB cached with `actions/cache`; + - `cosign attest --type vuln` the report on each built image; + - build the `dev.shelf.scan` JSON (per digest `critical`, `high`; `scanner`, `scannedAt`) and + pass it to `flux push artifact --annotations`. +- [x] 2.3 `build.yml`: `cosign sign` the deploy artifact digest after the push and before + `flux tag artifact`, so the channel tag only moves to a signed artifact. A failed signing fails + the run. +- [x] 2.4 `release.yml`: keyless `cosign sign` of the chart and the platform artifact by digest, + before the major tag moves (spec app-delivery "Release publication"). +- [x] 2.5 `examples/tenant/.github/workflows/deploy.yml`: `permissions: id-token: write` + (with `contents: read`, `packages: write`); README tenant section. Validate the YAML of all + three workflows. + +## 3. Signer setting and init + +- [x] 3.1 `internal/cluster`: + - signer settings `SHELF_SIGNER_MODE`, `SHELF_SIGNER_ISSUER` and `SHELF_SIGNER_SUBJECT` in + `shelf-config`, with the defaults of design decision 3; + - `shelf-system/signer-key` for the key form; + - copy it as `shelf-signer` into app and preview namespaces wherever the registry login is + copied. + + Table-driven tests for both forms (spec verification "Signer of a cluster", + cluster-init "System namespace and cluster settings"). +- [x] 3.2 `internal/cli` `init cluster`: + - flags `--signer-issuer`, `--signer-subject` and `--signer-key `, with the PEM checked + before anything is applied; + - golden output. +- [x] 3.3 FluxInstance: add `verify` to the sync OCIRepository through the existing kustomize + patch, in both signer forms (spec cluster-init "FluxInstance syncing the platform"). Unit test + of the rendered patch. +- [x] 3.4 Confirmation on upgrade: list the apps whose `deploy` OCIRepository has no + `SourceVerified=True`, with the sentence from the spec scenario "Apps without a verified + artifact". CLI test with the fake cluster. + +## 4. Platform + +- [x] 4.1 `platform/`: the policy-controller as a pinned HelmRepository plus HelmRelease in + `cosign-system` with `no-match-policy: allow`, its resources set and the replica count from + the spike; add it to the platform kustomization. kubeconform passes. +- [x] 4.2 ResourceSet `apps`: + - `verify` on `deploy` and `shelf-app`, keyless or with `secretRef: shelf-signer`, chosen + through `${SHELF_SIGNER_MODE}`; + - the namespace label `policy.sigstore.dev/include: "true"`; + - a ClusterImagePolicy `shelf-` for `ghcr.io///**` with the cluster's + signer. + + The package path comes from the provider input `url`. Extend the ResourceSet test in + `internal/cluster` to both signer forms. +- [x] 4.3 ResourceSet `previews`: the same `verify` and namespace label (spec branch-previews + "Preview contents"). Extend `TestPreviewsResourceSet`. +- [x] 4.4 `shelf app rm`: wait until the ClusterImagePolicy `shelf-` is gone too (spec + verification "Admission of built images", scenario "Removed with the app"). + +## 5. Verification state + +- [x] 5.1 `internal/cluster`: + - a `Verification` per app and preview from the OCIRepositories' `SourceVerified` conditions, + the Phase 9 configDigest comparison, the values' image digests and the `dev.shelf.scan` + metadata, with the reason order of design decision 7; + - table-driven tests: verified, unsigned, foreign signer, rollout pending, no scan, scan + missing a digest, chart unverified. +- [x] 5.2 Diagnosis: + - a failed verification in the deploy artifact stage gets a hint to push again with the + current workflow; + - a pod denied by the admission webhook (ReplicaSet `FailedCreate` with the webhook's message) + shows in the pods stage with a hint about the signature. + + Tests with fake events. +- [x] 5.3 `shelf app status`: + - the verification block (source, signed, scan per component, exposure) for an app and for + `--preview`; + - golden text output. +- [x] 5.4 `--json`: `AppStatus.verification` as in spec agent-interface "Verification in the + agent interface"; golden files `appstatus.json` and `appstatus-preview.json` extended, and a + new one for an unverified app. + +## 6. Exposure gate + +- [x] 6.1 `internal/ops`: the gate before every change that gives an app a tunnel it does not + have yet (`--cloudflare`, `--quick`), refusing with the reason and hint before any Cloudflare + call. Tests with the fake cluster and a Cloudflare fake that fails if it is called (spec + app-exposure "Only a verified app is exposed"). +- [x] 6.2 `shelf app add` with an exposure flag: add privately, wait for the rollout, check, + then expose; on refusal the app stays private and the command exits 1. ops and CLI tests for + both outcomes; the progress golden file gets the new lines. +- [x] 6.3 Admin UI: + - the app page shows the verification block, with a golden file per state; + - the access form's job fails with the gate's message, through the same ops call (spec admin-ui + "App page"). + +## 7. Dev environment and level 2 + +- [x] 7.1 `hack/cluster-up.sh`: create the dev key pair under `.local/cosign/` once, with an empty + password. `just init-cluster` passes `--signer-key .local/cosign/cosign.pub`. +- [x] 7.2 `hack/lib.sh` `sign_artifact ` (dev key, classic format, no tlog, plain-HTTP + registry, design decision 17), called by `platform-push.sh`, `chart-push.sh` and every `flux push artifact` in + `hack/smoke/*.sh`. The smoke tests init with `--signer-key`. Run `just smoke-apps` and + `just smoke-previews` to show that they still pass (level 2). +- [x] 7.3 `hack/smoke/verify.sh` and `just smoke-verify`, as in design decision 16: + - signed artifact with a signed built image is verified; + - unsigned artifact not applied; + - unsigned built image denied; + - no scan summary means not exposable; + - a signed preview is verified; + - prints the policy-controller's RAM. + + Run it on the dev cluster (level 2) and record the RAM in design.md under "Results". +- [ ] 7.4 CI: add `smoke-verify` to the full set in `ci.yml` and to `select-smokes.sh` + (`platform/*`, `internal/cluster/*`, `hack/lib.sh`, `hack/smoke/verify.sh`); every smoke job + creates its own key pair. Verify the selection on example paths and in the PR run. + +## 8. Documentation and checks + +- [x] 8.1 CLAUDE.md: + - the guardrail "Only a verified digest goes beyond a preview: `--cloudflare` and `--quick` + refuse an app whose running deploy artifact is not signed by the cluster's signer and + scanned"; + - the safety rule that the dev key under `.local/cosign/` signs dev artifacts only and never + leaves the devcontainer; + - conventions for `dev.shelf.scan`, `shelf-signer`, `shelf-` ClusterImagePolicy and the + signer settings; + - `smoke-verify` in the dev table; + - the cluster-up row mentions the dev key. +- [x] 8.2 README: + - a section on the verification chain that states exactly what is verified and what is not + (the shared workflow identity, third-party images by digest only, findings shown not + enforced, the public Rekor entries); + - the tenant permissions; + - the migration order; + - CLI table rows. +- [x] 8.3 `openspec/config.yaml` context: the verification chain in the architecture; the + exposure line names the gate. +- [x] 8.4 Run `just test` and `openspec validate --all --strict`; both pass. + +## 9. Acceptance (level 3, after release v0.8.0) + +- [ ] 9.1 With `tweinmann/shelf-hello`: + - add `id-token: write`, push, and check the keyless signatures and the attestation with + `cosign verify`; + - upgrade the dev cluster with the release platform: the confirmation lists `gretter`, and + `gretter` keeps running unverified until the next push, then is verified; + - push an unsigned artifact by hand under a branch tag: no preview applies it. +- [ ] 9.2 Exposure gate against the real Cloudflare connection: + - `shelf app credentials gretter --private`, then `--cloudflare tobile` succeeds while + `gretter` is verified; + - the same for an app whose artifact has no scan is refused. + + Record the CI time and push-to-Ready of a branch against Phase 10 (design decision 15) in + design.md under "Results". diff --git a/openspec/config.yaml b/openspec/config.yaml index 776554a..a5b681c 100644 --- a/openspec/config.yaml +++ b/openspec/config.yaml @@ -45,12 +45,21 @@ context: | `previews` runs each tag in a namespace `--` with a quota, limits and secrets the chart generates, never exposed. A branch build publishes the tag, deleting the branch deletes it, and the platform only reads the registry. + - Verification chain: the reusable workflow signs every built image and the deploy artifact + with keyless cosign (identity: shelf's build.yml) and writes a Trivy summary into the + artifact (`dev.shelf.scan`) before signing it; releases sign chart and platform. Flux + verifies every OCIRepository against the cluster's signer, and the Sigstore + policy-controller admits a built image of an app's package only with its signature, in app + and preview namespaces. Third-party images are pinned by digest in the signed artifact. + The dev cluster verifies with a dev key; there is no setting that turns verification off. - `shelf init` is split into idempotent layers: `init host` (Phase 13: provide the environment, macOS or Linux), `init cluster` (any kubecontext, no Colima or distribution assumptions) and the wrapper. - Three separate states per app and preview: build (built, scanned, signed), deployment (which digest runs where) and exposure (cluster only, port-forward, tunnel). Exposure is the - operator's decision, never part of app.yaml. Apps and previews are reached by + operator's decision, never part of app.yaml, and only a verified app can be exposed: the + gate in `ops.AddApp` refuses a tunnel to an app whose running artifact is not signed and + scanned. Apps and previews are reached by port-forward through the API server to Traefik (`shelf app open`); LAN access stays rejected. - Agents read the platform through `--json` documents and `shelf app status --wait --commit`, which only reads: the commit comes from the deploy artifact's revision annotation, never diff --git a/platform/apps/resourceset.yaml b/platform/apps/resourceset.yaml index 9deb9a9..c44bfd6 100644 --- a/platform/apps/resourceset.yaml +++ b/platform/apps/resourceset.yaml @@ -9,6 +9,12 @@ # login is copied into each of their namespaces. Namespace, secrets, tunnel and cloudflared are # every app's own. # +# Nothing runs unverified. The deploy artifact and the chart are applied only with a valid +# signature of the cluster's signer (verify), and the app's ClusterImagePolicy admits a built +# image of its package only with one; the policy-controller checks every namespace labelled +# policy.sigstore.dev/include. The signer comes from the cluster settings as JSON, which Flux +# substitutes as the YAML mapping it is: SHELF_SIGNER_VERIFY and SHELF_SIGNER_AUTHORITY. +# # ${SHELF_...} is substituted by Flux from the ConfigMap flux-system/shelf-config, which # `shelf init cluster` writes. << inputs.* >> is filled in by the Flux Operator per app. apiVersion: fluxcd.controlplane.io/v1 @@ -21,6 +27,13 @@ metadata: # reconcile.fluxcd.io/watch is picked up at once. fluxcd.controlplane.io/reconcileEvery: 10m spec: + # The ClusterImagePolicy type comes with the policy-controller. + dependsOn: + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: policy-controller + namespace: cosign-system + ready: true inputsFrom: - apiVersion: fluxcd.controlplane.io/v1 kind: ResourceSetInputProvider @@ -35,6 +48,7 @@ spec: name: << inputs.name >> labels: shelf.dev/app: << inputs.name >> + policy.sigstore.dev/include: "true" # Generated secret values, maintained by `shelf app add` in shelf-system/app-. - apiVersion: v1 @@ -57,6 +71,30 @@ spec: fluxcd.controlplane.io/copyFrom: "shelf-system/<< if inputs.registry >>connection-registry-<< inputs.registry >><< else >>registry-anonymous<< end >>" type: kubernetes.io/dockerconfigjson + # The signer's key for verify, empty for a keyless signer. + - apiVersion: v1 + kind: Secret + metadata: + name: shelf-signer + namespace: << inputs.name >> + annotations: + fluxcd.controlplane.io/copyFrom: shelf-system/signer-key + type: Opaque + + # Built images of the app's package are admitted only with the signer's signature, here and + # in the app's previews. + - apiVersion: policy.sigstore.dev/v1beta1 + kind: ClusterImagePolicy + metadata: + name: shelf-<< inputs.name >> + labels: + shelf.dev/app: << inputs.name >> + spec: + images: + - glob: '<< trimPrefix "oci://" inputs.url >>/**' + authorities: + - ${SHELF_SIGNER_AUTHORITY} + # The deploy Kustomization acts as this account, which may only manage ConfigMaps in the # app namespace. A deploy artifact with anything else in it fails instead of being applied. - apiVersion: v1 @@ -100,6 +138,7 @@ spec: insecure: << inputs.insecure >> secretRef: name: shelf-registry + verify: ${SHELF_SIGNER_VERIFY} - apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -134,6 +173,7 @@ spec: ref: tag: ${SHELF_CHART_TAG} insecure: ${SHELF_INSECURE_REGISTRY:=false} + verify: ${SHELF_SIGNER_VERIFY} layerSelector: mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip operation: copy diff --git a/platform/kustomization.yaml b/platform/kustomization.yaml index 58ff135..49b41f4 100644 --- a/platform/kustomization.yaml +++ b/platform/kustomization.yaml @@ -4,5 +4,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - traefik + - policy-controller - apps - previews diff --git a/platform/policy-controller/kustomization.yaml b/platform/policy-controller/kustomization.yaml new file mode 100644 index 0000000..f5a4a20 --- /dev/null +++ b/platform/policy-controller/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - release.yaml diff --git a/platform/policy-controller/namespace.yaml b/platform/policy-controller/namespace.yaml new file mode 100644 index 0000000..67bc049 --- /dev/null +++ b/platform/policy-controller/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cosign-system diff --git a/platform/policy-controller/release.yaml b/platform/policy-controller/release.yaml new file mode 100644 index 0000000..45b215e --- /dev/null +++ b/platform/policy-controller/release.yaml @@ -0,0 +1,47 @@ +# The Sigstore policy-controller admits a pod only if every image a ClusterImagePolicy matches +# is signed by the cluster's signer. The ResourceSet apps gives every app such a policy for its +# built images (ghcr.io///**); images no policy matches, such as postgres from +# Docker Hub, are admitted: their digests are fixed in the signed deploy artifact. +# +# Only namespaces labelled policy.sigstore.dev/include=true are checked, which are the app and +# preview namespaces. If the webhook does not answer, pods there are not created +# (failurePolicy: Fail), so it runs twice; one replica uses about 35 Mi. +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: sigstore + namespace: cosign-system +spec: + interval: 1h + url: https://sigstore.github.io/helm-charts +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: policy-controller + namespace: cosign-system +spec: + interval: 1h + chart: + spec: + chart: policy-controller + version: 0.10.8 + sourceRef: + kind: HelmRepository + name: sigstore + install: + crds: Create + upgrade: + crds: CreateReplace + values: + webhook: + replicaCount: 2 + failurePolicy: Fail + configData: + no-match-policy: allow + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 256Mi diff --git a/platform/previews/resourceset.yaml b/platform/previews/resourceset.yaml index f4d4ae0..1537d96 100644 --- a/platform/previews/resourceset.yaml +++ b/platform/previews/resourceset.yaml @@ -9,6 +9,10 @@ # secrets are generated by the chart (platform.preview), never copied from the app, and its # volumes are its own. A tag that disappears from the registry takes the preview with it. # +# A preview is verified like its app: its deploy artifact and the chart only with a valid +# signature of the cluster's signer, and its namespace is checked by the policy-controller +# against the app's ClusterImagePolicy, which the ResourceSet apps generates. +# # The slug was made by shelf (`shelf build-plan --branch`), so the name only strips the prefix. # It is spelled out wherever it is needed: Flux substitutes ${...} in the platform when it applies # it, and a template variable starting with $ would not survive that. @@ -36,6 +40,17 @@ spec: labels: shelf.dev/app: << inputs.app >> shelf.dev/preview: << trimPrefix "branch-" inputs.tag >> + policy.sigstore.dev/include: "true" + --- + # The signer's key for verify, empty for a keyless signer. + apiVersion: v1 + kind: Secret + metadata: + name: shelf-signer + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + annotations: + fluxcd.controlplane.io/copyFrom: shelf-system/signer-key + type: Opaque --- # The login of the app's registry connection, for the preview's images and deploy artifact. apiVersion: v1 @@ -119,6 +134,7 @@ spec: insecure: << inputs.insecure >> secretRef: name: shelf-registry + verify: ${SHELF_SIGNER_VERIFY} --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization @@ -152,6 +168,7 @@ spec: ref: tag: ${SHELF_CHART_TAG} insecure: ${SHELF_INSECURE_REGISTRY:=false} + verify: ${SHELF_SIGNER_VERIFY} layerSelector: mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip operation: copy