From 54b692912501334c0b3b2a7e282c0826b5fae24e Mon Sep 17 00:00:00 2001 From: tweinmann Date: Wed, 28 Oct 2026 17:01:01 +0000 Subject: [PATCH 1/2] Archive Phase 11: verification chain, with two follow-ups The level-3 acceptance passed: the release v0.8.0 is signed keyless, the first tenant build with the new workflow is signed and scanned, an unsigned artifact and an unsigned image are refused, and gretter was exposed only once it was verified. Two follow-ups from the acceptance: - shelf serve reads the kubeconfig again before every operation (ops.Ops.Reload), so a cluster reset no longer needs a restart of the admin UI. - The add form offers "Registry without TLS" unticked on a cluster with a dev registry; apps from GHCR failed when it was ticked. The deltas are merged into openspec/specs, with the new spec verification. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 12 +- README.md | 7 +- internal/cli/serve.go | 4 + internal/ops/ops.go | 16 ++- internal/ops/status.go | 9 +- internal/ops/status_test.go | 25 ++++ internal/server/actions.go | 1 - internal/server/server_test.go | 13 ++ internal/server/ui/new.html | 2 +- .../.openspec.yaml | 0 .../design.md | 36 ++++++ .../proposal.md | 0 .../specs/admin-ui/spec.md | 46 +++++++ .../specs/agent-interface/spec.md | 0 .../specs/app-delivery/spec.md | 0 .../specs/app-exposure/spec.md | 0 .../specs/branch-previews/spec.md | 0 .../specs/cluster-init/spec.md | 0 .../specs/dev-environment/spec.md | 0 .../specs/verification/spec.md | 0 .../tasks.md | 13 +- .../specs/admin-ui/spec.md | 24 ---- openspec/config.yaml | 6 +- openspec/specs/admin-ui/spec.md | 19 ++- openspec/specs/agent-interface/spec.md | 14 ++ openspec/specs/app-delivery/spec.md | 25 ++-- openspec/specs/app-exposure/spec.md | 26 ++++ openspec/specs/branch-previews/spec.md | 16 ++- openspec/specs/cluster-init/spec.md | 28 ++-- openspec/specs/dev-environment/spec.md | 14 +- openspec/specs/verification/spec.md | 122 ++++++++++++++++++ 31 files changed, 408 insertions(+), 70 deletions(-) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/.openspec.yaml (100%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/design.md (89%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/proposal.md (100%) create mode 100644 openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/admin-ui/spec.md rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/specs/agent-interface/spec.md (100%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/specs/app-delivery/spec.md (100%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/specs/app-exposure/spec.md (100%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/specs/branch-previews/spec.md (100%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/specs/cluster-init/spec.md (100%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/specs/dev-environment/spec.md (100%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/specs/verification/spec.md (100%) rename openspec/changes/{phase-11-verification-chain => archive/2026-10-28-phase-11-verification-chain}/tasks.md (95%) delete mode 100644 openspec/changes/phase-11-verification-chain/specs/admin-ui/spec.md create mode 100644 openspec/specs/verification/spec.md diff --git a/CLAUDE.md b/CLAUDE.md index 040dc17..4c70c90 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,7 +104,17 @@ in its own namespace `--` with a quota, limits and generated secrets, a new branch reached a Ready preview in about 1 min 45 s (CI 93 s, then 11 s), deleting it removed the preview within a minute with the workflow's `GITHUB_TOKEN`. Results in `openspec/changes/archive/2026-10-28-phase-10-branch-previews`. -Next: Phase 11 (verification chain). +Phase 11 complete (2026-10-28): the verification chain. The reusable workflow signs every built +image and the deploy artifact keyless with cosign and scans every image with Trivy (summary in +the annotation `dev.shelf.scan`); the release signs chart and platform. Flux verifies every +OCIRepository against the cluster's signer (`shelf init cluster`, keyless by default or +`--signer-key`), the Sigstore policy-controller admits an app's built images only when signed, +and `--cloudflare`/`--quick` refuse an app that is not verified; `shelf app status`, `--json` and +the app page show the chain. Release v0.8.0. Accepted with `tweinmann/shelf-hello`: the build takes +2 min 36 s (Phase 10: 1 min 33 s), an unsigned artifact and an unsigned image were refused, and +`gretter` was exposed only once verified. `shelf serve` now follows a rewritten kubeconfig. +Results in `openspec/changes/archive/2026-10-28-phase-11-verification-chain`. +Next: Phase 12 (hardening). ## Working agreements diff --git a/README.md b/README.md index 25d1545..3b19bb4 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ your own Cloudflare account, or a temporary `*.trycloudflare.com` name that need > shelf is a learning and reference project for platform engineering, and it is > **pre-production**: production is a separate cluster reached through eject (planned). It is > **still under construction**: today a push to your repository deploys your app, and the CLI or -> an admin UI in the browser adds, changes and removes apps. Branch previews, the verification -> chain, the host setup, the installer and eject are planned, not built. See [Status](#status). +> an admin UI in the browser adds, changes and removes apps, every branch runs as a preview, and +> only a signed and scanned app can be exposed. The host setup, the installer and eject are +> planned, not built. See [Status](#status). Where this is going: you run Claude Code (other agents later) in several worktrees at once. Every branch your agents push gets its own preview of the complete stack, database included, in a @@ -111,7 +112,7 @@ one starts. | 8d | Apps stay inside the cluster by default, quick tunnels, no cluster domain | ✅ done | | 9 | Agent interface: `--json`, `shelf app status --wait`, `shelf app open` | ✅ done | | 10 | Branch previews: one isolated preview per branch, discovered from the registry | ✅ done | -| 11 | Verification chain: signatures, scans, provenance; exposure only for verified digests | planned | +| 11 | Verification chain: signatures, scans, provenance; exposure only for verified digests | ✅ done | | 12 | Hardening: Pod Security, NetworkPolicies, smoke tests per preview | planned | | 13 | Host: macOS with Colima and Linux with k3s, `shelf doctor`, `shelf destroy` | planned | | 14 | Service and one-command installer | planned | diff --git a/internal/cli/serve.go b/internal/cli/serve.go index 1cf93c1..553a3b0 100644 --- a/internal/cli/serve.go +++ b/internal/cli/serve.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/tweinmann/shelf/internal/hostcfg" + "github.com/tweinmann/shelf/internal/ops" "github.com/tweinmann/shelf/internal/server" ) @@ -41,6 +42,9 @@ shows nothing but the page that asks for it.`, if err != nil { return err } + // A server outlives the cluster it started with: after a reset the kubeconfig holds + // a new certificate authority, and the old one would fail every request. + shelf.Reload = func() (ops.Target, error) { return ops.LoadTarget(target.kubeconfig, target.context) } home, err := o.shelfHome() if err != nil { return err diff --git a/internal/ops/ops.go b/internal/ops/ops.go index 567eaaf..f0b20d4 100644 --- a/internal/ops/ops.go +++ b/internal/ops/ops.go @@ -196,6 +196,10 @@ type FetchFunc func(ctx context.Context, ref string, auth authn.Authenticator, i // interfere with each other. type Ops struct { Target Target + // Reload, when set, reads the target again before every operation, so that a server that + // runs for days follows a kubeconfig rewritten under it, as when the cluster was recreated + // with a new certificate authority. When it fails, Target is used. + Reload func() (Target, error) Env Env Cluster Cluster @@ -216,8 +220,18 @@ func New(t Target, env Env) *Ops { } } +// target is the cluster the next operation works on. +func (o *Ops) target() Target { + if o.Reload != nil { + if t, err := o.Reload(); err == nil { + return t + } + } + return o.Target +} + // config is the cluster connection every operation hands to internal/cluster. -func (o *Ops) config() *rest.Config { return o.Target.Config } +func (o *Ops) config() *rest.Config { return o.target().Config } var appNameRE = regexp.MustCompile(schema.AppNamePattern) diff --git a/internal/ops/status.go b/internal/ops/status.go index ed6e2a2..0627350 100644 --- a/internal/ops/status.go +++ b/internal/ops/status.go @@ -108,11 +108,12 @@ type Status struct { // Status reads how the cluster is doing. func (o *Ops) Status(ctx context.Context) Status { - s := Status{Context: o.Target.Context} - if o.Target.Config != nil { - s.Server = o.Target.Config.Host + t := o.target() + s := Status{Context: t.Context} + if t.Config != nil { + s.Server = t.Config.Host } - settings, err := o.Cluster.Settings(ctx, o.config()) + settings, err := o.Cluster.Settings(ctx, t.Config) if err != nil { s.Error = err.Error() return s diff --git a/internal/ops/status_test.go b/internal/ops/status_test.go index e50420c..9379754 100644 --- a/internal/ops/status_test.go +++ b/internal/ops/status_test.go @@ -1,9 +1,12 @@ package ops import ( + "errors" "strings" "testing" + "k8s.io/client-go/rest" + "github.com/tweinmann/shelf/internal/cluster" ) @@ -150,3 +153,25 @@ func TestAppAddress(t *testing.T) { }) } } + +// TestReloadTarget covers a server whose kubeconfig is rewritten under it: every operation takes +// the target as it reads now, and the one it started with only while the file cannot be read. +func TestReloadTarget(t *testing.T) { + t.Parallel() + started := Target{Context: "k3d-shelf-dev", Config: &rest.Config{Host: "https://127.0.0.1:6445"}} + o := &Ops{Target: started} + if got := o.config(); got != started.Config { + t.Errorf("without Reload: got %v, want the started target", got) + } + + reset := Target{Context: "k3d-shelf-dev", Config: &rest.Config{Host: "https://127.0.0.1:6445"}} + var err error + o.Reload = func() (Target, error) { return reset, err } + if got := o.config(); got != reset.Config { + t.Errorf("after a reset: got %v, want the reloaded target", got) + } + err = errors.New("kubeconfig: half written") + if got := o.config(); got != started.Config { + t.Errorf("unreadable kubeconfig: got %v, want the started target", got) + } +} diff --git a/internal/server/actions.go b/internal/server/actions.go index 3d2a9f2..26ffe47 100644 --- a/internal/server/actions.go +++ b/internal/server/actions.go @@ -51,7 +51,6 @@ func (s *Server) newAppForm(w http.ResponseWriter, r *http.Request) { base: s.base(r, "Add an app"), Status: status, Connections: s.connections(ctx), - Insecure: status.Settings.InsecureRegistry, }) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index b7800a2..06dc215 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -484,6 +484,19 @@ func TestPages(t *testing.T) { golden(t, "new", get(h, "/apps/new", cookie)) }) + // A cluster with a dev registry offers to pull without TLS, but never ticks it: the + // artifacts most apps come from are on GHCR, which speaks nothing but TLS. + t.Run("add form, cluster with a dev registry", func(t *testing.T) { + t.Parallel() + dev := runningPlatform() + dev.status.Settings.InsecureRegistry = true + h, cookie := claimed(t, dev) + body := get(h, "/apps/new", cookie).Body.String() + if !strings.Contains(body, ``) { + t.Errorf("the form does not offer an unticked registry without TLS:\n%s", body) + } + }) + t.Run("quick tunnels", func(t *testing.T) { t.Parallel() h, cookie := claimed(t, quickPlatform()) diff --git a/internal/server/ui/new.html b/internal/server/ui/new.html index 26fd9a7..7917957 100644 --- a/internal/server/ui/new.html +++ b/internal/server/ui/new.html @@ -13,7 +13,7 @@

Add an app

{{if .Status.Settings.InsecureRegistry}} +Registry without TLS (a dev registry; leave off for GHCR) {{end}}
Connections diff --git a/openspec/changes/phase-11-verification-chain/.openspec.yaml b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/.openspec.yaml similarity index 100% rename from openspec/changes/phase-11-verification-chain/.openspec.yaml rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/.openspec.yaml diff --git a/openspec/changes/phase-11-verification-chain/design.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/design.md similarity index 89% rename from openspec/changes/phase-11-verification-chain/design.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/design.md index 5e27c57..a570f75 100644 --- a/openspec/changes/phase-11-verification-chain/design.md +++ b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/design.md @@ -466,3 +466,39 @@ Deviations found while implementing: registry)`: nothing scans in level 2, and the summary only lets a dev app count as verified. - **`smoke-tenant`.** It installs a released platform with keyless verification, because the tenant's artifacts are signed keyless and a cluster accepts one signer. + +Level 3 with the private tenant `tweinmann/shelf-hello` and release v0.8.0 on 2026-10-28; the dev +cluster runs the released platform with the default keyless signer: + +| Check | Result | +|---|---| +| Platform `v0.8.0` and chart `0.8.0` | keyless signatures of `release.yml`, accepted by `cosign verify` and by the cluster (`SourceVerified=True`) | +| First tenant build with the new workflow (commit `47c927b`) | CI run 2 min 36 s (Phase 10: 1 min 33 s, cache empty both times); artifact and built images signed, all four components scanned | +| `gretter` added from it | verified; built images admitted; exposure allowed | +| Unsigned artifact of the day before (`sha-f4a77c7`, built with v0.7.0) as a new app | `shelf app add` fails after 7 s with reason and hint; no pod ever ran | +| Unsigned built image of the day before | denied by the admission policy; the signed one of the same component admitted | +| `--cloudflare tobile` for that unverified app | refused before any Cloudflare call; no tunnel, no cloudflared | +| `--cloudflare tobile` for the verified `gretter` | deployed, verified, then tunnel and record created; `https://gretter.tobile.ch/` answers | + +Signing, scanning and attesting add about a minute to a tenant build. Most of it is Trivy and its +database on a cold cache; later runs restore the DB from the cache. + +Two things differ from the plan: +- **Unsigned artifact.** Task 9.1 planned an unsigned artifact pushed by hand under a branch tag. + The check used an unsigned artifact that already existed, as a whole app, which needs no + registry write access from here. The preview path is covered by `smoke-verify`. +- **Branch push.** No branch push was timed at level 3. A branch build runs the same steps as + `main`, and the cluster side was measured at level 2. + +Two follow-ups from the acceptance, fixed in this change: +- **`shelf serve` after a cluster reset.** The server kept the kubeconfig it started with, so a + recreated cluster with a new certificate authority failed every page with `x509: certificate + signed by unknown authority` until a restart. `ops.Ops` now has an optional `Reload`, which only + `shelf serve` sets: every operation reads the kubeconfig again, and keeps the one it started with + while the file cannot be read. client-go caches the transport by its TLS settings, so an + unchanged kubeconfig costs a file read, not a new connection. Alternatives: reload only after a + TLS error (needs a retry inside every call and still misses a changed server address), or watch + the file (a goroutine and a lock for what one read per request does). +- **"Registry without TLS" on the add form.** The form ticked it on every cluster installed with + `--insecure-registry`, as the dev cluster is, so an app added from GHCR there failed. It is now + offered unticked, as `--insecure-registry` of `shelf app add` defaults to off. diff --git a/openspec/changes/phase-11-verification-chain/proposal.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/proposal.md similarity index 100% rename from openspec/changes/phase-11-verification-chain/proposal.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/proposal.md diff --git a/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/admin-ui/spec.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/admin-ui/spec.md new file mode 100644 index 0000000..5c0de95 --- /dev/null +++ b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/admin-ui/spec.md @@ -0,0 +1,46 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Serve command and listener +The system SHALL provide `shelf serve`, which listens on `:7654` by default (overridable with +`--listen`), without TLS, prints the address of the UI, the cluster context and the state +directory, shuts down gracefully when its context ends (Ctrl-C or SIGTERM), and reads the +kubeconfig again before every operation, keeping the one it started with while the file cannot be +read. + +#### Scenario: Default start +- **WHEN** `shelf serve` is started without `--listen` +- **THEN** it listens on port 7654 on every address of the machine +- **AND** it prints `The admin UI is at http://localhost:7654/`, the cluster context and the + state directory + +#### Scenario: Health check +- **WHEN** `GET /healthz` is requested, with or without a session +- **THEN** the server answers 200 with the plain-text body `ok` + +#### Scenario: Cluster recreated under a running server +- **WHEN** the cluster is recreated with a new certificate authority and its kubeconfig rewritten + while `shelf serve` runs +- **THEN** the next page reads the cluster through the new kubeconfig, without a restart + +### Requirement: App page +The system SHALL show at `/apps/` the app's phase, the first stage that is not ready with +its message and hint, its address, artifact, revision and deploy time, its components each with +its phase and address (routed ones as links, the others marked `inside the cluster only` or `not +exposed`), its access settings, the names of its generated secrets, its verification as `shelf +app status` prints it, and the diagnosis chain under `How it gets here`. + +#### Scenario: Routed and internal components +- **WHEN** an app has a routed `web` component and an unrouted `db` component +- **THEN** `web` is shown with a link to its URL and `db` with its address and `inside the + cluster only` + +#### Scenario: Unknown app +- **WHEN** `/apps/` is requested for an app that does not exist or a name that is not a + valid app name +- **THEN** an error page with status 404 is shown + +#### Scenario: Unverified app +- **WHEN** the running artifact of an app is unsigned +- **THEN** the page says it is not verified and why, and that it cannot be exposed diff --git a/openspec/changes/phase-11-verification-chain/specs/agent-interface/spec.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/agent-interface/spec.md similarity index 100% rename from openspec/changes/phase-11-verification-chain/specs/agent-interface/spec.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/agent-interface/spec.md diff --git a/openspec/changes/phase-11-verification-chain/specs/app-delivery/spec.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/app-delivery/spec.md similarity index 100% rename from openspec/changes/phase-11-verification-chain/specs/app-delivery/spec.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/app-delivery/spec.md diff --git a/openspec/changes/phase-11-verification-chain/specs/app-exposure/spec.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/app-exposure/spec.md similarity index 100% rename from openspec/changes/phase-11-verification-chain/specs/app-exposure/spec.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/app-exposure/spec.md diff --git a/openspec/changes/phase-11-verification-chain/specs/branch-previews/spec.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/branch-previews/spec.md similarity index 100% rename from openspec/changes/phase-11-verification-chain/specs/branch-previews/spec.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/branch-previews/spec.md diff --git a/openspec/changes/phase-11-verification-chain/specs/cluster-init/spec.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/cluster-init/spec.md similarity index 100% rename from openspec/changes/phase-11-verification-chain/specs/cluster-init/spec.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/cluster-init/spec.md diff --git a/openspec/changes/phase-11-verification-chain/specs/dev-environment/spec.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/dev-environment/spec.md similarity index 100% rename from openspec/changes/phase-11-verification-chain/specs/dev-environment/spec.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/dev-environment/spec.md diff --git a/openspec/changes/phase-11-verification-chain/specs/verification/spec.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/verification/spec.md similarity index 100% rename from openspec/changes/phase-11-verification-chain/specs/verification/spec.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/specs/verification/spec.md diff --git a/openspec/changes/phase-11-verification-chain/tasks.md b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/tasks.md similarity index 95% rename from openspec/changes/phase-11-verification-chain/tasks.md rename to openspec/changes/archive/2026-10-28-phase-11-verification-chain/tasks.md index 9c3e6e8..a32080d 100644 --- a/openspec/changes/phase-11-verification-chain/tasks.md +++ b/openspec/changes/archive/2026-10-28-phase-11-verification-chain/tasks.md @@ -141,7 +141,7 @@ - prints the policy-controller's RAM. Run it on the dev cluster (level 2) and record the RAM in design.md under "Results". -- [ ] 7.4 CI: add `smoke-verify` to the full set in `ci.yml` and to `select-smokes.sh` +- [x] 7.4 CI: add `smoke-verify` to the full set in `ci.yml` and to `select-smokes.sh` (`platform/*`, `internal/cluster/*`, `hack/lib.sh`, `hack/smoke/verify.sh`); every smoke job creates its own key pair. Verify the selection on example paths and in the PR run. @@ -170,16 +170,23 @@ ## 9. Acceptance (level 3, after release v0.8.0) -- [ ] 9.1 With `tweinmann/shelf-hello`: +- [x] 9.1 With `tweinmann/shelf-hello`: - add `id-token: write`, push, and check the keyless signatures and the attestation with `cosign verify`; - upgrade the dev cluster with the release platform: the confirmation lists `gretter`, and `gretter` keeps running unverified until the next push, then is verified; - push an unsigned artifact by hand under a branch tag: no preview applies it. -- [ ] 9.2 Exposure gate against the real Cloudflare connection: +- [x] 9.2 Exposure gate against the real Cloudflare connection: - `shelf app credentials gretter --private`, then `--cloudflare tobile` succeeds while `gretter` is verified; - the same for an app whose artifact has no scan is refused. Record the CI time and push-to-Ready of a branch against Phase 10 (design decision 15) in design.md under "Results". + +## 10. Follow-ups from the acceptance + +- [x] 10.1 `shelf serve` reads the kubeconfig again before every operation (`ops.Ops.Reload`), so + a cluster reset needs no restart of the server; test in `internal/ops`. +- [x] 10.2 The add form offers "Registry without TLS" unticked on a cluster with a dev registry; + test in `internal/server`. diff --git a/openspec/changes/phase-11-verification-chain/specs/admin-ui/spec.md b/openspec/changes/phase-11-verification-chain/specs/admin-ui/spec.md deleted file mode 100644 index ed6e3b4..0000000 --- a/openspec/changes/phase-11-verification-chain/specs/admin-ui/spec.md +++ /dev/null @@ -1,24 +0,0 @@ -# Spec Delta - -## MODIFIED Requirements - -### Requirement: App page -The system SHALL show at `/apps/` the app's phase, the first stage that is not ready with -its message and hint, its address, artifact, revision and deploy time, its components each with -its phase and address (routed ones as links, the others marked `inside the cluster only` or `not -exposed`), its access settings, the names of its generated secrets, its verification as `shelf -app status` prints it, and the diagnosis chain under `How it gets here`. - -#### Scenario: Routed and internal components -- **WHEN** an app has a routed `web` component and an unrouted `db` component -- **THEN** `web` is shown with a link to its URL and `db` with its address and `inside the - cluster only` - -#### Scenario: Unknown app -- **WHEN** `/apps/` is requested for an app that does not exist or a name that is not a - valid app name -- **THEN** an error page with status 404 is shown - -#### Scenario: Unverified app -- **WHEN** the running artifact of an app is unsigned -- **THEN** the page says it is not verified and why, and that it cannot be exposed diff --git a/openspec/config.yaml b/openspec/config.yaml index a5b681c..22973b2 100644 --- a/openspec/config.yaml +++ b/openspec/config.yaml @@ -17,9 +17,9 @@ context: | artefact becomes reachable beyond a preview, and the same verified state goes to any Kubernetes cloud as a pull request (eject), without depending on shelf. shelf runs on any Linux host with k3s, with a one-step install on a Mac (Colima). Today it runs against any kubecontext, developed - on the dev cluster; previews, verification, the host setup and eject are planned. shelf is - pre-production: an app's "main" is its stable, verified state, not production. Learning and - reference project for platform engineering. + on the dev cluster, with branch previews and the verification chain; the host setup and eject + are planned. shelf is pre-production: an app's "main" is its stable, verified state, not + production. Learning and reference project for platform engineering. OpenSpec is the only planning source: `openspec/specs/` is what shelf does today, `openspec/changes/` what it is about to do (the roadmap is the `phase--` changes), and diff --git a/openspec/specs/admin-ui/spec.md b/openspec/specs/admin-ui/spec.md index 0eadac3..f6ef588 100644 --- a/openspec/specs/admin-ui/spec.md +++ b/openspec/specs/admin-ui/spec.md @@ -12,7 +12,9 @@ stored under `~/.shelf`. ### Requirement: Serve command and listener The system SHALL provide `shelf serve`, which listens on `:7654` by default (overridable with `--listen`), without TLS, prints the address of the UI, the cluster context and the state -directory, and shuts down gracefully when its context ends (Ctrl-C or SIGTERM). +directory, shuts down gracefully when its context ends (Ctrl-C or SIGTERM), and reads the +kubeconfig again before every operation, keeping the one it started with while the file cannot be +read. #### Scenario: Default start - **WHEN** `shelf serve` is started without `--listen` @@ -24,6 +26,11 @@ directory, and shuts down gracefully when its context ends (Ctrl-C or SIGTERM). - **WHEN** `GET /healthz` is requested, with or without a session - **THEN** the server answers 200 with the plain-text body `ok` +#### Scenario: Cluster recreated under a running server +- **WHEN** the cluster is recreated with a new certificate authority and its kubeconfig rewritten + while `shelf serve` runs +- **THEN** the next page reads the cluster through the new kubeconfig, without a restart + ### Requirement: Claim with a setup code The system SHALL generate a random setup code on the first start of an unclaimed instance, store it in `~/.shelf/admin.yaml` so that it survives restarts until used, print it on every start @@ -155,9 +162,9 @@ reason of the first broken step of its diagnosis. ### Requirement: App page The system SHALL show at `/apps/` the app's phase, the first stage that is not ready with its message and hint, its address, artifact, revision and deploy time, its components each with -its phase and address (the routed ones as links to their URL, the others with their in-cluster -address marked `inside the cluster only` or `not exposed`), its access settings, the names of its -generated secrets, and the whole diagnosis chain under `How it gets here`. +its phase and address (routed ones as links, the others marked `inside the cluster only` or `not +exposed`), its access settings, the names of its generated secrets, its verification as `shelf +app status` prints it, and the diagnosis chain under `How it gets here`. #### Scenario: Routed and internal components - **WHEN** an app has a routed `web` component and an unrouted `db` component @@ -169,6 +176,10 @@ generated secrets, and the whole diagnosis chain under `How it gets here`. valid app name - **THEN** an error page with status 404 is shown +#### Scenario: Unverified app +- **WHEN** the running artifact of an app is unsigned +- **THEN** the page says it is not verified and why, and that it cannot be exposed + ### Requirement: Changes run as jobs The system SHALL run every change made from the UI (add an app, deploy another artifact or tag, deploy again, change an app's connections and domain, remove an app, save or remove a connection) diff --git a/openspec/specs/agent-interface/spec.md b/openspec/specs/agent-interface/spec.md index 476cbe0..0151b6a 100644 --- a/openspec/specs/agent-interface/spec.md +++ b/openspec/specs/agent-interface/spec.md @@ -179,3 +179,17 @@ and `previewLimit` and `previewLimitReached` when the app has previews. #### Scenario: App without previews - **WHEN** an app has no previews switched on - **THEN** its `AppStatus` has no `previews`, `previewLimit` or `previewLimitReached` field + +### Requirement: Verification in the agent interface +The `AppStatus` of an app or preview SHALL hold `verification` with `verified`, `reason` and +`hint` when not verified, `source`, `signer`, `exposable`, and `scan`, one entry per component +with `component`, `digest`, `scanned`, `critical`, `high`, `scanner` and `scannedAt`; a +`WaitResult` with outcome `Ready` says nothing about verification. + +#### Scenario: Verified app as JSON +- **WHEN** a user runs `shelf app status hello --json` for a verified app with `web` and `db` +- **THEN** `verification.verified` and `verification.exposable` are true and `scan` has two entries + +#### Scenario: Agent reads why it cannot be exposed +- **WHEN** the running artifact has no scan summary +- **THEN** `verification.verified` is false and `reason` names the missing scan diff --git a/openspec/specs/app-delivery/spec.md b/openspec/specs/app-delivery/spec.md index b72e386..6ee4b09 100644 --- a/openspec/specs/app-delivery/spec.md +++ b/openspec/specs/app-delivery/spec.md @@ -86,9 +86,9 @@ branch. ### Requirement: Reusable tenant workflow The reusable workflow `.github/workflows/build.yml` SHALL validate the app.yaml, build every -`build:` component for `linux/arm64` as `ghcr.io///`, render the -app.yaml, and push the deploy artifact `ghcr.io//` tagged `sha-` and -`main` for the default branch or `branch-` for another branch, annotated with +`build:` component for the platforms of its input `platforms` (default `linux/arm64`) as +`ghcr.io///`, and push the rendered deploy artifact +`ghcr.io//` tagged `sha-` and `main` or `branch-`, annotated with `dev.shelf.package=` and the commit's source and revision. When a branch is deleted, it SHALL delete that branch's artifact. @@ -114,6 +114,10 @@ SHALL delete that branch's artifact. - **WHEN** the caller sets the input `shelf-version` - **THEN** the workflow installs that release's linux/arm64 binary after checking its checksum, and otherwise the latest release +#### Scenario: Two platforms +- **WHEN** the caller sets `platforms: linux/arm64,linux/amd64` +- **THEN** every built image is a multi-arch index of both, and the rendered values pin the index digest + ### Requirement: Build cache in the tenant workflow The reusable workflow SHALL reuse image layers between runs through a build cache in the registry, `ghcr.io///:buildcache`, which only runs that publish write. @@ -124,13 +128,18 @@ registry, `ghcr.io///:buildcache`, which only runs th ### Requirement: Release publication Pushing a `v*` tag MUST publish a shelf release: CLI binaries for linux and darwin (amd64 and -arm64) with checksums, the `shelf-app` chart, and the platform artifact, all at the release -version, and then move the major tag (for example `v0`) before marking the release as latest. +arm64) with checksums, the `shelf-app` chart and the platform artifact, both signed with keyless +cosign by `release.yml`, all at the release version, and then move the major tag (for example +`v0`) before marking the release as latest. #### Scenario: Tenant on the major tag - **WHEN** a tenant uses `tweinmann/shelf/.github/workflows/build.yml@v0` and a new v0 release is published - **THEN** the tenant's next build uses the new workflow together with the matching latest binary +#### Scenario: Signed chart and platform +- **WHEN** release `v0.8.0` is published +- **THEN** `ghcr.io/tweinmann/shelf/charts/shelf-app:0.8.0` and the platform artifact `v0.8.0` carry a signature whose subject is `release.yml` of `tweinmann/shelf` at that tag + ### Requirement: App naming The system SHALL name an app at `shelf app add `, independent of app.yaml's `name`, and MUST reject a name that is not a DNS label of at most 40 characters, that contains `--` (which separates @@ -321,9 +330,9 @@ state is that of the first stage that is not Ready. ### Requirement: App status command `shelf app status` SHALL list every app with its state and, when not Ready, the failing stage and reason in one line; `shelf app status ` SHALL print the app's address, artifact, registry -connection, running revision, the git commit that revision was built from, deploy time, -components, every stage with its message and, if not Ready, its hint, and the app's previews with -their branch tag, state and commit; `--preview ` SHALL print the same for one preview. +connection, running revision and commit, deploy time, components, its verification, every stage +with its message and, if not Ready, its hint, and its previews with tag, state and commit; +`--preview ` SHALL print the same for one preview. #### Scenario: No apps - **WHEN** the cluster has no registered apps diff --git a/openspec/specs/app-exposure/spec.md b/openspec/specs/app-exposure/spec.md index 356a6c5..a4a857d 100644 --- a/openspec/specs/app-exposure/spec.md +++ b/openspec/specs/app-exposure/spec.md @@ -292,3 +292,29 @@ so that the preview's routes and `stripPrefix` behave as the app's. #### Scenario: Branch without a preview - **WHEN** `hello` has no preview for the branch `gone` - **THEN** the command fails saying so and naming the previews `hello` has, before it listens on any port + +### Requirement: Only a verified app is exposed +`shelf app add` and `shelf app credentials` SHALL refuse `--cloudflare` and `--quick` for an app +that has no tunnel yet unless the app is verified, with the reason, before any Cloudflare call or +tunnel; a new app with such a flag SHALL be added privately and waited for first, and stays added +and private when the check fails. An app that is already exposed keeps its exposure. + +#### Scenario: Unverified app refused +- **WHEN** a user runs `shelf app credentials hello --quick` and the running artifact of `hello` is unsigned +- **THEN** the command fails with the reason and a hint to push again with the current workflow, and no cloudflared and no tunnel are created + +#### Scenario: New app exposed after its rollout +- **WHEN** a user runs `shelf app add hello oci://ghcr.io/acme/hello:main --cloudflare acme --domain hello.example.com` for a signed and scanned artifact +- **THEN** the app is added, rolls out privately, is found verified, and only then gets its tunnel and DNS record + +#### Scenario: New app that is not verified +- **WHEN** the same command runs for an artifact without a scan summary +- **THEN** the app is added and private, and the command exits 1 naming the missing scan + +#### Scenario: Same gate in the admin UI +- **WHEN** the access form of the app page asks for a quick tunnel for an unverified app +- **THEN** the job fails with the same reason, as it calls the same operation + +#### Scenario: Exposed app keeps its tunnel +- **WHEN** an exposed app gets a new artifact without a scan summary +- **THEN** it keeps its tunnel and its status shows it is not verified diff --git a/openspec/specs/branch-previews/spec.md b/openspec/specs/branch-previews/spec.md index 804bc83..6a87a24 100644 --- a/openspec/specs/branch-previews/spec.md +++ b/openspec/specs/branch-previews/spec.md @@ -6,6 +6,7 @@ user's own hardware, discovered from the registry, reachable only from the opera and removed when the branch is. ## Requirements + ### Requirement: Previews are the operator's decision `shelf app add` and `shelf app credentials` SHALL take `--previews[=N]`, which gives the app at most N previews (3 without a number, at most 10), and `--no-previews`, which removes them all; an app has @@ -75,9 +76,10 @@ answer at `--.shelf.internal` inside the cluster, and carry the label ### Requirement: Preview contents The platform SHALL give each preview the same deploy chain as an app (its deploy artifact at the -branch tag, checked every 30 seconds, the app values, the chart, the release), the app's registry login, a ResourceQuota -(requests of 1 CPU, 2Gi memory, 5Gi storage) and a LimitRange that gives containers without -requests a default; it MUST NOT give a preview a tunnel, cloudflared or a DNS record. +branch tag, checked every 30 seconds, the app values, the chart, the release), verified against +the cluster's signer, the app's registry login, a ResourceQuota (requests of 1 CPU, 2Gi memory, +5Gi storage) and a LimitRange that gives containers without requests a default; it MUST NOT give +a preview a tunnel, cloudflared or a DNS record. #### Scenario: Preview of an exposed app - **WHEN** an app exposed through a Cloudflare connection has a preview @@ -87,6 +89,14 @@ requests a default; it MUST NOT give a preview a tunnel, cloudflared or a DNS re - **WHEN** a preview's pods would request more than 2Gi memory together - **THEN** the pods beyond it are refused, and the preview's pods stage says so +#### Scenario: Unsigned branch artifact +- **WHEN** an artifact without a signature is pushed as `branch-feature-login` +- **THEN** the preview `hello--feature-login` does not apply it, and its deploy artifact stage says the verification failed + +#### Scenario: Built image of a preview +- **WHEN** a preview's values name an unsigned built image of the app's package +- **THEN** its pod is denied by the app's admission policy, as in the app's namespace + ### Requirement: A preview's own state A preview SHALL have its own volumes and its own secret values, generated in the cluster when the preview is first created and kept for its lifetime; it MUST NOT share a volume, a secret value or a diff --git a/openspec/specs/cluster-init/spec.md b/openspec/specs/cluster-init/spec.md index fdf1c02..bd25ee9 100644 --- a/openspec/specs/cluster-init/spec.md +++ b/openspec/specs/cluster-init/spec.md @@ -39,6 +39,10 @@ the host name pattern, and SHALL ask `Proceed? [y/N]` before applying anything; - **WHEN** `--yes` is passed - **THEN** no question is asked and the installation runs +#### Scenario: Apps without a verified artifact +- **WHEN** the platform is upgraded to one that verifies, and apps run artifacts that are not verified +- **THEN** the confirmation lists those apps and says that each needs one build with the current workflow before it gets new deploys, and that their running pods keep running + ### Requirement: Platform and chart artifacts The system SHALL take the platform artifact from `--platform` and the shelf-app chart from `--chart`, each as `oci:///:`; without the flags a release build MUST @@ -71,8 +75,8 @@ ready. The system SHALL apply a FluxInstance `flux` in `flux-system` with the pinned Flux distribution from `ghcr.io/fluxcd`, the components source-, kustomize-, helm- and notification-controller, network policies enabled, and a sync of the platform artifact as an OCIRepository at path `./` -every `1m`; the sync Kustomization MUST wait for everything it applied and substitute variables -from the ConfigMap `flux-system/shelf-config`. +every `1m`, verified against the cluster's signer; the sync Kustomization MUST wait for +everything it applied and substitute variables from `flux-system/shelf-config`. #### Scenario: Insecure dev registry - **WHEN** `--insecure-registry` is passed @@ -82,22 +86,28 @@ from the ConfigMap `flux-system/shelf-config`. - **WHEN** the FluxInstance has been applied - **THEN** the command waits until it reports Ready for its current generation +#### Scenario: Unsigned platform +- **WHEN** `shelf init cluster --platform` names a platform artifact without a valid signature +- **THEN** the platform OCIRepository reports a failed verification, and the command fails while waiting for the platform with that message + ### Requirement: Platform contents The platform artifact SHALL contain Traefik (Helm chart from `oci://ghcr.io/traefik/helm/traefik` -in namespace `traefik`, Service of type ClusterIP, ingress class `traefik` that is not the -default, Kubernetes Ingress and CRD providers enabled) and, in `shelf-system`, the ResourceSet -`apps`, which generates every app from the providers labelled `shelf.dev/app`, and the -ResourceSet `previews`, which generates every preview from those labelled `shelf.dev/previews`. +in namespace `traefik`, ClusterIP Service, ingress class `traefik` that is not the default, +Ingress and CRD providers enabled), the Sigstore policy-controller in `cosign-system`, and, in +`shelf-system`, the ResourceSet `apps`, which generates every app from the providers labelled +`shelf.dev/app`, and the ResourceSet `previews`, which generates every preview from those +labelled `shelf.dev/previews`. #### Scenario: Platform applied - **WHEN** the installation completes -- **THEN** Traefik runs in `traefik` reachable at `traefik.traefik.svc.cluster.local:80` and the ResourceSets `shelf-system/apps` and `shelf-system/previews` exist +- **THEN** Traefik runs in `traefik` reachable at `traefik.traefik.svc.cluster.local:80` the ResourceSets `shelf-system/apps` and `shelf-system/previews` exist, and the policy-controller's webhook is ready ### Requirement: System namespace and cluster settings The system SHALL apply the namespace `shelf-system` (labelled `app.kubernetes.io/managed-by: shelf`) and the ConfigMap `flux-system/shelf-config` with the keys -`SHELF_HOST_SUFFIX`, `SHELF_CHART_URL`, `SHELF_CHART_TAG` and `SHELF_INSECURE_REGISTRY`; the -cluster has no domain setting. +`SHELF_HOST_SUFFIX`, `SHELF_CHART_URL`, `SHELF_CHART_TAG`, `SHELF_INSECURE_REGISTRY` and the +signer keys `SHELF_SIGNER_MODE`, `SHELF_SIGNER_ISSUER` and `SHELF_SIGNER_SUBJECT`; the cluster has +no domain setting. #### Scenario: Settings written - **WHEN** `shelf init cluster --host-suffix -dev` completes diff --git a/openspec/specs/dev-environment/spec.md b/openspec/specs/dev-environment/spec.md index 268147b..09ea6dd 100644 --- a/openspec/specs/dev-environment/spec.md +++ b/openspec/specs/dev-environment/spec.md @@ -40,9 +40,9 @@ container's own host name. ### Requirement: Dev cluster `just cluster-up` SHALL create, or start if it exists, the k3d cluster `shelf-dev` with the k3s image pinned in the Dockerfile, without k3s's own Traefik and without k3d's load balancer, its API -server on the container's loopback, and the plain-HTTP registry `shelf-registry:5000`, reachable -by that name from pods and from the devcontainer, which allows deleting manifests. It MUST write the kubeconfig only to -`KUBECONFIG` and never change a default kubeconfig or switch a context. +server on the container's loopback, the plain-HTTP registry `shelf-registry.local:5000` that allows +deleting manifests, and a dev cosign key pair under `.local/cosign/`. It MUST write the +kubeconfig only to `KUBECONFIG` and never change a default kubeconfig or switch a context. #### Scenario: After a container restart - **WHEN** the devcontainer was rebuilt or restarted and `just cluster-up` runs @@ -53,9 +53,13 @@ by that name from pods and from the devcontainer, which allows deleting manifest - **THEN** cluster and registry are deleted and created empty, and `just init-cluster` pushes the platform and chart again before installing them #### Scenario: Deleting a tag -- **WHEN** a smoke test deletes a branch tag from `shelf-registry:5000` +- **WHEN** a smoke test deletes a branch tag from `shelf-registry.local:5000` - **THEN** the registry accepts the deletion, so a preview can be removed at level 2 as on GHCR +#### Scenario: Dev artifacts are signed +- **WHEN** `just init-cluster`, `just platform-push`, `just chart-push` or a smoke test pushes an artifact or a built image to `shelf-registry.local:5000` +- **THEN** it is signed with the dev key without a transparency log entry, and the cluster, installed with `--signer-key .local/cosign/cosign.pub`, verifies it + ### Requirement: Three test levels Checks SHALL be split into level 1 (no cluster: `just test`, which runs gofmt, `go vet`, `go test -race`, Helm lint, kubeconform, example validation and `openspec validate --all --strict`), @@ -76,7 +80,7 @@ Cloudflare), and a task that needs level 2 or 3 MUST say so. #### Scenario: Level 2 in full - **WHEN** a commit is pushed to `main`, or a pull request carries the label `level-2-full` -- **THEN** CI runs `smoke-secrets`, `smoke-chart`, `smoke-init`, `smoke-apps`, `smoke-agent` and `smoke-previews` with `SMOKE_FULL=1` +- **THEN** CI runs `smoke-secrets`, `smoke-chart`, `smoke-init`, `smoke-apps`, `smoke-agent`, `smoke-previews` and `smoke-verify` with `SMOKE_FULL=1` #### Scenario: main fails - **WHEN** CI fails on `main` diff --git a/openspec/specs/verification/spec.md b/openspec/specs/verification/spec.md new file mode 100644 index 0000000..fecd8f6 --- /dev/null +++ b/openspec/specs/verification/spec.md @@ -0,0 +1,122 @@ +# verification Specification + +## Purpose +Lets shelf and its operator know that what runs is what the reusable workflow built from a +known commit: signed by the expected identity, scanned, and verified by the cluster before it +runs, so that only a verified app goes beyond a preview. + +## Requirements + +### Requirement: Signing in the tenant workflow +When it publishes, the reusable workflow SHALL sign every built image and the deploy artifact by +digest with keyless cosign, using the job's GitHub OIDC token, so the caller's workflow MUST grant +`id-token: write`; a run that cannot sign MUST fail without moving the `main` or `branch-` +tag. + +#### Scenario: Published build +- **WHEN** the workflow publishes for a push to `main` +- **THEN** every built image and the deploy artifact have a cosign signature whose certificate names shelf's workflow as subject and `https://token.actions.githubusercontent.com` as issuer + +#### Scenario: Missing permission +- **WHEN** the caller's `deploy.yml` does not grant `id-token: write` +- **THEN** the run fails at signing, and the channel tag still points at the previous artifact + +#### Scenario: Pull request +- **WHEN** the workflow runs for a pull request +- **THEN** it signs nothing, as it publishes nothing + +### Requirement: Scan in the tenant workflow +When it publishes, the reusable workflow SHALL scan every image digest the rendered values name, +built and third-party, with a pinned Trivy, attach the full report to each built image as a +signed cosign attestation of type `vuln`, and write the annotation `dev.shelf.scan` to the deploy +artifact before it is signed: per image digest the CRITICAL and HIGH counts, the scanner version +and the scan time. + +#### Scenario: App with a third-party database +- **WHEN** an app has a built `web` and a `db` with `image: postgres:16` +- **THEN** `dev.shelf.scan` lists both digests with their counts, and only the `web` image has a `vuln` attestation + +#### Scenario: Findings do not fail the build +- **WHEN** an image has CRITICAL findings +- **THEN** the workflow still publishes, and the counts are in `dev.shelf.scan` + +### Requirement: Signer of a cluster +The cluster settings SHALL name the signer every verification expects: keyless by default with +`SHELF_SIGNER_ISSUER` and the subject regex `SHELF_SIGNER_SUBJECT` covering shelf's `build.yml` +and `release.yml` (`--signer-issuer`, `--signer-subject`), or a cosign public key given with +`--signer-key ` and kept in `shelf-system/signer-key`; there MUST be no setting that turns +verification off. + +#### Scenario: Default signer +- **WHEN** `shelf init cluster` runs without signer flags +- **THEN** `shelf-config` holds the GitHub Actions issuer and a subject regex matching `https://github.com/tweinmann/shelf/.github/workflows/build.yml@refs/tags/v0` + +#### Scenario: Key for a test cluster +- **WHEN** `shelf init cluster --signer-key cosign.pub` runs +- **THEN** `shelf-system/signer-key` holds the key, and every app and preview namespace gets it as `shelf-signer` + +#### Scenario: Unreadable key +- **WHEN** the file given to `--signer-key` is not a PEM public key +- **THEN** the command fails before it changes the cluster + +### Requirement: Artifacts verified by Flux +Every OCIRepository the platform creates for an app or a preview, the deploy artifact and the +`shelf-app` chart, SHALL verify the cosign signature against the cluster's signer, so that an +artifact without a valid signature is not applied and what already runs keeps running. + +#### Scenario: Unsigned deploy artifact +- **WHEN** an artifact without a signature is pushed under an app's tag +- **THEN** the app's `deploy` OCIRepository reports a failed verification, the release keeps the previous values, and the pods keep running + +#### Scenario: Signed by someone else +- **WHEN** an artifact is signed by an identity the subject regex does not match +- **THEN** it is not applied, as if unsigned + +### Requirement: Admission of built images +The platform SHALL run the Sigstore policy-controller and give every app a ClusterImagePolicy +`shelf-` that admits an image under `ghcr.io///` only with a valid signature +from the cluster's signer, in the namespaces of the app and its previews; images no policy +matches SHALL be admitted, as their digests are fixed in the signed deploy artifact. + +#### Scenario: Unsigned built image +- **WHEN** the values of an app name a built image of its package that has no signature +- **THEN** the pod is denied by the admission webhook, and the pods stage shows the denial with a hint about the signature + +#### Scenario: Third-party image +- **WHEN** an app runs `postgres:16@sha256:…` from Docker Hub +- **THEN** the pod is admitted + +#### Scenario: Removed with the app +- **WHEN** an app is removed with `shelf app rm` +- **THEN** its ClusterImagePolicy is gone + +### Requirement: Verified app +An app or preview SHALL count as verified when its deploy artifact and chart sources report a +verified signature for the revisions they hold, its release runs the values of that deploy +artifact revision, and the artifact's `dev.shelf.scan` names every image digest in those values; +otherwise the reason SHALL be the first of these that does not hold. + +#### Scenario: Verified +- **WHEN** a signed and scanned artifact has rolled out +- **THEN** the app is verified, and its scan summary per component is known + +#### Scenario: No scan summary +- **WHEN** a signed artifact without `dev.shelf.scan` runs +- **THEN** the app is not verified, with the reason that the artifact has no scan + +#### Scenario: Rollout still pending +- **WHEN** a new signed artifact is fetched but its values are not released yet +- **THEN** the app is not verified until the release runs them + +### Requirement: Verification in the status +`shelf app status ` and `--preview` SHALL print the artifact's source and commit, whether the +signature is verified and by which signer or why not, the CRITICAL and HIGH counts per component +with scanner and scan time, and whether the app may be exposed or why not. + +#### Scenario: Verified app +- **WHEN** a user runs `shelf app status hello` for a verified app +- **THEN** the output shows the source and commit, the signer, `verified yes`, a scan line per component, and `exposure allowed` + +#### Scenario: Unverified app +- **WHEN** the running artifact of `hello` is unsigned +- **THEN** the output shows `verified no` with the reason and a hint to push again with the current workflow, and `exposure refused` From d66b6b020d0aa241c81a3c53797bb72d0d820723 Mon Sep 17 00:00:00 2001 From: tweinmann Date: Wed, 28 Oct 2026 17:01:50 +0000 Subject: [PATCH 2/2] Phase 12: hardening Every app and preview namespace is fenced in, and "green" means the app's own tests passed: - Pod Security: namespaces enforce baseline and warn about restricted; shelf app status lists the pods restricted would refuse, read with a server-side dry run. The chart gives every pod seccomp RuntimeDefault and no privilege escalation. - NetworkPolicy shelf-isolation: pods reach their namespace, the cluster DNS and the internet only; only Traefik reaches in, and only cloudflared reaches Traefik. - LAN exceptions are the operator's: --allow-lan / --deny-lan on app add and credentials, and on the app page; never for a preview, never over the cluster's pods. - tests: in app.yaml become Helm test hooks run by helm-controller after every rollout. A failing test rolls an app back and leaves a preview on the failing version; the sixth stage "the tests" shows exit code and log, and --wait is Ready only once they pass. - shelf app report [--preview] prints a Markdown or JSON report for a pull request, with what a preview runs differently from its app. - just smoke-hardening joins level 2. The smoke tests start shelf app open as a binary, so kill no longer leaves forwarders behind. Level 2 passed in full on a fresh dev cluster; level 3 follows with the release. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/select-smokes.sh | 7 +- .github/workflows/ci.yml | 2 +- CLAUDE.md | 24 +- README.md | 63 ++- charts/shelf-app/templates/_helpers.tpl | 18 + charts/shelf-app/templates/tests.yaml | 71 ++++ charts/shelf-app/templates/workloads.yaml | 2 + charts/shelf-app/values.yaml | 3 +- examples/tenant/.claude/skills/shelf/SKILL.md | 25 +- examples/tenant/CLAUDE.md | 3 +- examples/tenant/app.yaml | 10 + hack/smoke/agent.sh | 5 +- hack/smoke/hardening.sh | 245 +++++++++++ hack/smoke/previews.sh | 3 +- .../testdata/empty-host-suffix.manifests.yaml | 15 + .../chart/testdata/exposed.manifests.yaml | 15 + internal/chart/testdata/features.app.yaml | 13 +- .../chart/testdata/features.manifests.yaml | 96 +++++ internal/chart/testdata/hello.manifests.yaml | 15 + .../chart/testdata/preview.manifests.yaml | 15 + internal/cli/app.go | 20 +- internal/cli/build.go | 58 +-- internal/cli/build_test.go | 37 ++ internal/cli/cli.go | 2 +- internal/cli/cli_test.go | 2 +- internal/cli/harness_test.go | 26 +- internal/cli/isolation_test.go | 64 +++ internal/cli/json.go | 36 ++ internal/cli/json_test.go | 9 +- internal/cli/previews_test.go | 2 +- internal/cli/report.go | 258 ++++++++++++ internal/cli/report_test.go | 42 ++ internal/cli/status.go | 31 +- internal/cli/tenant_test.go | 4 +- .../cli/testdata/json/appstatus-failing.json | 10 + .../cli/testdata/json/appstatus-preview.json | 10 + .../cli/testdata/json/appstatus-previews.json | 10 + .../testdata/json/appstatus-test-failed.json | 102 +++++ .../testdata/json/appstatus-unverified.json | 4 + internal/cli/testdata/json/appstatus.json | 10 + .../cli/testdata/json/report-preview.json | 75 ++++ internal/cli/testdata/report-app.md | 17 + internal/cli/testdata/report-preview.md | 27 ++ internal/cli/testdata/report-test-failed.md | 26 ++ internal/cli/tests_test.go | 62 +++ internal/cli/wait_test.go | 3 +- internal/cluster/apps.go | 21 + internal/cluster/cluster_test.go | 37 +- internal/cluster/isolation.go | 135 +++++++ internal/cluster/platform_test.go | 102 +++++ internal/cluster/rollout.go | 7 +- internal/cluster/rollout_test.go | 2 +- internal/cluster/state.go | 60 ++- internal/cluster/state_test.go | 19 +- internal/cluster/testdata/objects.yaml | 25 ++ internal/cluster/tests.go | 206 ++++++++++ internal/cluster/tests_test.go | 201 +++++++++ internal/cluster/verification.go | 18 +- internal/cluster/verification_test.go | 35 ++ internal/ops/apps.go | 88 ++++ internal/ops/ops.go | 12 + internal/ops/previews.go | 4 + internal/ops/report.go | 117 ++++++ internal/ops/report_test.go | 58 +++ internal/ops/status.go | 22 + internal/ops/wait.go | 24 +- internal/ops/wait_test.go | 26 +- internal/render/render.go | 89 +++- internal/render/render_test.go | 77 +++- internal/schema/jsonschema.go | 24 +- internal/schema/parse_test.go | 2 + internal/schema/types.go | 16 + internal/server/actions.go | 11 + internal/server/server_test.go | 34 +- internal/server/testdata/app-failed.html | 24 ++ internal/server/testdata/app-no-domain.html | 11 + internal/server/testdata/app-quick.html | 11 + internal/server/testdata/app-ready.html | 30 ++ internal/server/ui/app.html | 23 ++ internal/validate/validate.go | 88 ++-- internal/validate/validate_test.go | 23 ++ justfile | 4 + .../changes/phase-12-hardening/.openspec.yaml | 1 - openspec/changes/phase-12-hardening/design.md | 380 ++++++++++++++++++ .../changes/phase-12-hardening/proposal.md | 94 +++-- .../phase-12-hardening/specs/admin-ui/spec.md | 50 +++ .../specs/agent-interface/spec.md | 52 +++ .../specs/app-chart/spec.md | 31 ++ .../specs/app-delivery/spec.md | 61 +++ .../specs/app-manifest/spec.md | 78 ++++ .../specs/app-tests/spec.md | 52 +++ .../specs/branch-previews/spec.md | 31 ++ .../specs/dev-environment/spec.md | 33 ++ .../specs/isolation/spec.md | 79 ++++ openspec/changes/phase-12-hardening/tasks.md | 130 ++++++ openspec/config.yaml | 11 +- platform/apps/resourceset.yaml | 83 +++- platform/previews/resourceset.yaml | 53 +++ schema/app.schema.json | 79 ++++ 99 files changed, 4418 insertions(+), 163 deletions(-) create mode 100644 charts/shelf-app/templates/tests.yaml create mode 100755 hack/smoke/hardening.sh create mode 100644 internal/cli/isolation_test.go create mode 100644 internal/cli/report.go create mode 100644 internal/cli/report_test.go create mode 100644 internal/cli/testdata/json/appstatus-test-failed.json create mode 100644 internal/cli/testdata/json/report-preview.json create mode 100644 internal/cli/testdata/report-app.md create mode 100644 internal/cli/testdata/report-preview.md create mode 100644 internal/cli/testdata/report-test-failed.md create mode 100644 internal/cli/tests_test.go create mode 100644 internal/cluster/isolation.go create mode 100644 internal/cluster/tests.go create mode 100644 internal/cluster/tests_test.go create mode 100644 internal/ops/report.go create mode 100644 internal/ops/report_test.go create mode 100644 openspec/changes/phase-12-hardening/design.md create mode 100644 openspec/changes/phase-12-hardening/specs/admin-ui/spec.md create mode 100644 openspec/changes/phase-12-hardening/specs/agent-interface/spec.md create mode 100644 openspec/changes/phase-12-hardening/specs/app-chart/spec.md create mode 100644 openspec/changes/phase-12-hardening/specs/app-delivery/spec.md create mode 100644 openspec/changes/phase-12-hardening/specs/app-manifest/spec.md create mode 100644 openspec/changes/phase-12-hardening/specs/app-tests/spec.md create mode 100644 openspec/changes/phase-12-hardening/specs/branch-previews/spec.md create mode 100644 openspec/changes/phase-12-hardening/specs/dev-environment/spec.md create mode 100644 openspec/changes/phase-12-hardening/specs/isolation/spec.md create mode 100644 openspec/changes/phase-12-hardening/tasks.md diff --git a/.github/scripts/select-smokes.sh b/.github/scripts/select-smokes.sh index 0e4579c..882603f 100755 --- a/.github/scripts/select-smokes.sh +++ b/.github/scripts/select-smokes.sh @@ -7,7 +7,7 @@ # reason to skip. set -euo pipefail -all=(smoke-secrets smoke-chart smoke-init smoke-apps smoke-agent smoke-previews smoke-verify) +all=(smoke-secrets smoke-chart smoke-init smoke-apps smoke-agent smoke-previews smoke-verify smoke-hardening) declare -A need=() for_path() { @@ -20,15 +20,16 @@ for_path() { internal/server/* | internal/cloudflare/* | internal/github/*) ;; hack/smoke/secret-expansion*) echo smoke-secrets ;; hack/smoke/chart.sh) echo smoke-chart ;; - charts/* | internal/chart/*) echo smoke-chart smoke-apps smoke-agent smoke-previews ;; + charts/* | internal/chart/*) echo smoke-chart smoke-apps smoke-agent smoke-previews smoke-hardening ;; hack/smoke/init.sh | internal/cli/init.go | internal/cli/init_test.go) echo smoke-init ;; hack/smoke/apps.sh) echo smoke-apps ;; hack/smoke/agent.* | internal/cli/json* | internal/cli/open* | internal/cli/wait* | internal/cli/tenant_test.go) echo smoke-agent ;; hack/smoke/previews.sh | internal/cli/previews* | internal/ops/previews*) echo smoke-previews ;; hack/smoke/verify.sh | internal/cli/verification*) echo smoke-verify ;; + hack/smoke/hardening.sh | internal/cli/isolation* | internal/cli/tests_test.go | internal/cli/report*) echo smoke-hardening ;; internal/cli/* | internal/ops/* | internal/progress/* | internal/secrets/* | internal/hostcfg/*) - echo smoke-apps smoke-agent smoke-previews smoke-verify ;; + echo smoke-apps smoke-agent smoke-previews smoke-verify smoke-hardening ;; *) echo "${all[@]}" ;; esac } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 269a045..4985335 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} BASE: ${{ github.event.pull_request.base.sha }} run: | - all='["smoke-secrets","smoke-chart","smoke-init","smoke-apps","smoke-agent","smoke-previews","smoke-verify"]' + all='["smoke-secrets","smoke-chart","smoke-init","smoke-apps","smoke-agent","smoke-previews","smoke-verify","smoke-hardening"]' has() { jq -e --arg l "$1" 'index($l) != null' <<<"$LABELS" >/dev/null; } smokes='[]' full=0 if [[ "$EVENT" == push ]]; then diff --git a/CLAUDE.md b/CLAUDE.md index 4c70c90..afe483f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,7 +156,15 @@ Next: Phase 12 (hardening). - The platform is generic: no knowledge of databases or specific services. Everything is a component. - Apps are isolated from each other. The only thing apps share is a connection the user assigned - to several of them; namespace, secret values, tunnel and cloudflared are every app's own. + to several of them; namespace, secret values, tunnel and cloudflared are every app's own. The + platform enforces it: every app and preview namespace enforces Pod Security `baseline` (and + warns about `restricted`), and its NetworkPolicy `shelf-isolation` lets pods reach their own + namespace, the cluster DNS and the internet only — not other apps, the Kubernetes API, the LAN + or a metadata address — and only Traefik reach in. Only cloudflared may reach Traefik, which + routes by Host header and so could reach any app. +- What an app may reach beyond that is an operator decision too: `--allow-lan ` on + `shelf app add` / `shelf app credentials` (CLI or UI), never `app.yaml`, never for a preview, + and never a range that overlaps the cluster's pods. - Prefer standard building blocks used by larger platforms (Flux, Helm, Traefik, Cloudflare Tunnel) over custom code. - `shelf init cluster` must work against any kubecontext. No Colima, macOS or Linux @@ -198,6 +206,7 @@ it there when it belongs to one smoke test. | `just smoke-agent` | devcontainer | Phase 9: `--json` documents, `--wait --commit` (Ready only after the commit rolled out, Failed for an image that cannot be pulled, TimedOut for a commit that never arrives; with `SMOKE_FULL=1`, the default, also Ready again for a fix after a failure, which takes about 5 more minutes because Helm times out the broken upgrade first) and `shelf app open` with the app's Host and `stripPrefix` (needs Docker Hub) | | `just smoke-previews` | devcontainer | Phase 10: `examples/hello` with `--previews=2`; two branch tags run as isolated previews with their own database, volume and password; `open --preview`; a new commit keeps the data; the limit holds; a deleted tag removes its preview; `--no-previews` keeps the app; prints push-to-Ready times and RAM per preview (needs Docker Hub) | | `just smoke-verify` | devcontainer | Phase 11: a signed and scanned artifact with a signed built image is verified and exposable; the admission policy admits it and denies an unsigned image, in the app's and a preview's namespace; a preview is verified; an artifact without a scan runs but is not verified; an unsigned artifact is not applied and the wait fails with a hint; `rm` removes the image policy; prints the policy-controller's RAM (needs Docker Hub) | +| `just smoke-hardening` | devcontainer | Phase 12: from the app's and a preview's namespace, its own service, DNS and the internet are reachable, another namespace, the Kubernetes API and a private address (the dev registry's container) are not, and only Traefik reaches in; a LAN exception reaches its range but never the preview; a privileged pod is refused, postgres runs and is listed for `restricted`; a passing test gives Ready, a failing one keeps the app on the previous version and the preview on the failing one, both ending the wait with Failed at "the tests"; `shelf app report` names the changed image; prints push-to-Ready with and without a test (needs Docker Hub) | | `just smoke-tenant ` | devcontainer | Phase 4 acceptance with a real tenant repo and GHCR; installs a released platform and chart (`SHELF_RELEASE`, default the newest tag, v0.8.0 or later) with keyless verification, since the tenant's artifacts are signed keyless; asks for the GHCR login, waits for a push | | `just smoke-expose ` | devcontainer | Phase 8b acceptance: the app gets the Cloudflare connection `smoke`, its own tunnel, DNS record and HTTPS, then is taken off again; asks for the Cloudflare API token (set `CF_ACCOUNT_ID` if it sees several accounts); needs a cluster with a host suffix | | `just smoke-quick [app]` | devcontainer | Phase 8d acceptance: `examples/hello` as `hello-quick` with `--quick` gets a quick tunnel, answers with its own Host, gets a new name after a restart, `--private`/`--quick`, rm; keeps the cluster's host suffix; opens a public trycloudflare.com tunnel for a few minutes | @@ -259,7 +268,8 @@ it there when it belongs to one smoke test. resort. A machine running shelf as a service has no Docker config - Per app in `shelf-system`, all labelled `shelf.dev/app`: the provider `` (inputs `name`, `url`, `tag`, `insecure`, `domain`, `tunnel`, `quick`, `registry`, `cloudflare` — the last two - are connection names; `tunnel` and `quick` never both), `app-` (secret values), + are connection names; `tunnel` and `quick` never both — and `lan`, a list of CIDRs, read with + `index` because older providers lack it), `app-` (secret values), `tunnel-` (named tunnels only), and with previews `--previews` (label `shelf.dev/previews=`). A preview runs in namespace `--` (labels `shelf.dev/app`, `shelf.dev/preview`), so an app name never contains `--`; the slug is @@ -309,6 +319,16 @@ it there when it belongs to one smoke test. the deploy artifact's annotation `dev.shelf.scan` (per image digest the CRITICAL and HIGH counts), written before the artifact is signed. What "verified" means is decided in `cluster.verification` alone +- Tests: `tests:` in app.yaml become Helm test hooks, Pods `-test-` labelled + `shelf.dev/test` (never `shelf.dev/component`, so the pods stage leaves them out), run by + helm-controller after every install and upgrade (`test.enable`; `ignoreFailures` on previews + only). Their result is read from the HelmRelease history entry whose `configDigest` is the + last attempted one, by pod name; a rollback is an entry without tests. The stage "the tests" is + the sixth stage of the diagnosis, and the wait is Ready only when the awaited values' tests + passed. A test name never equals a component name, since a built test is + `ghcr.io///` +- Pod Security violations of `restricted` are read with a server-side dry run of the namespace + label (`cluster.AppIsolation`, warnings collected by a `WarningHandler`), never by changing it - Which values a Helm release holds is decided by its `configDigest`, which `cluster.ConfigDigest` computes as helm-controller does, never by timestamps: after a failed upgrade, its failure and its rollback are both newer than the values that replaced it diff --git a/README.md b/README.md index 3b19bb4..e7e3c34 100644 --- a/README.md +++ b/README.md @@ -366,6 +366,27 @@ get no new deploys until they are built again. A cluster that tests shelf verifies with a cosign key instead (`shelf init cluster --signer-key cosign.pub`), as the dev cluster does. +### Isolation + +A preview runs code an agent wrote, so every app and every preview is fenced in: + +- **Network.** Its pods reach each other, the cluster DNS and the internet, and nothing else: + not other apps, not the Kubernetes API, not your LAN, not a cloud metadata address. Only + Traefik reaches in, which is how routes, `shelf app open` and tunnels get there. +- **Pod Security.** Its namespace enforces the level `baseline`: no privileged containers, host + namespaces or `hostPath`. Images that start as root, such as `postgres:16`, still run. + `shelf app status` lists the pods that the stricter level `restricted` would refuse, without + refusing them. +- **LAN exceptions.** An app that needs a NAS or a printer gets it from you, never from its + `app.yaml`: `shelf app credentials --allow-lan 192.168.1.10/32`. Its previews never do. + `--deny-lan` takes it away. + +An app that relied on reaching another app or the LAN loses that path when the cluster gets this +platform; give it an exception, or talk to the other app through its public address. + +A preview counts as green only when its [tests](#tests) pass, and `shelf app report +--preview ` writes the report for its pull request. + ## Writing an `app.yaml` A complete example with a web frontend, an API and a Postgres database: @@ -404,6 +425,11 @@ components: volumes: data: { path: /var/lib/postgresql/data, size: 20Gi } +tests: + smoke: # runs after every rollout; exit 0 means it passed + image: curlimages/curl:8.10.1 + args: [-fsS, --retry, "5", --retry-connrefused, "http://${web.host}:${web.port}/healthz"] + secrets: db-password: { generate: true } ``` @@ -427,9 +453,27 @@ and other editors that support JSON Schema. `shelf schema` prints the same schem | `health` | HTTP probe: `{ path, port?, initialDelay? }`, with `initialDelay` in seconds. It tells Kubernetes (and your deploy) when the component is ready. | | `resources` | `{ cpu, memory }` requests. Memory is also the hard limit, which protects the other apps on the machine. | +### Tests + +`tests` declares containers that check the app after every rollout, of the app and of each of +its previews. Each test has `image` or `build`, and may have `command`, `args`, `env` and +`resources`, as a component does; it has no ports, route, volumes or health. Test names follow +the rules of component names and must differ from them, since a built test is published as +`ghcr.io///` next to the components. + +A test passes when its container exits with 0. It runs inside the app's namespace and reaches +the components by their references (`http://${web.host}:${web.port}/`), not through a route, +and it can read the app's secrets. Let it retry its first connection (`curl --retry 5 +--retry-connrefused`): for a moment after a pod starts, the network policy may not know it yet, +and refuses it. Until every test passed, the app is not Ready: + +- on the app, a failing test fails the upgrade, so the previous version comes back; +- on a preview, the failing version keeps running, so you can look at it with + `shelf app open --preview`. + ### References -Inside `env`, `command` and `args` you can use: +Inside `env`, `command` and `args` of components and tests you can use: | Reference | Becomes | |---|---| @@ -496,28 +540,29 @@ drift. | Command | What it does | |---|---| | `shelf validate ...` | Checks one or more files without network access. Errors and warnings show file, line and field. Exits with 1 on errors. | -| `shelf render ` | Validates, resolves images and prints the deploy manifest (a ConfigMap). `-o app` prints the resolved `app.yaml` instead. `--image =` supplies the image of a component with a `build` directory. Registry credentials come from `docker login`. | +| `shelf render ` | Validates, resolves images and prints the deploy manifest (a ConfigMap). `-o app` prints the resolved `app.yaml` instead. `--image =` supplies the image of a component or test with a `build` directory. Registry credentials come from `docker login`. | | `shelf schema` | Prints the JSON Schema for `app.yaml`. | -| `shelf build-plan ` | Prints the package name (`name` in `app.yaml`) and the components with a `build` directory as JSON. The workflow uses it to name the packages and to know what to build. `--branch ` adds `tag`, the tag a push to that branch publishes the deploy artifact under (`branch-`). | +| `shelf build-plan ` | Prints the package name (`name` in `app.yaml`) and the components and tests with a `build` directory as JSON. The workflow uses it to name the packages and to know what to build. `--branch ` adds `tag`, the tag a push to that branch publishes the deploy artifact under (`branch-`). | | `shelf init cluster` | Installs Flux and the platform (Traefik, the Sigstore policy-controller, app management) into the cluster of the current kubecontext, and waits until everything is ready. The cluster verifies everything it runs: by default keyless signatures of shelf's own workflows (`--signer-issuer`, `--signer-subject`), or with `--signer-key ` a cosign public key, for clusters that test shelf. The confirmation lists apps that do not run a verified artifact. `--host-suffix -dev` separates clusters that share a DNS zone: apps with a domain are then reachable at `-dev.`. Shows the target cluster and asks before changing anything (`--yes` skips the question); `--context` and `--kubeconfig` pick another cluster. Safe to run again. | | `shelf connection add registry ` | Defines a registry login for `ghcr.io` from `GHCR_USERNAME` and `GHCR_TOKEN`. It is stored in the cluster and copied only into the namespaces of the apps that chose it. | | `shelf connection add cloudflare ` | Defines a Cloudflare account from `CF_API_TOKEN` (and `CF_ACCOUNT_ID` when the token sees several). shelf checks the token and keeps it in `~/.shelf/connections/cloudflare/`, never in the cluster. | | `shelf connection list` / `rm ` | Lists the connections and the apps that use them; removes one no app uses. | | `shelf connection packages ` / `zones ` | Lists the deploy artifacts a registry connection can pull, or the zones of a Cloudflare connection's account. | -| `shelf app add ` | Deploys an app from a deploy artifact under the name `` and keeps it updated. The name is independent of the package, so one artifact can be added under several names. `--registry` picks the registry connection; `--cloudflare` with `--domain` puts the app on the internet under its own domain, `--quick` through a quick tunnel, both only once it runs a [verified](#verification-chain) artifact; without either it stays inside the cluster. Generates the app's secrets, stores them in the cluster and in `~/.shelf/apps//secrets.yaml`, and restores them from there after a cluster rebuild. Refuses at once if the cluster has no platform. Waits until the app is ready. `--previews[=N]` runs a [preview](#previews-of-branches) of each published branch, at most N (default 3). Safe to run again, e.g. after adding a secret. | -| `shelf app credentials ` | Changes the app's connections, domain or previews and deploys it again: `--registry`, `--no-registry`, `--cloudflare`, `--domain`, `--quick`, `--private`, `--previews[=N]`, `--no-previews`. | +| `shelf app add ` | Deploys an app from a deploy artifact under the name `` and keeps it updated. The name is independent of the package, so one artifact can be added under several names. `--registry` picks the registry connection; `--cloudflare` with `--domain` puts the app on the internet under its own domain, `--quick` through a quick tunnel, both only once it runs a [verified](#verification-chain) artifact; without either it stays inside the cluster. Generates the app's secrets, stores them in the cluster and in `~/.shelf/apps//secrets.yaml`, and restores them from there after a cluster rebuild. Refuses at once if the cluster has no platform. Waits until the app is ready. `--previews[=N]` runs a [preview](#previews-of-branches) of each published branch, at most N (default 3). `--allow-lan ` lets the app reach a private address range such as a NAS, which an app otherwise never reaches; its previews never do. Safe to run again, e.g. after adding a secret. | +| `shelf app credentials ` | Changes the app's connections, domain or previews and deploys it again: `--registry`, `--no-registry`, `--cloudflare`, `--domain`, `--quick`, `--private`, `--previews[=N]`, `--no-previews`, `--allow-lan ` (repeatable, replaces the ranges), `--deny-lan`. | | `shelf app open ` | Makes the app reachable from a browser on this machine at `http://127.0.0.1:/`, for as long as it runs: requests go through the Kubernetes API server to the cluster's Traefik with the app's own Host header, so routes and `stripPrefix` behave as through a tunnel. Works for apps that stay inside the cluster. Listens on 127.0.0.1 only; `--port` picks the port, `--browser` opens it. It is not an exposure and changes nothing about the app. `--preview ` opens that branch's preview. | +| `shelf app report ` | Prints a report for a pull request as Markdown: the commit, whether it is Ready or which step is not, the [tests](#tests), the signature and scan per component, and the isolation. `--preview ` reports on that branch's preview instead, with what it runs differently from the app: every component with another image, and the findings of both. Read-only. `shelf app report hello --preview "$(git branch --show-current)" \| gh pr comment --body-file -` posts it. | | `shelf app tags ` | Lists the tags of the app's deploy artifact, newest first. | | `shelf app redeploy ` | Fetches the app's artifact again under the tag it is registered with and rolls out what it finds. For a tag that moved, or a deploy worth another try. | | `shelf app secrets ` | Lists the names of the generated secrets; `--reveal` prints the values. | -| `shelf app status [app]` | Lists the apps with their state, or walks one app from its deploy artifact to its running pods and says which step is not ready. For one app it also shows the git commit the running artifact was built from, names its components and where each of them answers, and shows the [verification chain](#verification-chain): source, signer, verified or why not, findings per component, and whether the app may be exposed. Same chain as the admin UI shows. `--wait` waits until the app is Ready or a step failed and prints the verdict; `--commit ` first waits until the app runs the artifact built from that commit. Exit status 0 Ready, 1 failed, 2 timed out (`--timeout`, default 10m). For an app with previews it lists them; `--preview ` shows, or waits for, that preview instead. | +| `shelf app status [app]` | Lists the apps with their state, or walks one app from its deploy artifact to its running pods and says which step is not ready. For one app it also shows the git commit the running artifact was built from, names its components and where each of them answers, and shows the [verification chain](#verification-chain): source, signer, verified or why not, findings per component, and whether the app may be exposed; then the isolation: the Pod Security level, the pods the level `restricted` would refuse, and the private ranges it may reach. The chain ends with the app's [tests](#tests), with the end of the log of one that failed. Same chain as the admin UI shows. `--wait` waits until the app is Ready or a step failed and prints the verdict; `--commit ` first waits until the app runs the artifact built from that commit. Exit status 0 Ready, 1 failed, 2 timed out (`--timeout`, default 10m). For an app with previews it lists them; `--preview ` shows, or waits for, that preview instead. | | `shelf serve` | Runs the admin UI on port 7654 of the local network (`--listen 127.0.0.1:7654` keeps it off the network). The first start prints a setup code that claims the instance; after that a password protects it. | | `shelf app rm ` | Removes an app with its namespace, volumes, secrets, tunnel and DNS record, after asking. The secret backup stays. | | `shelf version` | Prints the version. | ### Output for scripts and agents -`shelf app status`, `shelf app tags`, `shelf connection list` and `shelf version` take `--json` +`shelf app status`, `shelf app report`, `shelf app tags`, `shelf connection list` and `shelf version` take `--json` and then print one JSON document on stdout and nothing else there: ```console @@ -534,8 +579,8 @@ $ shelf app status hello --json } ``` -Every document has `apiVersion` and `kind` (`AppList`, `AppStatus`, `WaitResult`, `TagList`, -`ConnectionList`, `Version`). A command that fails prints a document of kind `Error` with a +Every document has `apiVersion` and `kind` (`AppList`, `AppStatus`, `WaitResult`, `Report`, +`TagList`, `ConnectionList`, `Version`). A command that fails prints a document of kind `Error` with a `message`, and exits with status 1. Within one `apiVersion`, no field is removed, renamed or given another meaning; new fields may appear, so ignore the ones you do not know. Empty fields are left out rather than `null`. `shelf app secrets` has no `--json`, so that secret values never diff --git a/charts/shelf-app/templates/_helpers.tpl b/charts/shelf-app/templates/_helpers.tpl index 4308f40..7b3d20b 100644 --- a/charts/shelf-app/templates/_helpers.tpl +++ b/charts/shelf-app/templates/_helpers.tpl @@ -75,3 +75,21 @@ initialDelaySeconds: {{ . }} periodSeconds: 10 failureThreshold: {{ .failureThreshold }} {{- end -}} + +{{/* +The security settings every pod gets. They are what Pod Security restricted asks for that no +image depends on: the runtime's default seccomp profile, and no gaining privileges through setuid +binaries (an entrypoint that switches from root to a user with gosu or su-exec still works). What +restricted asks beyond that, a user other than root and no capabilities, depends on the image, so +it is left to it; the namespace warns about it. +*/}} +{{- define "shelf-app.podSecurity" -}} +securityContext: + seccompProfile: + type: RuntimeDefault +{{- end -}} + +{{- define "shelf-app.containerSecurity" -}} +securityContext: + allowPrivilegeEscalation: false +{{- end -}} diff --git a/charts/shelf-app/templates/tests.yaml b/charts/shelf-app/templates/tests.yaml new file mode 100644 index 0000000..37cf224 --- /dev/null +++ b/charts/shelf-app/templates/tests.yaml @@ -0,0 +1,71 @@ +{{- /* +One Pod per test, run by `helm test` after every install and upgrade; Flux's helm-controller +does that for the platform. The last pod of a test stays until the next run, so its log can be +read. Tests carry shelf.dev/test instead of shelf.dev/component: they are no part of the app's +pods, and nothing routes to them. +*/ -}} +{{- $secrets := .Values.secrets | default dict }} +{{- range $testName, $test := .Values.tests }} +{{- $resources := $test.resources | default dict }} +{{- $refs := include "shelf-app.referencedSecrets" (dict "comp" $test "secrets" $secrets) | fromJsonArray }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ $.Values.name }}-test-{{ $testName }} + labels: + shelf.dev/app: {{ $.Values.name }} + shelf.dev/test: {{ $testName }} + app.kubernetes.io/managed-by: {{ $.Release.Service }} + helm.sh/chart: {{ printf "%s-%s" $.Chart.Name $.Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} + annotations: + helm.sh/hook: test + helm.sh/hook-delete-policy: before-hook-creation +spec: + restartPolicy: Never + enableServiceLinks: false + automountServiceAccountToken: false + {{- include "shelf-app.podSecurity" . | nindent 2 }} + {{- with $.Values.platform.imagePullSecret }} + imagePullSecrets: + - name: {{ . }} + {{- end }} + containers: + - name: {{ $testName }} + image: {{ $test.image | quote }} + {{- include "shelf-app.containerSecurity" . | nindent 6 }} + {{- with $test.command }} + command: + {{- range . }} + - {{ include "shelf-app.expand" (dict "s" (toString .) "secrets" $secrets) | quote }} + {{- end }} + {{- end }} + {{- with $test.args }} + args: + {{- range . }} + - {{ include "shelf-app.expand" (dict "s" (toString .) "secrets" $secrets) | quote }} + {{- end }} + {{- end }} + {{- if or $refs $test.env }} + env: + {{- range $refs }} + - name: {{ include "shelf-app.secretEnv" . }} + valueFrom: + secretKeyRef: + name: shelf-secrets + key: {{ . }} + {{- end }} + {{- range $name, $value := $test.env }} + - name: {{ $name }} + value: {{ include "shelf-app.expand" (dict "s" (toString $value) "secrets" $secrets) | quote }} + {{- end }} + {{- end }} + resources: + requests: + cpu: {{ $resources.cpu | default "50m" | quote }} + memory: {{ $resources.memory | default "64Mi" | quote }} + {{- with $resources.memory }} + limits: + memory: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/charts/shelf-app/templates/workloads.yaml b/charts/shelf-app/templates/workloads.yaml index ca7c437..cb5481d 100644 --- a/charts/shelf-app/templates/workloads.yaml +++ b/charts/shelf-app/templates/workloads.yaml @@ -36,6 +36,7 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + {{- include "shelf-app.podSecurity" . | nindent 6 }} {{- with $.Values.platform.imagePullSecret }} imagePullSecrets: - name: {{ . }} @@ -43,6 +44,7 @@ spec: containers: - name: {{ $compName }} image: {{ $comp.image | quote }} + {{- include "shelf-app.containerSecurity" . | nindent 10 }} {{- with $comp.command }} command: {{- range . }} diff --git a/charts/shelf-app/values.yaml b/charts/shelf-app/values.yaml index c0c8826..4acdcba 100644 --- a/charts/shelf-app/values.yaml +++ b/charts/shelf-app/values.yaml @@ -1,5 +1,5 @@ # The app part of the values is a resolved app.yaml, as printed by `shelf render -o app` -# (apiVersion, name, components, secrets). Do not write it by hand: the chart relies on its +# (apiVersion, name, components, secrets, tests). Do not write it by hand: the chart relies on its # normalized shape and on every literal $ being escaped as $$. # # `name` is the app's name, which is also the release namespace. The resolved app.yaml carries a @@ -12,6 +12,7 @@ apiVersion: shelf.dev/v1alpha1 name: "" components: {} secrets: {} +tests: {} # Set by the platform, not by the app. platform: diff --git a/examples/tenant/.claude/skills/shelf/SKILL.md b/examples/tenant/.claude/skills/shelf/SKILL.md index 645f079..88ac0e5 100644 --- a/examples/tenant/.claude/skills/shelf/SKILL.md +++ b/examples/tenant/.claude/skills/shelf/SKILL.md @@ -60,9 +60,18 @@ The app's name on shelf is in `CLAUDE.md` (the operator chose it; it is not `nam in code. 5. For the whole picture, run `shelf app status --json` (or with `--preview `): - - every stage from the deploy artifact to the pods; + - every stage from the deploy artifact to the pods and the tests; - the running `commit`, the components and their addresses; + - `tests`, with the exit code and the end of the log of one that failed; - for the app, its `previews`. +6. When the branch has a pull request, post the report of its preview there: + + ```sh + shelf app report --preview | gh pr comment --body-file - + ``` + + It says whether the preview is Ready, how its tests went, whether it is signed and scanned, + and which images differ from `main`. ## What the stages mean @@ -74,16 +83,30 @@ The stages, in order: 4. `the app`: the Helm release; often a bad `app.yaml`. 5. `the pods`: image pull, crash loop, out of memory, missing secret, a preview's quota, or a built image the admission policy refused because it is not signed. +6. `the tests`: a test under `tests:` in `app.yaml` exited with something else than 0 against + this version. The message has its exit code and the end of its log. On `main` the previous + version runs again; on a branch the preview keeps the failing version, so the operator can + look at it. Fix the code or the test, and push again. The first stage that is not Ready is the cause; everything after it is a consequence. `shelf validate app.yaml` catches most app.yaml mistakes before a push. +## Tests + +`tests:` in `app.yaml` declares containers that run against the app after every rollout, inside +its namespace. A test reaches the components by their references, such as +`http://${web.host}:${web.port}/`, never through a route, and passes when it exits with 0. Add +one for what a change must not break; the wait counts the commit as Ready only once they pass. + ## What you cannot do - Make the app or a preview public. Exposure (a domain or a tunnel) is the operator's decision, made with `shelf app add` or `shelf app credentials` outside this repository. Nothing in `app.yaml` changes it, so do not try. Previews are never exposed. - Switch previews on or off; that is the operator's decision too. +- Reach other apps, the operator's LAN or the Kubernetes API from the app. The app reaches its + own components, DNS and the internet only; anything more is the operator's decision + (`--allow-lan`), and a preview never gets it. - Read secret values. `shelf app secrets` is for the operator; a preview's secrets are its own. - `shelf app open ` (with `--preview ` for a preview) makes the app reachable in a browser on the operator's machine. It is for the person, not for you; ask them to look if a diff --git a/examples/tenant/CLAUDE.md b/examples/tenant/CLAUDE.md index 07cf3af..77a1705 100644 --- a/examples/tenant/CLAUDE.md +++ b/examples/tenant/CLAUDE.md @@ -12,4 +12,5 @@ The app runs on shelf as **`greeter`**. (Operator: replace this with the name yo After every push, use the `shelf` skill: start `shelf app status greeter --wait --commit "$(git rev-parse HEAD)" --json` in the background (on a branch, with `--preview `), keep working, and fix what the verdict names. The work is -done when that command exits 0. +done when that command exits 0, which needs the tests under `tests:` in `app.yaml` to pass. On +a branch with a pull request, post `shelf app report greeter --preview ` there. diff --git a/examples/tenant/app.yaml b/examples/tenant/app.yaml index e8d409d..4053144 100644 --- a/examples/tenant/app.yaml +++ b/examples/tenant/app.yaml @@ -18,5 +18,15 @@ components: TOKEN: ${secrets.token} resources: { cpu: 10m, memory: 16Mi } +# Runs after every rollout, against the app or a preview, inside its namespace. The rollout counts +# only when curl exits with 0; on main a failing test brings the previous version back. +tests: + smoke: + image: curlimages/curl:8.10.1 + # A new pod's first connection can be refused for a moment, until the network policy knows + # it; --retry-connrefused waits that out, as against any service that just started. + args: [-fsS, --retry, "5", --retry-connrefused, --retry-delay, "1", "http://${web.host}:${web.port}/healthz"] + resources: { cpu: 10m, memory: 16Mi } + secrets: token: { generate: true } diff --git a/hack/smoke/agent.sh b/hack/smoke/agent.sh index fca5d27..bb67115 100755 --- a/hack/smoke/agent.sh +++ b/hack/smoke/agent.sh @@ -84,7 +84,7 @@ step "shelf app status --json" shelf app status "$app" --json >"$work/status.json" [[ "$(jq -r .kind "$work/status.json")" == AppStatus ]] || die "not an AppStatus: $(cat "$work/status.json")" [[ "$(jq -r .commit "$work/status.json")" == "$c1" ]] || die "status does not name commit $c1" -[[ "$(jq '.stages | length' "$work/status.json")" == 5 ]] || die "status does not hold five stages" +[[ "$(jq '.stages | length' "$work/status.json")" == 6 ]] || die "status does not hold six stages" shelf app status --json | jq -e --arg app "$app" '.kind == "AppList" and any(.items[]; .name == $app)' >/dev/null \ || die "the AppList lacks $app" shelf app status nope --json >"$work/error.json" 2>/dev/null && die "an unknown app was found" @@ -112,7 +112,8 @@ kubectl -n "$app" rollout status deploy/web --timeout=10s >/dev/null || die "Rea step "shelf app open reaches the app with its own Host and stripPrefix" port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')" -shelf app open "$app" --port "$port" >"$work/open.txt" 2>&1 & +# The binary itself, not the function shelf: $! has to be its PID, or kill leaves it running. +"$work/shelf" app open "$app" --port "$port" >"$work/open.txt" 2>&1 & open_pid=$! up() { curl -fsS "http://127.0.0.1:$port/" >/dev/null 2>&1; } for _ in $(seq 30); do up && break; sleep 1; done diff --git a/hack/smoke/hardening.sh b/hack/smoke/hardening.sh new file mode 100755 index 0000000..cd0068f --- /dev/null +++ b/hack/smoke/hardening.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +# Checks the Phase 12 hardening against the dev cluster: +# - the isolation, from the app's namespace and from a preview's: the app's own service is +# reachable, another namespace, the Kubernetes API and a private address (the dev registry's +# container, standing in for the LAN) are not, the internet is; another namespace cannot reach +# in, Traefik can (shelf app open); +# - a LAN exception lets the app reach that address, and never its preview; +# - Pod Security refuses a privileged pod, runs postgres:16, and lists it as violating +# restricted; +# - a passing test gives Ready; a failing test on the app ends the wait with Failed at "the tests" +# and keeps the previous version; a failing test on a preview keeps the preview on it, red; +# - shelf app report for that preview names the changed component. +# It prints the time from push to Ready with a test against the same app without one. Installs or +# updates the platform first (idempotent). Needs network access for images. +set -euo pipefail +source "$(dirname "$0")/../lib.sh" +require_devcontainer + +repo="$(cd "$(dirname "$0")/../.." && pwd)" +work="$(mktemp -d)" +app=hardening-smoke +other=hardening-smoke-other +# As on GHCR: the deploy artifact is /, a built image /. +artifact="oci://$SHELF_REGISTRY_HOST/smoke/$app" +image="$SHELF_REGISTRY_HOST/smoke/$app/web" +export SHELF_HOME="$work/home" + +shelf() { "$work/shelf" "$@"; } + +cleanup() { + if [[ "${KEEP:-}" == 1 ]]; then + echo "KEEP=1: leaving app $app and namespace $other in place" + return + fi + [[ -n "${open_pid:-}" ]] && kill "$open_pid" 2>/dev/null || true + shelf app rm "$app" --yes >/dev/null 2>&1 || true + kubectl delete namespace "$other" --ignore-not-found --wait=false >/dev/null 2>&1 || true + rm -rf "$work" +} + +step() { echo "--- $*"; } + +# commit: a random 40-digit commit hash, as the reusable workflow stamps on an artifact. +commit() { head -c 20 /dev/urandom | od -An -tx1 | tr -d ' \n'; } + +# render : writes the deploy manifest of the app with a web image +# built for it, postgres:16, and, unless the args are empty, a curl test. shelf render looks the +# built image up over TLS, which the dev registry has not, so it renders the Docker Hub image +# the dev one was copied from, and the reference is swapped afterwards. +render() { + local dir="$1" web="$2" test="$3" base + mkdir -p "$dir" + base="traefik/whoami:v1.11.0@$(crane digest traefik/whoami:v1.11.0)" + { + cat <<'EOF' +apiVersion: shelf.dev/v1alpha1 +name: hardening-smoke +components: + web: + build: ./web + port: 80 + route: / + health: { path: /health } + resources: { cpu: 10m, memory: 16Mi } + db: + image: postgres:16 + port: 5432 + env: + POSTGRES_PASSWORD: ${secrets.pw} + resources: { cpu: 50m, memory: 128Mi } +secrets: + pw: { generate: true } +EOF + if [[ -n "$test" ]]; then + printf 'tests:\n smoke:\n image: curlimages/curl:8.10.1\n args: [%s]\n resources: { cpu: 10m, memory: 16Mi }\n' "$test" + fi + } >"$work/app.yaml" + shelf render "$work/app.yaml" --image "web=$base" 2>/dev/null | sed "s|$base|$web|" >"$dir/configmap.yaml" + grep -qF "$web" "$dir/configmap.yaml" || die "the rendered manifest does not name $web" +} + +# probe : whether a pod in that namespace gets an answer from url. The pod is +# subject to the namespace's NetworkPolicy like every other one there. +probe() { + kubectl -n "$1" exec probe -- wget -q -T 4 -O /dev/null "$2" >/dev/null 2>&1 +} + +# probe_pod : starts the pod probe uses. +probe_pod() { + # The namespace warns that busybox would break restricted; that is expected, and noise here. + kubectl -n "$1" run probe --image=busybox:1.36 --restart=Never -- sleep 3600 >/dev/null 2>&1 + kubectl -n "$1" wait pod/probe --for=condition=Ready --timeout=120s >/dev/null || die "the probe in $1 did not start" +} + +# fenced : the isolation as seen from a namespace of the app or a preview. +fenced() { + local ns="$1" what="$2" + probe "$ns" "http://web.$ns" || die "$what: its own service is not reachable" + probe "$ns" "http://other.$other" && die "$what: another namespace is reachable" + kubectl -n "$ns" exec probe -- nc -z -w 4 kubernetes.default.svc 443 >/dev/null 2>&1 \ + && die "$what: the Kubernetes API is reachable" + probe "$ns" "http://$registry_ip:5000/v2/" && die "$what: the private address $registry_ip is reachable" + probe "$ns" "http://example.com" || die "$what: the internet is not reachable" + kubectl -n "$other" exec probe -- wget -q -T 4 -O /dev/null "http://web.$ns" >/dev/null 2>&1 \ + && die "$what: another namespace reaches in" + return 0 +} + +# wait_for [args...]: waits for the app (or, with --preview, a preview) to run a commit, +# and prints how long it took; the exit status is the verdict's. +wait_for() { + local sha="$1" + shift + shelf app status "$app" --wait --commit "$sha" --timeout 6m --json "$@" >"$work/verdict.json" +} + +for leftover in "namespace/$app" "namespace/$other" "-n shelf-system resourcesetinputprovider/$app"; do + # shellcheck disable=SC2086 + kubectl get $leftover >/dev/null 2>&1 && die "$leftover already exists; remove it first (shelf app rm $app)" +done +trap cleanup EXIT + +step "build shelf, push platform and chart, init cluster with the dev key" +(cd "$repo" && go build -o "$work/shelf" ./cmd/shelf) +"$repo/hack/platform-push.sh" >/dev/null 2>&1 +"$repo/hack/chart-push.sh" >/dev/null 2>&1 +shelf init cluster --yes --signer-key "$SHELF_SIGNER_PUB" --host-suffix "$(cluster_host_suffix)" --insecure-registry \ + --platform "oci://$SHELF_REGISTRY_HOST/shelf/platform:dev" \ + --chart "oci://$SHELF_REGISTRY_HOST/shelf/charts/shelf-app:0.0.0-dev" >/dev/null +kubectl -n cosign-system wait helmrelease/policy-controller --for=condition=Ready --timeout=300s >/dev/null \ + || die "the policy-controller is not ready" +# The registry's own address on the cluster's network stands in for a LAN device: here it is +# 127.0.0.1 through /etc/hosts, which inside a pod is the pod itself. +registry_ip="$(docker inspect -f "{{with index .NetworkSettings.Networks \"k3d-$SHELF_CLUSTER\"}}{{.IPAddress}}{{end}}" "$SHELF_REGISTRY")" +[[ "$registry_ip" =~ ^(10|172|192)\. ]] || die "cannot find the dev registry on the cluster's network: $registry_ip" + +step "signed web images for main and for a branch, and another namespace to reach for" +crane copy --insecure --platform linux/arm64 traefik/whoami:v1.11.0 "$image:base" >/dev/null 2>&1 \ + || die "cannot copy the base image" +for kind in main branch; do + crane mutate --insecure -l "dev.shelf.smoke=$kind-$(commit)" "$image:base" -t "$image:$kind" >/dev/null 2>&1 \ + || die "cannot make the $kind image" + sign_artifact "$image:$kind" +done +web_main="$image@$(crane digest --insecure "$image:main")" +web_branch="$image@$(crane digest --insecure "$image:branch")" +create_namespace "$other" +kubectl -n "$other" run other --image=traefik/whoami:v1.11.0 --port=80 --expose >/dev/null +probe_pod "$other" + +step "the app without a test, then with one: push to Ready" +render "$work/plain" "$web_main" "" +first="$(commit)" +push_artifact "$artifact:main" "$work/plain" "main@sha1:$first" >/dev/null +shelf app add "$app" "$artifact:main" --insecure-registry >/dev/null +wait_for "$first" || die "the app without a test did not become Ready: $(jq -c .stage "$work/verdict.json")" +again="$(commit)" +start=$(date +%s) +push_artifact "$artifact:main" "$work/plain" "main@sha1:$again" >/dev/null +wait_for "$again" || die "the app without a test did not become Ready again: $(jq -c .stage "$work/verdict.json")" +plain_seconds=$(($(date +%s) - start)) +render "$work/tested" "$web_main" '-fsS, --retry, "5", --retry-connrefused, --retry-delay, "1", "http://${web.host}:${web.port}/"' +second="$(commit)" +start=$(date +%s) +push_artifact "$artifact:main" "$work/tested" "main@sha1:$second" >/dev/null +wait_for "$second" || die "the app with a passing test did not become Ready: $(jq -c .stage "$work/verdict.json")" +tested_seconds=$(($(date +%s) - start)) +shelf app status "$app" --json | jq -e '.tests == [{"name": "smoke", "phase": "Ready"}] and .stages[-1].name == "the tests"' >/dev/null \ + || die "the passed test is not in the status: $(shelf app status "$app" --json | jq -c '{tests, stage: .stages[-1]}')" + +step "Pod Security: postgres runs, is listed for restricted, a privileged pod is refused" +shelf app status "$app" --json | jq -e '.isolation.podSecurity == "baseline" and + any(.isolation.restrictedViolations[]; .pod | startswith("db-"))' >/dev/null \ + || die "postgres is not listed as violating restricted: $(shelf app status "$app" --json | jq -c .isolation)" +kubectl -n "$app" run privileged --image=busybox:1.36 --restart=Never --overrides='{"spec":{"containers":[{"name":"privileged","image":"busybox:1.36","command":["sleep","60"],"securityContext":{"privileged":true}}]}}' \ + >/dev/null 2>"$work/privileged.txt" && die "a privileged pod was admitted" +grep -q 'violates PodSecurity "baseline' "$work/privileged.txt" || die "the privileged pod was not refused by Pod Security: $(cat "$work/privileged.txt")" + +step "the isolation of the app, and Traefik reaching in" +probe_pod "$app" +fenced "$app" "the app" +port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')" +# The binary itself, not the function shelf: $! has to be its PID, or kill leaves it running. +"$work/shelf" app open "$app" --port "$port" >"$work/open.txt" 2>&1 & +open_pid=$! +for _ in $(seq 1 30); do curl -fsS -o /dev/null "http://127.0.0.1:$port/" 2>/dev/null && break; sleep 1; done +curl -fsS -o /dev/null "http://127.0.0.1:$port/" || die "Traefik does not reach the app: $(cat "$work/open.txt")" +kill "$open_pid" +wait "$open_pid" 2>/dev/null || true +open_pid="" + +step "a LAN exception reaches its range, and the app's preview does not get it" +shelf app credentials "$app" --allow-lan "$registry_ip/32" --previews=1 >/dev/null +probe "$app" "http://$registry_ip:5000/v2/" || die "the app does not reach its LAN exception $registry_ip" +render "$work/branch" "$web_branch" '-fsS, --retry, "5", --retry-connrefused, --retry-delay, "1", "http://${web.host}:${web.port}/"' +branch="$(commit)" +push_artifact "$artifact:branch-check" "$work/branch" "check@sha1:$branch" >/dev/null +wait_for "$branch" --preview check || die "the preview did not become Ready: $(jq -c .stage "$work/verdict.json")" +probe_pod "$app--check" +fenced "$app--check" "the preview" +probe "$app--check" "http://web.$app" && die "the preview reaches the app" + +step "a failing test on a preview keeps the preview on it, red" +render "$work/branch-broken" "$web_branch" '-fsS, --max-time, "5", "http://${web.host}:81/"' +broken_branch="$(commit)" +push_artifact "$artifact:branch-check" "$work/branch-broken" "check@sha1:$broken_branch" >/dev/null +code=0 +wait_for "$broken_branch" --preview check || code=$? +[[ "$code" == 1 ]] || die "the wait for a preview with a failing test ended with $code, not 1" +jq -e '.stage.name == "the tests" and (.stage.message | test("exit code 7"))' "$work/verdict.json" >/dev/null \ + || die "the verdict does not name the test: $(jq -c .stage "$work/verdict.json")" +release="$(kubectl -n "$app--check" get helmrelease "$app--check" -o json)" +jq -e '.status.history[0].configDigest == .status.lastAttemptedConfigDigest' <<<"$release" >/dev/null \ + || die "the preview does not run the version whose test failed" + +step "shelf app report names the changed component" +shelf app report "$app" --preview check --json | jq -e '.comparison.comparable and + any(.comparison.changed[]; .component == "web") and any(.tests[]; .name == "smoke" and .phase == "Failed")' >/dev/null \ + || die "the report does not name web as changed: $(shelf app report "$app" --preview check --json | jq -c '{comparison, tests}')" +shelf app report "$app" --preview check | grep -q '| web | changed |' || die "the Markdown report lacks the changed web image" + +step "a failing test on the app keeps the previous version" +digest_before="$(kubectl -n "$app" get helmrelease "$app" -o jsonpath='{.status.history[0].configDigest}')" +render "$work/broken" "$web_main" '-fsS, --max-time, "5", "http://${web.host}:81/"' +broken="$(commit)" +push_artifact "$artifact:main" "$work/broken" "main@sha1:$broken" >/dev/null +code=0 +wait_for "$broken" || code=$? +[[ "$code" == 1 ]] || die "the wait for a failing test ended with $code, not 1" +jq -e '.stage.name == "the tests"' "$work/verdict.json" >/dev/null \ + || die "the verdict does not name the tests: $(jq -c .stage "$work/verdict.json")" +for _ in $(seq 1 90); do + [[ "$(kubectl -n "$app" get helmrelease "$app" -o jsonpath='{.status.history[0].configDigest}')" == "$digest_before" ]] && break + sleep 4 +done +[[ "$(kubectl -n "$app" get helmrelease "$app" -o jsonpath='{.status.history[0].configDigest}')" == "$digest_before" ]] \ + || die "the app did not go back to the version whose test passed" + +step "push to Ready" +echo " without a test ${plain_seconds}s" +echo " with a test ${tested_seconds}s" + +step "shelf app rm" +shelf app rm "$app" --yes >/dev/null +echo "smoke-hardening passed" diff --git a/hack/smoke/previews.sh b/hack/smoke/previews.sh index ccf8eea..a45cc51 100755 --- a/hack/smoke/previews.sh +++ b/hack/smoke/previews.sh @@ -125,7 +125,8 @@ hashes="$(printf '%s\n' "$(password "$app")" "$(password "$app--alpha")" "$(pass step "shelf app open --preview reaches the preview with its own Host" port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')" -shelf app open "$app" --preview alpha --port "$port" >"$work/open.txt" 2>&1 & +# The binary itself, not the function shelf: $! has to be its PID, or kill leaves it running. +"$work/shelf" app open "$app" --preview alpha --port "$port" >"$work/open.txt" 2>&1 & open_pid=$! up() { curl -fsS "http://127.0.0.1:$port/" >/dev/null 2>&1; } for _ in $(seq 30); do up && break; sleep 1; done diff --git a/internal/chart/testdata/empty-host-suffix.manifests.yaml b/internal/chart/testdata/empty-host-suffix.manifests.yaml index dadef36..d113ac4 100644 --- a/internal/chart/testdata/empty-host-suffix.manifests.yaml +++ b/internal/chart/testdata/empty-host-suffix.manifests.yaml @@ -91,9 +91,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: check image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false command: - "sh" - "-c" @@ -141,9 +146,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: web image: "traefik/whoami:v1.11.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false ports: - name: main containerPort: 80 @@ -200,9 +210,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: db image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false env: - name: SHELF_SECRET_DB_PASSWORD valueFrom: diff --git a/internal/chart/testdata/exposed.manifests.yaml b/internal/chart/testdata/exposed.manifests.yaml index fc05636..4e35a60 100644 --- a/internal/chart/testdata/exposed.manifests.yaml +++ b/internal/chart/testdata/exposed.manifests.yaml @@ -91,9 +91,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: check image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false command: - "sh" - "-c" @@ -141,9 +146,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: web image: "traefik/whoami:v1.11.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false ports: - name: main containerPort: 80 @@ -200,9 +210,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: db image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false env: - name: SHELF_SECRET_DB_PASSWORD valueFrom: diff --git a/internal/chart/testdata/features.app.yaml b/internal/chart/testdata/features.app.yaml index ca4de9e..e7411d7 100644 --- a/internal/chart/testdata/features.app.yaml +++ b/internal/chart/testdata/features.app.yaml @@ -1,6 +1,6 @@ # Exercises the chart paths the hello example does not: several ports, stripPrefix, probe # delay, secret references in command, args and env, literal $, a secret referenced only in -# escaped form, several instances with volumes, and a StatefulSet without ports. +# escaped form, several instances with volumes, a StatefulSet without ports, and tests. apiVersion: shelf.dev/v1alpha1 name: features @@ -32,6 +32,17 @@ components: cache: { path: /cache, size: 1Gi } resources: { cpu: 250m } +tests: + smoke: + image: curlimages/curl:8.10.1 + args: [-fsS, "http://${api.host}:${api.ports.http}/healthz"] + login: + image: ghcr.io/example/login-test:1.0 + env: + TOKEN: ${secrets.api-token} + DB: ${db.host}:${db.port} + resources: { cpu: 20m, memory: 32Mi } + secrets: api-token: { generate: true } session-key: { generate: true } diff --git a/internal/chart/testdata/features.manifests.yaml b/internal/chart/testdata/features.manifests.yaml index bdce3ff..8e65e36 100644 --- a/internal/chart/testdata/features.manifests.yaml +++ b/internal/chart/testdata/features.manifests.yaml @@ -112,11 +112,16 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault imagePullSecrets: - name: ghcr-pull containers: - name: api image: "ghcr.io/example/api:1.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false command: - "/bin/api" - "--token=$(SHELF_SECRET_API_TOKEN)" @@ -200,11 +205,16 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault imagePullSecrets: - name: ghcr-pull containers: - name: db image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false ports: - name: main containerPort: 5432 @@ -273,11 +283,16 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault imagePullSecrets: - name: ghcr-pull containers: - name: worker image: "ghcr.io/example/worker:1.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false resources: requests: cpu: "250m" @@ -339,3 +354,84 @@ spec: stripPrefix: prefixes: - "/api" +--- +# Source: shelf-app/templates/tests.yaml +apiVersion: v1 +kind: Pod +metadata: + name: features-test-login + labels: + shelf.dev/app: features + shelf.dev/test: login + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 + annotations: + helm.sh/hook: test + helm.sh/hook-delete-policy: before-hook-creation +spec: + restartPolicy: Never + enableServiceLinks: false + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault + imagePullSecrets: + - name: ghcr-pull + containers: + - name: login + image: "ghcr.io/example/login-test:1.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false + env: + - name: SHELF_SECRET_API_TOKEN + valueFrom: + secretKeyRef: + name: shelf-secrets + key: api-token + - name: DB + value: "db:5432" + - name: TOKEN + value: "$(SHELF_SECRET_API_TOKEN)" + resources: + requests: + cpu: "20m" + memory: "32Mi" + limits: + memory: "32Mi" + +--- +# Source: shelf-app/templates/tests.yaml +apiVersion: v1 +kind: Pod +metadata: + name: features-test-smoke + labels: + shelf.dev/app: features + shelf.dev/test: smoke + app.kubernetes.io/managed-by: Helm + helm.sh/chart: shelf-app-0.1.0 + annotations: + helm.sh/hook: test + helm.sh/hook-delete-policy: before-hook-creation +spec: + restartPolicy: Never + enableServiceLinks: false + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault + imagePullSecrets: + - name: ghcr-pull + containers: + - name: smoke + image: "curlimages/curl:8.10.1@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false + args: + - "-fsS" + - "http://api:8080/healthz" + resources: + requests: + cpu: "50m" + memory: "64Mi" + diff --git a/internal/chart/testdata/hello.manifests.yaml b/internal/chart/testdata/hello.manifests.yaml index dadef36..d113ac4 100644 --- a/internal/chart/testdata/hello.manifests.yaml +++ b/internal/chart/testdata/hello.manifests.yaml @@ -91,9 +91,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: check image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false command: - "sh" - "-c" @@ -141,9 +146,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: web image: "traefik/whoami:v1.11.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false ports: - name: main containerPort: 80 @@ -200,9 +210,14 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: db image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false env: - name: SHELF_SECRET_DB_PASSWORD valueFrom: diff --git a/internal/chart/testdata/preview.manifests.yaml b/internal/chart/testdata/preview.manifests.yaml index 70e1231..69bd033 100644 --- a/internal/chart/testdata/preview.manifests.yaml +++ b/internal/chart/testdata/preview.manifests.yaml @@ -105,11 +105,16 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault imagePullSecrets: - name: shelf-registry containers: - name: check image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false command: - "sh" - "-c" @@ -157,11 +162,16 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault imagePullSecrets: - name: shelf-registry containers: - name: web image: "traefik/whoami:v1.11.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false ports: - name: main containerPort: 80 @@ -218,11 +228,16 @@ spec: # Service links would inject variables such as DB_PORT=tcp://... into every container. enableServiceLinks: false automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault imagePullSecrets: - name: shelf-registry containers: - name: db image: "postgres:16@sha256:0000000000000000000000000000000000000000000000000000000000000000" + securityContext: + allowPrivilegeEscalation: false env: - name: SHELF_SECRET_DB_PASSWORD valueFrom: diff --git a/internal/cli/app.go b/internal/cli/app.go index 2af58db..8b10469 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -20,7 +20,7 @@ func newAppCmd(o Options) *cobra.Command { cmd.AddCommand( newAppAddCmd(o), newAppRmCmd(o), newAppStatusCmd(o), newAppRedeployCmd(o), newAppSecretsCmd(o), newAppCredentialsCmd(o), newAppTagsCmd(o), - newAppOpenCmd(o), + newAppOpenCmd(o), newAppReportCmd(o), ) return cmd } @@ -34,7 +34,11 @@ needs --domain, a zone of that account. Connections are defined with ` + "`shelf Without a Cloudflare connection, an app answers inside the cluster only. --quick exposes it through a quick tunnel instead: Cloudflare gives it a random https://.trycloudflare.com address, which needs no account and no domain, but changes whenever the tunnel restarts, and is -meant for trying things out. ` + "`shelf app status`" + ` shows the current one.` +meant for trying things out. ` + "`shelf app status`" + ` shows the current one. + +An app reaches its own components, the cluster DNS and the internet, and nothing else: not other +apps, not the Kubernetes API, not the LAN. --allow-lan lets it reach a private address range, +such as a NAS at 192.168.1.10/32; its previews never do. --deny-lan takes that away again.` // accessFlags choose what an app reaches its registry and the internet with. type accessFlags struct { @@ -47,6 +51,8 @@ type accessFlags struct { private bool previews int noPreviews bool + allowLAN []string + denyLAN bool // fs tells a --previews=0 that was given from one that was not. fs *pflag.FlagSet } @@ -66,6 +72,10 @@ func (f *accessFlags) register(fs *pflag.FlagSet, removable bool) { "previews are reachable from this machine only, with --preview", ops.DefaultPreviews, ops.MaxPreviews)) fs.Lookup("previews").NoOptDefVal = strconv.Itoa(ops.DefaultPreviews) fs.BoolVar(&f.noPreviews, "no-previews", false, "remove the app's previews, with their data") + fs.StringArrayVar(&f.allowLAN, "allow-lan", nil, + "let the app reach a private address range, such as 192.168.1.10/32 for a NAS; repeatable, "+ + "replaces the ranges it had; its previews never reach any") + fs.BoolVar(&f.denyLAN, "deny-lan", false, "take away every private address range the app may reach") if removable { fs.BoolVar(&f.noRegistry, "no-registry", false, "pull without a login") fs.BoolVar(&f.private, "private", false, @@ -90,7 +100,7 @@ func exactArgs(n int) cobra.PositionalArgs { // changes reports whether any flag asks for a change. func (f *accessFlags) changes() bool { return f.domain != "" || f.registry != "" || f.noRegistry || f.cloudflare != "" || f.noCloudflare || - f.quick || f.private || f.previews != 0 || f.noPreviews + f.quick || f.private || f.previews != 0 || f.noPreviews || len(f.allowLAN) > 0 || f.denyLAN } func (f *accessFlags) access() (ops.Access, error) { @@ -106,6 +116,8 @@ func (f *accessFlags) access() (ops.Access, error) { Private: f.private || f.noCloudflare, Previews: f.previews, NoPreviews: f.noPreviews, + LAN: f.allowLAN, + DenyLAN: f.denyLAN, } return a, a.Check() } @@ -190,7 +202,7 @@ the internet. } if !access.changes() { return fmt.Errorf("nothing to change; pass --domain, --registry, --cloudflare, " + - "--no-registry, --quick, --private, --previews or --no-previews") + "--no-registry, --quick, --private, --previews, --no-previews, --allow-lan or --deny-lan") } given, err := access.access() if err != nil { diff --git a/internal/cli/build.go b/internal/cli/build.go index 6d17068..23ae4a3 100644 --- a/internal/cli/build.go +++ b/internal/cli/build.go @@ -18,7 +18,7 @@ import ( var componentNameRE = regexp.MustCompile(schema.ComponentNamePattern) // buildPlan is what a CI needs before it can render an app.yaml: the package name, app.yaml's -// `name`, and the components it has to build. The app's name in shelf is not part of it: that +// `name`, and the components and tests it has to build. The app's name in shelf is not part of it: that // is chosen at `shelf app add`. type buildPlan struct { Package string `json:"package"` @@ -27,7 +27,8 @@ type buildPlan struct { Tag string `json:"tag,omitempty"` } -// buildItem is one component that has to be built before rendering. +// buildItem is one component or test that has to be built before rendering. A test is listed +// under component as well: the workflow builds and pushes both alike, as /. type buildItem struct { Component string `json:"component"` // Context is the build directory, relative to the working directory. @@ -38,14 +39,14 @@ func newBuildPlanCmd() *cobra.Command { var branch string cmd := &cobra.Command{ Use: "build-plan ", - Short: "Print the package name and the components that have to be built, as JSON", + Short: "Print the package name and the components and tests that have to be built, as JSON", Long: `Print {package, builds} as JSON: the package name (app.yaml's name), and every component -with a build directory. The build contexts are relative to the app.yaml, and printed relative to -the working directory. +and then every test with a build directory. The build contexts are relative to the app.yaml, and +printed relative to the working directory. A CI workflow publishes the deploy artifact as ghcr.io// and the images below -it: it builds each component, pushes the image and passes the result to -` + "`shelf render --image =`" + `. The app's name in shelf is chosen at +it: it builds each one, pushes the image and passes the result to +` + "`shelf render --image =`" + `. The app's name in shelf is chosen at ` + "`shelf app add`" + `, so one package can be deployed as several apps. With --branch, the plan also holds the tag a push to that branch publishes the deploy artifact @@ -63,16 +64,27 @@ under, branch-. An app with previews runs it as --.`, return fmt.Errorf("branch %q has no letters or digits to name a preview by", branch) } } + add := func(what, name, build string) error { + context := filepath.Join(filepath.Dir(file), build) + if info, err := os.Stat(context); err != nil || !info.IsDir() { + return fmt.Errorf("%s %s: build directory %s does not exist", what, name, context) + } + plan.Builds = append(plan.Builds, buildItem{Component: name, Context: context}) + return nil + } for _, compName := range slices.Sorted(maps.Keys(doc.App.Components)) { - comp := doc.App.Components[compName] - if comp == nil || !comp.IsBuilt() { - continue + if comp := doc.App.Components[compName]; comp != nil && comp.IsBuilt() { + if err := add("component", compName, comp.Build); err != nil { + return err + } } - context := filepath.Join(filepath.Dir(file), comp.Build) - if info, err := os.Stat(context); err != nil || !info.IsDir() { - return fmt.Errorf("component %s: build directory %s does not exist", compName, context) + } + for _, testName := range slices.Sorted(maps.Keys(doc.App.Tests)) { + if test := doc.App.Tests[testName]; test != nil && test.IsBuilt() { + if err := add("test", testName, test.Build); err != nil { + return err + } } - plan.Builds = append(plan.Builds, buildItem{Component: compName, Context: context}) } out, err := json.MarshalIndent(plan, "", " ") if err != nil { @@ -86,21 +98,21 @@ under, branch-. An app with previews runs it as --.`, return cmd } -// parseImages turns the --image flags (component=reference) into a map. +// parseImages turns the --image flags (name=reference, for a component or a test) into a map. func parseImages(flags []string) (map[string]string, error) { images := map[string]string{} for _, f := range flags { - component, ref, ok := strings.Cut(f, "=") + n, ref, ok := strings.Cut(f, "=") switch { - case !ok || component == "" || ref == "": - return nil, fmt.Errorf("--image %q must be =", f) - case !componentNameRE.MatchString(component): - return nil, fmt.Errorf("--image %q: %q is not a component name", f, component) + case !ok || n == "" || ref == "": + return nil, fmt.Errorf("--image %q must be =", f) + case !componentNameRE.MatchString(n): + return nil, fmt.Errorf("--image %q: %q is not a component or test name", f, n) } - if _, dup := images[component]; dup { - return nil, fmt.Errorf("--image: component %s is given twice", component) + if _, dup := images[n]; dup { + return nil, fmt.Errorf("--image: %s is given twice", n) } - images[component] = ref + images[n] = ref } return images, nil } diff --git a/internal/cli/build_test.go b/internal/cli/build_test.go index 883da66..ee7cf0d 100644 --- a/internal/cli/build_test.go +++ b/internal/cli/build_test.go @@ -69,6 +69,43 @@ func TestBuildPlanWithoutBuilds(t *testing.T) { } } +// A built test comes after the components, so the workflow builds and pushes it as +// / like a component. +func TestBuildPlanWithTests(t *testing.T) { + file := writeApp(t, `apiVersion: shelf.dev/v1alpha1 +name: shop +components: + web: { build: ./web, port: 8080 } +tests: + e2e: { build: ./e2e } + smoke: { image: curlimages/curl:8.10.1 } +`, "web", "e2e") + stdout, stderr, code := run(t, "build-plan", file) + if code != 0 { + t.Fatalf("exit code %d: %s", code, stderr) + } + var plan buildPlan + if err := json.Unmarshal([]byte(stdout), &plan); err != nil { + t.Fatalf("%v: %s", err, stdout) + } + dir := filepath.Dir(file) + want := []buildItem{{Component: "web", Context: filepath.Join(dir, "web")}, {Component: "e2e", Context: filepath.Join(dir, "e2e")}} + if len(plan.Builds) != 2 || plan.Builds[0] != want[0] || plan.Builds[1] != want[1] { + t.Errorf("plan %+v, want %+v", plan.Builds, want) + } + + file = writeApp(t, `apiVersion: shelf.dev/v1alpha1 +name: shop +components: + web: { image: nginx } +tests: + e2e: { build: ./e2e } +`) + if _, stderr, code := run(t, "build-plan", file); code == 0 || !strings.Contains(stderr, "test e2e: build directory") { + t.Errorf("exit code %d, stderr %q", code, stderr) + } +} + func TestBuildPlanMissingDirectory(t *testing.T) { file := writeApp(t, builtApp, "web") _, stderr, code := run(t, "build-plan", file) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 963f9ac..5921ba7 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -249,7 +249,7 @@ images first. Registry credentials come from the Docker config (docker login).`, cmd.Flags().StringVarP(&output, "output", "o", "configmap", "what to print: configmap (the deploy manifest) or app (the resolved app.yaml)") cmd.Flags().StringArrayVar(&images, "image", nil, - "image a built component was pushed as, =; repeatable") + "image a built component or test was pushed as, =; repeatable") return cmd } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 7871fc5..a00d9c6 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -64,7 +64,7 @@ func TestValidate(t *testing.T) { }{ {"valid", []string{"validate", hello}, 0, hello + ": valid\n", ""}, {"warning only", []string{"validate", warning}, 0, warning + ": valid (1 warning)\n", - warning + ":6: warning: secrets.pw: secret \"pw\" is not referenced by any component\n"}, + warning + ":6: warning: secrets.pw: secret \"pw\" is not referenced by any component or test\n"}, {"invalid", []string{"validate", invalid}, 1, invalid + ": invalid (1 error, 0 warnings)\n", invalid + ":4: error: components.web.route: route needs a port; add port or ports to the component\n"}, {"several files, one invalid", []string{"validate", hello, invalid}, 1, diff --git a/internal/cli/harness_test.go b/internal/cli/harness_test.go index fe395de..2b33b7f 100644 --- a/internal/cli/harness_test.go +++ b/internal/cli/harness_test.go @@ -56,6 +56,8 @@ type fakeCluster struct { states []cluster.AppState components []cluster.Component stages []cluster.Stage + // tests are the test results AppDiagnosis returns with the stages. + tests []cluster.TestResult // rollouts are what AppRollout returns, one per call; the last one is repeated. rollouts []cluster.Rollout rolloutCalls int @@ -65,6 +67,11 @@ type fakeCluster struct { // verifications are what AppVerification returns, by app or preview namespace; without an // entry, the app is verified and its one component scanned. verifications map[string]cluster.Verification + // isolations are what AppIsolation returns, by namespace; without an entry, the namespace + // enforces baseline and nothing violates restricted. + isolations map[string]cluster.Isolation + // podCIDRs are the address ranges of the cluster's pods; nil is k3s's default. + podCIDRs []string // traefik is the address ForwardTraefik hands out, empty when no Traefik pod is ready. traefik string found bool @@ -113,7 +120,7 @@ func (f *fakeCluster) AddApp(_ context.Context, _ *rest.Config, o cluster.AppOpt config := &cluster.AppConfig{ Artifact: o.Artifact, Insecure: o.Insecure, Domain: o.Domain, Registry: o.Registry, Cloudflare: o.Cloudflare, TunnelID: o.TunnelID, TunnelCredentials: o.TunnelCredentials, Quick: o.Quick, - Previews: o.Previews, + Previews: o.Previews, LAN: o.LAN, } if previous := f.configs[o.Name]; previous != nil && o.TunnelCredentials == nil && o.TunnelID != "" { config.TunnelCredentials = previous.TunnelCredentials @@ -157,7 +164,7 @@ func (f *fakeCluster) AppComponents(context.Context, *rest.Config, string) ([]cl } func (f *fakeCluster) AppDiagnosis(_ context.Context, _ *rest.Config, app string) (cluster.Diagnosis, error) { - return cluster.Diagnosis{Name: app, Stages: f.stages}, nil + return cluster.Diagnosis{Name: app, Stages: f.stages, Tests: f.tests}, nil } func (f *fakeCluster) AppRollout(_ context.Context, _ *rest.Config, app string) (cluster.Rollout, error) { @@ -189,6 +196,21 @@ func (f *fakeCluster) AppVerification(_ context.Context, _ *rest.Config, name st }, nil } +func (f *fakeCluster) AppIsolation(_ context.Context, _ *rest.Config, namespace string, lan []string) (cluster.Isolation, error) { + if iso, ok := f.isolations[namespace]; ok { + iso.LAN = lan + return iso, nil + } + return cluster.Isolation{PodSecurity: "baseline", LAN: lan}, nil +} + +func (f *fakeCluster) NodePodCIDRs(context.Context, *rest.Config) ([]string, error) { + if f.podCIDRs == nil { + return []string{"10.42.0.0/24"}, nil + } + return f.podCIDRs, nil +} + func (f *fakeCluster) ForwardTraefik(context.Context, *rest.Config) (*cluster.Forward, error) { if f.traefik == "" { return nil, errors.New("no Traefik pod is ready") diff --git a/internal/cli/isolation_test.go b/internal/cli/isolation_test.go new file mode 100644 index 0000000..4640161 --- /dev/null +++ b/internal/cli/isolation_test.go @@ -0,0 +1,64 @@ +package cli + +import ( + "slices" + "strings" + "testing" +) + +// TestLANExceptions covers the operator's decision to let one app reach private addresses: it is +// kept until it is changed, replaced as a whole, taken away with --deny-lan, and refused for +// anything that is not a private range or would reach the cluster's pods. +func TestLANExceptions(t *testing.T) { + t.Parallel() + h := newHarness(t) + h.app = appWithSecrets("hello") + h.mustRun(t, "app", "add", "hello", helloArtifact, "--insecure-registry") + if added := h.lastAdded(t); len(added.LAN) != 0 { + t.Errorf("a new app reaches %v, want no private range", added.LAN) + } + + stdout := h.mustRun(t, "app", "credentials", "hello", "--allow-lan", "192.168.1.10/32") + if added := h.lastAdded(t); !slices.Equal(added.LAN, []string{"192.168.1.10/32"}) { + t.Errorf("LAN %v", added.LAN) + } + if !strings.Contains(stdout, "LAN: may reach 192.168.1.10/32; its previews never do") { + t.Errorf("stdout:\n%s", stdout) + } + + // A deploy without the flags keeps the exception, as it keeps the connections. + h.mustRun(t, "app", "add", "hello", helloArtifact, "--insecure-registry") + if added := h.lastAdded(t); !slices.Equal(added.LAN, []string{"192.168.1.10/32"}) { + t.Errorf("a redeploy changed the LAN to %v", added.LAN) + } + + // Given again, the list is replaced, in canonical form and without repeats. + h.mustRun(t, "app", "credentials", "hello", "--allow-lan", "192.168.2.7/24", "--allow-lan", "192.168.2.0/24", "--allow-lan", "100.64.1.2/32") + if added := h.lastAdded(t); !slices.Equal(added.LAN, []string{"192.168.2.0/24", "100.64.1.2/32"}) { + t.Errorf("LAN %v", added.LAN) + } + + stdout = h.mustRun(t, "app", "credentials", "hello", "--deny-lan") + if added := h.lastAdded(t); len(added.LAN) != 0 || !strings.Contains(stdout, "LAN: none") { + t.Errorf("LAN %v after --deny-lan; stdout:\n%s", added.LAN, stdout) + } + + before := len(h.cluster.added) + for _, tt := range []struct { + args []string + want string + }{ + {[]string{"--allow-lan", "8.8.8.0/24"}, "8.8.8.0/24 is not a private range"}, + {[]string{"--allow-lan", "nas.local"}, `"nas.local" is not a CIDR; a LAN exception must be a private CIDR`}, + {[]string{"--allow-lan", "10.42.0.0/16"}, "10.42.0.0/16 overlaps 10.42.0.0/24, where the cluster runs its pods; it would reach other apps"}, + {[]string{"--allow-lan", "192.168.1.0/24", "--deny-lan"}, "cannot be given and taken away at once"}, + } { + _, stderr, code := h.run(t, append([]string{"app", "credentials", "hello"}, tt.args...)...) + if code == 0 || !strings.Contains(stderr, tt.want) { + t.Errorf("%v: exit %d, stderr %q, want %q", tt.args, code, stderr, tt.want) + } + } + if len(h.cluster.added) != before { + t.Error("a refused LAN exception changed the app") + } +} diff --git a/internal/cli/json.go b/internal/cli/json.go index 6996de0..8ba80a9 100644 --- a/internal/cli/json.go +++ b/internal/cli/json.go @@ -138,6 +138,33 @@ type appStatusDocument struct { PreviewLimitReached *bool `json:"previewLimitReached,omitempty"` // Verification is whether the app runs what the cluster's signer signed and CI scanned. Verification *verificationDocument `json:"verification,omitempty"` + // Tests are the app's tests for the values the release last tried, left out without any. + Tests []testDocument `json:"tests,omitempty"` + // Isolation is how the app is fenced in. + Isolation *isolationDocument `json:"isolation,omitempty"` +} + +type testDocument struct { + Name string `json:"name"` + Phase cluster.Phase `json:"phase"` + // ExitCode and Log, the end of its log, only for a test that failed. + ExitCode *int `json:"exitCode,omitempty"` + Log string `json:"log,omitempty"` +} + +type isolationDocument struct { + // PodSecurity is the level the namespace enforces, empty on a platform from before. + PodSecurity string `json:"podSecurity,omitempty"` + // RestrictedViolations are the pods that would break the level restricted, which is warned + // about but admitted. + RestrictedViolations []violationDocument `json:"restrictedViolations,omitempty"` + // LAN are the private ranges the app may reach; always empty for a preview. + LAN []string `json:"lan"` +} + +type violationDocument struct { + Pod string `json:"pod"` + Reason string `json:"reason"` } type verificationDocument struct { @@ -276,6 +303,15 @@ func appStatus(a ops.App, d cluster.Diagnosis) appStatusDocument { Name: c.Name, Phase: c.Phase, Path: c.Path, Address: c.Address, URL: c.URL, Internal: c.Internal, }) } + for _, t := range d.Tests { + doc.Tests = append(doc.Tests, testDocument{Name: t.Name, Phase: t.Phase, ExitCode: t.ExitCode, Log: t.Log}) + } + if iso := a.Isolation; iso != nil { + doc.Isolation = &isolationDocument{PodSecurity: iso.PodSecurity, LAN: append([]string{}, iso.LAN...)} + for _, v := range iso.Restricted { + doc.Isolation.RestrictedViolations = append(doc.Isolation.RestrictedViolations, violationDocument(v)) + } + } if v := a.Verification; v != nil { doc.Verification = &verificationDocument{Verified: v.Verified, Reason: v.Reason, Hint: v.Hint, Source: v.Source, Signer: v.Signer, Exposable: v.Exposable} diff --git a/internal/cli/json_test.go b/internal/cli/json_test.go index ffc6f26..35a6bc6 100644 --- a/internal/cli/json_test.go +++ b/internal/cli/json_test.go @@ -15,8 +15,8 @@ import ( const testCommit = "42a964eb0b4bd7248305d5f22233d6dbabcd56f3" -// fiveStages is the chain of a healthy app; failing replaces the pods stage. -func fiveStages(pods cluster.Stage) []cluster.Stage { +// chain is the chain of a healthy app whose one test passed; failing replaces the pods stage. +func chain(pods cluster.Stage) []cluster.Stage { return []cluster.Stage{ {Name: "the deploy artifact", Object: "OCIRepository hello/deploy", Phase: cluster.PhaseReady, Reason: "Succeeded", Message: "stored artifact for digest 'main@sha256:abc'"}, @@ -27,9 +27,12 @@ func fiveStages(pods cluster.Stage) []cluster.Stage { {Name: "the app", Object: "HelmRelease hello/hello", Phase: cluster.PhaseReady, Reason: "UpgradeSucceeded", Message: "Helm upgrade succeeded"}, pods, + testsPassed, } } +var testsPassed = cluster.Stage{Name: "the tests", Object: "Pods hello/hello-test-*", Phase: cluster.PhaseReady, Message: "1 passed: smoke"} + var ( podsReady = cluster.Stage{Name: "the pods", Object: "Pods", Phase: cluster.PhaseReady, Message: "2 running"} podsFailed = cluster.Stage{Name: "the pods", Object: "Pod hello/web-6d4f", Phase: cluster.PhaseFailed, @@ -107,7 +110,7 @@ func TestJSONAppStatus(t *testing.T) { t.Parallel() h := newHarness(t).public() h.cluster.states = []cluster.AppState{helloState(tt.phase)} - h.cluster.components, h.cluster.stages = components, fiveStages(tt.pods) + h.cluster.components, h.cluster.stages = components, chain(tt.pods) goldenJSON(t, name, runJSON(t, h, 0, "app", "status", "hello")) }) } diff --git a/internal/cli/previews_test.go b/internal/cli/previews_test.go index 88175e1..c63a919 100644 --- a/internal/cli/previews_test.go +++ b/internal/cli/previews_test.go @@ -111,7 +111,7 @@ func previewHarness(t *testing.T) *harness { h.cluster.configs = map[string]*cluster.AppConfig{"hello": {Artifact: hello.Artifact, Previews: 2}} h.cluster.components = []cluster.Component{{Name: "web", Ports: []cluster.Port{{Name: "main", Number: 80}}, Path: "/", Phase: cluster.PhaseReady}} - h.cluster.stages = fiveStages(podsReady) + h.cluster.stages = chain(podsReady) return h } diff --git a/internal/cli/report.go b/internal/cli/report.go new file mode 100644 index 0000000..4f30887 --- /dev/null +++ b/internal/cli/report.go @@ -0,0 +1,258 @@ +package cli + +import ( + "fmt" + "io" + "strings" + + "github.com/spf13/cobra" + + "github.com/tweinmann/shelf/internal/cluster" + "github.com/tweinmann/shelf/internal/ops" +) + +func newAppReportCmd(o Options) *cobra.Command { + var ( + target clusterFlags + asJSON bool + preview string + ) + cmd := &cobra.Command{ + Use: "report ", + Short: "Print a verification report of an app or a preview, as Markdown", + Long: `Print what a pull request needs to know about the version that runs: the commit, whether it +is Ready or which step is not, its tests, its signature and scan, and how it is fenced in. + +With --preview, the report is about the preview of that branch, and says what it runs +differently from the app: every component whose image differs, with the findings of both. +Everything comes from the cluster and the registry, never from Git; the diff of the code is the +pull request's own. + +The Markdown is meant to be posted as it is, for example with +` + "`gh pr comment --body-file -`" + `. With --json, the same comes as a Report document.`, + Example: ` shelf app report hello --preview "$(git branch --show-current)" | gh pr comment --body-file -`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if err := ops.CheckAppName(name); err != nil { + return err + } + shelf, err := target.load(o) + if err != nil { + return err + } + r, err := shelf.Report(cmd.Context(), name, preview) + if err != nil { + return err + } + if asJSON { + return printJSON(cmd.OutOrStdout(), reportDoc(name, r)) + } + printReport(cmd.OutOrStdout(), name, r) + return nil + }, + } + target.registerTarget(cmd.Flags()) + registerJSON(cmd, &asJSON) + cmd.Flags().StringVar(&preview, "preview", "", "the preview of this branch (its name or its slug) instead of the app") + return cmd +} + +// openCommand is how the reader of a report reaches what it is about. +func openCommand(app string, r ops.Report) string { + if r.Branch != "" { + return "shelf app open " + app + " --preview " + r.Branch + } + return "shelf app open " + app +} + +// printReport writes the report as Markdown, in the order a reviewer asks: does it run, did its +// tests pass, can it be trusted, what does it change. +func printReport(out io.Writer, app string, r ops.Report) { + a := r.App + if r.Branch != "" { + fmt.Fprintf(out, "### shelf: preview of %s (%s)\n\n", r.Branch, a.Name) + } else { + fmt.Fprintf(out, "### shelf: %s\n\n", a.Name) + } + state := fmt.Sprintf("**%s**", r.Diagnosis.Phase()) + if a.Commit != "" { + state += fmt.Sprintf(" at commit `%s`", a.Commit[:min(len(a.Commit), 12)]) + } + fmt.Fprintln(out, state+".") + if t := r.Diagnosis.Trouble(); t != nil { + fmt.Fprintf(out, "Not ready at %s: %s\n", t.Name, firstLine(t.Message)) + if t.Hint != "" { + fmt.Fprintf(out, "%s.\n", strings.TrimSuffix(t.Hint, ".")) + } + } + fmt.Fprintf(out, "Open it with `%s`.\n\n", openCommand(app, r)) + + printReportTests(out, r.Diagnosis.Tests) + printReportVerification(out, a.Verification) + if c := r.Comparison; c != nil { + printComparison(out, c) + } + if iso := a.Isolation; iso != nil { + level := iso.PodSecurity + if level == "" { + level = "none (the platform predates it)" + } + lan := "no private addresses" + if len(iso.LAN) > 0 { + lan = "private addresses " + strings.Join(iso.LAN, ", ") + } + fmt.Fprintf(out, "**Isolation:** Pod Security %s; reaches its own components, DNS, the internet and %s.\n", level, lan) + } +} + +func printReportTests(out io.Writer, tests []cluster.TestResult) { + if len(tests) == 0 { + fmt.Fprintln(out, "**Tests:** none declared in app.yaml.") + fmt.Fprintln(out) + return + } + fmt.Fprintln(out, "**Tests:**") + for _, t := range tests { + switch t.Phase { + case cluster.PhaseReady: + fmt.Fprintf(out, "- %s passed\n", t.Name) + case cluster.PhaseFailed: + line := "- " + t.Name + " failed" + if t.ExitCode != nil { + line += fmt.Sprintf(" with exit code %d", *t.ExitCode) + } + fmt.Fprintln(out, line) + if t.Log != "" { + fmt.Fprintf(out, "
end of its log\n\n ```\n") + for _, l := range strings.Split(t.Log, "\n") { + fmt.Fprintf(out, " %s\n", l) + } + fmt.Fprintf(out, " ```\n
\n") + } + default: + fmt.Fprintf(out, "- %s is still running\n", t.Name) + } + } + fmt.Fprintln(out) +} + +func printReportVerification(out io.Writer, v *ops.Verification) { + if v == nil { + return + } + if v.Verified { + fmt.Fprintf(out, "**Verification:** verified, signed by %s.\n", v.Signer) + } else { + fmt.Fprintf(out, "**Verification:** not verified: %s.\n", v.Reason) + } + if v.Scan != nil { + fmt.Fprintf(out, "\n**Scan** (%s, %s):\n\n| Component | Critical | High |\n|---|---|---|\n", v.Scan.Scanner, v.Scan.ScannedAt) + for _, img := range v.Scan.Images { + if img.Scanned { + fmt.Fprintf(out, "| %s | %d | %d |\n", img.Component, img.Critical, img.High) + } else { + fmt.Fprintf(out, "| %s | not scanned | |\n", img.Component) + } + } + } + fmt.Fprintln(out) +} + +func printComparison(out io.Writer, c *ops.Comparison) { + against := c.App + if c.Commit != "" { + against += fmt.Sprintf(" at commit `%s`", c.Commit[:min(len(c.Commit), 12)]) + } + if !c.Comparable { + fmt.Fprintf(out, "**Compared with %s:** cannot say, %s.\n\n", against, c.Reason) + return + } + if len(c.Changed) == 0 { + fmt.Fprintf(out, "**Compared with %s:** the same images.\n\n", against) + return + } + fmt.Fprintf(out, "**Compared with %s:**\n\n| Component | Image | Critical | High |\n|---|---|---|---|\n", against) + for _, ch := range c.Changed { + switch { + case ch.AppDigest == "": + fmt.Fprintf(out, "| %s | new | %d | %d |\n", ch.Component, ch.Critical, ch.High) + case ch.Digest == "": + fmt.Fprintf(out, "| %s | removed | | |\n", ch.Component) + default: + fmt.Fprintf(out, "| %s | changed | %d (was %d) | %d (was %d) |\n", + ch.Component, ch.Critical, ch.AppCritical, ch.High, ch.AppHigh) + } + } + fmt.Fprintf(out, "\nAll images: %d critical, %d high (%s: %d critical, %d high).\n\n", + c.Critical, c.High, c.App, c.AppCritical, c.AppHigh) +} + +type reportDocument struct { + header + Name string `json:"name"` + // PreviewOf and Branch are set for the report of a preview. + PreviewOf string `json:"previewOf,omitempty"` + Branch string `json:"branch,omitempty"` + Phase cluster.Phase `json:"phase"` + Host string `json:"host,omitempty"` + // Open is the command that reaches what the report is about. + Open string `json:"open"` + Commit string `json:"commit,omitempty"` + // Stage is the first stage that is not Ready, left out when everything is. + Stage *stageDocument `json:"stage,omitempty"` + Tests []testDocument `json:"tests,omitempty"` + Verification *verificationDocument `json:"verification,omitempty"` + Isolation *isolationDocument `json:"isolation,omitempty"` + // Comparison is what a preview runs differently from its app. + Comparison *comparisonDocument `json:"comparison,omitempty"` +} + +type comparisonDocument struct { + App string `json:"app"` + Commit string `json:"commit,omitempty"` + Comparable bool `json:"comparable"` + Reason string `json:"reason,omitempty"` + // Changed are the components whose image differs; a digest is left out on the side that + // does not have the component. + Changed []imageChangeDocument `json:"changed"` + Critical int `json:"critical"` + High int `json:"high"` + AppCritical int `json:"appCritical"` + AppHigh int `json:"appHigh"` +} + +type imageChangeDocument struct { + Component string `json:"component"` + Digest string `json:"digest,omitempty"` + AppDigest string `json:"appDigest,omitempty"` + Critical int `json:"critical"` + High int `json:"high"` + AppCritical int `json:"appCritical"` + AppHigh int `json:"appHigh"` +} + +func reportDoc(app string, r ops.Report) reportDocument { + status := appStatus(r.App, r.Diagnosis) + doc := reportDocument{ + header: newHeader("Report"), + Name: r.App.Name, Branch: r.Branch, Phase: r.Diagnosis.Phase(), Host: r.App.Host, + Open: openCommand(app, r), Commit: r.App.Commit, + Tests: status.Tests, Verification: status.Verification, Isolation: status.Isolation, + } + if r.Branch != "" { + doc.PreviewOf = app + } + if t := r.Diagnosis.Trouble(); t != nil { + s := stageDocument{Name: t.Name, Object: t.Object, Phase: t.Phase, Reason: t.Reason, Message: t.Message, Hint: t.Hint} + doc.Stage = &s + } + if c := r.Comparison; c != nil { + doc.Comparison = &comparisonDocument{App: c.App, Commit: c.Commit, Comparable: c.Comparable, Reason: c.Reason, + Changed: []imageChangeDocument{}, Critical: c.Critical, High: c.High, AppCritical: c.AppCritical, AppHigh: c.AppHigh} + for _, ch := range c.Changed { + doc.Comparison.Changed = append(doc.Comparison.Changed, imageChangeDocument(ch)) + } + } + return doc +} diff --git a/internal/cli/report_test.go b/internal/cli/report_test.go new file mode 100644 index 0000000..17ace96 --- /dev/null +++ b/internal/cli/report_test.go @@ -0,0 +1,42 @@ +package cli + +import ( + "path/filepath" + "testing" + + "github.com/tweinmann/shelf/internal/cluster" + "github.com/tweinmann/shelf/internal/testutil" +) + +// reportHarness is hello with a preview of feature/login that runs a new web image, and a test +// that passed on both. +func reportHarness(t *testing.T) *harness { + t.Helper() + h := previewHarness(t) + h.cluster.tests = []cluster.TestResult{{Name: "smoke", Phase: cluster.PhaseReady}} + h.cluster.verifications = map[string]cluster.Verification{"hello--feature-login": { + Verified: true, Source: "https://github.com/acme/hello", Commit: "3f2a9c1e5b7d9f1a3c5e7b9d1f3a5c7e9b1d3f5a", + Scan: &cluster.Scan{Scanner: "trivy 0.75.0", ScannedAt: "2026-10-28T13:00:00Z", Images: []cluster.ScannedImage{ + {Component: "smoke", Image: "curlimages/curl:8.10.1@sha256:c0ff", Digest: "sha256:c0ff", Scanned: true}, + {Component: "web", Image: "ghcr.io/o/hello/web:sha-3f2a9c1@sha256:3f2a", Digest: "sha256:3f2a", Scanned: true, High: 4}, + }}, + }} + return h +} + +func TestReport(t *testing.T) { + t.Parallel() + h := reportHarness(t) + testutil.Golden(t, filepath.Join("testdata", "report-preview.md"), + []byte(h.mustRun(t, "app", "report", "hello", "--preview", "feature/login"))) + testutil.Golden(t, filepath.Join("testdata", "report-app.md"), []byte(h.mustRun(t, "app", "report", "hello"))) + goldenJSON(t, "report-preview", runJSON(t, h, 0, "app", "report", "hello", "--preview", "feature-login")) +} + +// A failed test shows up in the report with the end of its log, and the first stage that is not +// Ready says where. +func TestReportOfAFailedTest(t *testing.T) { + t.Parallel() + testutil.Golden(t, filepath.Join("testdata", "report-test-failed.md"), + []byte(failedTestHarness(t).mustRun(t, "app", "report", "hello"))) +} diff --git a/internal/cli/status.go b/internal/cli/status.go index 491cba9..0e10757 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -6,6 +6,7 @@ import ( "io" "maps" "slices" + "strings" "time" "github.com/spf13/cobra" @@ -109,6 +110,7 @@ This is what the admin UI shows on its pages, in text.`, printApp(out, app) printComponents(out, app.Components, app.Public || app.Quick) printVerification(out, app.Verification, preview != "") + printIsolation(out, app.Isolation) printChain(out, diagnosis) printPreviews(out, app) return nil @@ -311,10 +313,37 @@ func componentAddress(c ops.Component, public bool) string { return c.Address + " (" + note + ")" } +// printIsolation says how the app is fenced in: the Pod Security level, the pods the level +// restricted would refuse, and the private ranges it may reach. +func printIsolation(out io.Writer, iso *cluster.Isolation) { + if iso == nil { + return + } + if iso.PodSecurity == "" { + fmt.Fprintln(out, " pod security none: the platform predates it; `shelf init cluster` installs the current one") + } else { + fmt.Fprintf(out, " pod security %s\n", iso.PodSecurity) + } + for _, v := range iso.Restricted { + fmt.Fprintf(out, " restricted would refuse %s: %s\n", v.Pod, v.Reason) + } + lan := "none" + if len(iso.LAN) > 0 { + lan = strings.Join(iso.LAN, ", ") + } + fmt.Fprintf(out, " lan %s\n", lan) +} + func printChain(out io.Writer, d cluster.Diagnosis) { for _, stage := range d.Stages { fmt.Fprintf(out, " %-9s %s\n", stage.Phase, stage.Name) - if stage.Message != "" { + switch { + // A failed test brings the end of its log, which is the point of showing it. + case stage.Name == "the tests" && stage.Phase == cluster.PhaseFailed: + for _, line := range strings.Split(stage.Message, "\n") { + fmt.Fprintf(out, " %s\n", line) + } + case stage.Message != "": fmt.Fprintf(out, " %s\n", firstLine(stage.Message)) } if stage.Hint != "" && stage.Phase != cluster.PhaseReady { diff --git a/internal/cli/tenant_test.go b/internal/cli/tenant_test.go index 0b9f315..7e4a54a 100644 --- a/internal/cli/tenant_test.go +++ b/internal/cli/tenant_test.go @@ -26,7 +26,9 @@ func TestTenantAgentInstructions(t *testing.T) { t.Fatal(err) } for _, m := range shelfCommand.FindAllStringSubmatch(string(data), -1) { - fields := strings.Fields(m[1])[1:] + // What a shelf command is piped into is not shelf's. + command, _, _ := strings.Cut(m[1], " | ") + fields := strings.Fields(command)[1:] cmd, rest, err := root.Find(fields) if err != nil || cmd == root { t.Errorf("%s: %q is not a shelf command", file, m[1]) diff --git a/internal/cli/testdata/json/appstatus-failing.json b/internal/cli/testdata/json/appstatus-failing.json index 699d6a9..a86b0da 100644 --- a/internal/cli/testdata/json/appstatus-failing.json +++ b/internal/cli/testdata/json/appstatus-failing.json @@ -62,6 +62,12 @@ "reason": "ImagePullBackOff", "message": "Back-off pulling image", "hint": "the image cannot be pulled; check the reference and whether the cluster may read that registry" + }, + { + "name": "the tests", + "object": "Pods hello/hello-test-*", + "phase": "Ready", + "message": "1 passed: smoke" } ], "verification": { @@ -80,5 +86,9 @@ "scannedAt": "2026-10-28T12:00:00Z" } ] + }, + "isolation": { + "podSecurity": "baseline", + "lan": [] } } diff --git a/internal/cli/testdata/json/appstatus-preview.json b/internal/cli/testdata/json/appstatus-preview.json index f5fc09d..1e68901 100644 --- a/internal/cli/testdata/json/appstatus-preview.json +++ b/internal/cli/testdata/json/appstatus-preview.json @@ -53,6 +53,12 @@ "object": "Pods", "phase": "Ready", "message": "2 running" + }, + { + "name": "the tests", + "object": "Pods hello/hello-test-*", + "phase": "Ready", + "message": "1 passed: smoke" } ], "verification": { @@ -71,5 +77,9 @@ "scannedAt": "2026-10-28T12:00:00Z" } ] + }, + "isolation": { + "podSecurity": "baseline", + "lan": [] } } diff --git a/internal/cli/testdata/json/appstatus-previews.json b/internal/cli/testdata/json/appstatus-previews.json index fa84b7d..890b803 100644 --- a/internal/cli/testdata/json/appstatus-previews.json +++ b/internal/cli/testdata/json/appstatus-previews.json @@ -54,6 +54,12 @@ "object": "Pods", "phase": "Ready", "message": "2 running" + }, + { + "name": "the tests", + "object": "Pods hello/hello-test-*", + "phase": "Ready", + "message": "1 passed: smoke" } ], "previews": [ @@ -91,5 +97,9 @@ "scannedAt": "2026-10-28T12:00:00Z" } ] + }, + "isolation": { + "podSecurity": "baseline", + "lan": [] } } diff --git a/internal/cli/testdata/json/appstatus-test-failed.json b/internal/cli/testdata/json/appstatus-test-failed.json new file mode 100644 index 0000000..b2e0957 --- /dev/null +++ b/internal/cli/testdata/json/appstatus-test-failed.json @@ -0,0 +1,102 @@ +{ + "apiVersion": "shelf.dev/v1alpha1", + "kind": "AppStatus", + "name": "hello", + "phase": "Failed", + "host": "hello.shelf.internal", + "artifact": "oci://ghcr.io/o/hello:main", + "registry": "ghcr", + "exposure": { + "kind": "none" + }, + "revision": "main@sha256:abc", + "commit": "42a964eb0b4bd7248305d5f22233d6dbabcd56f3", + "deployed": "2026-10-27T13:03:00Z", + "components": [ + { + "name": "web", + "phase": "Ready", + "path": "/", + "address": "hello.shelf.internal/" + } + ], + "stages": [ + { + "name": "the deploy artifact", + "object": "OCIRepository hello/deploy", + "phase": "Ready", + "reason": "Succeeded", + "message": "stored artifact for digest 'main@sha256:abc'" + }, + { + "name": "the app values", + "object": "Kustomization hello/deploy", + "phase": "Ready", + "reason": "ReconciliationSucceeded", + "message": "Applied revision: main@sha256:abc" + }, + { + "name": "the chart", + "object": "OCIRepository hello/shelf-app", + "phase": "Ready", + "reason": "Succeeded", + "message": "stored artifact for digest '0.6.0@sha256:def'" + }, + { + "name": "the app", + "object": "HelmRelease hello/hello", + "phase": "Ready", + "reason": "UpgradeSucceeded", + "message": "Helm upgrade succeeded" + }, + { + "name": "the pods", + "object": "Pods", + "phase": "Ready", + "message": "2 running" + }, + { + "name": "the tests", + "object": "Pod hello/hello-test-smoke", + "phase": "Failed", + "reason": "TestFailed", + "message": "test smoke failed with exit code 22:\ncurl: (22) The requested URL returned error: 500\nretrying would not help", + "hint": "the app's own test smoke fails against this version; the previous version runs again. Its log says why" + } + ], + "verification": { + "verified": true, + "source": "https://github.com/acme/hello", + "signer": "keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$", + "exposable": true, + "scan": [ + { + "component": "web", + "digest": "sha256:2006", + "scanned": true, + "critical": 1, + "high": 26, + "scanner": "trivy 0.75.0", + "scannedAt": "2026-10-28T12:00:00Z" + } + ] + }, + "tests": [ + { + "name": "smoke", + "phase": "Failed", + "exitCode": 22, + "log": "curl: (22) The requested URL returned error: 500\nretrying would not help" + } + ], + "isolation": { + "podSecurity": "baseline", + "restrictedViolations": [ + { + "pod": "db-0", + "reason": "allowPrivilegeEscalation != false, runAsNonRoot != true" + } + ], + "lan": [] + } +} diff --git a/internal/cli/testdata/json/appstatus-unverified.json b/internal/cli/testdata/json/appstatus-unverified.json index 44db7f0..9e4a974 100644 --- a/internal/cli/testdata/json/appstatus-unverified.json +++ b/internal/cli/testdata/json/appstatus-unverified.json @@ -18,5 +18,9 @@ "source": "https://github.com/acme/hello", "signer": "keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$", "exposable": false + }, + "isolation": { + "podSecurity": "baseline", + "lan": [] } } diff --git a/internal/cli/testdata/json/appstatus.json b/internal/cli/testdata/json/appstatus.json index b88621e..9c7e4f5 100644 --- a/internal/cli/testdata/json/appstatus.json +++ b/internal/cli/testdata/json/appstatus.json @@ -60,6 +60,12 @@ "object": "Pods", "phase": "Ready", "message": "2 running" + }, + { + "name": "the tests", + "object": "Pods hello/hello-test-*", + "phase": "Ready", + "message": "1 passed: smoke" } ], "verification": { @@ -78,5 +84,9 @@ "scannedAt": "2026-10-28T12:00:00Z" } ] + }, + "isolation": { + "podSecurity": "baseline", + "lan": [] } } diff --git a/internal/cli/testdata/json/report-preview.json b/internal/cli/testdata/json/report-preview.json new file mode 100644 index 0000000..3a2b4dd --- /dev/null +++ b/internal/cli/testdata/json/report-preview.json @@ -0,0 +1,75 @@ +{ + "apiVersion": "shelf.dev/v1alpha1", + "kind": "Report", + "name": "hello--feature-login", + "previewOf": "hello", + "branch": "feature-login", + "phase": "Ready", + "host": "hello--feature-login.shelf.internal", + "open": "shelf app open hello --preview feature-login", + "commit": "3f2a9c1e5b7d9f1a3c5e7b9d1f3a5c7e9b1d3f5a", + "tests": [ + { + "name": "smoke", + "phase": "Ready" + } + ], + "verification": { + "verified": true, + "source": "https://github.com/acme/hello", + "signer": "keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\\.com/tweinmann/shelf/\\.github/workflows/(build|release)\\.yml@refs/(tags|heads)/.+$", + "exposable": false, + "scan": [ + { + "component": "smoke", + "digest": "sha256:c0ff", + "scanned": true, + "critical": 0, + "high": 0, + "scanner": "trivy 0.75.0", + "scannedAt": "2026-10-28T13:00:00Z" + }, + { + "component": "web", + "digest": "sha256:3f2a", + "scanned": true, + "critical": 0, + "high": 4, + "scanner": "trivy 0.75.0", + "scannedAt": "2026-10-28T13:00:00Z" + } + ] + }, + "isolation": { + "podSecurity": "baseline", + "lan": [] + }, + "comparison": { + "app": "hello", + "commit": "42a964eb0b4bd7248305d5f22233d6dbabcd56f3", + "comparable": true, + "changed": [ + { + "component": "smoke", + "digest": "sha256:c0ff", + "critical": 0, + "high": 0, + "appCritical": 0, + "appHigh": 0 + }, + { + "component": "web", + "digest": "sha256:3f2a", + "appDigest": "sha256:2006", + "critical": 0, + "high": 4, + "appCritical": 1, + "appHigh": 26 + } + ], + "critical": 0, + "high": 4, + "appCritical": 1, + "appHigh": 26 + } +} diff --git a/internal/cli/testdata/report-app.md b/internal/cli/testdata/report-app.md new file mode 100644 index 0000000..610f284 --- /dev/null +++ b/internal/cli/testdata/report-app.md @@ -0,0 +1,17 @@ +### shelf: hello + +**Ready** at commit `42a964eb0b4b`. +Open it with `shelf app open hello`. + +**Tests:** +- smoke passed + +**Verification:** verified, signed by keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\.com/tweinmann/shelf/\.github/workflows/(build|release)\.yml@refs/(tags|heads)/.+$. + +**Scan** (trivy 0.75.0, 2026-10-28T12:00:00Z): + +| Component | Critical | High | +|---|---|---| +| web | 1 | 26 | + +**Isolation:** Pod Security baseline; reaches its own components, DNS, the internet and no private addresses. diff --git a/internal/cli/testdata/report-preview.md b/internal/cli/testdata/report-preview.md new file mode 100644 index 0000000..090c47f --- /dev/null +++ b/internal/cli/testdata/report-preview.md @@ -0,0 +1,27 @@ +### shelf: preview of feature/login (hello--feature-login) + +**Ready** at commit `3f2a9c1e5b7d`. +Open it with `shelf app open hello --preview feature/login`. + +**Tests:** +- smoke passed + +**Verification:** verified, signed by keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\.com/tweinmann/shelf/\.github/workflows/(build|release)\.yml@refs/(tags|heads)/.+$. + +**Scan** (trivy 0.75.0, 2026-10-28T13:00:00Z): + +| Component | Critical | High | +|---|---|---| +| smoke | 0 | 0 | +| web | 0 | 4 | + +**Compared with hello at commit `42a964eb0b4b`:** + +| Component | Image | Critical | High | +|---|---|---|---| +| smoke | new | 0 | 0 | +| web | changed | 0 (was 1) | 4 (was 26) | + +All images: 0 critical, 4 high (hello: 1 critical, 26 high). + +**Isolation:** Pod Security baseline; reaches its own components, DNS, the internet and no private addresses. diff --git a/internal/cli/testdata/report-test-failed.md b/internal/cli/testdata/report-test-failed.md new file mode 100644 index 0000000..1f40c7f --- /dev/null +++ b/internal/cli/testdata/report-test-failed.md @@ -0,0 +1,26 @@ +### shelf: hello + +**Failed** at commit `42a964eb0b4b`. +Not ready at the tests: test smoke failed with exit code 22: … +the app's own test smoke fails against this version; the previous version runs again. Its log says why. +Open it with `shelf app open hello`. + +**Tests:** +- smoke failed with exit code 22 +
end of its log + + ``` + curl: (22) The requested URL returned error: 500 + retrying would not help + ``` +
+ +**Verification:** verified, signed by keyless, issuer https://token.actions.githubusercontent.com, subject ^https://github\.com/tweinmann/shelf/\.github/workflows/(build|release)\.yml@refs/(tags|heads)/.+$. + +**Scan** (trivy 0.75.0, 2026-10-28T12:00:00Z): + +| Component | Critical | High | +|---|---|---| +| web | 1 | 26 | + +**Isolation:** Pod Security baseline; reaches its own components, DNS, the internet and no private addresses. diff --git a/internal/cli/tests_test.go b/internal/cli/tests_test.go new file mode 100644 index 0000000..e8793d2 --- /dev/null +++ b/internal/cli/tests_test.go @@ -0,0 +1,62 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/tweinmann/shelf/internal/cluster" +) + +// failedTestHarness is hello with a test that failed after its pods came up, and a database +// that would break Pod Security restricted. +func failedTestHarness(t *testing.T) *harness { + t.Helper() + h := newHarness(t).public() + h.cluster.states = []cluster.AppState{helloState(cluster.PhaseFailed)} + h.cluster.components = []cluster.Component{ + {Name: "web", Ports: []cluster.Port{{Name: "main", Number: 80}}, Path: "/", Phase: cluster.PhaseReady}, + } + stages := chain(podsReady) + stages[5] = cluster.Stage{Name: "the tests", Object: "Pod hello/hello-test-smoke", Phase: cluster.PhaseFailed, + Reason: "TestFailed", Message: "test smoke failed with exit code 22:\ncurl: (22) The requested URL returned error: 500\nretrying would not help", + Hint: "the app's own test smoke fails against this version; the previous version runs again. Its log says why"} + h.cluster.stages = stages + code := 22 + h.cluster.tests = []cluster.TestResult{{Name: "smoke", Phase: cluster.PhaseFailed, ExitCode: &code, + Log: "curl: (22) The requested URL returned error: 500\nretrying would not help"}} + h.cluster.isolations = map[string]cluster.Isolation{"hello": {PodSecurity: "baseline", Restricted: []cluster.Violation{ + {Pod: "db-0", Reason: "allowPrivilegeEscalation != false, runAsNonRoot != true"}, + }}} + return h +} + +func TestStatusOfAFailedTest(t *testing.T) { + t.Parallel() + h := failedTestHarness(t) + stdout := h.mustRun(t, "app", "status", "hello") + for _, want := range []string{ + " Failed the tests\n" + + " test smoke failed with exit code 22:\n" + + " curl: (22) The requested URL returned error: 500\n" + + " retrying would not help\n" + + " the app's own test smoke fails against this version; the previous version runs again. Its log says why\n", + " pod security baseline\n restricted would refuse db-0: allowPrivilegeEscalation != false, runAsNonRoot != true\n", + " lan none\n", + } { + if !strings.Contains(stdout, want) { + t.Errorf("stdout lacks %q:\n%s", want, stdout) + } + } + goldenJSON(t, "appstatus-test-failed", runJSON(t, h, 0, "app", "status", "hello")) +} + +func TestStatusShowsLANExceptions(t *testing.T) { + t.Parallel() + h := newHarness(t).public() + h.app = appWithSecrets("hello") + h.mustRun(t, "app", "add", "hello", helloArtifact, "--insecure-registry", "--allow-lan", "192.168.1.10/32") + h.cluster.stages = chain(podsReady) + if stdout := h.mustRun(t, "app", "status", "hello"); !strings.Contains(stdout, " lan 192.168.1.10/32\n") { + t.Errorf("stdout:\n%s", stdout) + } +} diff --git a/internal/cli/wait_test.go b/internal/cli/wait_test.go index 4fa1ad7..8fd9650 100644 --- a/internal/cli/wait_test.go +++ b/internal/cli/wait_test.go @@ -10,10 +10,11 @@ import ( // rolledOut is hello after the artifact of testCommit rolled out. func rolledOut(pods cluster.Stage) cluster.Rollout { return cluster.Rollout{ - Diagnosis: cluster.Diagnosis{Name: "hello", Stages: fiveStages(pods)}, + Diagnosis: cluster.Diagnosis{Name: "hello", Stages: chain(pods)}, Revision: "main@sha256:abc", Commit: testCommit, AppliedRevision: "main@sha256:abc", AttemptedRevision: "main@sha256:abc", ValuesDigest: "sha256:values", ReleasedDigest: "sha256:values", AttemptedDigest: "sha256:values", + TestedDigest: "sha256:values", } } diff --git a/internal/cluster/apps.go b/internal/cluster/apps.go index 8f02ce3..bebbf19 100644 --- a/internal/cluster/apps.go +++ b/internal/cluster/apps.go @@ -69,6 +69,9 @@ type AppOptions struct { // Previews is how many branches of the app may run as previews at once; 0 switches previews // off and removes the ones that run. Previews int + // LAN are the private address ranges the app's pods may reach, as CIDRs; its previews never + // get them. + LAN []string // Secrets are all secret values of the app; they replace the stored ones. Secrets map[string]string Timeout time.Duration @@ -120,11 +123,21 @@ func AppProvider(o AppOptions) *unstructured.Unstructured { "quick": o.Quick, "registry": o.Registry, "cloudflare": o.Cloudflare, + "lan": lanInput(o.LAN), }, }, }} } +// lanInput is the LAN exceptions as an input: always a list, so the platform can range over it. +func lanInput(lan []string) []any { + out := make([]any, len(lan)) + for i, cidr := range lan { + out[i] = cidr + } + return out +} + // AppSecrets returns the stored secret values of an app; none if the app has no Secret yet. func AppSecrets(ctx context.Context, cfg *rest.Config, app string) (map[string]string, error) { c, err := newClient(cfg) @@ -215,6 +228,8 @@ type AppConfig struct { Quick bool // Previews is how many previews the app may have, 0 when it has none. Previews int + // LAN are the private address ranges the app may reach, none when it has no exceptions. + LAN []string } // ReadAppConfig returns how an app is registered, or nil when there is no such app. @@ -241,6 +256,12 @@ func ReadAppConfig(ctx context.Context, cfg *rest.Config, app string) (*AppConfi config.Quick, _ = values["quick"].(bool) config.Registry, _ = values["registry"].(string) config.Cloudflare, _ = values["cloudflare"].(string) + lan, _ := values["lan"].([]any) + for _, cidr := range lan { + if s, ok := cidr.(string); ok { + config.LAN = append(config.LAN, s) + } + } previews, err := c.get(ctx, ref{gvk: providerGVK, namespace: SystemNamespace, name: PreviewProviderName(app)}) if err != nil { diff --git a/internal/cluster/cluster_test.go b/internal/cluster/cluster_test.go index dd51eb9..9df75e2 100644 --- a/internal/cluster/cluster_test.go +++ b/internal/cluster/cluster_test.go @@ -230,7 +230,8 @@ func TestPlatformObjects(t *testing.T) { AppProvider(AppOptions{Name: "hello", Artifact: app, Domain: "example.com", TunnelID: "t-1", Registry: "ghcr", Cloudflare: "home"}), AppProvider(AppOptions{Name: "empty", Artifact: app}), - AppProvider(AppOptions{Name: "quick", Artifact: app, Quick: true})) + AppProvider(AppOptions{Name: "quick", Artifact: app, Quick: true}), + AppProvider(AppOptions{Name: "nas", Artifact: app, LAN: []string{"192.168.1.10/32"}})) var out []byte for _, obj := range objs { @@ -387,3 +388,37 @@ func TestMovingApps(t *testing.T) { }) } } + +func TestPodCIDRs(t *testing.T) { + t.Parallel() + node := func(spec map[string]any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{"kind": "Node", "spec": spec}} + } + got := podCIDRs([]*unstructured.Unstructured{ + node(map[string]any{"podCIDR": "10.42.0.0/24", "podCIDRs": []any{"10.42.0.0/24", "fd00:42::/64"}}), + node(map[string]any{"podCIDR": "10.42.1.0/24"}), + node(map[string]any{}), + }) + if strings.Join(got, ",") != "10.42.0.0/24,fd00:42::/64,10.42.1.0/24" { + t.Errorf("pod ranges %v", got) + } +} + +// TestViolations reads the warnings the API server gave for a dry run of enforce=restricted on +// the dev cluster (2026-10-28). +func TestViolations(t *testing.T) { + t.Parallel() + got := violations([]string{ + `existing pods in namespace "shop" violate the new PodSecurity enforce level "restricted:latest"`, + `db-0: allowPrivilegeEscalation != false, runAsNonRoot != true`, + `web-5d8f (and 2 other pods): allowPrivilegeEscalation != false, unrestricted capabilities, runAsNonRoot != true, seccompProfile`, + `something else entirely`, + }) + want := []Violation{ + {Pod: "db-0", Reason: "allowPrivilegeEscalation != false, runAsNonRoot != true"}, + {Pod: "web-5d8f (and 2 other pods)", Reason: "allowPrivilegeEscalation != false, unrestricted capabilities, runAsNonRoot != true, seccompProfile"}, + } + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Errorf("violations %+v, want %+v", got, want) + } +} diff --git a/internal/cluster/isolation.go b/internal/cluster/isolation.go new file mode 100644 index 0000000..5690e42 --- /dev/null +++ b/internal/cluster/isolation.go @@ -0,0 +1,135 @@ +package cluster + +import ( + "context" + "fmt" + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" +) + +var nodeGVK = schema.GroupVersionKind{Version: "v1", Kind: "Node"} + +// NodePodCIDRs returns the address ranges the cluster gives its pods, from every node. A LAN +// exception inside one of them would let an app reach the pods of other apps. +func NodePodCIDRs(ctx context.Context, cfg *rest.Config) ([]string, error) { + c, err := newClient(cfg) + if err != nil { + return nil, err + } + nodes, err := c.list(ctx, nodeGVK, "", "") + if err != nil { + return nil, err + } + return podCIDRs(nodes), nil +} + +// podCIDRs reads spec.podCIDRs of the nodes, and spec.podCIDR of a node that has only that. +func podCIDRs(nodes []*unstructured.Unstructured) []string { + var out []string + for _, n := range nodes { + cidrs, _, _ := unstructured.NestedStringSlice(n.Object, "spec", "podCIDRs") + if len(cidrs) == 0 { + if one, _, _ := unstructured.NestedString(n.Object, "spec", "podCIDR"); one != "" { + cidrs = []string{one} + } + } + out = append(out, cidrs...) + } + return out +} + +// Pod Security labels the platform sets on app and preview namespaces. +const ( + podSecurityEnforce = "pod-security.kubernetes.io/enforce" + // restrictedLevel is the level the namespaces warn about, and the one violations are + // reported against. + restrictedLevel = "restricted" +) + +// Isolation is how an app or preview is fenced in. +type Isolation struct { + // PodSecurity is the level its namespace enforces, empty for a namespace from a platform + // before Pod Security. + PodSecurity string + // Restricted are the pods that would violate the level restricted, which the namespace warns + // about but admits. + Restricted []Violation + // LAN are the private ranges it may reach, none for a preview. + LAN []string +} + +// Violation is one pod, or several alike, that breaks a Pod Security level, and why. +type Violation struct { + Pod string + Reason string +} + +// AppIsolation reads how an app or a preview (by its namespace) is fenced in. It asks the API +// server which pods would violate restricted with a dry run of that label: the API server answers +// with a warning per pod and keeps nothing. +func AppIsolation(ctx context.Context, cfg *rest.Config, namespace string, lan []string) (Isolation, error) { + c, err := newClient(cfg) + if err != nil { + return Isolation{}, err + } + ns, err := c.get(ctx, ref{gvk: namespaceGVK, name: namespace}) + if err != nil { + return Isolation{}, err + } + if ns == nil { + return Isolation{}, &NotFoundError{Name: namespace} + } + iso := Isolation{PodSecurity: ns.GetLabels()[podSecurityEnforce], LAN: lan} + if iso.PodSecurity == "" || iso.PodSecurity == restrictedLevel { + return iso, nil + } + warnings := &warningCollector{} + dry := rest.CopyConfig(cfg) + dry.WarningHandler = warnings + dyn, err := dynamic.NewForConfig(dry) + if err != nil { + return Isolation{}, err + } + patch := fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, podSecurityEnforce, restrictedLevel) + if _, err := dyn.Resource(corev1.SchemeGroupVersion.WithResource("namespaces")).Patch(ctx, namespace, + types.MergePatchType, []byte(patch), metav1.PatchOptions{DryRun: []string{metav1.DryRunAll}}); err != nil { + return Isolation{}, fmt.Errorf("checking the pods of %s against Pod Security %s: %w", namespace, restrictedLevel, err) + } + iso.Restricted = violations(warnings.messages) + return iso, nil +} + +// warningCollector keeps the warnings of the API server's responses. +type warningCollector struct { + mu sync.Mutex + messages []string +} + +func (w *warningCollector) HandleWarningHeader(_ int, _ string, message string) { + w.mu.Lock() + defer w.mu.Unlock() + w.messages = append(w.messages, message) +} + +// violations reads the warnings of a Pod Security label change. The first one says the level is +// violated at all; each further one is ": ", where pods that break it alike are named +// together, as "web-5d8f (and 2 other pods)". +func violations(warnings []string) []Violation { + var out []Violation + for _, w := range warnings { + pod, reason, ok := strings.Cut(w, ": ") + if !ok || strings.HasPrefix(w, "existing pods in namespace") || strings.Contains(pod, " violate") { + continue + } + out = append(out, Violation{Pod: pod, Reason: reason}) + } + return out +} diff --git a/internal/cluster/platform_test.go b/internal/cluster/platform_test.go index 4060c55..2e0b91e 100644 --- a/internal/cluster/platform_test.go +++ b/internal/cluster/platform_test.go @@ -169,3 +169,105 @@ func TestPlatformVerifies(t *testing.T) { }) } } + +// TestPlatformIsolates renders an app without and with LAN exceptions, an app with a tunnel, and +// a preview of an app with exceptions, and checks the Pod Security labels, the NetworkPolicy and +// how a failing test is treated. +func TestPlatformIsolates(t *testing.T) { + t.Parallel() + settings := Settings{Chart: Artifact{URL: "oci://ghcr.io/tweinmann/shelf/charts/shelf-app", Tag: "0.9.0"}, Signer: DefaultSigner()} + appInputs := func(extra map[string]any) map[string]any { + in := map[string]any{ + "name": "hello", "url": "oci://ghcr.io/acme/hello", "tag": "main", "insecure": false, + "domain": "", "tunnel": "", "quick": false, "registry": "ghcr", "cloudflare": "", + } + for k, v := range extra { + in[k] = v + } + return in + } + // A provider from before LAN exceptions has no input lan. + plain := platformObjects(t, "../../platform/apps/resourceset.yaml", settings, appInputs(nil)) + nas := platformObjects(t, "../../platform/apps/resourceset.yaml", settings, appInputs(map[string]any{ + "lan": []any{"192.168.1.10/32", "192.168.2.0/24"}, + })) + tunnel := platformObjects(t, "../../platform/apps/resourceset.yaml", settings, appInputs(map[string]any{ + "domain": "example.com", "tunnel": "t-1", "cloudflare": "home", "lan": []any{}, + })) + preview := platformObjects(t, "../../platform/previews/resourceset.yaml", settings, map[string]any{ + "app": "hello", "url": "oci://ghcr.io/acme/hello", "tag": "branch-feature-login", + "insecure": false, "registry": "ghcr", "lan": []any{"192.168.1.10/32"}, + }) + + // The extra ipBlocks of the rule that reaches the internet. + exceptions := func(name string, objs []*unstructured.Unstructured) []string { + t.Helper() + policy := find(objs, "NetworkPolicy", "shelf-isolation") + if policy == nil { + t.Fatalf("%s: no NetworkPolicy shelf-isolation", name) + } + selector, _, _ := unstructured.NestedMap(policy.Object, "spec", "podSelector") + types, _, _ := unstructured.NestedStringSlice(policy.Object, "spec", "policyTypes") + if len(selector) != 0 || strings.Join(types, ",") != "Ingress,Egress" { + t.Errorf("%s: the policy selects %v for %v, want every pod both ways", name, selector, types) + } + ingress, _, _ := unstructured.NestedSlice(policy.Object, "spec", "ingress") + if from := ingress[0].(map[string]any)["from"].([]any); len(ingress) != 1 || len(from) != 2 { + t.Errorf("%s: ingress %v, want the namespace and Traefik only", name, ingress) + } + egress, _, _ := unstructured.NestedSlice(policy.Object, "spec", "egress") + if len(egress) != 3 { + t.Fatalf("%s: egress %v, want namespace, DNS and internet", name, egress) + } + blocks := egress[2].(map[string]any)["to"].([]any) + v4 := blocks[0].(map[string]any)["ipBlock"].(map[string]any) + if v4["cidr"] != "0.0.0.0/0" || len(v4["except"].([]any)) != 5 { + t.Errorf("%s: internet rule %v", name, v4) + } + var out []string + for _, b := range blocks[2:] { + out = append(out, b.(map[string]any)["ipBlock"].(map[string]any)["cidr"].(string)) + } + return out + } + if got := exceptions("app", plain); len(got) != 0 { + t.Errorf("an app without exceptions reaches %v", got) + } + if got := exceptions("app with a NAS", nas); strings.Join(got, ",") != "192.168.1.10/32,192.168.2.0/24" { + t.Errorf("an app with exceptions reaches %v", got) + } + if got := exceptions("preview", preview); len(got) != 0 { + t.Errorf("a preview reaches %v, want no exceptions", got) + } + + // Only cloudflared reaches Traefik, and only where it runs. + if find(plain, "NetworkPolicy", "shelf-cloudflared") != nil || find(preview, "NetworkPolicy", "shelf-cloudflared") != nil { + t.Error("a policy for cloudflared without cloudflared") + } + cloudflared := find(tunnel, "NetworkPolicy", "shelf-cloudflared") + if cloudflared == nil { + t.Fatal("an app with a tunnel has no policy that lets cloudflared reach Traefik") + } + if sel, _, _ := unstructured.NestedStringMap(cloudflared.Object, "spec", "podSelector", "matchLabels"); sel["app.kubernetes.io/name"] != "cloudflared" { + t.Errorf("the cloudflared policy selects %v", sel) + } + + for name, objs := range map[string][]*unstructured.Unstructured{"app": plain, "preview": preview} { + var ns *unstructured.Unstructured + for _, o := range objs { + if o.GetKind() == "Namespace" { + ns = o + } + } + labels := ns.GetLabels() + if labels["pod-security.kubernetes.io/enforce"] != "baseline" || labels["pod-security.kubernetes.io/warn"] != "restricted" || + labels["pod-security.kubernetes.io/audit"] != "restricted" || labels["pod-security.kubernetes.io/enforce-version"] != "latest" { + t.Errorf("%s: namespace labels %v", name, labels) + } + release := find(objs, "HelmRelease", map[string]string{"app": "hello", "preview": "hello--feature-login"}[name]) + test, _, _ := unstructured.NestedMap(release.Object, "spec", "test") + if test["enable"] != true || test["ignoreFailures"] != (name == "preview") { + t.Errorf("%s: test %v, want enabled and failures ignored only in a preview", name, test) + } + } +} diff --git a/internal/cluster/rollout.go b/internal/cluster/rollout.go index 55653ea..5249c48 100644 --- a/internal/cluster/rollout.go +++ b/internal/cluster/rollout.go @@ -34,6 +34,9 @@ type Rollout struct { // Reconciling means helm-controller is in the middle of an install, upgrade, rollback or // retry, so its Ready condition is about to change. Reconciling bool + // TestedDigest is the configDigest the tests stage reports on, empty while it has no result + // for any values. + TestedDigest string } // AppRollout reads one app's chain and its progress, without changing anything. @@ -54,11 +57,13 @@ func AppRollout(ctx context.Context, cfg *rest.Config, app string) (Rollout, err if err != nil { return Rollout{}, err } - return rollout(app, objs, values), nil + r := rollout(app, objs, values) + return r, c.addTestLogs(ctx, app, &r.Diagnosis) } func rollout(app string, o *appObjects, values *unstructured.Unstructured) Rollout { r := Rollout{Diagnosis: diagnose(app, o)} + r.TestedDigest = r.Diagnosis.testedDigest if o.source != nil { r.Revision, _, _ = unstructured.NestedString(o.source.Object, "status", "artifact", "revision") r.Commit = artifactCommit(o.source) diff --git a/internal/cluster/rollout_test.go b/internal/cluster/rollout_test.go index 2e99aa6..979897c 100644 --- a/internal/cluster/rollout_test.go +++ b/internal/cluster/rollout_test.go @@ -87,7 +87,7 @@ func TestRollout(t *testing.T) { if !r.Reconciling { t.Error("a release with Reconciling=True is not reconciling") } - if len(r.Diagnosis.Stages) != 5 { + if len(r.Diagnosis.Stages) != 6 { t.Errorf("stages %d", len(r.Diagnosis.Stages)) } diff --git a/internal/cluster/state.go b/internal/cluster/state.go index af70957..67bbda2 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -13,6 +13,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" + + "github.com/tweinmann/shelf/internal/render" ) var ( @@ -55,6 +57,10 @@ type Stage struct { type Diagnosis struct { Name string `json:"name"` Stages []Stage `json:"stages"` + // Tests are the results of the app's tests for the values the release last tried. + Tests []TestResult `json:"tests,omitempty"` + // testedDigest is the configDigest those results are for, empty while there are none. + testedDigest string } // Trouble returns the first stage that is not ready, or nil when the app is healthy. @@ -121,7 +127,11 @@ type appObjects struct { kustomization *unstructured.Unstructured chart *unstructured.Unstructured release *unstructured.Unstructured - pods []*unstructured.Unstructured + // values is the ConfigMap with the app values of the newest deploy artifact. + values *unstructured.Unstructured + pods []*unstructured.Unstructured + // tests are the pods of the app's tests, kept apart from its pods. + tests []*unstructured.Unstructured // refusals are the FailedCreate events in the namespace: a pod the API server did not // create, because the admission webhook or a quota refused it, never shows up as a pod. refusals []*unstructured.Unstructured @@ -136,7 +146,7 @@ func (e *NotFoundError) Error() string { return "app " + e.Name + " does not exi func (o *appObjects) isApp() bool { return o.provider != nil } // AppStates returns every registered app with its current state and its previews. It reads the -// whole cluster in nine list calls, however many apps and previews there are, plus two for each +// whole cluster in ten list calls, however many apps and previews there are, plus two for each // app with a quick tunnel. func AppStates(ctx context.Context, cfg *rest.Config) ([]AppState, error) { c, err := newClient(cfg) @@ -190,7 +200,8 @@ func AppDiagnosis(ctx context.Context, cfg *rest.Config, app string) (Diagnosis, if !ok { return Diagnosis{}, &NotFoundError{Name: app} } - return diagnose(app, objs), nil + d := diagnose(app, objs) + return d, c.addTestLogs(ctx, app, &d) } // appObjects collects everything that belongs to an app, in one pass over the cluster. The @@ -247,10 +258,24 @@ func (c *client) appObjects(ctx context.Context) (map[string]*appObjects, error) return nil, err } for _, p := range pods { - if app, ok := apps[p.GetNamespace()]; ok { + app, ok := apps[p.GetNamespace()] + switch { + case !ok: + case p.GetLabels()[TestLabel] != "": + app.tests = append(app.tests, p) + default: app.pods = append(app.pods, p) } } + values, err := c.listFields(ctx, configMapGVK, "metadata.name="+render.ConfigMapName) + if err != nil { + return nil, err + } + for _, v := range values { + if app, ok := apps[v.GetNamespace()]; ok { + app.values = v + } + } refusals, err := c.listFields(ctx, eventGVK, "reason=FailedCreate") if err != nil { return nil, err @@ -374,14 +399,32 @@ func diagnose(name string, o *appObjects) Diagnosis { fluxStage("the chart", "OCIRepository "+name+"/"+chartName, o.chart, "the platform has not created the app yet", "shelf cannot read the shelf-app chart from the registry"), - fluxStage("the app", "HelmRelease "+name+"/"+name, o.release, - "the platform has not created the app yet", - "the app.yaml was rendered but the release did not succeed"), + releaseStage(name, o.release), ) d.Stages = append(d.Stages, podStage(o.pods, o.refusals)) + tests, results, digest := testsStage(name, o) + d.Stages = append(d.Stages, tests) + d.Tests, d.testedDigest = results, digest return d } +// releaseStage is the HelmRelease's own stage. A release whose only problem is a failed test is +// not where the trouble lies: Helm released the values, and the tests stage after the pods says +// what failed. On an app, helm-controller has rolled back by then, and the previous version runs. +func releaseStage(name string, release *unstructured.Unstructured) Stage { + s := fluxStage("the app", "HelmRelease "+name+"/"+name, release, + "the platform has not created the app yet", + "the app.yaml was rendered but the release did not succeed") + if s.Phase != PhaseFailed { + return s + } + released, tested := condition(release, "Released"), condition(release, "TestSuccess") + if released != nil && released["status"] == "True" && tested != nil && tested["status"] == "False" { + s.Phase, s.Hint = PhaseReady, "" + } + return s +} + // fluxStage reads the Ready condition of a Flux object, which every one of them reports the // same way. func fluxStage(name, object string, obj *unstructured.Unstructured, missing, failedHint string) Stage { @@ -535,6 +578,9 @@ func podHint(reason, message string) string { case strings.Contains(message, "policy.sigstore.dev"): return "a built image is not signed by the cluster's signer; push again, so that the " + "reusable workflow builds and signs it" + case strings.Contains(message, "violates PodSecurity"): + return "the pod breaks the namespace's Pod Security level baseline (privileged containers, " + + "host namespaces, hostPath volumes or added capabilities); shelf does not run such pods" case strings.Contains(message, "exceeded quota"): return "the preview's quota is used up; give the components fewer resources, or fewer instances" } diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index 6cd619a..b6927be 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -176,6 +176,23 @@ func TestDiagnose(t *testing.T) { wantPhase: PhaseWorking, wantStage: "the pods", }, + { + name: "a privileged pod is refused", + objects: func(o *appObjects) { + o.pods = nil + o.refusals = []*unstructured.Unstructured{{Object: map[string]any{ + "metadata": map[string]any{"namespace": "shop"}, + "involvedObject": map[string]any{"kind": "ReplicaSet", "name": "web-5d8f"}, + "message": `Error creating: pods "web-5d8f-x" is forbidden: violates PodSecurity "baseline:latest": privileged`, + "lastTimestamp": "2026-10-28T12:00:00Z", + }}} + }, + wantPhase: PhaseFailed, + wantStage: "the pods", + wantMessage: `Error creating: pods "web-5d8f-x" is forbidden: violates PodSecurity "baseline:latest": privileged`, + wantHint: "the pod breaks the namespace's Pod Security level baseline (privileged containers, " + + "host namespaces, hostPath volumes or added capabilities); shelf does not run such pods", + }, { name: "no pods yet", objects: func(o *appObjects) { o.pods = nil }, @@ -190,7 +207,7 @@ func TestDiagnose(t *testing.T) { tt.objects(objects) d := diagnose("shop", objects) - if len(d.Stages) != 5 { + if len(d.Stages) != 6 { t.Fatalf("%d stages, want the whole chain", len(d.Stages)) } if got := d.Phase(); got != tt.wantPhase { diff --git a/internal/cluster/testdata/objects.yaml b/internal/cluster/testdata/objects.yaml index e1f4d32..155f513 100644 --- a/internal/cluster/testdata/objects.yaml +++ b/internal/cluster/testdata/objects.yaml @@ -113,6 +113,7 @@ spec: cloudflare: home domain: example.com insecure: false + lan: [] name: hello quick: false registry: ghcr @@ -133,6 +134,7 @@ spec: cloudflare: "" domain: "" insecure: false + lan: [] name: empty quick: false registry: "" @@ -153,6 +155,7 @@ spec: cloudflare: "" domain: "" insecure: false + lan: [] name: quick quick: true registry: "" @@ -160,3 +163,25 @@ spec: tunnel: "" url: oci://ghcr.io/tweinmann/hello-deploy type: Static +--- +apiVersion: fluxcd.controlplane.io/v1 +kind: ResourceSetInputProvider +metadata: + labels: + shelf.dev/app: nas + name: nas + namespace: shelf-system +spec: + defaultValues: + cloudflare: "" + domain: "" + insecure: false + lan: + - 192.168.1.10/32 + name: nas + quick: false + registry: "" + tag: main + tunnel: "" + url: oci://ghcr.io/tweinmann/hello-deploy + type: Static diff --git a/internal/cluster/tests.go b/internal/cluster/tests.go new file mode 100644 index 0000000..0769a88 --- /dev/null +++ b/internal/cluster/tests.go @@ -0,0 +1,206 @@ +package cluster + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/yaml" + + "github.com/tweinmann/shelf/internal/render" +) + +// TestLabel marks the pods of an app's tests, which the chart renders as Helm test hooks. They +// carry the app's label as well, but are no part of its pods. +const TestLabel = "shelf.dev/test" + +// testLogLines is how much of a failed test's log the tests stage shows. +const testLogLines = 20 + +// TestResult is how one test of an app went for the values the release last tried. +type TestResult struct { + Name string `json:"name"` + Phase Phase `json:"phase"` + // ExitCode is the exit status of a test that failed. + ExitCode *int `json:"exitCode,omitempty"` + // Log is the end of a failed test's log; only read for one app, never for a list. + Log string `json:"log,omitempty"` + // pod is the test's pod, to read the log from. + pod string +} + +// testsStage reports on the tests of the values helm-controller last tried, which is what a new +// commit is waiting for: after a failed upgrade and its rollback, the release runs older values, +// whose tests passed long ago, while the result that matters is that of the attempt. +// +// helm-controller runs the tests after every install and upgrade and records them in the history +// entry of that Helm revision, by pod name. A rollback is a revision of its own without tests, +// but it has the configDigest of the revision it went back to. +func testsStage(name string, o *appObjects) (Stage, []TestResult, string) { + s := Stage{Name: "the tests", Object: "Pods " + name + "/" + name + "-test-*"} + if o.release == nil { + s.Phase, s.Message = PhaseMissing, "the platform has not created the app yet" + return s, nil, "" + } + if enabled, _, _ := unstructured.NestedBool(o.release.Object, "spec", "test", "enable"); !enabled { + s.Phase, s.Message = PhaseReady, "this platform does not run tests" + return s, nil, "" + } + declared := declaredTests(o.values) + attempted, _, _ := unstructured.NestedString(o.release.Object, "status", "lastAttemptedConfigDigest") + if len(declared) == 0 { + s.Phase, s.Message = PhaseReady, "the app declares no tests" + return s, nil, attempted + } + hooks, found := attemptedHooks(o.release, attempted) + if !found { + s.Phase, s.Message = PhaseWorking, "waiting for the release" + return s, nil, "" + } + + pods := map[string]*unstructured.Unstructured{} + for _, p := range o.tests { + pods[p.GetName()] = p + } + prefix := name + "-test-" + var results []TestResult + for _, test := range declared { + r := TestResult{Name: test, Phase: PhaseWorking, pod: prefix + test} + switch hooks[prefix+test] { + case "Succeeded": + r.Phase = PhaseReady + case "Failed": + r.Phase = PhaseFailed + r.ExitCode = exitCode(pods[r.pod]) + } + results = append(results, r) + } + + var running, passed []string + for _, r := range results { + switch r.Phase { + case PhaseFailed: + release := "the previous version runs again" + if ignore, _, _ := unstructured.NestedBool(o.release.Object, "spec", "test", "ignoreFailures"); ignore { + release = "the preview keeps running it, so it can be looked at with shelf app open --preview" + } + s.Phase, s.Reason, s.Object = PhaseFailed, "TestFailed", "Pod "+name+"/"+r.pod + s.Message = "test " + r.Name + " failed" + if r.ExitCode != nil { + s.Message += fmt.Sprintf(" with exit code %d", *r.ExitCode) + } + s.Hint = "the app's own test " + r.Name + " fails against this version; " + release + + ". Its log says why" + return s, results, attempted + case PhaseReady: + passed = append(passed, r.Name) + default: + running = append(running, r.Name) + } + } + if len(running) > 0 { + s.Phase, s.Message = PhaseWorking, "running: "+strings.Join(running, ", ") + return s, results, attempted + } + s.Phase, s.Message = PhaseReady, fmt.Sprintf("%d passed: %s", len(passed), strings.Join(passed, ", ")) + return s, results, attempted +} + +// attemptedHooks returns the test hooks helm-controller recorded for the newest Helm revision with +// the given configDigest that ran its tests, and whether there is such a revision yet. +func attemptedHooks(release *unstructured.Unstructured, digest string) (map[string]string, bool) { + if digest == "" { + return nil, false + } + history, _, _ := unstructured.NestedSlice(release.Object, "status", "history") + seen := false + for _, item := range history { + entry, ok := item.(map[string]any) + if !ok || entry["configDigest"] != digest { + continue + } + seen = true + hooks, _ := entry["testHooks"].(map[string]any) + if len(hooks) == 0 { + continue + } + out := map[string]string{} + for pod, h := range hooks { + if hook, ok := h.(map[string]any); ok { + out[pod], _ = hook["phase"].(string) + } + } + return out, true + } + // A revision with these values exists, but its tests have not been recorded yet. + return map[string]string{}, seen +} + +// declaredTests are the names of the tests in the app's values, sorted. +func declaredTests(values *unstructured.Unstructured) []string { + if values == nil { + return nil + } + data, _, _ := unstructured.NestedString(values.Object, "data", render.ConfigMapKey) + var app struct { + Tests map[string]any `json:"tests"` + } + if err := yaml.Unmarshal([]byte(data), &app); err != nil { + return nil + } + return slices.Sorted(maps.Keys(app.Tests)) +} + +// exitCode is the exit status of a test pod's container, nil when it has none. +func exitCode(pod *unstructured.Unstructured) *int { + if pod == nil { + return nil + } + statuses, _, _ := unstructured.NestedSlice(pod.Object, "status", "containerStatuses") + for _, item := range statuses { + cs, _ := item.(map[string]any) + if terminated, ok := nested(cs, "state", "terminated"); ok { + if code, ok := terminated["exitCode"].(int64); ok { + n := int(code) + return &n + } + } + } + return nil +} + +// addTestLogs puts the end of the log of every failed test into its result and into the tests +// stage, which then says why without anyone running kubectl logs. +func (c *client) addTestLogs(ctx context.Context, namespace string, d *Diagnosis) error { + var clientset *kubernetes.Clientset + for i := range d.Tests { + r := &d.Tests[i] + if r.Phase != PhaseFailed { + continue + } + if clientset == nil { + var err error + if clientset, err = kubernetes.NewForConfig(c.cfg); err != nil { + return err + } + } + lines := int64(testLogLines) + log, err := clientset.CoreV1().Pods(namespace).GetLogs(r.pod, &corev1.PodLogOptions{TailLines: &lines}).DoRaw(ctx) + if err != nil { + // The pod is gone, as after the next run started; the result stands without its log. + continue + } + r.Log = strings.TrimRight(string(log), "\n") + for j := range d.Stages { + if s := &d.Stages[j]; s.Name == "the tests" && s.Phase == PhaseFailed && strings.HasSuffix(s.Object, "/"+r.pod) && r.Log != "" { + s.Message += ":\n" + r.Log + } + } + } + return nil +} diff --git a/internal/cluster/tests_test.go b/internal/cluster/tests_test.go new file mode 100644 index 0000000..002eda8 --- /dev/null +++ b/internal/cluster/tests_test.go @@ -0,0 +1,201 @@ +package cluster + +import ( + "strings" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// testedRelease is a HelmRelease with tests switched on, its conditions and its history, newest +// first, as helm-controller reports them (seen on the dev cluster on 2026-10-28). +func testedRelease(ignoreFailures bool, attempted string, conditions []any, history ...any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{"test": map[string]any{"enable": true, "ignoreFailures": ignoreFailures}}, + "status": map[string]any{ + "conditions": conditions, + "history": history, + "lastAttemptedConfigDigest": attempted, + }, + }} +} + +func cond(kind, status, reason, message string) map[string]any { + return map[string]any{"type": kind, "status": status, "reason": reason, "message": message} +} + +func snapshot(version int64, digest string, hooks map[string]any) map[string]any { + entry := map[string]any{"version": version, "configDigest": digest} + if hooks != nil { + entry["testHooks"] = hooks + } + return entry +} + +func hook(phase string) map[string]any { return map[string]any{"phase": phase} } + +func valuesWithTests(tests ...string) *unstructured.Unstructured { + app := "components:\n web: { image: nginx }\n" + if len(tests) > 0 { + app += "tests:\n" + for _, t := range tests { + app += " " + t + ": { image: curl }\n" + } + } + return &unstructured.Unstructured{Object: map[string]any{"data": map[string]any{"app.yaml": app}}} +} + +func testPod(name string, phase string, exit int64) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{"name": name, "namespace": "shop", "labels": map[string]any{TestLabel: strings.TrimPrefix(name, "shop-test-")}}, + "status": map[string]any{"phase": phase, "containerStatuses": []any{ + map[string]any{"name": "smoke", "state": map[string]any{"terminated": map[string]any{"exitCode": exit}}}, + }}, + }} +} + +func TestTestsStage(t *testing.T) { + t.Parallel() + ready := []any{cond("Ready", "True", "TestSucceeded", "Helm test succeeded"), cond("Released", "True", "UpgradeSucceeded", "")} + // What an app looks like after an upgrade whose test failed: rolled back to v1, whose values + // came back as v3 without tests, while v2 holds the failed result. + rolledBack := testedRelease(false, "sha256:2", []any{ + cond("Ready", "False", "RollbackSucceeded", "Helm rollback to previous release shop/shop.v1 succeeded"), + cond("Released", "True", "UpgradeSucceeded", ""), + cond("TestSuccess", "False", "TestFailed", "Helm test failed for release shop/shop.v2"), + cond("Remediated", "True", "RollbackSucceeded", ""), + }, snapshot(3, "sha256:1", nil), snapshot(2, "sha256:2", map[string]any{"shop-test-smoke": hook("Failed")}), + snapshot(1, "sha256:1", map[string]any{"shop-test-smoke": hook("Succeeded")})) + + tests := []struct { + name string + objects func(o *appObjects) + wantPhase Phase + wantStage string // the first stage that is not Ready, "" for none + wantTests string // the stage's message + wantDigest string + wantResults string + }{ + { + name: "a platform from before tests", + objects: func(o *appObjects) {}, + wantPhase: PhaseReady, wantTests: "this platform does not run tests", + }, + { + name: "an app without tests", + objects: func(o *appObjects) { + o.values = valuesWithTests() + o.release = testedRelease(false, "sha256:1", ready, snapshot(1, "sha256:1", nil)) + }, + wantPhase: PhaseReady, wantTests: "the app declares no tests", wantDigest: "sha256:1", + }, + { + name: "the release has not got to the new values", + objects: func(o *appObjects) { + o.values = valuesWithTests("smoke") + o.release = testedRelease(false, "sha256:2", ready, snapshot(1, "sha256:1", map[string]any{"shop-test-smoke": hook("Succeeded")})) + }, + wantPhase: PhaseWorking, wantStage: "the tests", wantTests: "waiting for the release", + }, + { + name: "a test is running", + objects: func(o *appObjects) { + o.values = valuesWithTests("smoke", "login") + o.release = testedRelease(false, "sha256:2", []any{cond("Ready", "Unknown", "AwaitingTests", "")}, + snapshot(2, "sha256:2", map[string]any{"shop-test-login": hook("Succeeded"), "shop-test-smoke": hook("Running")})) + o.tests = []*unstructured.Unstructured{testPod("shop-test-smoke", "Running", 0)} + }, + wantPhase: PhaseWorking, wantStage: "the app", wantTests: "running: smoke", wantDigest: "sha256:2", + wantResults: "login=Ready smoke=Working", + }, + { + name: "all tests passed", + objects: func(o *appObjects) { + o.values = valuesWithTests("smoke", "login") + o.release = testedRelease(false, "sha256:2", ready, + snapshot(2, "sha256:2", map[string]any{"shop-test-login": hook("Succeeded"), "shop-test-smoke": hook("Succeeded")})) + o.tests = []*unstructured.Unstructured{testPod("shop-test-smoke", "Succeeded", 0), testPod("shop-test-login", "Succeeded", 0)} + }, + wantPhase: PhaseReady, wantTests: "2 passed: login, smoke", wantDigest: "sha256:2", + wantResults: "login=Ready smoke=Ready", + }, + { + name: "a failed test rolled the app back", + objects: func(o *appObjects) { + o.values = valuesWithTests("smoke") + o.release = rolledBack + o.tests = []*unstructured.Unstructured{testPod("shop-test-smoke", "Failed", 1)} + }, + wantPhase: PhaseFailed, wantStage: "the tests", wantTests: "test smoke failed with exit code 1", wantDigest: "sha256:2", + wantResults: "smoke=Failed", + }, + { + name: "a failed test on a preview", + objects: func(o *appObjects) { + o.values = valuesWithTests("smoke") + o.release = testedRelease(true, "sha256:2", []any{ + cond("Ready", "True", "UpgradeSucceeded", ""), cond("Released", "True", "UpgradeSucceeded", ""), + cond("TestSuccess", "False", "TestFailed", "Helm test failed"), + }, snapshot(2, "sha256:2", map[string]any{"shop-test-smoke": hook("Failed")})) + o.tests = []*unstructured.Unstructured{testPod("shop-test-smoke", "Failed", 7)} + }, + wantPhase: PhaseFailed, wantStage: "the tests", wantTests: "test smoke failed with exit code 7", wantDigest: "sha256:2", + wantResults: "smoke=Failed", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + objects := healthy() + tt.objects(objects) + d := diagnose("shop", objects) + if len(d.Stages) != 6 || d.Stages[5].Name != "the tests" { + t.Fatalf("stages %+v, want the tests last", d.Stages) + } + if d.Phase() != tt.wantPhase { + t.Errorf("phase %s, want %s", d.Phase(), tt.wantPhase) + } + if trouble := d.Trouble(); (trouble == nil) != (tt.wantStage == "") || (trouble != nil && trouble.Name != tt.wantStage) { + t.Errorf("trouble %+v, want stage %q", trouble, tt.wantStage) + } + if got := d.Stages[5].Message; got != tt.wantTests { + t.Errorf("tests stage %q, want %q", got, tt.wantTests) + } + if d.testedDigest != tt.wantDigest { + t.Errorf("tested digest %q, want %q", d.testedDigest, tt.wantDigest) + } + var results []string + for _, r := range d.Tests { + results = append(results, r.Name+"="+string(r.Phase)) + } + if got := strings.Join(results, " "); got != tt.wantResults { + t.Errorf("results %q, want %q", got, tt.wantResults) + } + }) + } + + // The release stage of a rolled-back app is not the trouble: the tests stage says what is. + o := healthy() + o.values, o.release = valuesWithTests("smoke"), rolledBack + if s := diagnose("shop", o).Stages[3]; s.Phase != PhaseReady || s.Hint != "" { + t.Errorf("release stage %+v, want Ready after a rollback caused by a test", s) + } + failed := diagnose("shop", func() *appObjects { + o := healthy() + o.values, o.release = valuesWithTests("smoke"), rolledBack + return o + }()).Stages[5] + if !strings.Contains(failed.Hint, "the previous version runs again") || failed.Object != "Pod shop/shop-test-smoke" { + t.Errorf("failed tests stage %+v", failed) + } +} + +// A test pod that ended, whether it passed or failed, is no pod of the app's. +func TestTestPodsAreNoAppPods(t *testing.T) { + t.Parallel() + o := healthy() + o.tests = []*unstructured.Unstructured{testPod("shop-test-smoke", "Failed", 1)} + if s := diagnose("shop", o).Stages[4]; s.Phase != PhaseReady { + t.Errorf("pods stage %+v, want Ready", s) + } +} diff --git a/internal/cluster/verification.go b/internal/cluster/verification.go index b0268e1..c472ad7 100644 --- a/internal/cluster/verification.go +++ b/internal/cluster/verification.go @@ -163,8 +163,9 @@ func signatureReason(what string, source *unstructured.Unstructured) string { return "" } -// valueImages returns the image of every component in the app values, sorted by component. -// shelf render pins each one as @sha256:. +// valueImages returns the image of every component and test in the app values, sorted by name; +// test names never clash with component names. shelf render pins each one as +// @sha256:. func valueImages(values *unstructured.Unstructured) ([]ScannedImage, error) { if values == nil { return nil, fmt.Errorf("the app values (%s) are not there yet", render.ConfigMapName) @@ -174,14 +175,23 @@ func valueImages(values *unstructured.Unstructured) ([]ScannedImage, error) { Components map[string]struct { Image string `json:"image"` } `json:"components"` + Tests map[string]struct { + Image string `json:"image"` + } `json:"tests"` } if err := yaml.Unmarshal([]byte(data), &app); err != nil { return nil, fmt.Errorf("the app values cannot be read: %w", err) } var out []ScannedImage + add := func(name, image string) { + _, digest, _ := strings.Cut(image, "@") + out = append(out, ScannedImage{Component: name, Image: image, Digest: digest}) + } for name, comp := range app.Components { - _, digest, _ := strings.Cut(comp.Image, "@") - out = append(out, ScannedImage{Component: name, Image: comp.Image, Digest: digest}) + add(name, comp.Image) + } + for name, test := range app.Tests { + add(name, test.Image) } sort.Slice(out, func(i, j int) bool { return out[i].Component < out[j].Component }) return out, nil diff --git a/internal/cluster/verification_test.go b/internal/cluster/verification_test.go index def923f..e5781e3 100644 --- a/internal/cluster/verification_test.go +++ b/internal/cluster/verification_test.go @@ -145,6 +145,41 @@ func TestVerification(t *testing.T) { } } +// A test's image runs in the app's namespace like a component's, so the scan has to cover it +// too. The values here are what the release would run; the digest stays that of verifiedValues, +// as the rollout is not what is being checked. +func TestVerificationCoversTests(t *testing.T) { + t.Parallel() + const testDigest = "sha256:3333333333333333333333333333333333333333333333333333333333333333" + withTest := verifiedValues + "tests:\n smoke:\n image: curlimages/curl:8.10.1@" + testDigest + "\n" + o, _ := verifiedApp(t, fullScan) + images, err := valueImages(&unstructured.Unstructured{Object: map[string]any{"data": map[string]any{"app.yaml": withTest}}}) + if err != nil { + t.Fatal(err) + } + if len(images) != 3 || images[2].Component != "web" || images[1].Component != "smoke" || images[1].Digest != testDigest { + t.Errorf("images %+v, want db, smoke and web", images) + } + + // Run the whole check with values that name the test, and a release that holds them. + inline := map[string]any{"name": "shop"} + digest, err := ConfigDigest(withTest, inline) + if err != nil { + t.Fatal(err) + } + _ = unstructured.SetNestedSlice(o.release.Object, []any{map[string]any{"version": int64(2), "configDigest": digest}}, "status", "history") + values := &unstructured.Unstructured{Object: map[string]any{"data": map[string]any{"app.yaml": withTest}}} + if v := verification("shop", o, values); v.Verified || v.Reason != "the scan does not cover the image of smoke" { + t.Errorf("verified %v, reason %q", v.Verified, v.Reason) + } + covered := strings.Replace(fullScan, `"images":{`, `"images":{"`+testDigest+`":{"critical":0,"high":0},`, 1) + o, _ = verifiedApp(t, covered) + _ = unstructured.SetNestedSlice(o.release.Object, []any{map[string]any{"version": int64(2), "configDigest": digest}}, "status", "history") + if v := verification("shop", o, values); !v.Verified { + t.Errorf("not verified with the test's image scanned: %s", v.Reason) + } +} + func refusal(at, kind, name, message string) *unstructured.Unstructured { return &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "v1", "kind": "Event", diff --git a/internal/ops/apps.go b/internal/ops/apps.go index a103b38..ff1f423 100644 --- a/internal/ops/apps.go +++ b/internal/ops/apps.go @@ -5,7 +5,10 @@ import ( "context" "errors" "fmt" + "net/netip" "os" + "slices" + "strings" "time" "github.com/google/go-containerregistry/pkg/authn" @@ -56,6 +59,68 @@ type Access struct { Previews int // NoPreviews removes the app's previews, with their namespaces and volumes. NoPreviews bool + // LAN replaces the private address ranges the app may reach, as CIDRs; nil leaves them as + // they are. Its previews never get them. + LAN []string + // DenyLAN takes every LAN exception away again. + DenyLAN bool +} + +// privateRanges are the addresses the isolation keeps every app away from: the LAN, carrier-grade +// NAT (where Tailscale puts home devices) and link-local, which holds cloud metadata. A LAN +// exception has to lie inside one of them; anything else is reachable anyway. +var privateRanges = []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("172.16.0.0/12"), + netip.MustParsePrefix("192.168.0.0/16"), + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("169.254.0.0/16"), + netip.MustParsePrefix("fc00::/7"), + netip.MustParsePrefix("fe80::/10"), +} + +// CheckLAN parses the LAN exceptions of an app and returns them in canonical form, such as +// 192.168.1.0/24 for 192.168.1.7/24. +func CheckLAN(cidrs []string) ([]string, error) { + out := make([]string, 0, len(cidrs)) + for _, c := range cidrs { + prefix, err := netip.ParsePrefix(strings.TrimSpace(c)) + if err != nil { + return nil, fmt.Errorf("%q is not a CIDR; a LAN exception must be a private CIDR such as 192.168.1.10/32", c) + } + prefix = prefix.Masked() + private := false + for _, r := range privateRanges { + private = private || (r.Bits() <= prefix.Bits() && r.Contains(prefix.Addr())) + } + if !private { + return nil, fmt.Errorf("%s is not a private range; a LAN exception must be a private CIDR "+ + "(inside 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10 or 169.254.0.0/16), "+ + "and every other address is reachable anyway", prefix) + } + if !slices.Contains(out, prefix.String()) { + out = append(out, prefix.String()) + } + } + return out, nil +} + +// checkPodRanges refuses LAN exceptions that overlap the address ranges of the cluster's pods: +// those would let the app reach the pods of every other app. +func checkPodRanges(lan, pods []string) error { + for _, c := range lan { + exception := netip.MustParsePrefix(c) + for _, p := range pods { + pod, err := netip.ParsePrefix(p) + if err != nil { + continue + } + if exception.Overlaps(pod) { + return fmt.Errorf("%s overlaps %s, where the cluster runs its pods; it would reach other apps", c, p) + } + } + } + return nil } // MaxPreviews is the most previews an app may have at once. @@ -105,6 +170,12 @@ func (a Access) Check() error { case a.Previews < 0 || a.Previews > MaxPreviews: return fmt.Errorf("an app may have between 1 and %d previews, not %d", MaxPreviews, a.Previews) } + if a.DenyLAN && len(a.LAN) > 0 { + return errors.New("LAN exceptions cannot be given and taken away at once") + } + if _, err := CheckLAN(a.LAN); err != nil { + return err + } return nil } @@ -166,6 +237,22 @@ func (o *Ops) AddApp(ctx context.Context, opts AddOptions, report progress.Repor previews = opts.Access.Previews rep.Report(progress.Info("previews: up to %d branches, inside the cluster only", previews)) } + lan := current.LAN + switch { + case opts.Access.DenyLAN: + lan = nil + rep.Report(progress.Info("LAN: none")) + case opts.Access.LAN != nil: + lan, _ = CheckLAN(opts.Access.LAN) + pods, err := o.Cluster.NodePodCIDRs(ctx, o.config()) + if err != nil { + return err + } + if err := checkPodRanges(lan, pods); err != nil { + return err + } + rep.Report(progress.Info("LAN: may reach %s; its previews never do", strings.Join(lan, ", "))) + } // Before anything is written, here or in Cloudflare: without the platform the app would never // be deployed, and the wait for it would only end at the timeout. needs := cluster.PlatformNeeds{Quick: quick, Previews: previews > 0} @@ -235,6 +322,7 @@ func (o *Ops) AddApp(ctx context.Context, opts AddOptions, report progress.Repor Domain: domain, Registry: registry, Previews: previews, + LAN: lan, Secrets: values, Timeout: opts.Timeout, Report: rep, diff --git a/internal/ops/ops.go b/internal/ops/ops.go index f0b20d4..b84596e 100644 --- a/internal/ops/ops.go +++ b/internal/ops/ops.go @@ -92,6 +92,10 @@ type Cluster interface { // AppVerification reads whether an app or a preview, by its namespace, runs a verified // deploy artifact. AppVerification(ctx context.Context, cfg *rest.Config, name string) (cluster.Verification, error) + // AppIsolation reads how an app or a preview, by its namespace, is fenced in; lan are the + // exceptions it has. + AppIsolation(ctx context.Context, cfg *rest.Config, namespace string, lan []string) (cluster.Isolation, error) + NodePodCIDRs(ctx context.Context, cfg *rest.Config) ([]string, error) ForwardTraefik(ctx context.Context, cfg *rest.Config) (*cluster.Forward, error) Settings(ctx context.Context, cfg *rest.Config) (cluster.Settings, error) // AppConfig returns how an app is registered, or nil when there is no such app. @@ -137,6 +141,14 @@ func (liveCluster) AppVerification(ctx context.Context, cfg *rest.Config, name s return cluster.AppVerification(ctx, cfg, name) } +func (liveCluster) AppIsolation(ctx context.Context, cfg *rest.Config, namespace string, lan []string) (cluster.Isolation, error) { + return cluster.AppIsolation(ctx, cfg, namespace, lan) +} + +func (liveCluster) NodePodCIDRs(ctx context.Context, cfg *rest.Config) ([]string, error) { + return cluster.NodePodCIDRs(ctx, cfg) +} + func (liveCluster) AppNames(ctx context.Context, cfg *rest.Config) ([]string, error) { return cluster.AppNames(ctx, cfg) } diff --git a/internal/ops/previews.go b/internal/ops/previews.go index a8a4e3b..97817ff 100644 --- a/internal/ops/previews.go +++ b/internal/ops/previews.go @@ -60,6 +60,10 @@ func (o *Ops) Preview(ctx context.Context, name, branch string) (App, error) { if preview.Verification, err = o.verify(ctx, p.Name, settings, true); err != nil { return App{}, err } + // A preview never gets the app's LAN exceptions. + if preview.Isolation, err = o.isolation(ctx, p.Name, nil); err != nil { + return App{}, err + } return preview, nil } missing := &PreviewNotFoundError{App: name, Branch: branch, Off: a.PreviewLimit == 0} diff --git a/internal/ops/report.go b/internal/ops/report.go new file mode 100644 index 0000000..ca18c56 --- /dev/null +++ b/internal/ops/report.go @@ -0,0 +1,117 @@ +package ops + +import ( + "context" + + "github.com/tweinmann/shelf/internal/cluster" +) + +// Report is what an agent puts into its pull request: how the branch's preview, or the app, +// is doing, with its tests, its verification and its isolation, and for a preview what it runs +// differently from the app. Everything comes from the cluster, never from Git. +type Report struct { + // App is the app or the preview, with its verification and its isolation. + App App + // Branch is the preview's branch, empty for the app. + Branch string + // Diagnosis is the chain with the tests. + Diagnosis cluster.Diagnosis + // Comparison is what the preview runs differently from its app, nil for the app. + Comparison *Comparison +} + +// Comparison sets the images of a preview against those of its app, by component. +type Comparison struct { + // App is the app the preview belongs to, and Commit the commit the app runs. + App string + Commit string + // Comparable is false when one side has no scan to read its images from; Reason says which. + Comparable bool + Reason string + // Changed are the components whose image differs, or that only one side has, by name. + Changed []ImageChange + // Critical and High are the findings of all of the preview's images, AppCritical and AppHigh + // those of the app's. + Critical, High int + AppCritical, AppHigh int +} + +// ImageChange is one component the preview runs another image of than its app. A component only +// one side has leaves the other side's digest empty. +type ImageChange struct { + Component string + Digest, AppDigest string + Critical, High int + AppCritical, AppHigh int +} + +// Report puts together the report of an app, or of the preview of one of its branches. It +// changes nothing. +func (o *Ops) Report(ctx context.Context, name, branch string) (Report, error) { + app, err := o.App(ctx, name) + if err != nil { + return Report{}, err + } + r := Report{App: app, Branch: branch} + if branch != "" { + preview, err := o.Preview(ctx, name, branch) + if err != nil { + return Report{}, err + } + r.App = preview + r.Comparison = compare(preview, app) + } + if r.Diagnosis, err = o.Cluster.AppDiagnosis(ctx, o.config(), r.App.Name); err != nil { + return Report{}, err + } + return r, nil +} + +// compare sets the scanned images of a preview against those of its app. +func compare(preview, app App) *Comparison { + c := &Comparison{App: app.Name, Commit: app.Commit} + scan := func(a App) *cluster.Scan { + if a.Verification == nil { + return nil + } + return a.Verification.Scan + } + ps, as := scan(preview), scan(app) + switch { + case ps == nil: + c.Reason = "the preview has no scan to read its images from" + return c + case as == nil: + c.Reason = "the app has no scan to read its images from" + return c + } + c.Comparable = true + theirs := map[string]cluster.ScannedImage{} + for _, img := range as.Images { + theirs[img.Component] = img + c.AppCritical += img.Critical + c.AppHigh += img.High + } + ours := map[string]bool{} + for _, img := range ps.Images { + ours[img.Component] = true + c.Critical += img.Critical + c.High += img.High + other, ok := theirs[img.Component] + if ok && other.Digest == img.Digest { + continue + } + change := ImageChange{Component: img.Component, Digest: img.Digest, Critical: img.Critical, High: img.High} + if ok { + change.AppDigest, change.AppCritical, change.AppHigh = other.Digest, other.Critical, other.High + } + c.Changed = append(c.Changed, change) + } + for _, img := range as.Images { + if !ours[img.Component] { + c.Changed = append(c.Changed, ImageChange{Component: img.Component, AppDigest: img.Digest, + AppCritical: img.Critical, AppHigh: img.High}) + } + } + return c +} diff --git a/internal/ops/report_test.go b/internal/ops/report_test.go new file mode 100644 index 0000000..9826244 --- /dev/null +++ b/internal/ops/report_test.go @@ -0,0 +1,58 @@ +package ops + +import ( + "testing" + + "github.com/tweinmann/shelf/internal/cluster" +) + +func scanned(images ...cluster.ScannedImage) App { + return App{ + AppState: cluster.AppState{Name: "hello", Commit: "9956993c371777310050f0c10d4f9b90aebcb969"}, + Verification: &Verification{Verification: cluster.Verification{Scan: &cluster.Scan{Images: images}}}, + } +} + +// TestCompare covers what a report says a preview runs differently from its app: the components +// whose image changed, those only one side has, and the findings of both. +func TestCompare(t *testing.T) { + t.Parallel() + app := scanned( + cluster.ScannedImage{Component: "db", Digest: "sha256:db", Critical: 2, High: 86}, + cluster.ScannedImage{Component: "web", Digest: "sha256:web-old", Critical: 1, High: 26}, + cluster.ScannedImage{Component: "worker", Digest: "sha256:worker", High: 3}, + ) + preview := scanned( + cluster.ScannedImage{Component: "db", Digest: "sha256:db", Critical: 2, High: 86}, + cluster.ScannedImage{Component: "smoke", Digest: "sha256:curl"}, + cluster.ScannedImage{Component: "web", Digest: "sha256:web-new", High: 4}, + ) + c := compare(preview, app) + if !c.Comparable || c.App != "hello" || c.Commit != "9956993c371777310050f0c10d4f9b90aebcb969" { + t.Fatalf("comparison %+v", c) + } + want := []ImageChange{ + {Component: "smoke", Digest: "sha256:curl"}, + {Component: "web", Digest: "sha256:web-new", High: 4, AppDigest: "sha256:web-old", AppCritical: 1, AppHigh: 26}, + {Component: "worker", AppDigest: "sha256:worker", AppHigh: 3}, + } + if len(c.Changed) != len(want) { + t.Fatalf("changed %+v, want %+v", c.Changed, want) + } + for i := range want { + if c.Changed[i] != want[i] { + t.Errorf("changed[%d] = %+v, want %+v", i, c.Changed[i], want[i]) + } + } + if c.Critical != 2 || c.High != 90 || c.AppCritical != 3 || c.AppHigh != 115 { + t.Errorf("findings %d/%d against %d/%d", c.Critical, c.High, c.AppCritical, c.AppHigh) + } + + unscanned := App{AppState: cluster.AppState{Name: "hello--x"}, Verification: &Verification{}} + if c := compare(unscanned, app); c.Comparable || c.Reason != "the preview has no scan to read its images from" { + t.Errorf("comparison without a scan %+v", c) + } + if c := compare(preview, unscanned); c.Comparable || c.Reason != "the app has no scan to read its images from" { + t.Errorf("comparison without the app's scan %+v", c) + } +} diff --git a/internal/ops/status.go b/internal/ops/status.go index 0627350..e5015d3 100644 --- a/internal/ops/status.go +++ b/internal/ops/status.go @@ -33,6 +33,8 @@ type App struct { // Verification says whether the app runs what the cluster's signer signed and CI scanned, // filled by App and Preview but not by Apps. Verification *Verification `json:"verification,omitempty"` + // Isolation is how the app is fenced in, filled by App and Preview but not by Apps. + Isolation *cluster.Isolation `json:"isolation,omitempty"` } // Verification is the chain of an app or a preview from the commit to what runs, and whether @@ -171,11 +173,31 @@ func (o *Ops) App(ctx context.Context, name string) (App, error) { if a.Verification, err = o.verify(ctx, name, settings, false); err != nil { return App{}, err } + config, err := o.Cluster.AppConfig(ctx, o.config(), name) + if err != nil { + return App{}, err + } + var lan []string + if config != nil { + lan = config.LAN + } + if a.Isolation, err = o.isolation(ctx, name, lan); err != nil { + return App{}, err + } return a, nil } return App{}, &NotFoundError{Name: name} } +// isolation reads how an app or a preview, by its namespace, is fenced in. +func (o *Ops) isolation(ctx context.Context, namespace string, lan []string) (*cluster.Isolation, error) { + iso, err := o.Cluster.AppIsolation(ctx, o.config(), namespace, lan) + if err != nil { + return nil, err + } + return &iso, nil +} + // Diagnose returns the chain from the deploy artifact to the running pods, so that a caller can // say which step is the reason an app does not work. func (o *Ops) Diagnose(ctx context.Context, name string) (cluster.Diagnosis, error) { diff --git a/internal/ops/wait.go b/internal/ops/wait.go index cc14a62..2ee7b02 100644 --- a/internal/ops/wait.go +++ b/internal/ops/wait.go @@ -134,11 +134,12 @@ func (o *Ops) Wait(ctx context.Context, app string, opts WaitOptions) (WaitResul // on an older artifact, Ready or Failed, is still on its way. The deploy artifact has to be the // awaited commit, the values have to be those of that artifact, and the release has to hold // those values. Before the awaited commit has arrived, only a failing deploy artifact ends the -// wait: the later stages are still about the previous one. +// wait: the later stages are still about the previous one. Last, the app's tests have to have +// passed for those values. func Verdict(app string, r cluster.Rollout, commit string) (WaitResult, bool) { result := WaitResult{App: app, Commit: r.Commit, Revision: r.Revision} stages := r.Diagnosis.Stages - if len(stages) < 5 { + if len(stages) < 6 { return result, false } failed := func(s cluster.Stage) (WaitResult, bool) { @@ -154,7 +155,7 @@ func Verdict(app string, r cluster.Rollout, commit string) (WaitResult, bool) { return pending(s) } - source, values, chart, release, pods := stages[0], stages[1], stages[2], stages[3], stages[4] + source, values, chart, release, pods, tests := stages[0], stages[1], stages[2], stages[3], stages[4], stages[5] switch { case source.Phase == cluster.PhaseFailed: return failed(source) @@ -184,6 +185,13 @@ func Verdict(app string, r cluster.Rollout, commit string) (WaitResult, bool) { return pending(chart) } + // A failed test is about the values it ran with, whatever the release went back to since: on + // an app the rollback follows at once, and the retries before it change nothing. + tested := r.ValuesDigest != "" && r.TestedDigest == r.ValuesDigest + if tested && tests.Phase == cluster.PhaseFailed { + return failed(tests) + } + // Which values a release holds only its digest says: after a failed upgrade, the failure // and the rollback both happen after newer values arrived, and still belong to the older. current := r.ValuesDigest != "" && r.ReleasedDigest == r.ValuesDigest && !r.Reconciling @@ -210,6 +218,16 @@ func Verdict(app string, r cluster.Rollout, commit string) (WaitResult, bool) { default: return pending(pods) } + + // A tests stage without a digest is one that has nothing to wait for: a platform that runs + // no tests. With one, it has to be about the awaited values. + switch { + case tests.Phase == cluster.PhaseReady && (tested || r.TestedDigest == ""): + case tests.Phase == cluster.PhaseReady: + return working(tests, "waiting for the tests of the new values") + default: + return pending(tests) + } result.Outcome = OutcomeReady return result, true } diff --git a/internal/ops/wait_test.go b/internal/ops/wait_test.go index ba98b8a..634552e 100644 --- a/internal/ops/wait_test.go +++ b/internal/ops/wait_test.go @@ -36,11 +36,11 @@ func settled() cluster.Rollout { Diagnosis: cluster.Diagnosis{Name: "hello", Stages: []cluster.Stage{ stage("the deploy artifact", cluster.PhaseReady), stage("the app values", cluster.PhaseReady), stage("the chart", cluster.PhaseReady), stage("the app", cluster.PhaseReady), - stage("the pods", cluster.PhaseReady), + stage("the pods", cluster.PhaseReady), stage("the tests", cluster.PhaseReady), }}, Revision: "main@sha256:new", Commit: newCommit, AppliedRevision: "main@sha256:new", AttemptedRevision: "main@sha256:new", - ValuesDigest: newValues, ReleasedDigest: newValues, AttemptedDigest: newValues, + ValuesDigest: newValues, ReleasedDigest: newValues, AttemptedDigest: newValues, TestedDigest: newValues, } } @@ -71,6 +71,28 @@ func TestVerdict(t *testing.T) { contains string }{ {name: "already Ready", rollout: settled(), done: true, outcome: ops.OutcomeReady}, + // The tests, as seen on the dev cluster on 2026-10-28. + {name: "tests running", rollout: with(settled(), phase(5, cluster.PhaseWorking)), + stage: "the tests", phase: cluster.PhaseWorking}, + {name: "a platform that runs no tests", rollout: with(settled(), func(r *cluster.Rollout) { r.TestedDigest = "" }), + done: true, outcome: ops.OutcomeReady}, + {name: "tests passed for the previous values only", rollout: with(settled(), func(r *cluster.Rollout) { r.TestedDigest = oldValues }), + stage: "the tests", phase: cluster.PhaseWorking, contains: "waiting for the tests of the new values"}, + {name: "test failed on a preview", rollout: with(settled(), phase(5, cluster.PhaseFailed)), + done: true, outcome: ops.OutcomeFailed, stage: "the tests", phase: cluster.PhaseFailed}, + {name: "test failed and the app rolled back", rollout: with(settled(), func(r *cluster.Rollout) { + r.ReleasedDigest, r.AttemptedDigest = oldValues, newValues + phase(5, cluster.PhaseFailed)(r) + }), done: true, outcome: ops.OutcomeFailed, stage: "the tests", phase: cluster.PhaseFailed}, + {name: "test failed while the upgrade retries", rollout: with(settled(), func(r *cluster.Rollout) { + r.Reconciling = true + phase(3, cluster.PhaseWorking)(r) + phase(5, cluster.PhaseFailed)(r) + }), done: true, outcome: ops.OutcomeFailed, stage: "the tests", phase: cluster.PhaseFailed}, + {name: "a failed test of the previous values", rollout: with(settled(), func(r *cluster.Rollout) { + r.TestedDigest = oldValues + phase(5, cluster.PhaseFailed)(r) + }), stage: "the tests", phase: cluster.PhaseFailed}, {name: "Ready at the awaited commit", rollout: settled(), commit: "42a964e", done: true, outcome: ops.OutcomeReady}, {name: "full commit", rollout: settled(), commit: newCommit, done: true, outcome: ops.OutcomeReady}, {name: "pods starting", rollout: with(settled(), phase(4, cluster.PhaseWorking)), diff --git a/internal/render/render.go b/internal/render/render.go index 3a257f4..86af003 100644 --- a/internal/render/render.go +++ b/internal/render/render.go @@ -40,7 +40,7 @@ type Resolver interface { } // Render validates the document, resolves its images and returns the resolved app. built holds -// the image reference for every component with a `build` directory, by component name; the CI +// the image reference for every component and test with a `build` directory, by name; the CI // pushes those images and passes what it got. Findings include validation findings and warnings // from the image lookup. On validation errors it returns ErrInvalid together with the findings. func Render(ctx context.Context, doc *schema.Document, resolver Resolver, built map[string]string) (*schema.App, validate.Findings, error) { @@ -55,18 +55,30 @@ func Render(ctx context.Context, doc *schema.Document, resolver Resolver, built return nil, findings, err } images := map[string]ImageInfo{} - for _, compName := range slices.Sorted(maps.Keys(src.Components)) { - ref := refs[compName] + resolve := func(what, name string) (ImageInfo, error) { + ref := refs[name] info, ok := images[ref] if !ok { var err error info, err = resolver.Resolve(ctx, ref) if err != nil { - return nil, findings, fmt.Errorf("component %s: %w", compName, err) + return ImageInfo{}, fmt.Errorf("%s %s: %w", what, name, err) } images[ref] = info } - findings = append(findings, checkExposedPorts(doc, compName, ref, info)...) + return info, nil + } + for _, compName := range slices.Sorted(maps.Keys(src.Components)) { + info, err := resolve("component", compName) + if err != nil { + return nil, findings, err + } + findings = append(findings, checkExposedPorts(doc, compName, refs[compName], info)...) + } + for _, testName := range slices.Sorted(maps.Keys(src.Tests)) { + if _, err := resolve("test", testName); err != nil { + return nil, findings, err + } } findings.Sort() @@ -79,38 +91,58 @@ func Render(ctx context.Context, doc *schema.Document, resolver Resolver, built for compName, comp := range src.Components { out.Components[compName] = resolveComponent(src, comp, refs[compName], images[refs[compName]]) } + if len(src.Tests) > 0 { + out.Tests = map[string]*schema.Test{} + for testName, test := range src.Tests { + out.Tests[testName] = resolveTest(src, test, refs[testName], images[refs[testName]]) + } + } return out, findings, nil } -// imageRefs returns the image reference of every component: the one from app.yaml, or the one -// the CI passed for a component built from a directory. +// imageRefs returns the image reference of every component and test: the one from app.yaml, or +// the one the CI passed for one built from a directory. Validation keeps test names apart from +// component names, so one map holds both. func imageRefs(app *schema.App, built map[string]string) (map[string]string, error) { - for _, compName := range slices.Sorted(maps.Keys(built)) { - comp, ok := app.Components[compName] + sources := map[string]imageSource{} + for n, comp := range app.Components { + sources[n] = imageSource{"component", comp.Image, comp.Build} + } + for n, test := range app.Tests { + sources[n] = imageSource{"test", test.Image, test.Build} + } + for _, n := range slices.Sorted(maps.Keys(built)) { + src, ok := sources[n] switch { case !ok: - return nil, fmt.Errorf("no component %s in this app.yaml, but an image was passed for it", compName) - case !comp.IsBuilt(): - return nil, fmt.Errorf("component %s has an image in app.yaml, so no image can be passed for it", compName) + return nil, fmt.Errorf("no component or test %s in this app.yaml, but an image was passed for it", n) + case src.build == "": + return nil, fmt.Errorf("%s %s has an image in app.yaml, so no image can be passed for it", src.what, n) } } refs := map[string]string{} - for _, compName := range slices.Sorted(maps.Keys(app.Components)) { - comp := app.Components[compName] - if !comp.IsBuilt() { - refs[compName] = comp.Image + for _, n := range slices.Sorted(maps.Keys(sources)) { + src := sources[n] + if src.build == "" { + refs[n] = src.image continue } - ref, ok := built[compName] + ref, ok := built[n] if !ok { - return nil, fmt.Errorf("component %s is built from %s; pass the image it was pushed as (--image %s=)", - compName, comp.Build, compName) + return nil, fmt.Errorf("%s %s is built from %s; pass the image it was pushed as (--image %s=)", + src.what, n, src.build, n) } - refs[compName] = ref + refs[n] = ref } return refs, nil } +// imageSource is where the image of a component or a test comes from. +type imageSource struct { + what string // "component" or "test", for messages + image, build string +} + // checkExposedPorts warns about declared ports the image does not EXPOSE. Images without any // EXPOSE are not checked. func checkExposedPorts(doc *schema.Document, compName, image string, info ImageInfo) validate.Findings { @@ -182,6 +214,23 @@ func resolveComponent(app *schema.App, comp *schema.Component, image string, inf return out } +// resolveTest pins the test's image and resolves its references as for a component. +func resolveTest(app *schema.App, test *schema.Test, image string, info ImageInfo) *schema.Test { + out := &schema.Test{ + Image: pinImage(image, info.Digest), + Command: resolveList(app, test.Command), + Args: resolveList(app, test.Args), + Resources: test.Resources, + } + if test.Env != nil { + out.Env = map[string]string{} + for k, v := range test.Env { + out.Env[k] = resolveString(app, v) + } + } + return out +} + // soleOr returns portName, or the only port's name if portName is empty. func soleOr(ports map[string]int, portName string) string { if portName != "" || len(ports) != 1 { diff --git a/internal/render/render_test.go b/internal/render/render_test.go index 7723eb1..17dc700 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -407,7 +407,7 @@ components: wantErr string }{ {"no image for a built component", nil, "pass the image it was pushed as"}, - {"image for an unknown component", map[string]string{"web": "ghcr.io/o/shop-web:main", "nope": "x"}, "no component nope"}, + {"image for an unknown component", map[string]string{"web": "ghcr.io/o/shop-web:main", "nope": "x"}, "no component or test nope"}, {"image for a component with an image", map[string]string{"web": "ghcr.io/o/shop-web:main", "db": "x"}, "component db has an image"}, } for _, tt := range tests { @@ -419,3 +419,78 @@ components: }) } } + +// Tests are pinned like components, with their references resolved, whether their image is +// ready-made or built by the CI. +func TestRenderTests(t *testing.T) { + const src = head + ` +components: + web: { image: "traefik/whoami:v1.11.0", port: 80, env: { TOKEN: "${secrets.token}" } } +tests: + smoke: { image: "curlimages/curl:8.10.1", args: [-fsS, "http://${web.host}:${web.port}/"] } + e2e: { build: ./e2e, env: { TOKEN: "${secrets.token}", PRICE: "5$" } } +secrets: + token: { generate: true } +` + const e2eRef = "ghcr.io/o/shop/e2e:sha-abc1234" + images := fakeResolver{ + "traefik/whoami:v1.11.0": {Digest: digest('a'), ExposedPorts: []int{80}}, + "curlimages/curl:8.10.1": {Digest: digest('c')}, + e2eRef: {Digest: digest('d')}, + } + app, _ := renderApp(t, src, images, map[string]string{"e2e": e2eRef}) + + smoke := app.Tests["smoke"] + if smoke.Image != "curlimages/curl:8.10.1@"+digest('c') { + t.Errorf("smoke image %q", smoke.Image) + } + if got := strings.Join(smoke.Args, " "); got != "-fsS http://web:80/" { + t.Errorf("smoke args %q", got) + } + e2e := app.Tests["e2e"] + if e2e.Image != e2eRef+"@"+digest('d') || e2e.Build != "" { + t.Errorf("e2e image %q, build %q", e2e.Image, e2e.Build) + } + if e2e.Env["TOKEN"] != "${secrets.token}" || e2e.Env["PRICE"] != "5$$" { + t.Errorf("e2e env %v", e2e.Env) + } + + // Without tests, the resolved app.yaml has none, so the values of existing apps stay as + // they were. + app, _ = renderApp(t, head+"components:\n web: { image: \"traefik/whoami:v1.11.0\" }\n", images, nil) + if app.Tests != nil { + t.Errorf("tests %v, want none", app.Tests) + } +} + +func TestRenderTestErrors(t *testing.T) { + const src = head + ` +components: + web: { image: postgres:16 } +tests: + e2e: { build: ./e2e } + smoke: { image: curl } +` + images := fakeResolver{"postgres:16": {Digest: digest('b')}, "curl": {Digest: digest('c')}, "e2e:1": {Digest: digest('d')}} + tests := []struct { + name string + built map[string]string + wantErr string + }{ + {"no image for a built test", nil, "test e2e is built from ./e2e; pass the image"}, + {"image for a test with an image", map[string]string{"e2e": "e2e:1", "smoke": "x"}, "test smoke has an image"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, err := Render(context.Background(), parse(t, "app.yaml", []byte(src)), images, tt.built) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error %v, want one containing %q", err, tt.wantErr) + } + }) + } + // An image that cannot be looked up names the test. + _, _, err := Render(context.Background(), parse(t, "app.yaml", []byte(src)), fakeResolver{"postgres:16": {Digest: digest('b')}, "e2e:1": {Digest: digest('d')}}, map[string]string{"e2e": "e2e:1"}) + if err == nil || !strings.Contains(err.Error(), "test smoke") { + t.Errorf("error %v, want one naming test smoke", err) + } +} diff --git a/internal/schema/jsonschema.go b/internal/schema/jsonschema.go index 3c2550a..203a248 100644 --- a/internal/schema/jsonschema.go +++ b/internal/schema/jsonschema.go @@ -59,15 +59,25 @@ func (App) JSONSchemaExtend(s *jsonschema.Schema) { components.PropertyNames = namePattern(ComponentNamePattern, MaxComponentNameLength) components.MinProperties = ptr(uint64(1)) prop(s, "secrets").PropertyNames = namePattern(SecretNamePattern, MaxSecretNameLength) + prop(s, "tests").PropertyNames = namePattern(ComponentNamePattern, MaxTestNameLength) } -func (Component) JSONSchemaExtend(s *jsonschema.Schema) { - // YAML users write `PORT: 8080`; the parser accepts any scalar and keeps its text. - env := prop(s, "env") +// envSchema lets YAML users write `PORT: 8080`; the parser accepts any scalar and keeps its text. +func envSchema(env *jsonschema.Schema) { env.PropertyNames = namePattern(EnvNamePattern, 253) env.AdditionalProperties = &jsonschema.Schema{AnyOf: []*jsonschema.Schema{ {Type: "string"}, {Type: "number"}, {Type: "boolean"}, }} +} + +// imageSource requires exactly one of image and build. +func imageSource(s *jsonschema.Schema) { + prop(s, "build").Pattern = BuildPathPattern + s.OneOf = []*jsonschema.Schema{{Required: []string{"image"}}, {Required: []string{"build"}}} +} + +func (Component) JSONSchemaExtend(s *jsonschema.Schema) { + envSchema(prop(s, "env")) port := prop(s, "port") port.Minimum, port.Maximum = "1", "65535" ports := prop(s, "ports") @@ -77,8 +87,12 @@ func (Component) JSONSchemaExtend(s *jsonschema.Schema) { prop(s, "instances").Minimum = "1" prop(s, "volumes").PropertyNames = namePattern(VolumeNamePattern, MaxVolumeNameLength) s.Not = &jsonschema.Schema{Required: []string{"port", "ports"}} - prop(s, "build").Pattern = BuildPathPattern - s.OneOf = []*jsonschema.Schema{{Required: []string{"image"}}, {Required: []string{"build"}}} + imageSource(s) +} + +func (Test) JSONSchemaExtend(s *jsonschema.Schema) { + envSchema(prop(s, "env")) + imageSource(s) } func (Route) JSONSchema() *jsonschema.Schema { diff --git a/internal/schema/parse_test.go b/internal/schema/parse_test.go index 92ee5a2..c73b1bd 100644 --- a/internal/schema/parse_test.go +++ b/internal/schema/parse_test.go @@ -63,6 +63,8 @@ func TestParse(t *testing.T) { {name: "comment only", yaml: "# nothing\n", wantErr: "file is empty"}, {name: "unknown top-level field", yaml: "nmae: a\n", wantErr: "field nmae not found"}, {name: "unknown component field", yaml: "components:\n web:\n replicas: 2\n", wantErr: "field replicas not found"}, + {name: "route on a test", yaml: "tests:\n smoke:\n image: curl\n route: /\n", wantErr: "field route not found"}, + {name: "port on a test", yaml: "tests:\n smoke:\n image: curl\n port: 80\n", wantErr: "field port not found"}, {name: "unknown route field", yaml: "components:\n web:\n route: { path: /, strip: true }\n", wantErr: "field strip not found in route"}, {name: "route of wrong type", yaml: "components:\n web:\n route: [/]\n", wantErr: "route must be a path or a mapping"}, {name: "route as number", yaml: "components:\n web:\n route: 80\n", wantErr: "route must be a path or a mapping"}, diff --git a/internal/schema/types.go b/internal/schema/types.go index 7785d4e..d3dfa07 100644 --- a/internal/schema/types.go +++ b/internal/schema/types.go @@ -18,6 +18,7 @@ const SecretEnvPrefix = "SHELF_SECRET_" const ( MaxAppNameLength = 40 MaxComponentNameLength = 40 + MaxTestNameLength = 40 MaxVolumeNameLength = 40 MaxSecretNameLength = 63 ) @@ -45,6 +46,7 @@ type App struct { Name string `yaml:"name" jsonschema_description:"Package name: the workflow publishes the deploy artifact as ghcr.io// and built images below it. Not the app's name in shelf, which is given at shelf app add."` Components map[string]*Component `yaml:"components" jsonschema_description:"The containers that make up the app, by name."` Secrets map[string]*Secret `yaml:"secrets,omitempty" jsonschema_description:"Secrets referenced as ${secrets.}, by name."` + Tests map[string]*Test `yaml:"tests,omitempty" jsonschema_description:"Containers run against the app after every rollout, by name; the rollout counts only when each exits with 0."` } // Component is one container image with its runtime settings. @@ -63,6 +65,20 @@ type Component struct { Resources *Resources `yaml:"resources,omitempty" jsonschema_description:"Resource requests. Memory is also the limit."` } +// Test is a container run once against the app after every rollout. It reaches the components +// by their references, ${.host} and ${.port}, inside the app's namespace. +type Test struct { + Image string `yaml:"image,omitempty" jsonschema_description:"Image reference of a ready-made image. Tags are pinned to a digest by shelf render. Mutually exclusive with build."` + Build string `yaml:"build,omitempty" jsonschema_description:"Directory with the Dockerfile of this test, relative to app.yaml. The CI builds it and shelf render pins the result. Mutually exclusive with image."` + Command []string `yaml:"command,omitempty" jsonschema_description:"Overrides the image entrypoint. Supports ${...} references; write $$ for a literal $."` + Args []string `yaml:"args,omitempty" jsonschema_description:"Overrides the image command. Supports ${...} references; write $$ for a literal $."` + Env map[string]string `yaml:"env,omitempty" jsonschema_description:"Environment variables. Values support ${...} references; write $$ for a literal $."` + Resources *Resources `yaml:"resources,omitempty" jsonschema_description:"Resource requests. Memory is also the limit."` +} + +// IsBuilt reports whether the test's image is built from this repository. +func (t *Test) IsBuilt() bool { return t.Build != "" } + // Route exposes a component over HTTP. In app.yaml it is either a path or a mapping. type Route struct { Path string `yaml:"path" jsonschema_description:"Path prefix, starting with /. Passed through unchanged unless stripPrefix is set."` diff --git a/internal/server/actions.go b/internal/server/actions.go index 26ffe47..038310f 100644 --- a/internal/server/actions.go +++ b/internal/server/actions.go @@ -8,6 +8,7 @@ import ( "net/http" "strconv" "strings" + "unicode" "github.com/tweinmann/shelf/internal/cluster" "github.com/tweinmann/shelf/internal/ops" @@ -111,6 +112,16 @@ func accessFromForm(r *http.Request) (ops.Access, error) { default: a.Cloudflare = choice } + // The access form shows the LAN exceptions the app has, so what it sends is the whole list: + // an empty field takes them away. The add form has no such field and keeps them. + if _, ok := r.PostForm["lan"]; ok { + lan := strings.FieldsFunc(r.PostFormValue("lan"), func(c rune) bool { return c == ',' || unicode.IsSpace(c) }) + if len(lan) == 0 { + a.DenyLAN = true + } else { + a.LAN = lan + } + } return a, a.Check() } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 06dc215..cad968e 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "reflect" "regexp" "strings" "sync" @@ -208,6 +209,8 @@ func runningPlatform() *fakePlatform { Public: true, URL: "https://greeter-dev.example.com/", CloudflareAccount: "acc-1", + Isolation: &cluster.Isolation{PodSecurity: "baseline", LAN: []string{"192.168.1.10/32"}, + Restricted: []cluster.Violation{{Pod: "db-0", Reason: "allowPrivilegeEscalation != false, runAsNonRoot != true"}}}, Verification: &ops.Verification{ Verification: cluster.Verification{ Verified: true, Source: "https://github.com/tweinmann/shelf-hello", @@ -253,6 +256,7 @@ func runningPlatform() *fakePlatform { Host: "shop-dev.shop.ch", Public: true, CloudflareMissing: true, + Isolation: &cluster.Isolation{PodSecurity: "baseline"}, Verification: &ops.Verification{ Verification: cluster.Verification{Reason: "the deploy artifact has no scan (dev.shelf.scan)"}, Signer: "keyless, issuer https://token.actions.githubusercontent.com", @@ -290,6 +294,7 @@ func runningPlatform() *fakePlatform { {Name: "the app", Object: "HelmRelease greeter/greeter", Phase: cluster.PhaseReady, Reason: "InstallSucceeded", Message: "Helm install succeeded"}, {Name: "the pods", Object: "Pods", Phase: cluster.PhaseReady, Message: "2 running"}, + {Name: "the tests", Object: "Pods greeter/greeter-test-*", Phase: cluster.PhaseReady, Message: "1 passed: smoke"}, }}, "shop": {Name: "shop", Stages: []cluster.Stage{ {Name: "the deploy artifact", Object: "OCIRepository shop/deploy", Phase: cluster.PhaseFailed, @@ -302,6 +307,9 @@ func runningPlatform() *fakePlatform { {Name: "the app", Object: "HelmRelease shop/shop", Phase: cluster.PhaseReady, Reason: "InstallSucceeded", Message: "Helm install succeeded"}, {Name: "the pods", Object: "Pods", Phase: cluster.PhaseReady, Message: "1 running"}, + {Name: "the tests", Object: "Pod shop/shop-test-smoke", Phase: cluster.PhaseFailed, Reason: "TestFailed", + Message: "test smoke failed with exit code 22:\ncurl: (22) The requested URL returned error: 500", + Hint: "the app's own test smoke fails against this version; the previous version runs again. Its log says why"}, }}, }, } @@ -976,6 +984,16 @@ func TestAccessForms(t *testing.T) { form: url.Values{"registry": {"ghcr"}, "cloudflare": {""}}, want: ops.Access{Registry: "ghcr", Private: true}, }, + { + name: "let an app reach a NAS", path: "/apps/greeter/access", + form: url.Values{"registry": {"ghcr"}, "cloudflare": {"home"}, "lan": {"192.168.1.10/32, 192.168.2.0/24"}}, + want: ops.Access{Registry: "ghcr", Cloudflare: "home", LAN: []string{"192.168.1.10/32", "192.168.2.0/24"}}, + }, + { + name: "take the LAN away", path: "/apps/greeter/access", + form: url.Values{"registry": {"ghcr"}, "cloudflare": {"home"}, "lan": {" "}}, + want: ops.Access{Registry: "ghcr", Cloudflare: "home", DenyLAN: true}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -987,12 +1005,26 @@ func TestAccessForms(t *testing.T) { t.Fatalf("status %d\n%s", rec.Code, rec.Body) } waitForJob(t, h, cookie, "job-1") - if got := platform.lastAccess(); got != tt.want { + if got := platform.lastAccess(); !reflect.DeepEqual(got, tt.want) { t.Errorf("access %+v, want %+v", got, tt.want) } }) } + t.Run("an invalid LAN range", func(t *testing.T) { + t.Parallel() + platform := runningPlatform() + h, cookie := claimed(t, platform) + form := url.Values{"registry": {"ghcr"}, "cloudflare": {"home"}, "lan": {"8.8.8.0/24"}} + rec := post(h, "/apps/greeter/access", withCSRF(t, h, cookie, form), cookie) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "8.8.8.0/24 is not a private range") { + t.Errorf("status %d\n%s", rec.Code, rec.Body) + } + if got := platform.lastAccess(); !reflect.DeepEqual(got, ops.Access{}) { + t.Errorf("a refused range started a change: %+v", got) + } + }) + t.Run("an invalid domain", func(t *testing.T) { t.Parallel() platform := runningPlatform() diff --git a/internal/server/testdata/app-failed.html b/internal/server/testdata/app-failed.html index a661c4e..6fba507 100644 --- a/internal/server/testdata/app-failed.html +++ b/internal/server/testdata/app-failed.html @@ -86,6 +86,20 @@

Verification

+
+

Isolation

+
+
Pod Security
baseline
+
LAN
none
+
+ +

The app reaches its own components, the cluster DNS and the internet, and nothing +else: not other apps, not the Kubernetes API, not the LAN, except the ranges given under Access. +Only Traefik reaches in. Its previews never reach the LAN.

+
+ + +

Deploy

@@ -130,6 +144,8 @@

Access

+ +

The connections themselves are defined on the connections @@ -183,6 +199,14 @@

How it gets here

Pods

+
  • +
    the testsFailed
    +
    test smoke failed with exit code 22:
    +curl: (22) The requested URL returned error: 500
    +

    the app's own test smoke fails against this version; the previous version runs again. Its log says why

    +

    Pod shop/shop-test-smoke · TestFailed

    +
  • +
    diff --git a/internal/server/testdata/app-no-domain.html b/internal/server/testdata/app-no-domain.html index d1738f1..fba1bc4 100644 --- a/internal/server/testdata/app-no-domain.html +++ b/internal/server/testdata/app-no-domain.html @@ -65,6 +65,8 @@

    Components

    + +

    Deploy

    @@ -104,6 +106,8 @@

    Access

    + +

    The connections themselves are defined on the connections @@ -157,6 +161,13 @@

    How it gets here

    Pods

    +
  • +
    the testsReady
    +
    1 passed: smoke
    + +

    Pods greeter/greeter-test-*

    +
  • +
    diff --git a/internal/server/testdata/app-quick.html b/internal/server/testdata/app-quick.html index 095c755..b58a003 100644 --- a/internal/server/testdata/app-quick.html +++ b/internal/server/testdata/app-quick.html @@ -66,6 +66,8 @@

    Components

    + +

    Deploy

    @@ -109,6 +111,8 @@

    Access

    + +

    The connections themselves are defined on the connections @@ -162,6 +166,13 @@

    How it gets here

    Pods

    +
  • +
    the testsReady
    +
    1 passed: smoke
    + +

    Pods greeter/greeter-test-*

    +
  • +
    diff --git a/internal/server/testdata/app-ready.html b/internal/server/testdata/app-ready.html index a1188bc..07c13cd 100644 --- a/internal/server/testdata/app-ready.html +++ b/internal/server/testdata/app-ready.html @@ -99,6 +99,27 @@

    Verification

    +
    +

    Isolation

    +
    +
    Pod Security
    baseline
    +
    LAN
    192.168.1.10/32
    +
    + +

    The stricter level restricted would refuse:

    +
      + +
    • db-0allowPrivilegeEscalation != false, runAsNonRoot != true
    • + +
    + +

    The app reaches its own components, the cluster DNS and the internet, and nothing +else: not other apps, not the Kubernetes API, not the LAN, except the ranges given under Access. +Only Traefik reaches in. Its previews never reach the LAN.

    +
    + + +

    Deploy

    @@ -143,6 +164,8 @@

    Access

    + +

    The connections themselves are defined on the connections @@ -196,6 +219,13 @@

    How it gets here

    Pods

    +
  • +
    the testsReady
    +
    1 passed: smoke
    + +

    Pods greeter/greeter-test-*

    +
  • +
    diff --git a/internal/server/ui/app.html b/internal/server/ui/app.html index 62a544b..aa7645a 100644 --- a/internal/server/ui/app.html +++ b/internal/server/ui/app.html @@ -76,6 +76,27 @@

    Verification

    {{end}} +{{with .App.Isolation}} +
    +

    Isolation

    +
    +
    Pod Security
    {{with .PodSecurity}}{{.}}{{else}}none: the platform predates it; install the current one with shelf init cluster{{end}}
    +
    LAN
    {{if .LAN}}{{range $i, $c := .LAN}}{{if $i}}, {{end}}{{$c}}{{end}}{{else}}none{{end}}
    +
    +{{if .Restricted}} +

    The stricter level restricted would refuse:

    +
      +{{range .Restricted}} +
    • {{.Pod}}{{.Reason}}
    • +{{end}} +
    +{{end}} +

    The app reaches its own components, the cluster DNS and the internet, and nothing +else: not other apps, not the Kubernetes API, not the LAN, except the ranges given under Access. +Only Traefik reaches in. Its previews never reach the LAN.

    +
    +{{end}} + {{if .Running}}

    A change is running: {{.Running.Title}}. The buttons below wait until it is done.

    @@ -125,6 +146,8 @@

    Access

    {{end}} + +

    The connections themselves are defined on the connections diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 3de524f..c78477f 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -70,6 +70,9 @@ func Validate(doc *schema.Document) Findings { for _, compName := range slices.Sorted(maps.Keys(app.Components)) { c.checkComponent(app, compName) } + for _, testName := range slices.Sorted(maps.Keys(app.Tests)) { + c.checkTest(app, testName) + } c.checkRouteCollisions(app) c.checkSecrets(app) @@ -113,7 +116,7 @@ func (c *checker) checkComponent(app *schema.App, compName string) { return } - c.checkImage(p, comp) + c.checkImage(p, comp.Image, comp.Build) c.checkPorts(p, comp) c.checkRoute(p, comp) @@ -122,15 +125,46 @@ func (c *checker) checkComponent(app *schema.App, compName string) { } c.checkVolumes(p, comp) c.checkHealth(p, comp) - c.checkResources(p, comp) + c.checkResources(p, comp.Resources) + c.checkRuntime(app, p, comp.Command, comp.Args, comp.Env) +} + +// checkTest checks one entry of tests. A test is a container like a component, without anything +// that would make it reachable or keep it running; its image is published next to the +// components' as /, so the names must not clash. +func (c *checker) checkTest(app *schema.App, testName string) { + p := []string{"tests", testName} + test := app.Tests[testName] + + switch { + case len(testName) > schema.MaxTestNameLength: + c.errorf(p, "test name must be at most %d characters", schema.MaxTestNameLength) + case !componentNameRE.MatchString(testName): + c.errorf(p, "test name %q must consist of lowercase letters, digits and '-', start "+ + "with a letter and end with a letter or digit", testName) + } + if _, ok := app.Components[testName]; ok { + c.errorf(p, "test name %q is already the name of a component", testName) + } + if test == nil { + c.errorf(p, "test is empty; image or build is required") + return + } + c.checkImage(p, test.Image, test.Build) + c.checkResources(p, test.Resources) + c.checkRuntime(app, p, test.Command, test.Args, test.Env) +} - for i, s := range comp.Command { +// checkRuntime checks what a component and a test share: command, args and env, with their +// references. +func (c *checker) checkRuntime(app *schema.App, p []string, command, args []string, env map[string]string) { + for i, s := range command { c.checkTemplate(app, at(p, "command", fmt.Sprint(i)), s) } - for i, s := range comp.Args { + for i, s := range args { c.checkTemplate(app, at(p, "args", fmt.Sprint(i)), s) } - for _, k := range slices.Sorted(maps.Keys(comp.Env)) { + for _, k := range slices.Sorted(maps.Keys(env)) { ep := at(p, "env", k) switch { case !envNameRE.MatchString(k): @@ -139,24 +173,24 @@ func (c *checker) checkComponent(app *schema.App, compName string) { c.errorf(ep, "the prefix %s is reserved for secrets; use ${secrets.} instead", schema.SecretEnvPrefix) } - c.checkTemplate(app, ep, comp.Env[k]) + c.checkTemplate(app, ep, env[k]) } } // checkImage requires exactly one of image (a ready-made image) and build (a directory in the // repository, which the CI builds and shelf render pins). -func (c *checker) checkImage(p []string, comp *schema.Component) { +func (c *checker) checkImage(p []string, image, build string) { switch { - case comp.Image == "" && comp.Build == "": + case image == "" && build == "": c.errorf(p, "needs image or build") - case comp.Image != "" && comp.Build != "": + case image != "" && build != "": c.errorf(at(p, "build"), "cannot be used together with image") - case comp.Image != "": - if _, err := name.ParseReference(comp.Image); err != nil { - c.errorf(at(p, "image"), "invalid image reference %q: %v", comp.Image, err) + case image != "": + if _, err := name.ParseReference(image); err != nil { + c.errorf(at(p, "image"), "invalid image reference %q: %v", image, err) } default: - c.checkBuildPath(at(p, "build"), comp.Build) + c.checkBuildPath(at(p, "build"), build) } } @@ -338,8 +372,7 @@ func (c *checker) checkHealth(p []string, comp *schema.Component) { } } -func (c *checker) checkResources(p []string, comp *schema.Component) { - r := comp.Resources +func (c *checker) checkResources(p []string, r *schema.Resources) { if r == nil { return } @@ -418,17 +451,22 @@ func (c *checker) checkRef(app *schema.App, p []string, ref *schema.Ref) { func (c *checker) checkSecrets(app *schema.App) { used := map[string]bool{} + var strs []string for _, comp := range app.Components { - if comp == nil { - continue + if comp != nil { + strs = slices.Concat(strs, comp.Command, comp.Args, slices.Collect(maps.Values(comp.Env))) } - strs := slices.Concat(comp.Command, comp.Args, slices.Collect(maps.Values(comp.Env))) - for _, s := range strs { - tokens, _ := schema.ParseTemplate(s) - for _, t := range tokens { - if t.Ref != nil && t.Ref.Kind == schema.RefSecret { - used[t.Ref.Secret] = true - } + } + for _, test := range app.Tests { + if test != nil { + strs = slices.Concat(strs, test.Command, test.Args, slices.Collect(maps.Values(test.Env))) + } + } + for _, s := range strs { + tokens, _ := schema.ParseTemplate(s) + for _, t := range tokens { + if t.Ref != nil && t.Ref.Kind == schema.RefSecret { + used[t.Ref.Secret] = true } } } @@ -446,7 +484,7 @@ func (c *checker) checkSecrets(app *schema.App) { c.errorf(sp, "only generated secrets are supported; set generate: true") } if !used[secretName] { - c.add(Warning, sp, "secret %q is not referenced by any component", secretName) + c.add(Warning, sp, "secret %q is not referenced by any component or test", secretName) } } } diff --git a/internal/validate/validate_test.go b/internal/validate/validate_test.go index 2b75611..9b7ba08 100644 --- a/internal/validate/validate_test.go +++ b/internal/validate/validate_test.go @@ -54,6 +54,20 @@ components: components: web: { build: ./web, port: 8080, route: / } api: { build: services/api } +`, + "test against a component": head + ` +components: + web: { image: nginx, port: 80 } +tests: + smoke: { image: curlimages/curl:8.10.1, args: [-fsS, "http://${web.host}:${web.port}/"] } +`, + "built test with a secret": head + ` +components: + web: { image: nginx, port: 80, env: { TOKEN: "${secrets.token}" } } +tests: + e2e: { build: ./e2e, env: { TOKEN: "${secrets.token}", URL: "http://${web.host}" }, resources: { memory: 64Mi } } +secrets: + token: { generate: true } `, "digest-pinned image": head + ` components: @@ -243,6 +257,15 @@ func TestErrors(t *testing.T) { {"unknown named port", head + "components:\n web: { image: nginx, env: { A: '${api.ports.c}' } }\n api: { image: nginx, ports: { a: 80 } }\n", "components.web.env.A", `unknown port "c"`}, {"unknown secret", head + "components:\n web: { image: nginx, env: { A: '${secrets.pw}' } }\n", "components.web.env.A", `unknown secret "pw"`}, + // tests + {"test named like a component", head + "components:\n web: { image: nginx }\ntests:\n web: { image: curl }\n", "tests.web", "already the name of a component"}, + {"empty test", head + "components:\n web: { image: nginx }\ntests:\n smoke:\n", "tests.smoke", "test is empty"}, + {"test without image", head + "components:\n web: { image: nginx }\ntests:\n smoke: { args: [a] }\n", "tests.smoke", "needs image or build"}, + {"test name with underscore", head + "components:\n web: { image: nginx }\ntests:\n my_test: { image: curl }\n", "tests.my_test", "test name"}, + {"test name too long", head + "components:\n web: { image: nginx }\ntests:\n " + strings.Repeat("t", 41) + ": { image: curl }\n", "tests." + strings.Repeat("t", 41), "at most 40"}, + {"test refers to an unknown component", head + "components:\n web: { image: nginx }\ntests:\n smoke: { image: curl, args: ['${api.host}'] }\n", "tests.smoke.args.0", `unknown component "api"`}, + {"test with an invalid memory", head + "components:\n web: { image: nginx }\ntests:\n smoke: { image: curl, resources: { memory: lots } }\n", "tests.smoke.resources.memory", "invalid quantity"}, + // secrets {"secret without generate", head + "components:\n web: { image: nginx, env: { A: '${secrets.pw}' } }\nsecrets:\n pw: {}\n", "secrets.pw", "generate: true"}, {"secret with generate false", head + "components:\n web: { image: nginx, env: { A: '${secrets.pw}' } }\nsecrets:\n pw: { generate: false }\n", "secrets.pw", "generate: true"}, diff --git a/justfile b/justfile index 66cf6d7..7ab1fb4 100644 --- a/justfile +++ b/justfile @@ -113,6 +113,10 @@ smoke-previews: smoke-verify: hack/smoke/verify.sh +# Phase 12: isolation of app and preview, LAN exception, Pod Security, tests on app and preview, report +smoke-hardening: + hack/smoke/hardening.sh + # Phase 8b acceptance: expose an app through its own Cloudflare tunnel (asks for the API token) smoke-expose app domain: hack/smoke/expose.sh {{app}} {{domain}} diff --git a/openspec/changes/phase-12-hardening/.openspec.yaml b/openspec/changes/phase-12-hardening/.openspec.yaml index ad9e5be..97e3642 100644 --- a/openspec/changes/phase-12-hardening/.openspec.yaml +++ b/openspec/changes/phase-12-hardening/.openspec.yaml @@ -1,3 +1,2 @@ schema: spec-driven created: 2026-10-27 -skip_specs: true diff --git a/openspec/changes/phase-12-hardening/design.md b/openspec/changes/phase-12-hardening/design.md new file mode 100644 index 0000000..f8fc7b1 --- /dev/null +++ b/openspec/changes/phase-12-hardening/design.md @@ -0,0 +1,380 @@ +# Design + +## Context + +What a preview can do today: + +- **Namespaces.** Every app and every preview has a namespace of its own, created by the + ResourceSets `apps` and `previews`. The namespaces have the labels `shelf.dev/*` and + `policy.sigstore.dev/include`, and nothing that limits which pods they admit. +- **Network.** There is no NetworkPolicy anywhere. Any pod reaches any other pod and service in + the cluster, the Kubernetes API, the host's network and the internet. Traefik, in the namespace + `traefik`, routes by Host header, so a pod that sends `Host: greeter.shelf.internal` to Traefik + reaches `greeter` even across namespaces. +- **Green.** `shelf app status --wait --commit` says Ready when the release runs the commit's + values and every pod is ready. Nothing of the app's own is run against it. +- **Releases.** The HelmRelease of an app and of a preview have install and upgrade remediation + with three retries; upgrade remediation rolls back after the last one + (`remediateLastFailure: true`). +- **Cluster.** k3s, also in k3d, ships kube-router's NetworkPolicy controller and enables it by + default. The spike below shows that it enforces what this design needs. + +## Goals / Non-Goals + +**Goals:** + +- **Fenced in.** A preview reaches neither another app, nor the Kubernetes API, nor the LAN, nor a + cloud metadata address. It still reaches the internet, and the app's own components reach each + other. +- **Green means tested.** An app's own tests decide whether a rollout counts, for the app and for + its previews, in the wait verdict too. +- **One report.** An agent gets one document per branch to put into its pull request. +- **Ejectable.** Everything is plain Kubernetes (namespace labels, NetworkPolicy) and plain Helm + (test hooks): an ejected app keeps it without shelf. + +**Non-Goals:** + +- No `restricted` opt-in yet (see decision 1). +- No egress allow-list for the internet: an app that calls an API keeps working. Exfiltration + through the internet is not prevented, only the way into the home network. +- No identity on preview traffic (Cloudflare Access stays in "Later"). +- No test history beyond the running revision, and no test retries of shelf's own. +- No CNI-specific policies (Cilium, Calico): the policy has to work on any CNI that implements + NetworkPolicy. + +## Decisions + +### 1. Pod Security: `baseline` enforced, `restricted` warned and audited + +*Asked and decided by the maintainer.* Namespace labels: +- `pod-security.kubernetes.io/enforce: baseline`; +- `warn: restricted` and `audit: restricted`; +- every `*-version` label is `latest`. + +`baseline` refuses what breaks out of a container: privileged containers, host namespaces, +`hostPath`, added capabilities beyond the default set. It admits images that start as root, +which `postgres:16` and most stock database images do. + +`latest` follows the cluster's Kubernetes version. Pinning a version would keep the rules +stable across a cluster upgrade, but nobody would move the pin. + +Alternatives: +- **`restricted` enforced:** breaks stock images that start as root. +- **An opt-in `security: restricted` in app.yaml:** safe for a tenant to choose, since it can only + tighten. It was left for later, because nobody needs it yet. + +### 1b. Two security defaults in the chart (found during apply) + +*Asked and decided by the maintainer.* The chart set no `securityContext`, so every pod violated +`restricted` the same four ways (`allowPrivilegeEscalation != false, unrestricted capabilities, +runAsNonRoot != true, seccompProfile`), and the list in the status would only repeat them. The +chart now sets what no image depends on: +- `seccompProfile: RuntimeDefault` on every pod; +- `allowPrivilegeEscalation: false` on every container. + +An entrypoint that drops from root to a user with `gosu` or `su-exec`, as Postgres does, still +works: changing the user is a syscall, not a setuid binary. What is left in the list is what the +image decides, root and capabilities, and that is what the operator wants to see. + +Alternatives: +- **Leave the chart alone:** no change of behaviour, but the list is noise. +- **Also drop all capabilities and require a non-root user:** that would be `restricted` + enforced through the back door, breaking the images decision 1 keeps working. + +### 2. `restricted` violations are read with a server-side dry run + +The `warn` label sends its warnings to whoever creates the pod. That is helm-controller, so +nobody sees them. shelf reads them itself instead. A server-side dry-run patch of the namespace +label `enforce: restricted` makes the API server answer with one warning per violating pod. A +client-go `WarningHandler` collects them, and nothing is changed: a dry run is not persisted. +`cluster` parses the warning lines (`: `) into `restrictedViolations`. + +Alternatives: +- **The Pod Security admission library (`k8s.io/pod-security-admission`) in shelf:** a large + dependency whose version would have to follow the cluster's, and the API server already does + the same evaluation. +- **Reading the audit log:** k3s does not write it by default. + +### 3. One NetworkPolicy per namespace: `shelf-isolation` + +Rendered by the ResourceSets, for the app namespace and for each preview namespace: + +```yaml +spec: + podSelector: {} + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: {} + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: traefik}} + egress: + - to: [{podSelector: {}}] + - to: [{namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}, + podSelector: {matchLabels: {k8s-app: kube-dns}}}] + ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}] + - to: + - ipBlock: {cidr: 0.0.0.0/0, except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, + 100.64.0.0/10, 169.254.0.0/16]} + - ipBlock: {cidr: ::/0, except: [fc00::/7, fe80::/10]} + # an app's LAN exceptions, one ipBlock each (decision 4) +``` + +Why each part: +- **Pod IPs are excluded.** Pod and service IPs of any usual cluster lie in the excluded ranges, + so the internet rule does not open other namespaces. Policies match after the service address + is translated, so a ClusterIP counts as the pod behind it. +- **The Kubernetes API is excluded.** It is a private node address behind `10.43.0.1`. +- **DNS is the only thing allowed outside the namespace inside the cluster.** +- **`100.64.0.0/10`** is where Tailscale and carrier-grade NAT put home devices. +- **`169.254.0.0/16`** holds the cloud metadata address, which matters on an ejected or Hetzner + host. + +**Spike on the dev cluster (2026-10-28)**, two throwaway namespaces with this policy in one: + +| From the isolated namespace to | Without policy | With policy | +|---|---|---| +| a service in its own namespace | open | open | +| a service in another namespace | open | blocked | +| the registry container `172.19.0.3:5000`, standing in for the LAN | open | blocked | +| `https://10.43.0.1` (Kubernetes API) | — | blocked | +| `http://example.com` (DNS and internet) | open | open | +| from the other namespace into the isolated one | open | blocked | +| API server pod proxy into the isolated namespace (how shelf reads a quick tunnel's name) | open | open | + +So level 2 proves the LAN rule on k3d as it holds on k3s on a host. + +**cloudflared (found during apply).** cloudflared runs in the app namespace and forwards to +`traefik.traefik.svc`, which the isolation would block, taking every exposed app off the +internet. A second policy `shelf-cloudflared` therefore lets only the pods labelled +`app.kubernetes.io/name: cloudflared` reach the Traefik namespace. It is rendered only for an app +with a tunnel or a quick tunnel, never for a preview. App pods cannot carry that label, since the +chart sets their labels, so app code still cannot reach Traefik. Policies add up, so this needs +no change to `shelf-isolation`. + +Alternatives: +- **Egress to Traefik:** would let tests use routes. Rejected: any pod could then reach any app + through Traefik by its Host header. Tests call components directly (decision 6). +- **Denying the internet too:** breaks every app that calls an API, and package downloads at + start. +- **CNI-specific policies with FQDN rules:** not generic, and a CRD in the runtime output. + +### 4. LAN exceptions: CIDRs set by the operator + +*Asked and decided by the maintainer: isolation for every app, with an exception per app.* + +The exception is a list of ranges, not "LAN open": +- The excluded private ranges also hold the cluster's own pods, `10.42.0.0/16` on k3s. Opening + them would reopen every other app. +- A generic platform does not know a cluster's ranges. + +The operator names what the app needs: +- `--allow-lan 192.168.1.10/32`, repeatable; the flags replace the list, and `--deny-lan` clears + it; +- neither flag keeps the list, as `--domain` and the connections are kept. + +The list is the provider input `lan` (a list of strings), and the `apps` ResourceSet adds one +`ipBlock` per entry. The `previews` ResourceSet does not read it. + +`ops` checks every range: +- it must parse as a CIDR; +- it must lie inside one of the excluded private ranges (anything else is already allowed); +- it must not overlap a node's pod range (`spec.podCIDRs` of the nodes, readable through the API + on any cluster). + +The service range cannot be read generically. A range covering it is not refused, but it opens +nothing: services are reached through their pods. + +The UI's access form gets a text field, ranges separated by spaces or commas, and calls the same +`ops` function. + +Alternatives: +- **A boolean `--allow-lan`:** reopens the cluster's pods, as above. +- **An exception in app.yaml:** a tenant could open the LAN, against the guardrail that the + operator decides what reaches beyond the app. + +### 5. Tests are Helm test hooks run by Flux + +*Asked and decided by the maintainer.* + +Rendering: +- `tests:` in app.yaml becomes one Pod per test in the chart, with `helm.sh/hook: test`; +- `before-hook-creation` keeps the last pod, and its log, until the next run; +- the label `shelf.dev/test: ` marks it; +- the pods stage leaves it out. + +Running: the HelmRelease gets `test.enable: true`, so helm-controller runs the tests after every +install and upgrade, within the release timeout (5 minutes). + +What a failure does: +- **App:** `test.ignoreFailures: false`. A failed test fails the upgrade, the remediation retries + it, and after the last retry it rolls back, so the previous version runs. +- **First install:** install remediation has no rollback, so the failing version keeps running, + as the spec says. +- **Preview:** `test.ignoreFailures: true`. The release stays Ready and the version that failed + keeps running. The result is in `status.history[0].testHooks`. + +`cluster` reads the tests stage: +- from the latest history entry's `testHooks`, by revision; +- with the `TestSuccess` condition; +- with the last 20 lines of the failed pod's log, through the pod log API. + +A test with `build:` is built, signed and scanned like a component: +- `shelf build-plan` lists it; +- `render --image` pins it; +- the workflow scans every digest in the values; +- the admission policy covers `/**`; +- `cluster.verification` includes test images in "the scan covers every image". + +Alternatives: +- **A Job started by shelf:** shelf would become a second rollout engine, and the tests would not + run when nobody runs shelf. +- **Flux Kustomization health checks:** they check readiness, not the app's behaviour. +- **Tests only for previews:** "green" would mean two things. + +### 6. Tests reach the app inside its namespace + +A test calls components by their service names, through the references it already has: +`http://${web.host}:${web.port}/`. The isolation admits that. Traefik is not reachable +(decision 3), so a route's `stripPrefix` is not part of what a test sees. The test containers get +the app's secrets in the same way, so a test can log in. + +### 7. The wait counts tests + +The tests stage comes after the pods. `Verdict` says Ready only when the tests of the awaited +commit's revision passed, or there are none. A failed test is `Failed` at once: it does not wait +for the three upgrade retries, so an agent hears it as soon as it happened. + +### 8. `shelf app report` + +*Asked and decided by the maintainer: a command of its own.* + +`ops.Report` puts together, read-only: +- the `AppStatus` parts (state, commit, tests, verification, isolation); +- for a preview, the comparison with its app: component digests from both values ConfigMaps, and + the CRITICAL and HIGH counts from both scan summaries. + +`internal/cli` prints Markdown by default, meant for `gh pr comment --body-file -`, and with +`--json` a `Report` document. + +"What changed" means the running images, not a Git diff: +- the platform never talks to Git; +- the agent has the diff already. + +Alternatives: +- **Only in `app status`:** no text an agent can paste. +- **A job summary of the workflow:** the workflow cannot see the cluster. + +### 9. Turnaround and RAM + +There is no new controller: kube-router is part of k3s, and helm-controller runs the tests. + +A test adds its own run time to a rollout. On a failed upgrade it adds the three retries before +the rollback, but not to the verdict (decision 7). + +`smoke-hardening` prints the time from push to Ready for an app with one curl test, against the +same app without it. + +### 10. `smoke-hardening` + +New level-2 smoke test, with `examples/hello` plus a curl test. It checks: +- the isolation table of the spike, run from the app and from a preview; +- a LAN exception reaches the registry container, while the preview still cannot; +- a privileged pod is refused; +- `postgres:16` runs and is listed as violating `restricted`; +- a passing test gives Ready; +- a failing test on main keeps the previous version and ends `--wait` with Failed at "the tests"; +- a failing test on a preview keeps the preview running and red; +- `shelf app report` for that preview names the changed component. + +`select-smokes.sh` lists the paths it covers. + +## Risks / Trade-offs + +- **[Risk] Kubelet health probes come from the node, which a policy might block** → kube-router + admits traffic from the node to its own pods. `smoke-chart` and `smoke-hardening` run apps + with probes under the policy. If a probe fails there, the policy gets an ingress rule for the + node addresses. +- **[Risk, seen during apply] A new pod's first connection inside its namespace is refused** → + kube-router lets a pod in through the ingress rule `podSelector: {}` only once its IP is in the + rule's ipset, which happens a moment after the pod starts. A curl test that connects at once + got `Connection refused` after 1 ms; one retry 100 ms later got through (dev cluster, + 2026-10-28). Tests retry their first connection (`curl --retry 5 --retry-connrefused` in the + example and the README), as against any service that just started; the same holds for app + components that connect at start, which retry anyway. Alternatives: an init container that + sleeps (an extra image in every app, and a guessed duration), or a rule by address range + (not generic: a cluster's pod range is not known). +- **[Risk] An existing app that reaches the LAN or another app breaks at the platform + upgrade** → named in the release notes. The operator adds `--allow-lan`; apps were never meant + to reach each other. +- **[Risk] A pod that violates `baseline` is refused at its next start after the upgrade** → the + running pods stay, since labels do not evict. The pods stage names the refusal with a Pod + Security hint. Before upgrading, `shelf app status` shows the `restricted` violations, a + superset. +- **[Risk] A failed test on main retries the upgrade three times before the rollback** → about + three times the test run until the old version is back; the old pods keep serving meanwhile, + as an upgrade replaces pods only when the new ones are ready. +- **[Trade-off] The internet stays open** → an agent's code can still send data out. Closing it + needs an allow-list per app, which no app declares today. +- **[Trade-off] IPv6 is covered by the policy but untested** → k3d is IPv4 only. The rule is + written for dual-stack hosts and checked at level 1 by rendering. +- **[Risk] A dry run of the namespace label needs patch rights on namespaces** → shelf already + has them. The dry run is never persisted, so `--wait` and status stay read-only in effect. + +## Migration Plan + +1. Release with the new platform and chart. `shelf init cluster` upgrades existing clusters; the + ResourceSets add the labels, the policy and `test.enable` to every app and preview. +2. Apps without tests: the tests stage is Ready with nothing to run, and nothing else changes, + apart from the isolation. +3. Apps that need the LAN: `shelf app credentials --allow-lan `. +4. Rollback: an older platform removes the policies and labels again, since the ResourceSets + prune what they no longer render. + +## Open Questions + +None. + +## Results + +### Level 2 (dev cluster, 2026-10-28) + +Every smoke test passed against a fresh dev cluster, with `SMOKE_FULL=1`: + +| Smoke test | Time | +|---|---| +| `smoke-init` | 208 s | +| `smoke-secrets` | 7 s | +| `smoke-chart` (health probes under the policy) | 128 s | +| `smoke-apps` (apps without tests are Ready) | 261 s | +| `smoke-agent` (`shelf app open` through Traefik) | 392 s | +| `smoke-previews` | 344 s | +| `smoke-verify` | 218 s | +| `smoke-hardening` | 366 s | + +`smoke-quick` was not run: it opens a public tunnel, and is left to its own acceptance. + +Measured by `smoke-hardening` and `smoke-previews`: + +| Check | Result | +|---|---| +| Push to Ready without a test | 52 s | +| Push to Ready with one curl test | 63 s | +| First push of a branch to Ready | 27 s (Phase 10: 23 s) | +| Later commit on a branch to Ready | 13 s (Phase 10: 11 s) | +| RAM of a preview / of the app | 29 Mi / 32 Mi | +| Deleted branch tag to preview gone | 29 s | + +Every isolation check has a control: from a namespace without the policy, the Kubernetes API +and the registry container are reachable, from the app's and the preview's not. + +What apply found: +- **cloudflared and Traefik.** See decision 3; without `shelf-cloudflared`, every exposed app + would have gone offline at the upgrade. +- **A new pod's first connection.** See the risk above; the tests retry it. +- **The `restricted` list.** See decision 1b; without the two defaults, it named every pod alike. +- **Leaked forwarders.** `smoke-agent`, `smoke-previews` and the new `smoke-hardening` started + `shelf app open` through a shell function, so `kill $!` stopped the subshell and left the + forwarder running, holding its port into the next run. They start the binary now. + +Also confirmed: `shelf serve` kept working through two `cluster-reset`s without a restart, the +follow-up of Phase 11. diff --git a/openspec/changes/phase-12-hardening/proposal.md b/openspec/changes/phase-12-hardening/proposal.md index e75e179..603192b 100644 --- a/openspec/changes/phase-12-hardening/proposal.md +++ b/openspec/changes/phase-12-hardening/proposal.md @@ -1,45 +1,85 @@ # Proposal -> Draft. This change holds the roadmap text only and is marked `skip_specs: true`. -> `/opsx:propose` turns it into a full change: it writes the delta specs, `design.md` and -> `tasks.md`, revises this proposal, and removes `skip_specs` from `.openspec.yaml`. - ## Why -A preview runs code an agent wrote. It must not reach other apps or the home LAN, and a preview -should count as green only when the app's own checks pass against it. +A preview runs code an agent wrote, and since Phase 11 shelf knows that code was built by its +workflow, but not that it behaves. Today a preview can reach every other app in the cluster, the +home LAN and the cloud metadata address, and it counts as green as soon as its pods are ready. +Before agents push branches unattended, a preview must be fenced in, and "green" must mean that +the app's own checks pass against it. ## What Changes -- Pod Security labels on app and preview namespaces. -- Default-deny NetworkPolicies per app namespace: traffic only within the namespace and from - Traefik; egress to the internet allowed, egress to private address ranges outside the cluster - (the home LAN) denied. -- Smoke tests by convention: an optional test component that runs against the preview and reports - by exit code; a preview counts as green only when it passes. -- A verification report per branch: preview address, smoke test result, scan result, what changed. +- **Pod Security.** App and preview namespaces enforce the Pod Security level `baseline` and warn + and audit `restricted`. Pods that would violate `restricted` are listed in `shelf app status`, + without blocking them, so stock images that start as root (Postgres) keep working. The chart + gives every pod the two settings no image depends on (seccomp `RuntimeDefault`, no privilege + escalation), so that list shows what the images decide. +- **Network isolation.** Every app and preview namespace gets a default-deny NetworkPolicy: + - ingress only from its own namespace and from Traefik; + - egress to its own namespace, to the cluster DNS and to the internet; + - egress to other namespaces, the Kubernetes API, private address ranges (the LAN) and + link-local addresses (cloud metadata) denied. +- **LAN exceptions, operator only.** `shelf app add|credentials --allow-lan ` (repeatable, + `--deny-lan` clears it) lets one app reach named private addresses, for example a NAS. Like + exposure, it is never part of `app.yaml`, and previews never get it. +- **Tests in app.yaml.** A new optional section `tests:` declares test containers (`image` or + `build`, `command`, `args`, `env` with references such as `${web.host}`). The chart renders them + as Helm test hooks, and Flux runs them after every install and upgrade. A failing test: + - on an app, fails the upgrade, so the previous version keeps running; + - on a preview, keeps the failing version running, so it can be looked at; + - in both cases marks the app red: a new sixth stage "the tests", and `--wait` ends `Failed`. +- **Built tests.** `shelf build-plan`, `shelf render --image` and the reusable workflow build, + sign and scan test images like component images. +- **Report.** A new command `shelf app report [--preview ]` prints a verification + report as Markdown, or `--json` (kind `Report`): + - the commit, the address and the verdict; + - the test results; + - signature and scan; + - for a preview, what differs from the app: components with another digest, and the change in + CRITICAL and HIGH findings. + + Everything comes from the cluster and the registry, never from Git. +- **Admin UI.** The app page shows the tests, the isolation and its LAN exceptions; the connection + form edits the exceptions through the same function as the CLI. +- **`just smoke-hardening`** joins level 2. ## Capabilities ### New Capabilities -To be decided at `/opsx:propose`; possibly `isolation` and `preview-checks`. +- `isolation`: Pod Security levels, the NetworkPolicy of app and preview namespaces, LAN + exceptions as an operator decision, and how isolation is reported. +- `app-tests`: running an app's tests after each rollout, how a failing test affects an app and a + preview, and how test results are reported. ### Modified Capabilities -To be decided at `/opsx:propose`; likely `app-chart`, `app-manifest`, `branch-previews`. +- `app-manifest`: the `tests` section, test names, `--image` and `shelf build-plan` for built tests. +- `app-chart`: test hooks rendered from `tests`. +- `app-delivery`: the diagnosis gets the stage "the tests". +- `agent-interface`: `AppStatus` carries tests and isolation; the new `Report` document. +- `branch-previews`: a preview's contents include the isolation, without LAN exceptions. +- `admin-ui`: the app page shows tests and isolation, and edits LAN exceptions. +- `dev-environment`: `smoke-hardening` in level 2. ## Impact -`charts/shelf-app`, `platform/apps`, `internal/schema`, `internal/ops`, CLI. Level 2. - -## Roadmap text (from the re-cut of 2026-10-27) - -**Open:** `restricted` breaks many stock images (e.g. Postgres starting as root); likely -`baseline` enforced plus `restricted` warned, with a per-app opt-in to `restricted`. - -**Open:** whether the dev cluster's k3d network policy controller enforces the LAN egress rule the -same way k3s on a host does, i.e. what level 2 can prove. - -**Acceptance (level 2):** a preview cannot reach another app's namespace or a LAN address; a -failing smoke test marks the preview red. +- **Chart and platform:** + - `charts/shelf-app`: test hook pods; + - `platform/apps` and `platform/previews`: namespace labels, NetworkPolicy, HelmRelease `test`, + provider input `lan`. +- **Code:** + - `internal/schema`: `tests`; + - `internal/render`, `internal/cli` build-plan; + - `internal/cluster`: the tests stage, isolation state, restricted findings, LAN input; + - `internal/ops`: report, LAN option; + - `internal/cli`: `--allow-lan`, `report`, JSON; + - `internal/server`: app page, connection form. +- **Workflow:** `build.yml` needs no new step, since it builds what `shelf build-plan` lists and + scans every digest in the values. +- **Existing apps:** keep running. An app that reaches another namespace or the LAN loses that + path at the platform upgrade, until the operator adds an exception. A pod that violates + `baseline` is refused at its next start. Both are named in the upgrade notes. +- **Levels:** level 2 (k3d enforces NetworkPolicies, checked in a spike); level 3 with + `shelf-hello` and a test. diff --git a/openspec/changes/phase-12-hardening/specs/admin-ui/spec.md b/openspec/changes/phase-12-hardening/specs/admin-ui/spec.md new file mode 100644 index 0000000..bc31b30 --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/admin-ui/spec.md @@ -0,0 +1,50 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: App page +The system SHALL show at `/apps/` the app's phase, the first stage that is not ready with +its message and hint, its address, artifact, revision and deploy time, its components each with +its phase and address (routed ones as links, the others marked `inside the cluster only` or `not +exposed`), its access settings, the names of its generated secrets, its verification and its +isolation as `shelf app status` prints them, and the diagnosis chain with the tests under `How +it gets here`. + +#### Scenario: Routed and internal components +- **WHEN** an app has a routed `web` component and an unrouted `db` component +- **THEN** `web` is shown with a link to its URL and `db` with its address and `inside the + cluster only` + +#### Scenario: Unknown app +- **WHEN** `/apps/` is requested for an app that does not exist or a name that is not a + valid app name +- **THEN** an error page with status 404 is shown + +#### Scenario: Unverified app +- **WHEN** the running artifact of an app is unsigned +- **THEN** the page says it is not verified and why, and that it cannot be exposed + +#### Scenario: Failing test +- **WHEN** the test `smoke` of an app failed +- **THEN** the page shows the tests stage as Failed with the test's name and the end of its log + +### Requirement: Deploy, redeploy and connection changes on the app page +The system SHALL offer on the app page a form that deploys the app from another artifact or tag +(`POST /apps//deploy`), a button that deploys the same tag again +(`POST /apps//redeploy`), and a form that changes the registry connection, the Cloudflare +connection or quick tunnel, the domain and the LAN exceptions (`POST /apps//access`), each +running as a job. + +#### Scenario: Roll back to a commit tag +- **WHEN** the deploy form is submitted with `oci://ghcr.io/owner/greeter:sha-905acce` +- **THEN** a job `Deploy from oci://ghcr.io/owner/greeter:sha-905acce` points the app at + that tag + +#### Scenario: Take an app off the internet +- **WHEN** the access form is submitted with the Cloudflare choice `none` +- **THEN** a job `Change the connections of ` makes the app private; the choice `@quick` + asks for a quick tunnel instead + +#### Scenario: LAN exception from the browser +- **WHEN** the access form is submitted with the LAN exceptions `192.168.1.10/32` +- **THEN** the job gives the app that exception, as `shelf app credentials --allow-lan 192.168.1.10/32` does; an empty field takes the exceptions away, and an invalid range is answered with status 400 and the reason, without a job diff --git a/openspec/changes/phase-12-hardening/specs/agent-interface/spec.md b/openspec/changes/phase-12-hardening/specs/agent-interface/spec.md new file mode 100644 index 0000000..a0255c0 --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/agent-interface/spec.md @@ -0,0 +1,52 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Tests and isolation in the agent interface +The `AppStatus` of an app or preview SHALL hold `tests`, one entry per test of the running +revision with `name`, `phase` and, when it failed, `exitCode` and `log` (its last lines), and +`isolation` with `podSecurity`, `restrictedViolations` (each with `pod` and `reason`) and +`lan` (the exceptions). + +#### Scenario: Failed test as JSON +- **WHEN** an agent runs `shelf app status hello --preview feature-login --json` and the test `smoke` failed +- **THEN** `tests` holds `smoke` with phase `Failed`, its exit code and its log, and the stage "the tests" is Failed + +### Requirement: Verification report +The system SHALL provide `shelf app report [--preview ]`, read-only, which prints a +Markdown report and with `--json` a `Report` document: the app or preview, its address and how to +open it, the commit, the state with the first stage that is not Ready, the tests, the +verification with the scan per component, and, for a preview, the comparison with the app: each +component whose image digest differs, and the CRITICAL and HIGH counts of the preview against +the app's. + +#### Scenario: Report for a pull request +- **WHEN** an agent runs `shelf app report hello --preview feature-login` for a Ready, verified preview whose `web` image differs from the app's +- **THEN** the Markdown names the commit, says Ready, lists the passed tests, the signature and scan, and lists `web` as changed with the findings of both + +#### Scenario: Report of an app +- **WHEN** the report is printed without `--preview` +- **THEN** it has no comparison section + +#### Scenario: Report as JSON +- **WHEN** `--json` is given +- **THEN** a `Report` document is printed with the same content, and no Markdown + +## MODIFIED Requirements + +### Requirement: Commands with JSON output +`--json` SHALL be offered by `shelf app status` (kind `AppList` without a name, `AppStatus` with +one, `WaitResult` with `--wait`), `shelf app report` (`Report`), `shelf app tags` (`TagList`), +`shelf connection list` (`ConnectionList`) and `shelf version` (`Version`). + +#### Scenario: One app in detail +- **WHEN** a user runs `shelf app status hello --json` for a healthy app +- **THEN** the `AppStatus` holds the app's name, phase, address, artifact, registry connection, exposure, revision, commit, deploy time, components with their addresses, and the six stages each with name, object, phase and, where present, reason, message and hint + +#### Scenario: Tags +- **WHEN** a user runs `shelf app tags hello --json` +- **THEN** the `TagList` lists the tags newest first, and the tag the app runs with has `current: true` + +#### Scenario: Version +- **WHEN** a user runs `shelf version --json` +- **THEN** the `Version` document holds the shelf version and the app.yaml `apiVersion` it reads diff --git a/openspec/changes/phase-12-hardening/specs/app-chart/spec.md b/openspec/changes/phase-12-hardening/specs/app-chart/spec.md new file mode 100644 index 0000000..796b287 --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/app-chart/spec.md @@ -0,0 +1,31 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Tests as Helm test hooks +The chart SHALL render one Pod per entry in `tests`, named `-test-` and labelled +`shelf.dev/app` and `shelf.dev/test` but not `shelf.dev/component`, the annotations `helm.sh/hook: test` and +`helm.sh/hook-delete-policy: before-hook-creation`, `restartPolicy: Never`, the test's image, +command, args, env with references and secrets expanded as for a component, its resources, and +the image pull secret; it MUST NOT render a Service, Ingress or volume for a test. + +#### Scenario: Test pod +- **WHEN** the values of `hello` declare a test `smoke` with `image` and `args` +- **THEN** the chart renders the Pod `hello-test-smoke` with the hook annotations and `restartPolicy: Never`, and no other object for `smoke` + +#### Scenario: Test reads a secret +- **WHEN** the test `smoke` sets `env: { TOKEN: ${secrets.token} }` +- **THEN** the Pod reads `TOKEN` from the Secret `shelf-secrets`, as a component does + +### Requirement: Security defaults of every pod +The chart SHALL give every pod it renders, components and tests, the seccomp profile +`RuntimeDefault` and every container `allowPrivilegeEscalation: false`, and SHALL NOT set a user, +group or capabilities, which depend on the image. + +#### Scenario: Component +- **WHEN** the chart renders the component `web` +- **THEN** its pod template has `securityContext.seccompProfile.type: RuntimeDefault` and its container `securityContext.allowPrivilegeEscalation: false` + +#### Scenario: Image that starts as root +- **WHEN** an app runs `postgres:16`, whose entrypoint switches from root to the user postgres +- **THEN** it runs with these defaults, and Pod Security still lists it for `restricted`, because it runs as root diff --git a/openspec/changes/phase-12-hardening/specs/app-delivery/spec.md b/openspec/changes/phase-12-hardening/specs/app-delivery/spec.md new file mode 100644 index 0000000..22e9302 --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/app-delivery/spec.md @@ -0,0 +1,61 @@ +# Spec Delta + +## RENAMED Requirements + +- FROM: `### Requirement: Five-stage diagnosis` +- TO: `### Requirement: Stage diagnosis` + +## MODIFIED Requirements + +### Requirement: Stage diagnosis +The system SHALL diagnose an app as six ordered stages: the deploy artifact (OCIRepository +`deploy`), the app values (Kustomization `deploy`), the chart (OCIRepository `shelf-app`), the +app (HelmRelease), the pods, and the tests; each stage is Ready, Working, Failed or Missing, and +the app's state is that of the first stage that is not Ready. + +#### Scenario: Image cannot be pulled +- **WHEN** all Flux stages are Ready and a pod waits in `ImagePullBackOff` +- **THEN** the pods stage is Failed with a hint that the image cannot be pulled, and the app is Failed + +#### Scenario: Secret missing from the app's Secret +- **WHEN** a pod fails with `CreateContainerConfigError` because a key is missing in `shelf-secrets` +- **THEN** the hint names the secret and says to deploy the app again so shelf generates it + +#### Scenario: Not yet created +- **WHEN** the platform has not created the app's objects yet +- **THEN** the first stage is Missing with the message that the platform has not created the app yet + +#### Scenario: Failing test +- **WHEN** every pod is ready and the test `smoke` exited with 1 +- **THEN** the tests stage is Failed naming `smoke`, and the app is Failed + +### Requirement: App status command +`shelf app status` SHALL list every app with its state and, when not Ready, the failing stage and +reason in one line; `shelf app status ` SHALL print the app's address, artifact, registry +connection, running revision and commit, deploy time, components, its verification, its +isolation, every stage with its message and, if not Ready, its hint, and its previews with tag, +state and commit; `--preview ` SHALL print the same for one preview. + +#### Scenario: No apps +- **WHEN** the cluster has no registered apps +- **THEN** `shelf app status` prints "No apps yet." + +#### Scenario: One app in detail +- **WHEN** a user runs `shelf app status hello` for a healthy app +- **THEN** the output shows `hello: Ready`, the artifact reference, the revision, and all six stages as Ready + +#### Scenario: Commit of the running artifact +- **WHEN** the app's deploy artifact was pushed by the reusable workflow for commit `42a964eb0b4b…` +- **THEN** `shelf app status hello` shows that commit, read from the artifact's revision annotation as the cluster reports it, without calling a forge API + +#### Scenario: Artifact without a commit +- **WHEN** the deploy artifact carries no git revision +- **THEN** the commit line is left out + +#### Scenario: App with previews +- **WHEN** `hello` has previews for `feature/login` and `fix-typo` +- **THEN** `shelf app status hello` lists both with their tags, states and commits, and the list of all apps shows them under `hello`, not as apps of their own + +#### Scenario: One preview +- **WHEN** a user runs `shelf app status hello --preview feature/login` +- **THEN** the output shows the preview's namespace `hello--feature-login`, its host, artifact tag, commit, components and six stages diff --git a/openspec/changes/phase-12-hardening/specs/app-manifest/spec.md b/openspec/changes/phase-12-hardening/specs/app-manifest/spec.md new file mode 100644 index 0000000..41c40d5 --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/app-manifest/spec.md @@ -0,0 +1,78 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Tests section +The system SHALL accept an optional top-level `tests` map; each test name MUST be a DNS-1035 label +of at most 40 characters that is not also a component name, and each test MUST have exactly one of +`image` or `build` and MAY have `command`, `args`, `env` and `resources`, with the same rules and +references as a component; a test MUST NOT have ports, a route, volumes, health or instances. + +#### Scenario: Test against the web component +- **WHEN** an app.yaml has `tests: { smoke: { image: curlimages/curl:8.10.1, args: [-fsS, "http://${web.host}:${web.port}/healthz"] } }` +- **THEN** validation passes, and `shelf render` pins the test image to its digest like a component image + +#### Scenario: Name taken by a component +- **WHEN** a test and a component are both named `web` +- **THEN** validation reports an error at `tests.web` + +#### Scenario: Test with a route +- **WHEN** a test declares `route: /` +- **THEN** parsing fails, as `route` is not a field of a test + +## MODIFIED Requirements + +### Requirement: Document structure and strict parsing +The system SHALL accept an app.yaml only as a single YAML document with the top-level fields +`apiVersion`, `name`, `components` and optional `secrets` and `tests`, and SHALL reject unknown +fields, duplicate keys, wrong types, empty files and multiple documents as parse errors. + +#### Scenario: Unknown field +- **WHEN** an app.yaml contains a field that is not part of the schema, at any level including inside a long-form `route` +- **THEN** parsing fails with an error that names the file +- **AND** no findings are produced for the document + +#### Scenario: Scalar env values +- **WHEN** a component sets `env: { PORT: 8080, DEBUG: true }` +- **THEN** the values are taken as the text `8080` and `true` + +#### Scenario: Wrong apiVersion +- **WHEN** `apiVersion` is anything other than `shelf.dev/v1alpha1` +- **THEN** validation reports an error at `apiVersion` + +### Requirement: Built components and --image +The system SHALL take the image of every component and test with `build` from a repeatable flag +`--image =` and SHALL fail when a built component or test has no `--image`, when +`--image` names an unknown component or test or one with `image`, when a name is given twice, or +when the flag is not of the form `=`. + +#### Scenario: Built image passed back +- **WHEN** `shelf render --image web=ghcr.io/owner/greeter/web:sha-abc1234` is run for `examples/tenant/app.yaml` +- **THEN** the resolved `web` image is `ghcr.io/owner/greeter/web:sha-abc1234@sha256:...` + +#### Scenario: Missing --image +- **WHEN** a component has `build: ./web` and no `--image web=...` is given +- **THEN** `shelf render` fails with an error telling the user to pass `--image web=` + +#### Scenario: Built test +- **WHEN** a test `e2e` has `build: ./e2e` and `--image e2e=ghcr.io/owner/greeter/e2e:sha-abc1234` is given +- **THEN** the resolved test `e2e` has that image pinned to its digest + +### Requirement: shelf build-plan +The system SHALL provide `shelf build-plan ` that prints JSON +`{"package": , "builds": [{"component", "context"}]}` listing every component and test with +`build`, components first, each in name order, with each context relative to the working +directory, an empty list when nothing is built, and an error when a build directory does not +exist. + +#### Scenario: Tenant example +- **WHEN** `shelf build-plan examples/tenant/app.yaml` is run +- **THEN** it prints `package` `greeter` and one build with `component` `web` and `context` `examples/tenant/web` + +#### Scenario: Missing build directory +- **WHEN** a component's `build` directory does not exist +- **THEN** the command fails with an error naming the component and the directory + +#### Scenario: Built test +- **WHEN** an app.yaml has a built component `web` and a built test `e2e` +- **THEN** the plan lists `web`, then `e2e`, so the workflow builds and pushes `ghcr.io///e2e` diff --git a/openspec/changes/phase-12-hardening/specs/app-tests/spec.md b/openspec/changes/phase-12-hardening/specs/app-tests/spec.md new file mode 100644 index 0000000..75dda35 --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/app-tests/spec.md @@ -0,0 +1,52 @@ +# Spec Delta + +## Purpose + +Lets an app check itself after every rollout: the tests its `app.yaml` declares run against the +app or preview in its own namespace, and an app counts as green only when they pass. + +## ADDED Requirements + +### Requirement: Tests run after every rollout +The platform SHALL run every test an app declares after each install and upgrade of the app's or +preview's release, in its namespace, one pod per test with the app's resources defaults and a +timeout of 5 minutes for all of them together; a test passes when its container exits with 0. + +#### Scenario: New commit with a test +- **WHEN** a new artifact of an app with a test `smoke` rolls out +- **THEN** the pod for `smoke` runs after the components are ready, and its result is recorded for that release revision + +#### Scenario: App without tests +- **WHEN** an app declares no tests +- **THEN** its rollout runs no test, and the tests stage is Ready + +### Requirement: A failing test on an app keeps the previous version +When a test fails after an upgrade of an app, the platform SHALL count the upgrade as failed, so +the release returns to the last revision whose tests passed; after a failed first install the +failing version keeps running. + +#### Scenario: Broken commit on main +- **WHEN** a new artifact of `hello` makes its test `smoke` exit with 1 +- **THEN** `hello` runs its previous version again, and the tests stage is Failed naming `smoke` + +### Requirement: A failing test on a preview keeps the preview +When a test fails on a preview, the platform SHALL keep the version that failed running, so the +branch can be looked at with `shelf app open --preview`. + +#### Scenario: Broken branch +- **WHEN** the artifact of `branch-feature-login` makes the test `smoke` fail +- **THEN** the preview runs that artifact, and its tests stage is Failed naming `smoke` + +### Requirement: Tests in the status and the wait +The tests stage SHALL follow the pods stage: Ready when every test of the running revision passed +or there are none, Working while they run, Failed when one failed, with the failing test's name, +its exit code and the last lines of its log as the message; `shelf app status --wait --commit` +SHALL end with `Ready` only after the tests of that commit passed. + +#### Scenario: Waiting for a commit whose test fails +- **WHEN** an agent runs `shelf app status hello --preview feature-login --wait --commit ` and the test `smoke` fails for that commit +- **THEN** the verdict is `Failed` with the stage "the tests", the test name and the end of its log, and the exit status is 1 + +#### Scenario: Tests still running +- **WHEN** the pods are ready and the test `smoke` still runs +- **THEN** the app is Working at the stage "the tests" diff --git a/openspec/changes/phase-12-hardening/specs/branch-previews/spec.md b/openspec/changes/phase-12-hardening/specs/branch-previews/spec.md new file mode 100644 index 0000000..8a71002 --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/branch-previews/spec.md @@ -0,0 +1,31 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Preview contents +The platform SHALL give each preview the same deploy chain as an app (its deploy artifact at the +branch tag, checked every 30 seconds, the app values, the chart, the release, the tests), +verified against the cluster's signer, the app's registry login, the isolation of an app without +LAN exceptions, a ResourceQuota (requests of 1 CPU, 2Gi memory, 5Gi storage) and a LimitRange that +gives containers without requests a default; it MUST NOT give a preview a tunnel, cloudflared or a +DNS record. + +#### Scenario: Preview of an exposed app +- **WHEN** an app exposed through a Cloudflare connection has a preview +- **THEN** the preview namespace has no cloudflared, and nothing outside the cluster reaches it + +#### Scenario: Over the quota +- **WHEN** a preview's pods would request more than 2Gi memory together +- **THEN** the pods beyond it are refused, and the preview's pods stage says so + +#### Scenario: Unsigned branch artifact +- **WHEN** an artifact without a signature is pushed as `branch-feature-login` +- **THEN** the preview `hello--feature-login` does not apply it, and its deploy artifact stage says the verification failed + +#### Scenario: Built image of a preview +- **WHEN** a preview's values name an unsigned built image of the app's package +- **THEN** its pod is denied by the app's admission policy, as in the app's namespace + +#### Scenario: Preview of an app with a LAN exception +- **WHEN** `hello` may reach `192.168.1.10/32` and has the preview `hello--feature-login` +- **THEN** the preview's pods cannot reach `192.168.1.10`, nor the pods of `hello` diff --git a/openspec/changes/phase-12-hardening/specs/dev-environment/spec.md b/openspec/changes/phase-12-hardening/specs/dev-environment/spec.md new file mode 100644 index 0000000..4e7137e --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/dev-environment/spec.md @@ -0,0 +1,33 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: Three test levels +Checks SHALL be split into level 1 (no cluster: `just test`, which runs gofmt, `go vet`, +`go test -race`, Helm lint, kubeconform, example validation and `openspec validate --all --strict`), +level 2 (the k3d cluster: the `just smoke-*` recipes), and level 3 (the Mac mini and real +Cloudflare), and a task that needs level 2 or 3 MUST say so. + +#### Scenario: Level 1 in CI +- **WHEN** a commit is pushed to `main` or a pull request is opened or pushed to +- **THEN** CI runs `just test` inside the devcontainer image on an arm64 runner + +#### Scenario: Level 2 in CI +- **WHEN** a pull request is opened or reopened as ready, leaves draft, or is pushed to or labelled while it carries the label `level-2` +- **THEN** CI also runs the smoke tests its changed paths need, chosen by `.github/scripts/select-smokes.sh`, each against a fresh k3d cluster of its own, side by side, with `SMOKE_FULL=0` + +#### Scenario: Documentation only +- **WHEN** a pull request changes only planning, documentation or examples +- **THEN** CI runs no smoke test, and a path the selection does not know runs all of them + +#### Scenario: Level 2 in full +- **WHEN** a commit is pushed to `main`, or a pull request carries the label `level-2-full` +- **THEN** CI runs `smoke-secrets`, `smoke-chart`, `smoke-init`, `smoke-apps`, `smoke-agent`, `smoke-previews`, `smoke-verify` and `smoke-hardening` with `SMOKE_FULL=1` + +#### Scenario: main fails +- **WHEN** CI fails on `main` +- **THEN** it opens an issue labelled `ci-main-red`, or comments on the one that is open, with the commit and the run + +#### Scenario: Release gate +- **WHEN** a release tag is pushed for a commit whose CI run on `main` did not succeed +- **THEN** `release.yml` stops before it publishes anything diff --git a/openspec/changes/phase-12-hardening/specs/isolation/spec.md b/openspec/changes/phase-12-hardening/specs/isolation/spec.md new file mode 100644 index 0000000..d1bcd8f --- /dev/null +++ b/openspec/changes/phase-12-hardening/specs/isolation/spec.md @@ -0,0 +1,79 @@ +# Spec Delta + +## Purpose + +Fences in what an app or preview runs: which pods its namespace admits, and which addresses its +pods can reach, so that code an agent wrote reaches neither other apps nor the operator's LAN +unless the operator allowed it for one app. + +## ADDED Requirements + +### Requirement: Pod Security of app and preview namespaces +The platform SHALL label every app and preview namespace to enforce the Pod Security level +`baseline` and to warn and audit `restricted`, both at the version `latest`. + +#### Scenario: Privileged pod +- **WHEN** an app's values would start a privileged container or one with a `hostPath` volume +- **THEN** the pod is refused, and the pods stage shows the refusal with a hint about Pod Security + +#### Scenario: Image that starts as root +- **WHEN** an app runs `postgres:16`, which starts as root +- **THEN** the pod runs + +### Requirement: Network isolation of app and preview namespaces +The platform SHALL give every app and preview namespace a NetworkPolicy that selects all its pods +and allows ingress only from pods in the same namespace and from the Traefik namespace, and +egress only to pods in the same namespace, to the cluster DNS on port 53, and to any address +outside `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `100.64.0.0/10`, `169.254.0.0/16`, +`fc00::/7` and `fe80::/10`. + +#### Scenario: Another app +- **WHEN** a pod of the preview `hello--feature-login` connects to the service of the app `greeter` +- **THEN** the connection fails + +#### Scenario: Traffic within the app +- **WHEN** the `web` pod of `hello` connects to `db:5432` in its own namespace +- **THEN** the connection succeeds + +#### Scenario: LAN and metadata +- **WHEN** a pod connects to `192.168.1.10` or `169.254.169.254` +- **THEN** the connection fails, and a connection to a public address succeeds + +#### Scenario: Routed and exposed +- **WHEN** a request reaches the app through Traefik, or through cloudflared in the app namespace +- **THEN** it is answered as before + +### Requirement: LAN exceptions are the operator's decision +`shelf app add` and `shelf app credentials` SHALL take `--allow-lan `, repeatable, which +replaces the app's list of private address ranges it may reach, and `--deny-lan`, which clears +the list; each range MUST lie inside one of the private ranges of the isolation, and the +exceptions MUST NOT be part of `app.yaml` and MUST NOT apply to the app's previews. + +#### Scenario: NAS +- **WHEN** an operator runs `shelf app credentials hello --allow-lan 192.168.1.10/32` +- **THEN** pods of `hello` reach `192.168.1.10` and no other private address, and pods of its previews reach none + +#### Scenario: Not a private range +- **WHEN** `--allow-lan 8.8.8.0/24` or `--allow-lan nas.local` is given +- **THEN** the command fails before it changes anything, saying the value must be a private CIDR + +#### Scenario: Range of the cluster's pods +- **WHEN** `--allow-lan 10.42.0.0/16` is given on a cluster whose nodes have pod ranges inside it +- **THEN** the command fails before it changes anything, saying the range would reach other apps + +#### Scenario: Unchanged by redeploy +- **WHEN** an app with an exception is deployed again or pointed at another tag without `--allow-lan` +- **THEN** it keeps its exception + +### Requirement: Isolation in the status +`shelf app status ` and `--preview` SHALL show the Pod Security level, the pods that would +violate `restricted` with the first reason for each, and the LAN exceptions, or `none`; reading +them MUST NOT change the cluster. + +#### Scenario: Image that starts as root +- **WHEN** `hello` runs `postgres:16` in its `db` component +- **THEN** the status shows `pod security baseline`, and lists the `db` pod as violating `restricted` with its reason + +#### Scenario: No exceptions +- **WHEN** an app has no LAN exceptions +- **THEN** the status shows `lan none` diff --git a/openspec/changes/phase-12-hardening/tasks.md b/openspec/changes/phase-12-hardening/tasks.md new file mode 100644 index 0000000..d5f0ad6 --- /dev/null +++ b/openspec/changes/phase-12-hardening/tasks.md @@ -0,0 +1,130 @@ +# Tasks + +## 1. Tests in app.yaml + +- [x] 1.1 `internal/schema`: `tests` map with `image|build`, `command`, `args`, `env`, `resources`. + Validation: + - test names: DNS-1035, at most 40 characters, not a component name; + - references and secrets as for components; + - unknown fields refused (`route`, `ports`, `volumes`, `health`, `instances`). + + Verify: table tests for the scenarios of "Tests section"; `just golden` refreshes + `schema/app.schema.json`. +- [x] 1.2 `shelf render`: pin test images, take `--image =` for built tests, and + write `tests` into the normalized app.yaml. Verify: render tests for an image test and a built + test, plus the `--image` errors for an unknown or doubled name. +- [x] 1.3 `shelf build-plan` lists built tests after the components. Verify: a build-plan test with + `web` and `e2e`. +- [x] 1.4 `examples/tenant/app.yaml` gets a curl test against `/healthz`. Verify: `just test` + validates the example. Document `tests` in the README's app.yaml reference. + +## 2. Chart + +- [x] 2.1 `charts/shelf-app`: one hook Pod `-test-` per test: + - annotations `helm.sh/hook: test` and `before-hook-creation`; + - `restartPolicy: Never`, labels with `shelf.dev/test`; + - env, references and secrets as for a component; + - resources and the pull secret. + + Verify: golden files for a test with an image and with a secret; `helm lint` and kubeconform in + `just test`. + +## 3. Platform + +- [x] 3.1 `platform/apps` and `platform/previews`: + - namespace labels: `enforce: baseline`, `warn` and `audit: restricted`, `*-version: latest`; + - the NetworkPolicy `shelf-isolation` of design decision 3, in every app and preview namespace; + - in `apps` only, one `ipBlock` per entry of the input `lan`. + + Verify: `platform_test.go` renders an app without and with `lan` and a preview of an app with + `lan`, and checks the policy and the labels. +- [x] 3.2 HelmRelease `test.enable: true`, with `ignoreFailures: false` for apps and `true` for + previews. Verify: rendered in `platform_test.go`. + +## 4. Cluster state + +- [x] 4.1 `internal/cluster`: + - the provider input `lan` in `AppOptions` and `AppConfig`; + - `NodePodCIDRs` reads `spec.podCIDRs` of the nodes. + + Verify: unit tests against the fake client. +- [x] 4.2 The stage "the tests" after the pods: + - read from the HelmRelease history's `testHooks` and the condition `TestSuccess`, for the + revision that runs; + - the last 20 log lines of a failed test pod; + - test pods (`shelf.dev/test`) left out of the pods stage. + + Verify: state tests for none, running, passed, failed on an app (rolled back) and failed on a + preview; the diagnosis golden files gain the sixth stage. +- [x] 4.3 Isolation state: + - the Pod Security level from the namespace labels; + - `restricted` violations through a server-side dry-run patch with a `WarningHandler`; + - the LAN exceptions. + + Also a Pod Security hint for a `FailedCreate` refused by `baseline`. Verify: unit tests for + parsing the warnings and for the hint. +- [x] 4.4 `cluster.verification` counts test images under "the scan covers every image". Verify: a + verification test where the scan lacks the test's digest. + +## 5. Operations + +- [x] 5.1 `ops`: the LAN option of add and credentials: + - kept when not given, replaced by the given list, cleared by deny; + - checked as a private CIDR that does not overlap a node's pod range. + + Verify: `internal/cli` tests for the scenarios of "LAN exceptions are the operator's decision". +- [x] 5.2 `Verdict`: Ready only when the awaited revision's tests passed; Failed at once for a + failed test. Verify: wait tests for both, and for an app without tests. +- [x] 5.3 `ops.Report`: status parts plus, for a preview, the comparison with the app by component + digest and by findings. Verify: unit tests for an app and for a preview with one changed + component. + +## 6. CLI + +- [x] 6.1 `--allow-lan ` (repeatable) and `--deny-lan` on `app add` and `app credentials`. + Verify: harness tests for keep, replace, clear and each refusal. +- [x] 6.2 `shelf app status`: the tests and isolation lines, six stages. JSON: `tests` and + `isolation` in `AppStatus`. Verify: refreshed text and JSON golden files, plus one for a failed + test. +- [x] 6.3 `shelf app report [--preview ] [--json]`: Markdown and `Report`. Verify: + golden files for an app, a preview and JSON; the README CLI table. + +## 7. Admin UI + +- [x] 7.1 App page: the tests stage in `How it gets here` and an isolation card. Access form: a LAN + field that calls the same `ops` function, with the form shown again on an invalid range. + Verify: page golden files, and a server test for a valid and an invalid range. + +## 8. Dev environment and docs + +- [x] 8.1 `hack/smoke/hardening.sh` as in design decision 10, plus a `just smoke-hardening` + recipe, an entry in `select-smokes.sh` and in `ci.yml`. Verify: it passes on a fresh dev cluster + (level 2). +- [x] 8.2 Docs: + - CLAUDE.md: guardrail "Apps are isolated" enforced by the policy, LAN exceptions as an operator + decision, conventions (`lan` input, `shelf-isolation`, `shelf.dev/test`), dev table row; + - `openspec/config.yaml` context; + - README: isolation, tests, report; + - `examples/tenant`: `CLAUDE.md` and the skill name the tests stage and `shelf app report`. + + Verify: `just test`. + +## 9. Acceptance + +- [x] 9.1 Level 2: run every smoke test against a fresh dev cluster: + - all of `select-smokes.sh` in full; + - `smoke-chart` and `smoke-agent` check that probes still pass under the policy; + - `smoke-quick` checks that the quick tunnel's name is still read through the pod proxy. + + Record the turnaround with and without a test in design.md under "Results". +- [ ] 9.2 Level 3 with `tweinmann/shelf-hello`: + - add a curl test and push to main: Ready; + - a branch whose test fails: a red preview that keeps running, then `shelf app report` for it; + - merge a failing test: `gretter` keeps the previous version. + + Record the times in design.md. + +## Workflow follow-up + +- Archive with `/opsx:archive phase-12-hardening`; update CLAUDE.md Status, the README Status row + 12 and the context. diff --git a/openspec/config.yaml b/openspec/config.yaml index 22973b2..ba40231 100644 --- a/openspec/config.yaml +++ b/openspec/config.yaml @@ -52,6 +52,13 @@ context: | policy-controller admits a built image of an app's package only with its signature, in app and preview namespaces. Third-party images are pinned by digest in the signed artifact. The dev cluster verifies with a dev key; there is no setting that turns verification off. + - Hardening: app and preview namespaces enforce Pod Security `baseline` (warn and audit + `restricted`), and the NetworkPolicy `shelf-isolation` lets pods reach their namespace, the + cluster DNS and the internet only; only Traefik reaches in, and only cloudflared reaches + Traefik. The operator may give one app LAN ranges (`--allow-lan`), never its previews. Tests + from `tests:` in app.yaml are Helm test hooks run by helm-controller after every rollout; a + failing test rolls an app back and leaves a preview on the failing version, and the wait is + Ready only once they pass. `shelf app report` is the per-branch report for a pull request. - `shelf init` is split into idempotent layers: `init host` (Phase 13: provide the environment, macOS or Linux), `init cluster` (any kubecontext, no Colima or distribution assumptions) and the wrapper. @@ -81,7 +88,9 @@ context: | - The registry is the only interface between build and platform; the cluster never talks to Git or forge APIs. - Every admin UI action exists as a CLI command; both call the same function in internal/ops. - - The platform is generic (everything is a component) and apps are isolated from each other. + - The platform is generic (everything is a component) and apps are isolated from each other, + enforced by Pod Security and a NetworkPolicy per namespace; LAN access is per app and an + operator decision. - Layering: internal/cluster talks to the API server only (client-go, server-side apply); orchestration in internal/ops, flags in internal/cli, routes and templates in internal/server; progress through progress.Reporter; test seams are fields of ops.Ops / ops.Env, never diff --git a/platform/apps/resourceset.yaml b/platform/apps/resourceset.yaml index c44bfd6..46ac3a1 100644 --- a/platform/apps/resourceset.yaml +++ b/platform/apps/resourceset.yaml @@ -1,6 +1,6 @@ # Every app is a ResourceSetInputProvider in shelf-system, labelled shelf.dev/app, created by -# `shelf app add` with the inputs name, url, tag, insecure, domain, tunnel, quick, registry and -# cloudflare. For each one this +# `shelf app add` with the inputs name, url, tag, insecure, domain, tunnel, quick, registry, +# cloudflare and lan. For each one this # ResourceSet generates the app namespace and everything Flux needs to deploy the app from its # deploy artifact, and, for an app with a tunnel or a quick tunnel, the cloudflared that carries # it to the internet. Deleting the provider deletes all of it, including the namespace. @@ -9,6 +9,12 @@ # login is copied into each of their namespaces. Namespace, secrets, tunnel and cloudflared are # every app's own. # +# Apps are fenced in. The namespace enforces the Pod Security level baseline and warns about +# restricted, and the NetworkPolicy shelf-isolation lets its pods talk to each other, the cluster +# DNS and the internet, and nothing else: not other namespaces, not the Kubernetes API, not the +# LAN or a metadata address. Only Traefik reaches in. The operator can let one app reach named +# private ranges (lan, set with shelf app credentials --allow-lan); its previews never get them. +# # Nothing runs unverified. The deploy artifact and the chart are applied only with a valid # signature of the cluster's signer (verify), and the app's ClusterImagePolicy admits a built # image of its package only with one; the policy-controller checks every namespace labelled @@ -49,6 +55,12 @@ spec: labels: shelf.dev/app: << inputs.name >> policy.sigstore.dev/include: "true" + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest # Generated secret values, maintained by `shelf app add` in shelf-system/app-. - apiVersion: v1 @@ -203,6 +215,11 @@ spec: # the host name with a "" in it. hostSuffix: "<< if inputs.domain >>${SHELF_HOST_SUFFIX:=}<< end >>" imagePullSecret: shelf-registry + # The app's tests run after every install and upgrade. A failing test fails the upgrade, + # so the remediation rolls back to the version whose tests passed. + test: + enable: true + ignoreFailures: false install: remediation: retries: 3 @@ -221,7 +238,53 @@ spec: # the Host header to the app's own host name, which is what Traefik routes by. The input is read # with index, which is empty for a provider from before quick tunnels, where inputs.quick would # fail the whole ResourceSet. + # + # The isolation is rendered here rather than under resources because of the LAN exceptions, one + # ipBlock each; lan is read with index for the same reason as quick. cloudflared is the one pod + # that reaches Traefik, which routes by Host header and so could reach any app: app pods cannot + # set its label, since the chart decides their labels. resourcesTemplate: | + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: shelf-isolation + namespace: << inputs.name >> + spec: + podSelector: {} + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: {} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + egress: + - to: + - podSelector: {} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10, 169.254.0.0/16] + - ipBlock: + cidr: ::/0 + except: [fc00::/7, fe80::/10] + <<- range (index inputs "lan") >> + - ipBlock: + cidr: << . >> + <<- end >> <<- if inputs.tunnel >> --- apiVersion: v1 @@ -249,6 +312,22 @@ spec: <<- end >> <<- if or inputs.tunnel (index inputs "quick") >> --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: shelf-cloudflared + namespace: << inputs.name >> + spec: + podSelector: + matchLabels: + app.kubernetes.io/name: cloudflared + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + --- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/platform/previews/resourceset.yaml b/platform/previews/resourceset.yaml index 1537d96..b072129 100644 --- a/platform/previews/resourceset.yaml +++ b/platform/previews/resourceset.yaml @@ -9,6 +9,10 @@ # secrets are generated by the chart (platform.preview), never copied from the app, and its # volumes are its own. A tag that disappears from the registry takes the preview with it. # +# A preview is fenced in as an app is: Pod Security baseline, and the NetworkPolicy +# shelf-isolation, which lets its pods reach each other, the cluster DNS and the internet, and +# only Traefik reach in. It never gets the app's LAN exceptions. +# # A preview is verified like its app: its deploy artifact and the chart only with a valid # signature of the cluster's signer, and its namespace is checked by the policy-controller # against the app's ClusterImagePolicy, which the ResourceSet apps generates. @@ -41,6 +45,50 @@ spec: shelf.dev/app: << inputs.app >> shelf.dev/preview: << trimPrefix "branch-" inputs.tag >> policy.sigstore.dev/include: "true" + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + --- + # As for an app, without exceptions: code from a branch never reaches a private address. + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: shelf-isolation + namespace: << inputs.app >>--<< trimPrefix "branch-" inputs.tag >> + spec: + podSelector: {} + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: {} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + egress: + - to: + - podSelector: {} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10, 169.254.0.0/16] + - ipBlock: + cidr: ::/0 + except: [fc00::/7, fe80::/10] --- # The signer's key for verify, empty for a keyless signer. apiVersion: v1 @@ -194,6 +242,11 @@ spec: hostSuffix: "" imagePullSecret: shelf-registry preview: true + # The tests run after every install and upgrade. A failing test leaves the preview on the + # version that failed, so the branch can be looked at; shelf reads the result. + test: + enable: true + ignoreFailures: true install: remediation: retries: 3 diff --git a/schema/app.schema.json b/schema/app.schema.json index 3ad3070..7631110 100644 --- a/schema/app.schema.json +++ b/schema/app.schema.json @@ -203,6 +203,73 @@ "generate" ] }, + "Test": { + "oneOf": [ + { + "required": [ + "image" + ] + }, + { + "required": [ + "build" + ] + } + ], + "properties": { + "image": { + "type": "string", + "description": "Image reference of a ready-made image. Tags are pinned to a digest by shelf render. Mutually exclusive with build." + }, + "build": { + "type": "string", + "pattern": "^\\.?/?[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$", + "description": "Directory with the Dockerfile of this test, relative to app.yaml. The CI builds it and shelf render pins the result. Mutually exclusive with image." + }, + "command": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Overrides the image entrypoint. Supports ${...} references; write $$ for a literal $." + }, + "args": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Overrides the image command. Supports ${...} references; write $$ for a literal $." + }, + "env": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "propertyNames": { + "type": "string", + "maxLength": 253, + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "type": "object", + "description": "Environment variables. Values support ${...} references; write $$ for a literal $." + }, + "resources": { + "$ref": "#/$defs/Resources", + "description": "Resource requests. Memory is also the limit." + } + }, + "additionalProperties": false, + "type": "object" + }, "Volume": { "properties": { "path": { @@ -259,6 +326,18 @@ }, "type": "object", "description": "Secrets referenced as ${secrets.\u003cname\u003e}, by name." + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/Test" + }, + "propertyNames": { + "type": "string", + "maxLength": 40, + "pattern": "^[a-z]([-a-z0-9]*[a-z0-9])?$" + }, + "type": "object", + "description": "Containers run against the app after every rollout, by name; the rollout counts only when each exits with 0." } }, "additionalProperties": false,