diff --git a/.agents/skills/compoza/SKILL.md b/.agents/skills/compoza/SKILL.md index 5a85bfe..05db7dd 100644 --- a/.agents/skills/compoza/SKILL.md +++ b/.agents/skills/compoza/SKILL.md @@ -70,6 +70,20 @@ relative to deploy.yaml after destination selection. Credentials stay in Caddy e placeholders, never literal template text. Shared config templates apply only during provisioning; they do not overwrite operator-managed configuration. +## Roles + +Each key under `roles` in deploy.yaml is a Compose service. `targets` and the +`${_IMAGE}` variables no longer exist: a config that still has `targets` fails +until it is migrated. A role with `build` is built as `-:`, and compoza +sets that service's `image`. Its `target`, `workingDirectory`, `dockerfile` and +`context` fall back to the top-level fields. A role's `environment` holds literal +values and its `secrets` names variables from the secret source; both reach only that +service's container. The top-level `environment` and `secrets` stay Compose +interpolation inputs. A name set in a role must not also be in the service's Compose +`environment:`, and role secrets need a secret source other than `none`. `check` +reports these problems without fetching secrets. The generated override, +`compoza.roles.yaml`, holds `${NAME}` references, never secret values. + ## Recover with a known image tag ```sh @@ -79,13 +93,17 @@ compoza status infra ``` `--tag` skips Git/builds and reuses images with the current local configuration and -currently selected secret source. A missing source checkout is allowed for `--tag`, -`targets: []`, status, logs and lifecycle commands. There is no release rollback command, saved -configuration history or backup folder. Check out an earlier app revision separately -when its configuration is required. Never imply that database changes are reversed. +currently selected secret source. `--tag ` applies to every role with `build`, and +`--tag =` overrides it for one role; repeat the flag as needed. Every role +with `build` needs a tag. A missing source checkout is allowed for `--tag`, stacks whose +roles build nothing, status, logs and lifecycle commands. There is no release rollback +command, saved configuration history or backup folder. Check out an earlier app +revision separately when its configuration is required. Never imply that database +changes are reversed. SSH stacks contain `compose.yaml`, private `.env`, declared assets and one nonsecret -`.compoza.json` lifecycle metadata file. A live host overlay may also be present. +`.compoza.json` lifecycle metadata file, which records each role's tag. The roles +override is merged into the flat `compose.yaml`. A live host overlay may also be present. Runtime scripts and filters are sent with each CLI operation, not installed on the host. Upload and validation failures leave deployed files unchanged. Later replacement, Compose, routing or health failures can leave changed files or containers; recover diff --git a/.agents/skills/deploy/SKILL.md b/.agents/skills/deploy/SKILL.md index bebc1b9..2f241bd 100644 --- a/.agents/skills/deploy/SKILL.md +++ b/.agents/skills/deploy/SKILL.md @@ -49,9 +49,13 @@ compoza logs infra --tail 50 - Explain the target, image transport, and proposed changes from the preview. An SSH dry run does not stage files, build images, lock the stack, or validate the full host candidate. Do not present it as proof that deployment will succeed. -- Built targets normally require a clean committed checkout in the resolved source directory. Do not commit changes or - add `--allow-dirty` without authorization. `targets: []` and `--tag` need no source checkout or Dockerfile. - Status, logs and lifecycle operations also work without that source directory. +- Each role with `build` normally requires a clean committed checkout of its source + repository, and is tagged with that repository's commit. Do not commit changes or add + `--allow-dirty` without authorization. Stacks whose roles build nothing, and `--tag` + deploys, need no source checkout or Dockerfile. Status, logs and lifecycle operations + also work without that source directory. +- A config that still has `targets` fails to load. Propose the migration to `roles` + (docs/configuration.md, "Migrating from 0.6.0") instead of editing it unasked. - Once deployment is authorized, continue through verification; do not request the same permission again. Report phase failures rather than retrying blindly. - Success requires the final `Deployed` message and expected service state. Only @@ -94,15 +98,16 @@ Never overwrite operator data to bypass an ownership error. Upload and validation failure preserve deployed files. Replacement is in place, and later failure can leave changed files, containers or routes. Recover using -`deploy --tag ` with current local configuration and the current -credential source. There is no saved release configuration or rollback command. +`deploy --tag `, adding `--tag =` where roles differ, with +the current local configuration and credential source. There is no saved release configuration or rollback command. Do not promise automatic recovery or reversal of database changes. Lifecycle uses the deployed files without fetching a provider. Do not remove a lock without establishing that its owner has exited. ## Handoff -Report the destination, image tag, verification result, and URL when configured. +Report the destination, image tag (per role when they differ), verification result, +and URL when configured. Identify any remaining action, such as publishing a content section. Do not claim success when a phase failed or cleanup is still required. diff --git a/.claude/skills/compoza/SKILL.md b/.claude/skills/compoza/SKILL.md index 5a85bfe..05db7dd 100644 --- a/.claude/skills/compoza/SKILL.md +++ b/.claude/skills/compoza/SKILL.md @@ -70,6 +70,20 @@ relative to deploy.yaml after destination selection. Credentials stay in Caddy e placeholders, never literal template text. Shared config templates apply only during provisioning; they do not overwrite operator-managed configuration. +## Roles + +Each key under `roles` in deploy.yaml is a Compose service. `targets` and the +`${_IMAGE}` variables no longer exist: a config that still has `targets` fails +until it is migrated. A role with `build` is built as `-:`, and compoza +sets that service's `image`. Its `target`, `workingDirectory`, `dockerfile` and +`context` fall back to the top-level fields. A role's `environment` holds literal +values and its `secrets` names variables from the secret source; both reach only that +service's container. The top-level `environment` and `secrets` stay Compose +interpolation inputs. A name set in a role must not also be in the service's Compose +`environment:`, and role secrets need a secret source other than `none`. `check` +reports these problems without fetching secrets. The generated override, +`compoza.roles.yaml`, holds `${NAME}` references, never secret values. + ## Recover with a known image tag ```sh @@ -79,13 +93,17 @@ compoza status infra ``` `--tag` skips Git/builds and reuses images with the current local configuration and -currently selected secret source. A missing source checkout is allowed for `--tag`, -`targets: []`, status, logs and lifecycle commands. There is no release rollback command, saved -configuration history or backup folder. Check out an earlier app revision separately -when its configuration is required. Never imply that database changes are reversed. +currently selected secret source. `--tag ` applies to every role with `build`, and +`--tag =` overrides it for one role; repeat the flag as needed. Every role +with `build` needs a tag. A missing source checkout is allowed for `--tag`, stacks whose +roles build nothing, status, logs and lifecycle commands. There is no release rollback +command, saved configuration history or backup folder. Check out an earlier app +revision separately when its configuration is required. Never imply that database +changes are reversed. SSH stacks contain `compose.yaml`, private `.env`, declared assets and one nonsecret -`.compoza.json` lifecycle metadata file. A live host overlay may also be present. +`.compoza.json` lifecycle metadata file, which records each role's tag. The roles +override is merged into the flat `compose.yaml`. A live host overlay may also be present. Runtime scripts and filters are sent with each CLI operation, not installed on the host. Upload and validation failures leave deployed files unchanged. Later replacement, Compose, routing or health failures can leave changed files or containers; recover diff --git a/.claude/skills/deploy/SKILL.md b/.claude/skills/deploy/SKILL.md index 4d120dc..37bdf52 100644 --- a/.claude/skills/deploy/SKILL.md +++ b/.claude/skills/deploy/SKILL.md @@ -50,9 +50,13 @@ compoza logs infra --tail 50 - Explain the target, image transport, and proposed changes from the preview. An SSH dry run does not stage files, build images, lock the stack, or validate the full host candidate. Do not present it as proof that deployment will succeed. -- Built targets normally require a clean committed checkout in the resolved source directory. Do not commit changes or - add `--allow-dirty` without authorization. `targets: []` and `--tag` need no source checkout or Dockerfile. - Status, logs and lifecycle operations also work without that source directory. +- Each role with `build` normally requires a clean committed checkout of its source + repository, and is tagged with that repository's commit. Do not commit changes or add + `--allow-dirty` without authorization. Stacks whose roles build nothing, and `--tag` + deploys, need no source checkout or Dockerfile. Status, logs and lifecycle operations + also work without that source directory. +- A config that still has `targets` fails to load. Propose the migration to `roles` + (docs/configuration.md, "Migrating from 0.6.0") instead of editing it unasked. - Once deployment is authorized, continue through verification; do not request the same permission again. Report phase failures rather than retrying blindly. - Success requires the final `Deployed` message and expected service state. Only @@ -95,15 +99,16 @@ Never overwrite operator data to bypass an ownership error. Upload and validation failure preserve deployed files. Replacement is in place, and later failure can leave changed files, containers or routes. Recover using -`deploy --tag ` with current local configuration and the current -credential source. There is no saved release configuration or rollback command. +`deploy --tag `, adding `--tag =` where roles differ, with +the current local configuration and credential source. There is no saved release configuration or rollback command. Do not promise automatic recovery or reversal of database changes. Lifecycle uses the deployed files without fetching a provider. Do not remove a lock without establishing that its owner has exited. ## Handoff -Report the destination, image tag, verification result, and URL when configured. +Report the destination, image tag (per role when they differ), verification result, +and URL when configured. Identify any remaining action, such as publishing a content section. Do not claim success when a phase failed or cleanup is still required. diff --git a/README.md b/README.md index cc92eec..474d9d9 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,6 @@ my-app/ ```yaml name: hello-web # the Compose project name on the server host: juniper # a host from hosts.yaml -targets: [] # images to build; none, because nginx is pulled as is execution: ssh # run Compose on the server from a stack folder composeFiles: - compose.yaml @@ -146,7 +145,7 @@ Stack deployment complete Deployed hello-web third-party to juniper; stack folder updated. ``` -The deploy writes `compose.yaml`, `.env` and a small metadata file into `stackDir`, runs `docker compose up --wait` on the server, and succeeds only once `health.url` answers. `third-party` is the image tag of an app that builds nothing; apps with `targets` are tagged with the commit. +The deploy writes `compose.yaml`, `.env` and a small metadata file into `stackDir`, runs `docker compose up --wait` on the server, and succeeds only once `health.url` answers. `third-party` is the image tag of an app that builds nothing; each role with a `build` is tagged with the commit of its repository. ### 5. Look at it @@ -159,7 +158,7 @@ hello-web-web-1 nginx:alpine "/docker-entrypoint.…" web 2 seconds $ compoza logs infra --tail 20 --follow $ ssh deploy@juniper.example.com web-1 | 2026/11/03 10:06:20 [notice] 1#1: start worker process 40 -web-1 | 192.168.158.1 - - [03/Nov/2026:10:06:21 +0000] "GET / HTTP/1.1" 200 896 "-" "curl/8.7.1" "-" +web-1 | 192.0.2.1 - - [03/Nov/2026:10:06:21 +0000] "GET / HTTP/1.1" 200 896 "-" "curl/8.7.1" "-" ``` `--follow` keeps printing new lines until you press Ctrl-C, each prefixed with the container that wrote it. To see the page itself, ask the server: `ssh deploy@juniper.example.com curl -s http://127.0.0.1:8080/`. @@ -179,12 +178,12 @@ Deleted hello-web; data, secrets and stack files stay ### Next steps - **Serve it on a domain** through a shared [Caddy proxy or a Cloudflare Tunnel](docs/ingress.md): configure the provider on the host, run `compoza setup-proxy juniper` once, and add `ingress` with the app's domain to `deploy.yaml`. -- **Build your own images** by listing Dockerfile `targets`; each is built, tagged with the commit and shipped to the server. See [Apps](docs/configuration.md#apps) and [Image transports](docs/configuration.md#image-transports). +- **Build your own images** by giving a Compose service a [role](docs/configuration.md#roles) with `build`; compoza builds it, tags it with the commit, ships it to the server and sets the service's `image`. Roles also carry per-service environment and secrets. See [Image transports](docs/configuration.md#image-transports). - **Add secrets** from a dotenv file or 1Password without putting values in configuration. For 1Password, save your desktop account once with `compoza config set 1password.account ""`. See [Secrets](docs/secrets.md). - **Share hosts across machines or teammates** by keeping a `hosts.yaml` in an infrastructure repository and running `compoza hosts link `. See [Linked hosts files](docs/configuration.md#linked-hosts-files). - **Run Compose from your machine instead** by leaving `execution` out: the default, client execution, runs Compose locally against `docker --host ssh://` and needs Docker on your machine. See [Execution and image transport](#execution-and-image-transport). - **Keep deployment configuration out of the app repository.** For `Repositories/deployments/foo/deploy.yaml`, set `workingDirectory: ../../foo`; Git checks and builds then use `Repositories/foo`, while Compose files, assets and templates stay beside `deploy.yaml`. See [separate source and deployment folders](docs/configuration.md#separate-source-and-deployment-folders). -- **Recover** from a bad deploy with `compoza deploy infra --tag `. +- **Recover** from a bad deploy with `compoza deploy infra --tag `, or `--tag =` for one role. ## Execution and image transport @@ -199,17 +198,17 @@ Two independent settings decide how a deploy runs: ## Commands -| Task | Command | -| ---------------------- | ----------------------------------------------------------------------------- | -| Check readiness | `compoza check [--json] [--secrets]` | -| Preview / deploy | `compoza deploy [--dry-run]` | -| Reuse an earlier image | `compoza deploy --tag ` | -| Inspect services | `compoza status ` | -| Read or follow logs | `compoza logs [--service ] [--tail ] [--follow]` | -| Stop / resume | `compoza stop ` / `compoza start ` | -| Remove a stack | `compoza delete [--volumes]` | -| Mirror content | `compoza publish
` | -| Set up host ingress | `compoza setup-proxy [--ingress caddy\|cloudflare-tunnel] [--dry-run]` | +| Task | Command | +| -------------------- | ----------------------------------------------------------------------------- | +| Check readiness | `compoza check [--json] [--secrets]` | +| Preview / deploy | `compoza deploy [--dry-run]` | +| Reuse earlier images | `compoza deploy --tag [--tag =]` | +| Inspect services | `compoza status ` | +| Read or follow logs | `compoza logs [--service ] [--tail ] [--follow]` | +| Stop / resume | `compoza stop ` / `compoza start ` | +| Remove a stack | `compoza delete [--volumes]` | +| Mirror content | `compoza publish
` | +| Set up host ingress | `compoza setup-proxy [--ingress caddy\|cloudflare-tunnel] [--dry-run]` | `` holds `deploy.yaml` (or is the file itself) and defaults to the working directory, except for `publish`, which requires it. Every app command accepts `--destination `, and `stop`, `start`, `delete`, `publish` and `setup-proxy` accept `--dry-run`. `delete` is also available as `down`. diff --git a/docs/caddy-production.md b/docs/caddy-production.md index 77b9a09..271a6a0 100644 --- a/docs/caddy-production.md +++ b/docs/caddy-production.md @@ -2,7 +2,7 @@ [README](../README.md) · [Commands](commands.md) · [Ingress](ingress.md) · [Caddy templates](caddy-templates.md) -This guide covers a hardened **single-host** Caddy deployment. A host, Docker engine or network outage still interrupts every app on that host. The extension does not provide automatic failover, capacity planning, backups or an external monitoring service. +These defaults harden Caddy on a **single host**. A host, Docker engine or network outage still interrupts every app on that host. The extension does not provide automatic failover, capacity planning, backups or an external monitoring service. ## Defaults and their purpose diff --git a/docs/caddy-templates.md b/docs/caddy-templates.md index 6f128e7..110f88d 100644 --- a/docs/caddy-templates.md +++ b/docs/caddy-templates.md @@ -2,7 +2,7 @@ [README](../README.md) · [Commands](commands.md) · [Configuration](configuration.md) · [Ingress](ingress.md) · [Secrets](secrets.md) · [SSH deployment](ssh-deployment.md) · [Development](development.md) -Use templates when a shared Caddyfile or an app route needs custom directives. This guide covers file locations, placeholders and overrides; see [Ingress](ingress.md#shared-caddy-proxy) for host setup and provisioning. +Use templates when a shared Caddyfile or an app route needs custom directives. Host setup and provisioning are in [Ingress](ingress.md#shared-caddy-proxy). - [Select template sources](#select-template-sources) - [Shared Caddyfile template](#shared-caddyfile-template) diff --git a/docs/commands.md b/docs/commands.md index bec8d78..8f88150 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -61,7 +61,7 @@ A read-only readiness report. It changes nothing on the host and fetches no secr | `--secrets` | Also read the selected secret source and report whether it is accessible, with variable names only. | | `--local` | Skip the remote checks. With `--secrets`, check the `local.secrets` source instead of the deployment's. | -What is checked depends on the app and host: DNS for the app's domain, the ingress proxy and its network, registry login when images are pushed, and, for `execution: ssh`, the host's Docker, Compose, jq, curl and stack folder (see [host prerequisites](ssh-deployment.md#host-prerequisites)). Each line names the check and what was found. The last line counts the failures, and says how many block a deploy when that is not all of them: +What is checked depends on the app and host: DNS for the app's domain, the ingress proxy and its network, registry login when images are pushed, the roles (each role is a service in the selected Compose files, and its environment and secrets do not clash), and, for `execution: ssh`, the host's Docker, Compose, jq, curl and stack folder (see [host prerequisites](ssh-deployment.md#host-prerequisites)). Each line names the check and what was found. The last line counts the failures, and says how many block a deploy when that is not all of them: ```text ok docker: docker ready for SSH execution @@ -75,35 +75,45 @@ See [Checks](configuration.md#checks) and [Verifying secret access](secrets.md#v ## deploy ```sh -compoza deploy [] [--dry-run] [--tag ] [--allow-dirty] [--local] +compoza deploy [] [--dry-run] [--tag ]... [--allow-dirty] [--local] ``` Checks readiness, builds and ships the images, runs Compose, routes traffic, and waits for the app to answer. Progress is printed as numbered phases, each with its duration, and the last line names the deployed tag. -| Flag | Effect | -| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--dry-run` | Run the read-only checks and print every command and host script instead of running it. Nothing is built, shipped, written or started. | -| `--tag ` | Reuse images built earlier with this tag, skipping Git and builds. The **current local configuration** is used; there is no saved configuration history. | -| `--allow-dirty` | Build from a checkout with uncommitted changes and tag the images `-dirty`. Without it, a build requires a clean checkout. | -| `--local` | Build into this machine's Docker and start the stack here, with the `local` overrides; no remote checks, transport, routes or wait. | +| Flag | Effect | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--dry-run` | Run the read-only checks and print every command and host script instead of running it. Nothing is built, shipped, written or started. | +| `--tag ` | Reuse images built earlier instead of building, skipping Git. Repeatable: `` for every role with a `build`, `=` for one role. See [Reusing images](#reusing-images). | +| `--allow-dirty` | Build from a checkout with uncommitted changes and tag the images `-dirty`. Without it, a build requires a clean checkout. | +| `--local` | Build into this machine's Docker and start the stack here, with the `local` overrides; no remote checks, transport, routes or wait. | The phases run in this order, and a phase with nothing to do is left out: 1. **Checks**: the same readiness checks as `check`. A blocking failure stops the deploy before anything changes. 2. **Folders** (client execution): creates the app's `directories` and `content` folders on the host. -3. **Build**: builds each target and ships it through the host's transport (registry, `ssh-load` or `remote-build`). There is no build with `targets: []` or `--tag`. +3. **Build**: builds each role that has a `build` and ships it through the host's transport (registry, `ssh-load` or `remote-build`). There is no build when no role has a `build`, or with `--tag`. 4. **Compose**: `docker compose up --wait` on the host. With `execution: ssh` this one phase also stages and validates the flat stack folder, creates the folders, writes the route and probes `health.url` (or the app's public URL) from the host. 5. **Route** (client execution, with ingress): writes the app's Caddy site or Cloudflare route. 6. **Wait** (client execution, with ingress): waits for the app's public URL to answer with a status below 500. -Images are tagged with the commit's first twelve characters, or `third-party` when the app builds nothing. Reuse an earlier image with `--tag`: +Each role's image is tagged with the first twelve characters of its repository's commit, so roles from one repository share a tag. The deploy is tagged `third-party` when no role has a `build`. ```sh compoza deploy infra --dry-run compoza deploy infra +``` + +### Reusing images + +`--tag ` sets the tag of every role with a `build`, and `--tag =` overrides it for one role: + +```sh compoza deploy infra --tag 1a2b3c4d5e6f +compoza deploy infra --tag 1a2b3c4d5e6f --tag worker=9f8e7d6c5b4a ``` +Every role with a `build` must end up with a tag. Naming an unknown role, a role without `build` or the same role twice fails, and so does a second bare tag. A tag follows Docker's rules: letters, digits, underscores, dots and dashes, at most 128 characters, not starting with a dot or dash. That rejects `--tag web:abc`, a likely typo for `--tag web=abc`. The **current local configuration** is used; there is no saved configuration history. + A failed deploy can leave new containers or routes running: there is no automatic rollback. Inspect `status` and `logs`, then deploy a known tag. See [Images and tags](configuration.md#images-and-tags), [Local runs](configuration.md#local-runs) and, for SSH execution, the [deploy transaction](ssh-deployment.md#deploy-transaction). ## status diff --git a/docs/configuration.md b/docs/configuration.md index c8a3cd1..9ca6056 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,6 +9,8 @@ compoza reads three kinds of files: global, linked and repository-local `hosts.y - [Linked hosts files](#linked-hosts-files) - [Settings](#settings) - [Apps](#apps) + - [Roles](#roles) + - [Images and tags](#images-and-tags) - [Destinations](#destinations) - [Execution and transport](#execution-and-transport) - [Client execution](#client-execution) @@ -150,15 +152,19 @@ Prefer the commands to editing the file by hand: they validate the result before ## Apps -An app opts in with a folder holding `deploy.yaml` and its Compose files. Its source and Dockerfile can live there or in a separate `workingDirectory`. A typical deployment folder is `infra/` or `apps//deploy/`. This example uses the default client execution with two built targets and Caddy routing: +An app opts in with a folder holding `deploy.yaml` and its Compose files. Its source and Dockerfile can live there or in a separate `workingDirectory`. A typical deployment folder is `infra/` or `apps//deploy/`. This example uses the default client execution with two built services and Caddy routing: ```yaml name: sample-app host: atlas dockerfile: Dockerfile -targets: - - worker - - web +roles: + web: + build: + target: web + worker: + build: + target: worker composeFiles: - compose.yaml - compose.production.yaml @@ -182,26 +188,26 @@ local: none: {} ``` -| Field | Description | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `name` | Compose project name and image prefix. | -| `host` | Host key in `hosts.yaml`. | -| `targets` | Images to build from the Dockerfile, one per target. Use `[]` for third-party-only stacks. | -| `workingDirectory` | Source directory for Git and builds; defaults to the deploy folder. Accepts absolute, `~/`, or config-relative paths. | -| `dockerfile`, `context` | Relative to `workingDirectory`. Default to `Dockerfile` there and the source Git root. | -| `composeFiles` | Compose files to apply, in order; relative to the YAML folder or explicit absolute source paths. | -| `profiles` | Compose profiles to enable. | -| `execution` | `client` (default) or `ssh`; see [Execution and transport](#execution-and-transport). | -| `ingress` | `caddy`, `cloudflare-tunnel` or `none` (default); see [Ingress](#ingress). | -| `health` | SSH execution: optional `url` and `timeoutSeconds`, probed on the host; see [SSH deployment](ssh-deployment.md#health-and-readiness). | -| `directories` | Absolute host folders created before Compose runs. | -| `content` | Absolute host folders for content sections, created before Compose runs; see [Publishing content](#publishing-content). | -| `environment` | Non-secret variables passed to Compose. | -| `secrets` | Exactly one secret source: `env-file`, `1password` or `none`; see [Secrets](secrets.md). | -| `local` | Overrides for `--local` runs; see [Local runs](#local-runs). | -| `assets` | SSH execution only: declared read-only files uploaded to the stack. | -| `remote` | SSH execution only: `stackDir`, `overlayFile`, `waitTimeoutSeconds`. | -| `destinations` | Named alternative deployments of the same app; see [Destinations](#destinations). | +| Field | Description | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Compose project name and image prefix. | +| `host` | Host key in `hosts.yaml`. | +| `roles` | Per Compose service: what to build, and the container environment and secrets it gets; see [Roles](#roles). Omit for third-party-only stacks. | +| `workingDirectory` | Default source directory for Git and builds; defaults to the deploy folder. Accepts absolute, `~/`, or config-relative paths. | +| `dockerfile`, `context` | Defaults for every role with `build`. Relative to `workingDirectory`; default to `Dockerfile` there and the source Git root. | +| `composeFiles` | Compose files to apply, in order; relative to the YAML folder or explicit absolute source paths. | +| `profiles` | Compose profiles to enable. | +| `execution` | `client` (default) or `ssh`; see [Execution and transport](#execution-and-transport). | +| `ingress` | `caddy`, `cloudflare-tunnel` or `none` (default); see [Ingress](#ingress). | +| `health` | SSH execution: optional `url` and `timeoutSeconds`, probed on the host; see [SSH deployment](ssh-deployment.md#health-and-readiness). | +| `directories` | Absolute host folders created before Compose runs. | +| `content` | Absolute host folders for content sections, created before Compose runs; see [Publishing content](#publishing-content). | +| `environment` | Non-secret variables passed to Compose for interpolation. | +| `secrets` | Exactly one secret source: `env-file`, `1password` or `none`; see [Secrets](secrets.md). Provides Compose interpolation and role secrets. | +| `local` | Overrides for `--local` runs; see [Local runs](#local-runs). | +| `assets` | SSH execution only: declared read-only files uploaded to the stack. | +| `remote` | SSH execution only: `stackDir`, `overlayFile`, `waitTimeoutSeconds`. | +| `destinations` | Named alternative deployments of the same app; see [Destinations](#destinations). | SSH-specific fields are described in [SSH deployment](ssh-deployment.md). @@ -224,18 +230,21 @@ Repositories/ name: foo host: atlas workingDirectory: ../../foo -targets: [web] +roles: + web: + build: + target: web composeFiles: [compose.yaml] secrets: env-file: path: ./app.env ``` -Git checks and image tags use the repository containing `workingDirectory`. Build commands run there; relative `dockerfile` and `context` paths resolve there, and the default context remains that repository's root. Absolute build paths remain absolute. `workingDirectory` itself resolves relative to the selected YAML file, including an explicitly named file such as `production.yaml`; it also accepts absolute paths and `~/`. +Git checks and image tags use the repository containing a role's working directory (here the top-level `workingDirectory`). Build commands run there; relative `dockerfile` and `context` paths resolve there, and the default context remains that repository's root. Absolute build paths remain absolute. `workingDirectory` itself resolves relative to the selected YAML file, including an explicitly named file such as `production.yaml`; it also accepts absolute paths and `~/`. Relative Compose files and the project directory, assets, dotenv paths and app templates use the deployment YAML folder. Host paths still refer to the remote host. Host inventory discovery continues to use the CLI's current working directory, independently of both folders. -All destinations share `workingDirectory`; it cannot be overridden per destination. A real build requires that directory to exist. `--tag`, `targets: []`, status, logs and lifecycle commands do not require the source checkout. +All destinations share `workingDirectory` and the roles' `build` settings; they cannot be overridden per destination. A real build requires the working directories to exist. `--tag`, stacks without a `build`, status, logs and lifecycle commands do not require the source checkout. To use the app repository's Compose files directly, select their absolute workstation paths: @@ -251,12 +260,164 @@ composeFiles: SSH execution reads those files for each deployment and stages them under their basenames, then merges them in the listed order into the host's flat `compose.yaml`. Selected files must have distinct upload paths: two external files named `compose.yaml`, or an external filename colliding with a relative Compose file or asset, are rejected. Source symlinks, symlinked ancestors and dot traversal are rejected. Assets and any relative file references inside Compose still belong beside `deploy.yaml`; absolute Compose paths do not change that base. Keep persistent data and secret file references as absolute **host** paths. Lifecycle operations use the already deployed stack and do not need the source files. +### Roles + +A role configures one Compose service: the image compoza builds for it, and the environment variables and secrets its container receives. The top-level `environment` and `secrets` are Compose interpolation inputs instead: one pool for the whole project, which each service maps by hand (`FOO: ${FOO}`). + +```yaml +name: sample-app +host: atlas +workingDirectory: /Users/you/Repositories/sample-app +dockerfile: infra/Dockerfile +secrets: + 1password: + environmentId: fictional-atlas-environment-id +environment: + SAMPLE_APP_DATA_DIR: /srv/sample-app/data +roles: + web: + build: + target: server + secrets: + - SAMPLE_APP_API_TOKEN + worker: + build: + target: worker + dockerfile: infra/worker.Dockerfile + environment: + MY_API_BASE_URL: https://api.example.com/v1 + postgres: + secrets: + - POSTGRES_PASSWORD +``` + +Role names are Compose service names, and a role name becomes a component of an image name, so it must match Docker's rule without dots: lowercase letters and digits joined by single or double underscores or runs of dashes (`^[a-z0-9]+(?:(?:_|__|-+)[a-z0-9]+)*$`). `web_` and `a_-b` are rejected. Every role must be a service in at least one selected Compose file. + +| Field | Meaning | +| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `build` | Optional. Present means compoza builds an image for the service. A role without `build` only sets environment and secrets, like the third-party `postgres` above. | +| `environment` | Optional map of container environment variables for this service only. Values are literal: nothing is interpolated. | +| `secrets` | Optional list of variable names read from the stack's secret source. Each is delivered to this service under the same name. Secret values never enter the generated override. | + +compoza sets each built service's `image:` itself, so the Compose files leave it out. A role's `environment` and `secrets` reach the container without a `${NAME}` mapping in the Compose file; see [How roles reach services](#how-roles-reach-services). + +#### Build per role + +`build` accepts these fields, all optional: + +| Field | Meaning | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------ | +| `target` | Dockerfile stage. Without it no `--target` flag is passed and the final stage is built. | +| `workingDirectory` | Source directory of the role. Resolved like the top-level field: relative to the deploy YAML file, `~/` and absolute paths accepted. | +| `dockerfile` | Relative to the role's working directory; absolute paths are kept. Default `Dockerfile`. | +| `context` | Relative to the role's working directory; absolute paths are kept. Default the Git root of the role's working directory. | + +A field that is unset falls back to the top-level field of the same name, then to the default. Build commands for a role run in its working directory. + +Two Dockerfiles in one repository, each role building from its own file: + +```yaml +workingDirectory: ../../sample-app +context: . +roles: + web: + build: + dockerfile: docker/web.Dockerfile + worker: + build: + dockerfile: docker/worker.Dockerfile +``` + +Both roles build from one repository, so it is checked once and both images get the same tag. They also share the context, and BuildKit applies the context's `.dockerignore` to every build. To give one image its own ignore rules, put a file named after its Dockerfile beside it, such as `docker/web.Dockerfile.dockerignore`. BuildKit uses it instead of the context's `.dockerignore` for that build. + +A role built from another repository: + +```yaml +workingDirectory: ../../sample-app +roles: + web: + build: + target: server + jobs: + build: + workingDirectory: ../../sample-jobs +``` + +`jobs` builds from the `Dockerfile` at the root of `sample-jobs` and is tagged with that repository's commit, so the two roles usually have different tags. Each repository is checked separately: both need a commit and a clean checkout. + +#### Environment and secrets + +Names in a role's `environment` and `secrets` must be valid environment identifiers, and `environment` values cannot contain carriage returns, line feeds or NUL. Role secrets also join the interpolation pool, so they follow the same naming rules as any secret. + +A role fails validation, before anything is built, when: + +- a name is in both its `environment` and its `secrets`; +- a secret name is reserved, or is also a key of the top-level `environment` (or `local.environment` for `--local`); +- a name is also set in the service's own `environment:` in a selected Compose file, in map or list form, because Compose's precedence would otherwise pick a winner silently (see below); +- it takes secrets but the run has no secret source, or `none`; +- the secret source lacks one of its secrets. This is known only once values are read, so it is checked by a real deploy and `check --secrets`. + +The other rules need no secret values, so `check` and `--dry-run` report them too. + +The duplicate check follows the selected Compose files the way Compose merges them: + +- A later file's `environment: !reset` clears the earlier entries and `!override` replaces them. A per-key `!reset` removes one key. A key removed this way no longer clashes. +- A service that uses `extends`, in the same file or another one, also carries the base service's keys. +- Services from `include:` files count, for client execution and `--local` runs. SSH execution rejects `include:`. + +Role secrets work with `env-file` and `1password`; see [Secrets](secrets.md#role-secrets). + +#### What stays stack-wide + +`workingDirectory`, `dockerfile` and `context` are defaults that roles inherit. The top-level `environment` and `secrets` stay stack-wide, because Compose interpolates the whole project from one pool: `${SAMPLE_APP_DATA_DIR}` in a volume path, a published port or a third-party image tag cannot belong to one service. Roles set container environment, which can. A value needed both for interpolation and inside a container is declared in both places. + +#### How roles reach services + +With roles, compoza generates one extra Compose file, `compoza.roles.yaml`, and applies it as the last file. For each role it contains the `image` of a building role and the role's `environment`: literal values (with `$` written as `$$`) and secrets as `${NAME}` references. It never contains secret values; those stay in the interpolation pool (the stack `.env` for SSH execution, the process environment for client execution) under their own names. + +- Client execution and `--local`: written as a `0600` file in a private temporary folder, passed as the last `--file` of `compose up` and removed afterwards. +- SSH execution: uploaded with the bundle and merged into the flat `compose.yaml`. It is not installed as a separate file on the host. A Compose file or asset that uses the upload name `compoza.roles.yaml` is rejected. + +`--dry-run` prints the file in every execution mode. + +Without `roles` there is no extra file and nothing changes. + +#### Migrating from 0.6.0 + +`targets` was removed in favor of `roles`. A configuration that still has it fails with a dedicated message that names the replacement. + +1. Give each target a role named after the Compose service that runs it, with the target as `build.target`. Delete `targets: []` outright. + + ```yaml + # 0.6.0: the server stage ran as the web service + targets: [server, worker] + + # now + roles: + web: + build: + target: server + worker: + build: + target: worker + ``` + +2. Remove `image: ${SERVER_IMAGE}` and the other `_IMAGE` lines from Compose files. compoza sets each built service's image, and those variables no longer exist. A fallback such as `${WEB_IMAGE:-web:local}` can stay: compoza's image overrides it. +3. Images are now named `-:`. Where the role name differs from the old target name (`server` above), the image name changes and earlier tags cannot be reused with `--tag`. Build that role once from Git. +4. On SSH hosts the next deploy rewrites `.env` without `*_IMAGE` variables and `.compoza.json` with `tags` per role instead of one `tag`. + ### Images and tags -- Each target is named `-:`, prefixed with the registry for registry transport, and handed to Compose as `_IMAGE`. Compose files read `${WEB_IMAGE}`, `${WORKER_IMAGE}` and so on. -- A build requires a clean checkout of the source repository with at least one commit and tags images with the commit's first twelve characters. `--allow-dirty` permits a dirty checkout and tags `-dirty`. -- `--tag ` reuses images built earlier and skips Git and builds, so it works from a deploy folder outside any checkout. It always uses the **current local configuration**; there is no saved Compose configuration history. -- With `targets: []` there is no Git, Dockerfile or build-context requirement, and no build or application-registry authentication check. Compose still pulls missing third-party images. +- The image of a role with `build` is `-:`, prefixed with the registry for registry transport. compoza sets it as the service's `image`. +- A build requires a clean checkout with at least one commit in each role's repository. Each repository is checked once, and its roles are tagged with the first twelve characters of its commit. `--allow-dirty` permits a dirty checkout and tags `-dirty`. +- `--tag` reuses images built earlier and skips Git and builds, so it works from a deploy folder outside any checkout. `--tag ` applies to every role with `build`, and `--tag =` overrides it for one role; repeat the flag as needed. Every role with `build` must end up with a tag. `--tag` always uses the **current local configuration**; there is no saved Compose configuration history. +- Progress messages show the single tag when all roles share it, and `web=abc worker=def` otherwise. +- Without any role that has `build` there is no Git, Dockerfile or build-context requirement, and no build or application-registry authentication check. Compose still pulls missing third-party images, and the deploy is tagged `third-party`. + +```sh +compoza deploy infra --tag 1a2b3c4d5e6f +compoza deploy infra --tag 1a2b3c4d5e6f --tag worker=9f8e7d6c5b4a +``` ### Ingress @@ -270,7 +431,7 @@ Compose resolves relative bind mounts on the host, so host paths must be absolut ### Local runs -`local` overrides `composeFiles`, `environment` and the complete secret source for `--local` runs. When the app declares a production secret source, an explicit `local.secrets` is required, including `none: {}` when local Compose needs no secrets. +`local` overrides `composeFiles`, `environment` and the complete secret source for `--local` runs. `local.roles` sets `environment` and `secrets` per role and merges over the app's roles the way [destinations](#destinations) do; `build` cannot be changed. When the app declares a production secret source, an explicit `local.secrets` is required, including `none: {}` when local Compose needs no secrets. `--local` builds into this machine's Docker and starts the stack here, without remote readiness checks, transport or routes. It keeps this client build-and-run behavior regardless of `execution`. @@ -281,12 +442,14 @@ One `deploy.yaml` can deploy to several hosts or projects. Select one with `--de ```yaml name: sample-app host: atlas -targets: - - web composeFiles: - compose.yaml environment: PUBLIC_TITLE: Sample app +roles: + web: + build: + target: web destinations: atlas: {} cedar: @@ -301,6 +464,7 @@ destinations: - When supplied, a destination **replaces** `host`, `projectName`, `composeFiles`, `execution`, the entire `remote` object, `profiles`, `ingress`, `health`, `directories`, `content` and the complete `secrets` source. - A destination's `environment` **merges** over the app defaults. +- A destination's `roles` take `environment` and `secrets` per role, without `build`: destinations share the source. For each role, `environment` merges key by key over the app role's, and `secrets` replaces the app role's list. A role named only in the destination is allowed and sets environment only. - `name` remains the image prefix; `projectName` selects the Compose project and the Caddy site-file key. - Unknown destination names fail. @@ -323,10 +487,10 @@ Execution and image transport are independent: Client execution is the default mode. A `deploy`: -1. Resolves the image tag from Git, or uses `--tag`. +1. Resolves the image tags from Git, or uses `--tag`. 2. Runs the readiness checks and stops if DNS, the proxy, the network or the registry login is not ready. 3. Creates the host folders. -4. Builds each target for the host's platform and ships it by the host's transport. +4. Builds each role with a `build` for the host's platform and ships it by the host's transport. 5. Runs `docker --host ssh:// compose up --detach --remove-orphans --no-build`, with `--pull always` for a `registry` host and `--pull missing` for `ssh-load` hosts and `--local`. For `ssh-load` and local runs, already-loaded images are reused while missing third-party images such as `postgres` are fetched. The explicit `--host` overrides any active Docker context, so a `DOCKER_CONTEXT` on this machine never redirects a deploy. 6. For `caddy`, atomically replaces the site file under a shared proxy lock and uses Caddy reload to validate and activate it; a rejected reload restores the file and attempts a recovery reload. Long-lived connections depend on the configured drain and streaming policy. Then waits up to three minutes for the URL to answer. @@ -343,6 +507,7 @@ compoza check infra --secrets --local --json ``` - `check` is read-only and fetches no secrets. +- A blocking `roles` check validates the roles against the selected Compose files: unknown services, clashing environment keys, reserved or colliding secret names and secrets without a secret source. `check --local` checks `local.roles` and `local.composeFiles`. It runs without fetching secrets. - `--secrets` explicitly resolves the selected secret source and reports status and variable names only. - `--secrets --local` checks only the local source and skips remote readiness. `--local` alone neither resolves secrets nor checks local Docker. - A valid hosts configuration is still needed to resolve the app. diff --git a/docs/examples.md b/docs/examples.md index 9064016..6fd27be 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -64,7 +64,7 @@ One host file can describe both ingress capabilities on the same physical machin ```text config/ ├── hosts.yaml -├── macmini-server/ +├── linden/ │ └── host.yaml └── devbox/ └── host.yaml @@ -73,16 +73,16 @@ config/ Complete `config/hosts.yaml`: ```yaml -macmini-server: - $ref: ./macmini-server/host.yaml +linden: + $ref: ./linden/host.yaml devbox: $ref: ./devbox/host.yaml ``` -Complete `config/macmini-server/host.yaml`: +Complete `config/linden/host.yaml`: ```yaml -ssh: deploy@macmini.example.com +ssh: deploy@linden.example.com platform: linux/arm64 transport: mode: ssh-load @@ -118,8 +118,8 @@ These paths and SSH destinations are illustrative; use host paths appropriate to Preview each provider separately: ```sh -compoza --hosts config/hosts.yaml setup-proxy macmini-server --ingress caddy --dry-run -compoza --hosts config/hosts.yaml setup-proxy macmini-server --ingress cloudflare-tunnel --dry-run +compoza --hosts config/hosts.yaml setup-proxy linden --ingress caddy --dry-run +compoza --hosts config/hosts.yaml setup-proxy linden --ingress cloudflare-tunnel --dry-run ``` An ambiguous setup without `--ingress` fails without SSH or credential-provider access. Setup never implicitly recreates both shared services. Single-provider hosts can omit the flag. @@ -127,7 +127,7 @@ An ambiguous setup without `--ingress` fails without SSH or credential-provider For an app using Caddy, a **partial deploy mapping** selects: ```yaml -host: macmini-server +host: linden ingress: caddy: domain: direct.example.com @@ -137,7 +137,7 @@ ingress: Another app on that same host can select Cloudflare instead: ```yaml -host: macmini-server +host: linden ingress: cloudflare-tunnel: domain: tunnel.example.com @@ -155,7 +155,6 @@ Complete `static-site/deploy.yaml`: ```yaml name: static-site host: cedar -targets: [] execution: ssh composeFiles: [compose.yaml] assets: [nginx.conf] @@ -218,7 +217,7 @@ services: `assets` are explicit, deploy-relative regular files; relative bind mounts must be declared and read-only. Do not put dotenv files or other credentials in assets. Persistent content uses an absolute host path and is not uploaded with the stack. The host overlay remains live and operator-owned; omit `overlayFile` if you do not need one. -`targets: []` needs no Git checkout, Dockerfile or build context. Compose still pulls missing third-party images. `waitTimeoutSeconds` controls Compose readiness; `health` adds a separate HTTP probe executed on the host. Any HTTP status below 500 counts as reachable, including a 404 while the content folder is empty; this is not a content assertion. +Without a role that has `build`, the stack needs no Git checkout, Dockerfile or build context. Compose still pulls missing third-party images. `waitTimeoutSeconds` controls Compose readiness; `health` adds a separate HTTP probe executed on the host. Any HTTP status below 500 counts as reachable, including a 404 while the content folder is empty; this is not a content assertion. After creating a local `public/index.html`, the intended flow is: @@ -257,7 +256,13 @@ host: atlas workingDirectory: ../../foo dockerfile: docker/Dockerfile context: . -targets: [web, worker] +roles: + web: + build: + target: web + worker: + build: + target: worker execution: ssh composeFiles: [compose.yaml] profiles: [jobs] @@ -305,7 +310,6 @@ Complete `Repositories/deployments/foo/compose.yaml`: ```yaml services: web: - image: ${WEB_IMAGE} environment: APP_MODE: ${APP_MODE} APP_TOKEN: ${APP_TOKEN:-} @@ -313,7 +317,6 @@ services: edge: aliases: ['${PUBLIC_ALIAS}'] worker: - image: ${WORKER_IMAGE} profiles: [jobs] environment: APP_MODE: ${APP_MODE} @@ -325,7 +328,7 @@ networks: The Caddy setup below creates the external `caddy-edge` network on `atlas`. The staging override below instead gives the private staging stack its own Compose-managed network. The unique alias makes the production route resolve `foo-production-web:80`, even when other apps also have a service named `web`. -Complete `Repositories/foo/docker/Dockerfile`, with both configured build targets: +Complete `Repositories/foo/docker/Dockerfile`, with the two build targets named by the roles: ```dockerfile FROM nginx:alpine AS web @@ -335,7 +338,7 @@ FROM nginx:alpine AS worker CMD ["sh", "-c", "while :; do sleep 60; done"] ``` -Add a `public/index.html` in the source repository and commit it with the Dockerfile. This minimal image pair demonstrates target wiring; replace the worker with the application's real job process. `APP_TOKEN` demonstrates explicit container environment mapping; these example images do not consume it. Keep `production.env` private and out of version control; its values are not shown here. Exclude any secrets inside a build context with `.dockerignore`. +Add a `public/index.html` in the source repository and commit it with the Dockerfile. This minimal image pair demonstrates role wiring; replace the worker with the application's real job process. `APP_TOKEN` demonstrates explicit container environment mapping; these example images do not consume it. Keep `production.env` private and out of version control; its values are not shown here. Exclude any secrets inside a build context with `.dockerignore`. The Compose files declare no `image:` for `web` and `worker`: compoza sets `foo-web:` and `foo-worker:` from the roles. To give each image its own Dockerfile, set `build.dockerfile` per role; see [Roles](configuration.md#roles). `workingDirectory` resolves relative to `deploy.yaml`, accepts absolute and `~/` paths, and defaults to the deploy folder. Git checks and image tags use the repository containing that directory. Relative `dockerfile` and `context` paths resolve there; omitting `context` uses the source Git root. Build commands run in `workingDirectory`. @@ -348,12 +351,10 @@ The previous deploy file defines both variations. Complete `compose.local.yaml`, ```yaml services: web: - image: ${WEB_IMAGE} ports: ['127.0.0.1:8081:80'] environment: APP_MODE: ${APP_MODE} worker: - image: ${WORKER_IMAGE} profiles: [jobs] ``` @@ -376,7 +377,7 @@ compoza deploy Repositories/deployments/foo --destination staging --dry-run `--local` builds and runs on the CLI machine, without remote checks, transport or ingress, even when `execution` is `ssh`. It replaces `composeFiles` and the secret source with the local choices and merges local `environment`. A declared production source requires explicit `local.secrets`; `none` prevents production credential fallback. -A destination replaces the selected arrays and objects rather than merging their contents. For example, staging's `remote` includes its own `stackDir`, and `profiles: []` disables the jobs profile. `environment` alone merges by key. `secrets` replaces the entire source, and `ingress.none` clears the public route selection. Destinations can also replace `execution`, `health`, `directories` and `content`. `name`, `workingDirectory`, build settings, targets and assets remain shared. +A destination replaces the selected arrays and objects rather than merging their contents. For example, staging's `remote` includes its own `stackDir`, and `profiles: []` disables the jobs profile. `environment` alone merges by key. `secrets` replaces the entire source, and `ingress.none` clears the public route selection. Destinations can also replace `execution`, `health`, `directories` and `content`. `name`, `workingDirectory`, the roles' `build` settings and assets remain shared. A destination or `local` can set `roles..environment` and `secrets`, merging environment per key and replacing the secret list. `projectName` changes Compose and route identity; it does not change the `foo-web` and `foo-worker` image prefixes. Keep `--destination staging` on subsequent check, status, logs, lifecycle and publish commands. @@ -384,7 +385,7 @@ A destination replaces the selected arrays and objects rather than merging their The inventory deliberately shows all three transports: -| Host | Transport | Result for built targets | +| Host | Transport | Result for built roles | | ---------- | -------------- | ------------------------------------------------------------------------------ | | `atlas` | `registry` | Build locally and push to `registry.example.com`; the host pulls images. | | `cedar` | `ssh-load` | Build locally, then stream compressed images into the host daemon. | @@ -398,7 +399,13 @@ For a small **client execution** variation, save this complete deploy file in th name: foo-client host: cedar dockerfile: docker/Dockerfile -targets: [web, worker] +roles: + web: + build: + target: web + worker: + build: + target: worker execution: client composeFiles: [compose.local.yaml] environment: @@ -571,7 +578,6 @@ Complete `tunnel-site/deploy.yaml`: ```yaml name: tunnel-site host: relay -targets: [] execution: ssh composeFiles: [compose.yaml] remote: @@ -691,4 +697,4 @@ compoza deploy Repositories/deployments/foo --destination staging --tag abc123de compoza deploy Repositories/deployments/foo --destination staging --tag abc123def456 ``` -`--tag` skips Git and builds; it does not restore earlier configuration, secrets or database state. A missing `workingDirectory` is allowed for `--tag`, `targets: []`, status, logs and lifecycle operations. `stop`, `start` and `delete` operate on the deployed stack; delete retains host files and persistent bind data, and removes named volumes only when `--volumes` is explicitly selected. +`--tag` skips Git and builds; it does not restore earlier configuration, secrets or database state. A missing `workingDirectory` is allowed for `--tag`, stacks without a `build`, status, logs and lifecycle operations. `stop`, `start` and `delete` operate on the deployed stack; delete retains host files and persistent bind data, and removes named volumes only when `--volumes` is explicitly selected. diff --git a/docs/ingress.md b/docs/ingress.md index 6425948..2af376f 100644 --- a/docs/ingress.md +++ b/docs/ingress.md @@ -41,8 +41,8 @@ Each extension validates its required settings. Empty selections, unknown extens When a host configures both providers, setup requires an explicit selection: ```sh -compoza setup-proxy macmini-server --ingress caddy --dry-run -compoza setup-proxy macmini-server --ingress cloudflare-tunnel --dry-run +compoza setup-proxy linden --ingress caddy --dry-run +compoza setup-proxy linden --ingress cloudflare-tunnel --dry-run ``` Remove `--dry-run` only when ready to set up that provider. Omitting `--ingress` on a host with both fails before SSH or credential-provider access. On a single-provider host the flag is optional. Apps select their matching host settings automatically; no app-command selector is needed. See [one host with both providers](examples.md#one-host-with-caddy-and-cloudflare-tunnel). diff --git a/docs/secrets.md b/docs/secrets.md index 548ad98..af4b33c 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -63,12 +63,12 @@ secrets: 1password: environmentId: fictional-atlas-environment-id variables: - - PROMPTSUB_API_KEY + - SAMPLE_APP_API_KEY ``` -`variables` accepts environment identifiers, matched exactly and case-sensitively, with no wildcards or renaming. A missing requested variable fails the secret read before builds or host writes. An empty list imports nothing and skips the SDK read; omitting the field imports every variable. A variable with an empty value is still imported; applications can require a nonempty value in Compose with `${PROMPTSUB_API_KEY:?required}`. +`variables` accepts environment identifiers, matched exactly and case-sensitively, with no wildcards or renaming. A missing requested variable fails the secret read before builds or host writes. The effective allowlist is `variables` plus every [role secret](#role-secrets) of the run, so a role's secrets are imported without repeating them here. An empty list with no role secrets imports nothing and skips the SDK read; `variables: []` with role secrets reads exactly those secrets. Omitting the field imports every variable. A variable with an empty value is still imported; applications can require a nonempty value in Compose with `${SAMPLE_APP_API_KEY:?required}`. -The SDK still reads the complete Environment into CLI memory. Compoza discards excluded entries before validating deployment variables or producing commands and payloads, so excluded names and values are absent from the generated `.env`, SSH upload and secret-access report. This applies to client and SSH execution, destinations and explicit local sources. Other providers are unchanged. Ordinary nonsecret settings and image references are still written to `.env`. +The SDK still reads the complete Environment into CLI memory. Compoza discards excluded entries before validating deployment variables or producing commands and payloads, so excluded names and values are absent from the generated `.env`, SSH upload and secret-access report. This applies to client and SSH execution, destinations and explicit local sources. Other providers are unchanged. Ordinary nonsecret settings are still written to `.env`. ### Authentication @@ -119,7 +119,7 @@ compoza check infra --secrets --local --json ## Names and dotenv format -Secret names must be valid environment identifiers and unique. Collisions with configured non-secret variables, image variables, Compose/Docker/Buildx controls and process-control variables fail instead of overriding them. +Secret names must be valid environment identifiers and unique. Collisions with configured non-secret variables, Compose/Docker/Buildx controls and process-control variables fail instead of overriding them. Dotenv files use simple `KEY=value` lines: @@ -130,7 +130,7 @@ Dotenv files use simple `KEY=value` lines: ## Using secrets in Compose -Provider variables supply Compose **interpolation** inputs. They are not injected into every container; each service must explicitly map the keys it needs: +Provider variables supply Compose **interpolation** inputs. They are not injected into every container; each service must explicitly map the keys it needs, or receive them as [role secrets](#role-secrets): ```yaml services: @@ -139,13 +139,36 @@ services: API_TOKEN: '${API_TOKEN:?required}' ``` +- A role secret needs no Compose mapping. Do not also map the same key in the service's `environment:`: a key set in both places is rejected. - With client execution, resolved values are added only to Compose, never to build or transport commands. - Explicit sources, including `none`, disable Compose's implicit `.env` discovery, so missing keys cannot fall back to a production dotenv file. - Secret-bearing Compose output is suppressed because third-party diagnostics can repeat values. Application logs still stream through `logs`, and redacting them is the application's responsibility. +### Role secrets + +A [role](configuration.md#roles) can list variable names from the secret source in `secrets`. compoza delivers each to that service's container under the same name, through the generated roles override, so the Compose file needs no `${NAME}` mapping for it: + +```yaml +secrets: + 1password: + environmentId: fictional-atlas-environment-id + variables: [] +roles: + web: + secrets: + - SAMPLE_APP_API_TOKEN +``` + +- Only services whose role lists a secret receive it. Any other service that needs the value still maps it in Compose. +- The override holds a `${NAME}` reference, never the value. The value stays in the interpolation pool: the stack `.env` over SSH, the process environment for client execution. +- With `variables: []` as above, the 1Password read returns exactly the role secrets. +- `destinations..roles..secrets` replaces that role's list, and `local.roles` does the same for `--local` runs, which read the local source. + +A role secret that is missing from the source, reserved, or also set in the service's Compose `environment:` fails validation before anything is built. See [Environment and secrets](configuration.md#environment-and-secrets) for every rule. + ## SSH execution -With a managed source (`env-file` or `1password`), the selected values travel over SSH stdin and are combined with generated nonsecret variables and image references in the stack `.env`, mode `0600`. Compose validates the staged files before replacing the flat stack. No resolved credentials enter `compose.yaml`, `.compoza.json`, process arguments or deployment logs; private validation diagnostics are discarded. +With a managed source (`env-file` or `1password`), the selected values travel over SSH stdin and are combined with generated nonsecret variables in the stack `.env`, mode `0600`. Compose validates the staged files before replacing the flat stack. No resolved credentials enter `compose.yaml`, `.compoza.json`, process arguments or deployment logs; private validation diagnostics are discarded. Upload or validation failure preserves the deployed `.env`. Later replacement, Compose or readiness failure may leave updated files or containers; recover by redeploying. No earlier credential copies or deployment backups are retained. @@ -154,6 +177,6 @@ Values that must never leave the host belong in a separate operator-owned dotenv ## Recovery and lifecycle - `status`, `logs`, `stop`, `start` and `delete` read deployed host files without fetching 1Password or client dotenv. -- `deploy --tag ` reuses earlier images with current local configuration and freshly resolves the selected source. It does not restore earlier passwords or archived configuration. +- `deploy --tag ` reuses earlier images with current local configuration and freshly resolves the selected source, including role secrets. It does not restore earlier passwords or archived configuration. - Deploying `none` writes the generated `.env` without managed secret values; there is no fallback to an earlier source. - Delete retains the stack `.env` and other files; remove sensitive files separately only when that cleanup is explicitly intended. diff --git a/docs/ssh-deployment.md b/docs/ssh-deployment.md index 7c51c6f..15f989d 100644 --- a/docs/ssh-deployment.md +++ b/docs/ssh-deployment.md @@ -43,10 +43,12 @@ borealis: ```yaml name: sample-app host: atlas -targets: - - web dockerfile: Dockerfile context: .. +roles: + web: + build: + target: web execution: ssh composeFiles: - compose.yaml @@ -94,7 +96,6 @@ destinations: ```yaml services: web: - image: ${WEB_IMAGE:?built image required} profiles: [web] volumes: - ./config/nginx.conf:/etc/nginx/conf.d/default.conf:ro @@ -144,8 +145,8 @@ A deployed stack contains: ```text / - compose.yaml # selected Compose files merged without interpolation - .env # generated variables, image tags and selected managed secrets (0600) + compose.yaml # selected Compose files and the roles override, merged without interpolation + .env # generated variables and selected managed secrets (0600) .compoza.json # small nonsecret lifecycle metadata file (0600) config/nginx.conf # explicitly declared assets retain their relative paths compose.host.yaml # optional live operator-owned overlay @@ -153,7 +154,7 @@ A deployed stack contains: There are no release, activation, current/previous, backup or archived-secret directories. The transient `.deploy-lock` holds upload files and disappears when the operation exits. No runtime shell scripts or jq filters are installed in the stack. -`.compoza.json` records the deployed project, tag, profiles, declared files, owned file paths, the overlay path, credential policy and ingress removal operation. Lifecycle commands use these settings rather than undeployed local edits. It contains no resolved credentials or previous deployment snapshots. +`.compoza.json` records the deployed project, the image tag of each role (`tags`), profiles, declared files, owned file paths, the overlay path, credential policy and ingress removal operation. Lifecycle commands use these settings rather than undeployed local edits. It contains no resolved credentials or previous deployment snapshots. The CLI replaces only its own stack files. It refuses symlinked stack ancestors and outputs, invalid deployed metadata, and files it does not own. A `.env` in a stack folder without lifecycle metadata is treated as operator data: move it to a separate host file and reference that file with an absolute `env_file:` path in Compose or the host overlay, or choose a fresh `stackDir`. The overlay path must be separate from generated outputs. @@ -165,7 +166,7 @@ The CLI uploads only the selected `composeFiles` and explicitly declared nonsecr `composeFiles` may name explicit absolute workstation paths, allowing a central deployment YAML to use the app repository's own Compose files. They are read directly and staged under their basenames before normalization; merge order is preserved, and basename collisions with other sources or assets fail before upload. Symlinks and symlinked source ancestors are rejected. Assets remain deploy-relative, including relative file references from external Compose files; credentials and persistent mounts still use absolute host paths. -Bundle paths must be deploy-relative and contain no dot components or dotfiles. `.env`, `.compoza.json` and `.deploy-lock` are reserved. Symlinked local files and parent directories are rejected. The generated `compose.yaml` cannot also be an asset. +Bundle paths must be deploy-relative and contain no dot components or dotfiles. `.env`, `.compoza.json`, `.deploy-lock` and `compoza.roles.yaml` are reserved. Symlinked local files and parent directories are rejected. The generated `compose.yaml` cannot also be an asset. All relative file references resolve from the stack root, including when a selected source Compose file lives in a subdirectory. Relative config files and bind mounts must be declared assets; relative binds must be read-only. Persistent data mounts use absolute host paths. `include` and external `extends.file` are unsupported: select explicit Compose overlays instead. @@ -175,7 +176,7 @@ Service `env_file` and `secrets.file` must use absolute host paths, with optiona Compose runs with a clean shell environment, retaining only home, path and Docker connection settings. Arbitrary SSH-session values cannot override application settings. -- Compose interpolation reads only the private generated `.env`. It supplies configured `environment`, selected managed secrets and the image references for this deployment. +- Compose interpolation reads only the private generated `.env`. It supplies configured `environment` and selected managed secrets. Image references are not variables: [roles](configuration.md#roles) set each built service's `image` in the generated roles override. - Host-only values reach containers through absolute service `env_file:` paths in Compose or the host overlay. Compose reads those files on the host when it creates the containers; a service's `environment` entries take precedence over them. An explicit `secrets.none` writes no managed credentials. The CLI never adopts an operator `.env` in the stack root. Managed values are encoded literally, never evaluated as shell code, and stay out of process arguments and lifecycle metadata. @@ -184,6 +185,8 @@ An explicit `secrets.none` writes no managed credentials. The CLI never adopts a The host normalizes selected Compose files with `--no-interpolate --no-env-resolution --no-path-resolution`. This preserves variable expressions, raw service env-file declarations and relative asset paths. The operator overlay remains separate and live. +With [roles](configuration.md#how-roles-reach-services), the generated `compoza.roles.yaml` is normalized last, so its `image` and `environment` entries end up in the flat `compose.yaml`. It holds `${NAME}` references for role secrets, never their values, and is not installed as a separate stack file. + Before replacing the deployed files or running `up`, Compose validates the normalized file with the selected env and overlay. A private JSON model check rejects undeclared relative files, unsafe writable asset binds and relative secret/env paths. Resolved Compose configuration and third-party diagnostics are suppressed because they can contain secrets; no validation log is left in the stack. Managed status projects only service/state/health/exit-code fields and redacts selected dotenv values. Requested application logs can still contain secrets; applications are responsible for their logging policy. @@ -211,7 +214,7 @@ compoza deploy infra --destination borealis --tag compoza status infra --destination borealis ``` -`--tag` skips Git and builds and reuses available images. It uses the current local Compose/configuration and resolves the currently selected credential source again. There is no release rollback command or saved configuration history. Check out an earlier application revision yourself when its configuration is also required, and review database changes before redeploying. +`--tag` skips Git and builds and reuses available images; pass `--tag =` to name a tag per role (see [Images and tags](configuration.md#images-and-tags)). It uses the current local Compose/configuration and resolves the currently selected credential source again. There is no release rollback command or saved configuration history. Check out an earlier application revision yourself when its configuration is also required, and review database changes before redeploying. ## Lifecycle on SSH execution @@ -227,4 +230,4 @@ Normal SSH termination releases the stack lock. After power loss or SIGKILL, ins ## Third-party-only stacks -Use `targets: []` for stacks that run published images. Git, a Dockerfile, a build context and application-registry build authentication are unnecessary. Compose still pulls missing third-party images; the same stack validation and readiness rules apply. The [README quickstart](../README.md#quickstart) is a complete example. +Omit `roles`, or give roles no `build`, for stacks that run published images. Git, a Dockerfile, a build context and application-registry build authentication are unnecessary. Compose still pulls missing third-party images; the same stack validation and readiness rules apply. The [README quickstart](../README.md#quickstart) is a complete example. diff --git a/scripts/package-smoke.mjs b/scripts/package-smoke.mjs index b8bd3aa..6e7d93c 100644 --- a/scripts/package-smoke.mjs +++ b/scripts/package-smoke.mjs @@ -181,7 +181,10 @@ writeFileSync( `name: external host: fixture workingDirectory: ../app source -targets: [web] +roles: + web: + build: + target: web composeFiles: [compose.yaml] `, ); @@ -204,14 +207,14 @@ const flat = deploymentScript( config: { name: 'fixture', host: 'fixture', - targets: [], + roles: {}, composeFiles: ['compose.yaml'], execution: 'ssh', remote: { stackDir: '/srv/fixture-stack' }, }, }, { ssh: 'fixture', platform: 'linux/arm64', transport: { mode: 'ssh-load' } }, - 'known', + {}, [{ path: 'compose.yaml', encoded: Buffer.from('services: {}').toString('base64') }], ); assert.match(flat.script, /normalized\.yaml/); diff --git a/src/bundle.test.ts b/src/bundle.test.ts index fd0cf7a..c730310 100644 --- a/src/bundle.test.ts +++ b/src/bundle.test.ts @@ -24,7 +24,6 @@ function withBundle( config: { name: 'app', host: 'host', - targets: [], composeFiles: ['compose.yaml'], assets: ['settings.txt'], }, @@ -260,6 +259,27 @@ describe('bounded Compose bundle', () => { ), ); + it.effect('reserves the roles override name for the generated file', () => + withBundle((app) => + Effect.gen(function* () { + yield* Effect.promise(() => writeFile(join(app.dir, 'compoza.roles.yaml'), 'services: {}')); + yield* Effect.promise(() => writeFile(join(app.dir, 'compose.yaml'), 'services: {}')); + + const asset = yield* readBundle({ + ...app, + config: { ...app.config, assets: ['compoza.roles.yaml'] }, + }).pipe(Effect.flip); + assert.include(asset.message, 'compoza.roles.yaml is reserved for the generated roles'); + + const compose = yield* readBundle({ + ...app, + config: { ...app.config, composeFiles: ['compose.yaml', 'compoza.roles.yaml'] }, + }).pipe(Effect.flip); + assert.include(compose.message, 'reserved'); + }), + ), + ); + it.effect('rejects symlinked deploy folders and non-file assets as typed failures', () => withBundle((app) => Effect.gen(function* () { @@ -352,7 +372,7 @@ describe('bounded Compose bundle', () => { const fs = FileSystem.makeNoop({ lstat: () => Effect.die(bug) }); const app: App = { dir: '/unused', - config: { name: 'app', host: 'host', targets: [], composeFiles: ['compose.yaml'] }, + config: { name: 'app', host: 'host', composeFiles: ['compose.yaml'] }, }; const filesystem = yield* readBundle(app).pipe( Effect.provideService(FileSystem.FileSystem, fs), diff --git a/src/bundle.ts b/src/bundle.ts index 918accd..33ec891 100644 --- a/src/bundle.ts +++ b/src/bundle.ts @@ -2,7 +2,7 @@ import type { PlatformError } from 'effect/PlatformError'; import { Effect, FileSystem } from 'effect'; import { basename, isAbsolute, join, parse, resolve } from 'node:path'; -import { parseDocument } from 'yaml'; +import { type Document, parseDocument } from 'yaml'; import type { App } from './config/deploy.ts'; @@ -63,6 +63,17 @@ const LINE_BREAK_OR_NUL = /[\r\n\0]/; */ const LEADING_DOT_SLASH = /^\.\//; +/** + * The upload name reserved for the generated roles override, which the SSH transaction passes to + * Compose last. Compose files and assets cannot be uploaded under it. + */ +export const ROLES_OVERRIDE_FILE = 'compoza.roles.yaml'; + +/** + * The error for a Compose file or asset uploaded under `ROLES_OVERRIDE_FILE`. + */ +export const ROLES_OVERRIDE_RESERVED = `${ROLES_OVERRIDE_FILE} is reserved for the generated roles override`; + /** * Keeps explicit external Compose sources inside the upload folder without exposing workstation * paths to the host. @@ -214,7 +225,7 @@ function isRelativeBind(source: unknown): source is string { /** * Narrows parsed YAML mappings before inspecting their fields. */ -function isRecord(value: unknown): value is Record { +export function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } @@ -226,13 +237,29 @@ function assetPath(reference: string): string { } /** - * Parses one Compose file into a plain mapping, reporting syntax errors, excessive alias expansion - * and non-mapping documents without exposing the file's contents. + * One parsed Compose file: the YAML document, which keeps custom tags such as `!reset`, and its + * plain mapping. */ -function parseComposeModel( +export interface ComposeFile { + /** + * The parsed document, for readers that need node tags. + */ + readonly document: Document.Parsed; + + /** + * The document as a plain mapping, with merge keys and aliases expanded. + */ + readonly model: Record; +} + +/** + * Parses one Compose file, reporting syntax errors, excessive alias expansion and non-mapping + * documents without exposing the file's contents. + */ +export function parseComposeFile( text: string, file: string, -): Effect.Effect, DeployError> { +): Effect.Effect { return Effect.gen(function* () { const document = yield* Effect.try({ try: () => parseDocument(text, { merge: true, prettyErrors: false }), @@ -250,10 +277,20 @@ function parseComposeModel( return yield* composeFileError(file, 'Compose document must be a mapping'); } - return model; + return { document, model }; }); } +/** + * Parses one Compose file into a plain mapping; see `parseComposeFile`. + */ +export function parseComposeModel( + text: string, + file: string, +): Effect.Effect, DeployError> { + return parseComposeFile(text, file).pipe(Effect.map(({ model }) => model)); +} + /** * Walks Compose file references, keeping production secrets outside the explicit asset bundle. * @@ -464,6 +501,10 @@ function declaredUploadPaths( ): Effect.Effect, DeployError> { return Effect.gen(function* () { const staged = paths.map((path) => uploadPath(path, compose)); + if (staged.includes(ROLES_OVERRIDE_FILE)) { + return yield* new DeployError({ message: ROLES_OVERRIDE_RESERVED }); + } + const declared = new Set(yield* Effect.forEach(staged, bundlePath)); if (declared.size !== paths.length) { return yield* new DeployError({ diff --git a/src/checks.test.ts b/src/checks.test.ts index 6a57f2f..800ba91 100644 --- a/src/checks.test.ts +++ b/src/checks.test.ts @@ -13,7 +13,7 @@ const app: App = { config: { name: 'demo', host: 'atlas', - targets: ['web'], + roles: { web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml'], ingress: { caddy: { domain: 'demo.example.com', upstream: 'demo-web:80' } }, }, diff --git a/src/checks.ts b/src/checks.ts index 3beb78b..5506bd6 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -7,6 +7,7 @@ import { type HostConfig, registryOf } from './config/hosts.ts'; import { type DeployError, tryPlan } from './errors.ts'; import { ingressFor, tryIngressFor } from './extensions/ingress.ts'; import { parseCheck, registryProbe } from './probe.ts'; +import { buildingRoles } from './roles.ts'; import { Remote } from './services/Remote.ts'; import { sshReadinessScript } from './ssh-plan.ts'; @@ -163,7 +164,7 @@ export const runChecks = Effect.fn('runChecks')(function* ( * pushes nothing. */ function requiredRegistry(app: App, host: HostConfig): string | undefined { - return app.config.targets.length === 0 ? undefined : registryOf(host); + return buildingRoles(app.config).length === 0 ? undefined : registryOf(host); } /** diff --git a/src/cli.test.ts b/src/cli.test.ts index b9f569e..36c3983 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -155,6 +155,80 @@ describe('compoza hosts list', { timeout: CLI_TEST_TIMEOUT_MS }, () => { }); }); +describe('compoza roles', () => { + /** + * Writes a deploy folder with two building roles and a hosts file, and returns the folder and the + * environment that selects the hosts file. + */ + function rolesFixture(directory: string, extra = '') { + writeFileSync( + join(directory, 'hosts.yaml'), + 'host:\n ssh: deploy@host.example.com\n platform: linux/amd64\n' + + ' transport:\n mode: ssh-load\n', + ); + writeFileSync( + join(directory, 'compose.yaml'), + 'services: {web: {image: a}, worker: {image: b}}', + ); + writeFileSync( + join(directory, 'deploy.yaml'), + 'name: app\nhost: host\ncomposeFiles: [compose.yaml]\nroles:\n' + + ' web:\n build: {target: server}\n' + + ' worker:\n build: {}\n environment: {MODE: fast}\n' + + extra, + ); + return { COMPOZA_HOSTS: join(directory, 'hosts.yaml') }; + } + + it('reuses images with a default tag and a per-role tag from the repeatable --tag flag', () => { + using temporary = mkdtempDisposableSync(join(tmpdir(), 'compoza-roles-')); + const env = rolesFixture(temporary.path); + const result = runCli( + ['deploy', temporary.path, '--local', '--dry-run', '--tag', 'abc', '--tag', 'worker=def'], + env, + ); + assert.strictEqual(result.status, 0, result.stderr); + assert.include(result.stdout, 'Deploying app web=abc worker=def to local Docker'); + assert.include(result.stdout, 'image: "app-web:abc"'); + assert.include(result.stdout, 'image: "app-worker:def"'); + assert.include(result.stdout, 'MODE: "fast"'); + }); + + it('rejects --tag values that name an unknown role or leave a role untagged', () => { + using temporary = mkdtempDisposableSync(join(tmpdir(), 'compoza-roles-')); + const env = rolesFixture(temporary.path); + const unknown = runCli( + ['deploy', temporary.path, '--local', '--dry-run', '--tag', 'ghost=abc'], + env, + ); + assert.notStrictEqual(unknown.status, 0); + assert.include(unknown.stderr, 'Unknown role "ghost" in --tag; roles: web, worker'); + + const untagged = runCli( + ['deploy', temporary.path, '--local', '--dry-run', '--tag', 'web=abc'], + env, + ); + assert.notStrictEqual(untagged.status, 0); + assert.include(untagged.stderr, 'No tag for role "worker"'); + }); + + it('reports the roles check in compoza check', () => { + using temporary = mkdtempDisposableSync(join(tmpdir(), 'compoza-roles-')); + const env = rolesFixture(temporary.path); + const ok = runCli(['check', temporary.path, '--local'], env); + assert.strictEqual(ok.status, 0, ok.stderr); + assert.include(ok.stdout, 'ok roles: Roles match the Compose files'); + + const bad = rolesFixture(temporary.path, ' wbe:\n environment: {A: b}\n'); + const failed = runCli(['check', temporary.path, '--local'], bad); + assert.strictEqual(failed.status, 2, failed.stderr); + assert.include( + failed.stdout, + 'FAIL roles: Role "wbe" is not a service in the Compose files; services: web, worker', + ); + }); +}); + describe('compoza logs --follow', { timeout: CLI_TEST_TIMEOUT_MS }, () => { /** * Waits until `condition` holds, checking every 50 ms, and fails after `CLI_TIMEOUT_MS`. @@ -199,7 +273,7 @@ describe('compoza logs --follow', { timeout: CLI_TEST_TIMEOUT_MS }, () => { ); writeFileSync( join(dir, 'deploy.yaml'), - 'name: app\nhost: host\ntargets: []\nexecution: ssh\ncomposeFiles: [compose.yaml]\n' + + 'name: app\nhost: host\nexecution: ssh\ncomposeFiles: [compose.yaml]\n' + 'remote:\n stackDir: /srv/app\n', ); diff --git a/src/cli.ts b/src/cli.ts index 3033e35..c89d417 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,6 +6,7 @@ import type { Resolved } from './deploy.ts'; import { blocking, type Check, formatChecks, runChecks } from './checks.ts'; import { command } from './command.ts'; +import { rolesOf } from './config/deploy.ts'; import { hostLinks, linkHosts, unlinkHosts } from './config/host-links.ts'; import { hostNamed, @@ -30,6 +31,7 @@ import { closestNames, helpText, IgnoredSharedFlags } from './help.ts'; import { setupOperationInput } from './ingress-setup.ts'; import { lifecycle, projectArgs, remove, requireProjectService } from './lifecycle.ts'; import { publish } from './publish.ts'; +import { validateRoles } from './roles.ts'; import { Remote } from './services/Remote.ts'; import { DryRun, Shell } from './services/Shell.ts'; import { type InspectOptions, inspectSsh } from './ssh-deploy.ts'; @@ -187,6 +189,24 @@ function secretsCheck(app: App, local: boolean) { ); } +/** + * The `roles` check: whether each role's settings are valid and its service exists in the selected + * Compose files without setting the same variables itself. Reads only local files. + */ +function rolesCheck(app: App, local: boolean) { + return validateRoles(app, { local }).pipe( + Effect.map((): Check => ({ + name: 'roles', + ok: true, + detail: 'Roles match the Compose files', + fix: 'manual', + })), + Effect.catchTag('DeployError', (error): Effect.Effect => + Effect.succeed({ name: 'roles', ok: false, detail: error.message, fix: 'manual' }), + ), + ); +} + /** * `check`: the read-only readiness report, ending with a one-line count of failing checks. Exit * code 2 when a check other than the route fails. @@ -201,7 +221,8 @@ const check = Command.make( ), local: switchFlag( 'local', - 'Skip the remote checks; with --secrets, check the local.secrets source instead', + 'Skip the remote checks and check local.roles and local.composeFiles; ' + + 'with --secrets, check the local.secrets source instead', ), json: switchFlag('json', 'Print { app, host, checks } as JSON instead of one line per check'), }, @@ -209,6 +230,10 @@ const check = Command.make( const { app, host } = yield* resolveFolder(folder); const checks: Array = [...(local ? [] : yield* runChecks(app, host))]; + if (Object.keys(rolesOf(app.config, { local })).length > 0) { + checks.push(yield* rolesCheck(app, local)); + } + if (secrets) { checks.push(yield* secretsCheck(app, local)); } @@ -266,8 +291,11 @@ const deployCommand = Command.make( 'Build a checkout with uncommitted changes, tagging the images -dirty', ), tag: Flag.String('tag').pipe( - Flag.withDescription('Reuse the images built earlier with this tag instead of building'), - Flag.optional, + Flag.withDescription( + 'Reuse images built earlier instead of building: for every built role, ' + + '= for one; repeatable', + ), + Flag.atLeast(0), ), }, Effect.fn(function* ({ folder, dryRun, local, allowDirty, tag }) { @@ -277,7 +305,7 @@ const deployCommand = Command.make( dryRun, local, allowDirty, - tag, + tags: tag, hosts, destination: Option.getOrUndefined(destination), }).pipe(Effect.provideService(DryRun, dryRun)); @@ -291,8 +319,10 @@ const deployCommand = Command.make( '--wait, the route, and a wait until the public URL answers. A blocking check failure ' + 'stops the deploy before anything changes.', 'Images are tagged with the first twelve characters of the commit, so a build needs a ' + - 'clean checkout unless --allow-dirty is given. --tag reuses earlier images instead of ' + - 'building, with the current local configuration. --local runs the stack in this ' + + 'clean checkout unless --allow-dirty is given. Each role with a build is tagged with ' + + 'the commit of its own repository. --tag reuses earlier images instead of building, ' + + 'with the current local configuration: --tag names the tag of every built role ' + + 'and --tag = overrides it for one role (repeat the flag as needed). --local runs the stack in this ' + "machine's Docker with the local overrides, without remote checks, routes or wait.", 'A failed deploy is not rolled back: inspect "compoza status" and "compoza logs", then ' + 'deploy a known tag.', @@ -312,6 +342,10 @@ const deployCommand = Command.make( command: 'compoza deploy infra --tag 1a2b3c4d5e6f', description: 'Redeploy earlier images with the current local config', }, + { + command: 'compoza deploy infra --tag 1a2b3c4d5e6f --tag worker=9f8e7d6c5b4a', + description: 'Redeploy earlier images, with another tag for the worker role', + }, ]), ); diff --git a/src/client-destination.test.ts b/src/client-destination.test.ts index 9c463f9..a4ce049 100644 --- a/src/client-destination.test.ts +++ b/src/client-destination.test.ts @@ -118,7 +118,6 @@ it.effect( YAML.stringify({ name: 'app', host: 'host', - targets: [], execution: 'client', composeFiles: ['compose.yaml'], ingress: { caddy }, @@ -135,7 +134,7 @@ it.effect( dryRun: true, local: false, allowDirty: false, - tag: Option.none(), + tags: [], destination: 'dev', }); diff --git a/src/compose-environment.ts b/src/compose-environment.ts new file mode 100644 index 0000000..abb9680 --- /dev/null +++ b/src/compose-environment.ts @@ -0,0 +1,404 @@ +import { Effect, FileSystem } from 'effect'; +import { dirname, relative, resolve } from 'node:path'; +import { type Document, isAlias, isMap, isScalar, isSeq } from 'yaml'; + +import type { App, RunOptions } from './config/deploy.ts'; + +import { type ComposeFile, parseComposeFile } from './bundle.ts'; +import { composeFilesOf } from './config/deploy.ts'; +import { DeployError } from './errors.ts'; + +/** + * The YAML tag that removes a value, or a whole `environment:`, set by an earlier file. + */ +const RESET_TAG = '!reset'; + +/** + * The YAML tag that replaces a value set by an earlier file instead of merging into it. + */ +const OVERRIDE_TAG = '!override'; + +/** + * How deep merge keys and aliases are followed before a mapping is treated as empty, which only a + * cyclic document can reach. + */ +const MAX_MERGE_DEPTH = 32; + +/** + * The variable names every service sets in its `environment:` after Compose merges the selected + * files, each with the file that last set it. + */ +export type ComposeEnvironment = ReadonlyMap>; + +/** + * What one service of one file does to the `environment:` accumulated from earlier files. + */ +interface Contribution { + /** + * Whether earlier files' variables are dropped first (`!reset` or `!override`). + */ + readonly replace: boolean; + + /** + * The variables set, each with the file that sets it. + */ + readonly keys: ReadonlyMap; + + /** + * The variables removed with a per-key `!reset`. + */ + readonly removed: ReadonlySet; +} + +/** + * A Compose file read for this check, with the name used in messages. + */ +interface Loaded { + /** + * The absolute path, which also keys the cache and detects cycles. + */ + readonly path: string; + + /** + * The name shown in messages: as written in `composeFiles`, else relative to the deploy folder. + */ + readonly name: string; + + /** + * The parsed file. + */ + readonly file: ComposeFile; +} + +/** + * Reads the selected Compose files the way Compose merges them and returns, for every service, the + * variables it sets in `environment:`. + * + * Later files merge into earlier ones; `environment: !reset` clears what came before, `environment: + * !override` replaces it, and a `!reset` value removes one variable. A service's `extends` (same + * file, or another file relative to the declaring file) contributes its variables first. Under + * client execution `include:` entries are read relative to the including file, and the services + * they declare count; SSH execution rejects `include:` elsewhere, so it is ignored here unless the + * run is local, because a local run always uses client-side Compose. Service fields are read + * through `<<` merge keys. A file named by `extends` or `include` that cannot be read is skipped, + * because Compose reports it. Fails when a selected file cannot be read or parsed. + */ +export const readComposeEnvironment = Effect.fn('readComposeEnvironment')(function* ( + app: App, + options: RunOptions, +): Effect.fn.Return { + const fs = yield* FileSystem.FileSystem; + const cache = new Map(); + const follow = app.config.execution !== 'ssh' || options.local === true; + + const load = Effect.fn('loadComposeFile')(function* ( + path: string, + name: string, + required: boolean, + ): Effect.fn.Return { + const known = cache.get(path); + if (known !== undefined || (!required && cache.has(path))) { + return known; + } + + const text = yield* fs + .readFileString(path) + .pipe(Effect.orElseSucceed((): string | undefined => undefined)); + if (text === undefined && required) { + return yield* new DeployError({ message: `Cannot read Compose file ${name}` }); + } + + const loaded = + text === undefined ? undefined : { path, name, file: yield* parseComposeFile(text, name) }; + cache.set(path, loaded); + + return loaded; + }); + + const contributionOf = Effect.fn('contributionOf')(function* ( + loaded: Loaded, + node: unknown, + visiting: ReadonlySet, + ): Effect.fn.Return { + const service = resolveNode(node, loaded.file.document); + if (!isMap(service)) { + return emptyContribution(); + } + + const fields = mappingEntries(service, loaded.file.document); + const own = environmentContribution(fields.get('environment'), loaded); + const base = yield* inheritedContribution(loaded, fields.get('extends'), visiting); + return combine(base, own); + }); + + const inheritedContribution = Effect.fn('inheritedContribution')(function* ( + loaded: Loaded, + extension: unknown, + visiting: ReadonlySet, + ): Effect.fn.Return { + const reference = extendsReference(extension, loaded.file.document); + if (reference === undefined) { + return emptyContribution(); + } + + const target = + reference.file === undefined + ? loaded + : yield* load( + resolve(dirname(loaded.path), reference.file), + relative(app.dir, resolve(dirname(loaded.path), reference.file)), + false, + ); + const key = `${target?.path}#${reference.service}`; + const node = target === undefined ? undefined : servicesOf(target).get(reference.service); + if (target === undefined || node === undefined || visiting.has(key)) { + return emptyContribution(); + } + + return yield* contributionOf(target, node, new Set(visiting).add(key)); + }); + + const apply = Effect.fn('applyFile')(function* ( + loaded: Loaded, + services: Map>, + inProgress: ReadonlySet, + ): Effect.fn.Return { + const path = new Set(inProgress).add(loaded.path); + if (follow) { + for (const include of includedFiles(loaded)) { + if (path.has(include)) { + continue; + } + + const included = yield* load(include, relative(app.dir, include), false); + if (included !== undefined) { + yield* apply(included, services, path); + } + } + } + + for (const [name, node] of servicesOf(loaded)) { + if (tagOf(node) === RESET_TAG) { + services.delete(name); + continue; + } + + const contribution = yield* contributionOf(loaded, node, new Set([`${loaded.path}#${name}`])); + const base = tagOf(node) === OVERRIDE_TAG ? undefined : services.get(name); + services.set(name, merge(base, contribution)); + } + }); + + const services = new Map>(); + for (const name of composeFilesOf(app.config, options)) { + const loaded = yield* load(resolve(app.dir, name), name, true); + if (loaded !== undefined) { + yield* apply(loaded, services, new Set()); + } + } + + return services; +}); + +/** + * A contribution that sets and removes nothing. + */ +function emptyContribution(): Contribution { + return { replace: false, keys: new Map(), removed: new Set() }; +} + +/** + * The services declared by one file, by name, with their YAML nodes. + */ +function servicesOf(loaded: Loaded): ReadonlyMap { + return mappingEntries(loaded.file.document.get('services', true), loaded.file.document); +} + +/** + * The absolute paths named by a file's top-level `include:`, each written as a string or as a + * mapping whose `path` is a string or a list of strings, relative to the including file. + */ +function includedFiles(loaded: Loaded): ReadonlyArray { + const document = loaded.file.document; + const include = resolveNode(document.get('include', true), document); + if (!isSeq(include)) { + return []; + } + + const paths = include.items.flatMap((item) => { + const entry = resolveNode(item, document); + if (isScalar(entry)) { + return [entry.value]; + } + + const path = resolveNode(mappingEntries(entry, document).get('path'), document); + return isSeq(path) ? path.items.map((part) => resolveNode(part, document)) : [path]; + }); + return paths + .map((path) => (isScalar(path) ? path.value : path)) + .filter((path): path is string => typeof path === 'string') + .map((path) => resolve(dirname(loaded.path), path)); +} + +/** + * The service, and optionally the file, a service's `extends` names, written as a string or a + * mapping. `undefined` when it is absent or malformed. + */ +function extendsReference( + node: unknown, + document: Document.Parsed, +): { readonly service: string; readonly file: string | undefined } | undefined { + const extension = resolveNode(node, document); + if (isScalar(extension)) { + return scalarString(extension) === undefined + ? undefined + : { service: String(extension.value), file: undefined }; + } + + const fields = mappingEntries(extension, document); + const service = scalarString(resolveNode(fields.get('service'), document)); + const file = scalarString(resolveNode(fields.get('file'), document)); + return service === undefined ? undefined : { service, file }; +} + +/** + * The string value of a scalar node, else `undefined`. + */ +function scalarString(node: unknown): string | undefined { + return isScalar(node) && typeof node.value === 'string' ? node.value : undefined; +} + +/** + * What an `environment:` node does: its variables, written as a mapping or a list of `KEY=value` + * and `KEY` entries, and whether its tag clears earlier values. A whole-node `!reset` discards its + * payload as well, so it sets no variables; `!override` keeps its payload as the replacement. + */ +function environmentContribution(node: unknown, loaded: Loaded): Contribution { + const document = loaded.file.document; + const environment = resolveNode(node, document); + const keys = new Map(); + const removed = new Set(); + const tag = tagOf(environment); + if (tag === RESET_TAG) { + return { replace: true, keys, removed }; + } + + if (isMap(environment)) { + for (const [key, value] of mappingEntries(environment, document)) { + if (tagOf(resolveNode(value, document)) === RESET_TAG) { + removed.add(key); + } else { + keys.set(key, loaded.name); + } + } + } else if (isSeq(environment)) { + for (const item of environment.items) { + const entry = scalarString(resolveNode(item, document)); + if (entry !== undefined) { + keys.set(entry.split('=')[0] ?? entry, loaded.name); + } + } + } + + return { replace: tag === OVERRIDE_TAG, keys, removed }; +} + +/** + * The contribution of a service that inherits `base` through `extends` and sets `own`. + */ +function combine(base: Contribution, own: Contribution): Contribution { + if (own.replace) { + return own; + } + + const keys = new Map(base.keys); + for (const key of own.removed) { + keys.delete(key); + } + + for (const [key, file] of own.keys) { + keys.set(key, file); + } + + const removed = new Set([...base.removed, ...own.removed]); + for (const key of own.keys.keys()) { + removed.delete(key); + } + + return { replace: base.replace, keys, removed }; +} + +/** + * The variables of a service after a later file's `contribution` is merged into `base`. + */ +function merge( + base: ReadonlyMap | undefined, + contribution: Contribution, +): Map { + const next = new Map(contribution.replace ? undefined : base); + for (const key of contribution.removed) { + next.delete(key); + } + + for (const [key, file] of contribution.keys) { + next.set(key, file); + } + + return next; +} + +/** + * The node an alias points at, or the node itself. + */ +function resolveNode(node: unknown, document: Document.Parsed): unknown { + return isAlias(node) ? node.resolve(document) : node; +} + +/** + * The explicit YAML tag of a node, such as `!reset`, or `undefined` for an untagged one. + */ +function tagOf(node: unknown): string | undefined { + return isScalar(node) || isSeq(node) || isMap(node) ? node.tag : undefined; +} + +/** + * The entries of a mapping node by key, following aliases and `<<` merge keys. Explicit keys win + * over merged ones, and earlier merge sources win over later ones. Anything that is not a mapping + * has no entries. + */ +function mappingEntries( + node: unknown, + document: Document.Parsed, + depth = 0, +): ReadonlyMap { + const mapping = resolveNode(node, document); + const entries = new Map(); + if (!isMap(mapping) || depth > MAX_MERGE_DEPTH) { + return entries; + } + + const inherited = new Map(); + for (const pair of mapping.items) { + const key: unknown = isScalar(pair.key) ? pair.key.value : pair.key; + if (typeof key === 'symbol') { + const sources = isSeq(pair.value) ? pair.value.items : [pair.value]; + for (const source of sources) { + for (const [name, value] of mappingEntries(source, document, depth + 1)) { + if (!inherited.has(name)) { + inherited.set(name, value); + } + } + } + } else if (typeof key === 'string') { + entries.set(key, pair.value); + } + } + + for (const [name, value] of inherited) { + if (!entries.has(name)) { + entries.set(name, value); + } + } + + return entries; +} diff --git a/src/config/config-file.ts b/src/config/config-file.ts index 8176da0..de55bb5 100644 --- a/src/config/config-file.ts +++ b/src/config/config-file.ts @@ -25,6 +25,13 @@ interface ReadConfigOptions { * details, which can quote the file's contents. `false` when left out. */ readonly privateInput?: boolean; + + /** + * Top-level keys that no longer exist, each with the message that replaces the schema's generic + * unknown-field error. The message is shown even for private input, because it quotes no + * contents. + */ + readonly removedKeys?: Readonly>; } /** @@ -44,7 +51,7 @@ export const readConfigFile = Effect.fn('readConfigFile')(function* ( file: string, schema: Schema.Decoder, missingMessage: string, - { privateInput = false }: ReadConfigOptions = {}, + { privateInput = false, removedKeys = {} }: ReadConfigOptions = {}, ): Effect.fn.Return { if (!(yield* fileExists(file))) { return yield* new DeployError({ message: missingMessage }); @@ -58,6 +65,11 @@ export const readConfigFile = Effect.fn('readConfigFile')(function* ( try: () => parseConfig(text), catch: (cause) => invalidYaml(file, privateInput ? undefined : String(cause)), }); + const removed = removedKeyMessage(value, removedKeys); + if (removed !== undefined) { + return yield* invalidConfiguration(file, removed); + } + return yield* Schema.decodeUnknownEffect(schema, { onExcessProperty: 'error' })(value).pipe( Effect.mapError((error) => invalidConfiguration(file, privateInput ? PRIVATE_DECODING_HINT : error.message), @@ -104,6 +116,22 @@ export function parseConfigDocument(text: string): Document { return document; } +/** + * The replacement message for the first removed key that `value` still has at its top level, or + * `undefined` when it has none. + */ +function removedKeyMessage( + value: unknown, + removedKeys: Readonly>, +): string | undefined { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + return undefined; + } + + const key = Object.keys(removedKeys).find((name) => Object.hasOwn(value, name)); + return key === undefined ? undefined : removedKeys[key]; +} + /** * The failure for a file that is not valid YAML, with the parser's `detail` appended when there is * one to show. diff --git a/src/config/deploy.test.ts b/src/config/deploy.test.ts index c76ab4f..f1dd550 100644 --- a/src/config/deploy.test.ts +++ b/src/config/deploy.test.ts @@ -6,6 +6,7 @@ import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import * as YAML from 'yaml'; +import { buildingRoles } from '../roles.ts'; import { composeFilesOf, DeployConfig, @@ -13,6 +14,8 @@ import { environmentOf, ingressOf, loadApp, + rolesOf, + selectDestination, workingDirectoryOf, } from './deploy.ts'; @@ -33,7 +36,10 @@ function withTempDir( const valid = { name: 'demo', host: 'atlas', - targets: ['builder', 'web'], + roles: { + builder: { build: { target: 'builder' } }, + web: { build: { target: 'web' }, environment: { MODE: 'a' }, secrets: ['TOKEN'] }, + }, composeFiles: ['compose.yaml', 'compose.production.yaml'], ingress: { caddy: { domain: 'demo.example.com', upstream: 'demo-web:80', healthPath: '/healthz' }, @@ -137,7 +143,10 @@ describe('DeployConfig', () => { { ...valid, ingress: { caddy: { ...caddy, healthPath: '/health z' } } }, ], ['directories', { ...valid, directories: ['relative/path'] }], - ['targets', { ...valid, targets: ['Invalid'] }], + ['role build target', { ...valid, roles: { web: { build: { target: 'Bad' } } } }], + ['role environment value', { ...valid, roles: { web: { environment: { A: 1 } } } }], + ['role secrets', { ...valid, roles: { web: { secrets: 'TOKEN' } } }], + ['role workingDirectory', { ...valid, roles: { web: { build: { workingDirectory: ' ' } } } }], ['environment', { ...valid, environment: { DEMO_PORT: 8090 } }], ])('rejects an invalid %s', ([, input]) => Effect.gen(function* () { @@ -147,6 +156,156 @@ describe('DeployConfig', () => { ); }); +describe('roles', () => { + const decode = (input: unknown) => Schema.decodeUnknownEffect(DeployConfig)(input); + + it.effect.each([ + ['an invalid role name', { Web: {} }], + ['a role name ending in a separator', { web_: {} }], + ['a role name mixing separators', { 'a_-b': {} }], + ['a role name starting with a separator', { '-web': {} }], + ['an unknown role field', { web: { ports: [] } }], + ['an unknown build field', { web: { build: { image: 'x' } } }], + ] as const)('rejects %s', ([, roles]) => + Effect.gen(function* () { + const decodeStrict = Schema.decodeUnknownEffect(DeployConfig, { onExcessProperty: 'error' }); + const error = yield* decodeStrict({ ...valid, roles }).pipe(Effect.flip); + assert.strictEqual(error._tag, 'SchemaError'); + }), + ); + + it.effect.each([ + ['destination', { destinations: { dev: { roles: { web: { build: {} } } } } }], + ['local', { local: { roles: { web: { build: {} } } } }], + ] as const)('rejects a build in %s roles', ([, input]) => + Effect.gen(function* () { + const decodeStrict = Schema.decodeUnknownEffect(DeployConfig, { onExcessProperty: 'error' }); + const error = yield* decodeStrict({ ...valid, ...input }).pipe(Effect.flip); + assert.strictEqual(error._tag, 'SchemaError'); + }), + ); + + it.effect('accepts build fields, roles without a build and destination or local settings', () => + Effect.gen(function* () { + const config = yield* decode({ + ...valid, + workingDirectory: '../repo', + dockerfile: 'infra/Dockerfile', + context: '.', + roles: { + ...valid.roles, + worker: { + build: { + target: 'worker', + workingDirectory: '../other', + dockerfile: 'worker.Dockerfile', + context: 'src', + }, + }, + postgres: { environment: { POSTGRES_USER: 'demo' } }, + 'my_role-1': {}, + }, + destinations: { dev: { roles: { web: { environment: { MODE: 'b' }, secrets: [] } } } }, + local: { roles: { postgres: { secrets: ['PG_PASSWORD'] } } }, + }); + assert.deepStrictEqual(Object.keys(config.roles ?? {}), [ + 'builder', + 'web', + 'worker', + 'postgres', + 'my_role-1', + ]); + assert.strictEqual(config.roles?.worker?.build?.dockerfile, 'worker.Dockerfile'); + }), + ); + + it.effect('does not require roles', () => + Effect.gen(function* () { + const { roles: _roles, ...bare } = valid; + const config = yield* decode(bare); + assert.deepStrictEqual(rolesOf(config), {}); + }), + ); + + it.effect('rolls the working directory of a role over the top-level one', () => + Effect.gen(function* () { + const app = { + dir: '/deployment', + config: yield* decode({ + ...valid, + workingDirectory: '../repo', + roles: { web: { build: { workingDirectory: '~/other' } }, api: { build: {} } }, + }), + }; + assert.strictEqual(workingDirectoryOf(app), '/repo'); + assert.strictEqual(workingDirectoryOf(app, '~/other'), join(homedir(), 'other')); + }), + ); + + it.effect('merges destination roles over the app roles', () => + Effect.gen(function* () { + const config = yield* decode({ + ...valid, + destinations: { + dev: { + roles: { + web: { environment: { EXTRA: 'x' }, secrets: ['OTHER'] }, + builder: { environment: { ONLY: 'dest' } }, + sidecar: { environment: { SIDE: '1' } }, + }, + }, + }, + }); + const { config: selected } = selectDestination({ dir: '/d', config }, 'dev'); + assert.deepStrictEqual(selected.roles?.web, { + build: { target: 'web' }, + environment: { MODE: 'a', EXTRA: 'x' }, + secrets: ['OTHER'], + }); + assert.deepStrictEqual(selected.roles?.builder, { + build: { target: 'builder' }, + environment: { ONLY: 'dest' }, + }); + assert.deepStrictEqual(selected.roles?.sidecar, { environment: { SIDE: '1' } }); + assert.deepStrictEqual(buildingRoles(selected), ['builder', 'web']); + }), + ); + + it.effect('keeps the app roles when the destination sets none', () => + Effect.gen(function* () { + const config = yield* decode({ ...valid, destinations: { dev: { host: 'other' } } }); + const { config: selected } = selectDestination({ dir: '/d', config }, 'dev'); + assert.deepStrictEqual(selected.roles, config.roles); + }), + ); + + it.effect( + 'merges local roles over the roles of the run, with an empty list clearing secrets', + () => + Effect.gen(function* () { + const config = yield* decode({ + ...valid, + local: { + roles: { + web: { environment: { MODE: 'local' }, secrets: [] }, + sidecar: { secrets: ['SIDE_TOKEN'] }, + }, + }, + }); + assert.deepStrictEqual(rolesOf(config).web?.secrets, ['TOKEN']); + assert.deepStrictEqual(rolesOf(config, { local: true }).web, { + build: { target: 'web' }, + environment: { MODE: 'local' }, + secrets: [], + }); + assert.deepStrictEqual(rolesOf(config, { local: true }).sidecar, { + secrets: ['SIDE_TOKEN'], + }); + assert.deepStrictEqual(buildingRoles(config), ['builder', 'web']); + }), + ); +}); + describe('loadApp', () => { it.effect('reads a folder or the file itself and resolves the folder', () => withTempDir((dir) => @@ -191,6 +350,25 @@ describe('loadApp', () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.effect('fails with the dedicated message when targets is still configured', () => + withTempDir((dir) => + Effect.gen(function* () { + yield* Effect.promise(() => + writeFile(join(dir, 'deploy.yaml'), YAML.stringify({ ...valid, targets: ['web'] })), + ); + + const error = yield* loadApp(dir).pipe(Effect.flip); + assert.strictEqual(error._tag, 'DeployError'); + assert.strictEqual( + error.message, + `${join(dir, 'deploy.yaml')}: targets was replaced by roles: give each built Compose ` + + 'service a role with build.target, and remove image: ${_IMAGE} from Compose ' + + "files; compoza now sets each built service's image", + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.effect('discovers deploy.yml and prefers deploy.yaml when both exist', () => withTempDir((dir) => Effect.gen(function* () { diff --git a/src/config/deploy.ts b/src/config/deploy.ts index 3799792..0f3c102 100644 --- a/src/config/deploy.ts +++ b/src/config/deploy.ts @@ -11,7 +11,7 @@ import { } from '../extensions/config.ts'; import { fileExists, readConfigFile } from './config-file.ts'; import { expandHome } from './hosts.ts'; -import { AbsolutePath, Name, Target } from './schema.ts'; +import { AbsolutePath, Name, RoleName, Target } from './schema.ts'; export { HealthPath, Ingress, SecretsConfig } from '../extensions/config.ts'; export type { CaddyIngress, NoIngress } from '../extensions/config.ts'; @@ -32,6 +32,14 @@ const ALTERNATE_DEPLOY_FILE = 'deploy.yml'; */ export const DEFAULT_HEALTH_PATH = '/'; +/** + * The message for a `deploy.yaml` that still has `targets`, which `roles` replaced. + */ +const TARGETS_REMOVED_MESSAGE = + 'targets was replaced by roles: give each built Compose service a role with build.target, ' + + 'and remove image: ${_IMAGE} from Compose files; ' + + "compoza now sets each built service's image"; + /** * A target that already names a YAML file rather than a folder. */ @@ -59,6 +67,81 @@ const HEALTH_URL_PATTERN = /^https?:\/\/[^\s@]+$/; */ const DIRECTORY_PATH_PATTERN = /^(?=.*\S)[^\p{Cc}]+$/u; +/** + * A directory path that is not blank and holds no control characters, as `workingDirectory` takes. + */ +const DirectoryPath = Schema.String.pipe( + Schema.check( + Schema.isPattern(DIRECTORY_PATH_PATTERN, { + message: 'expected a nonempty directory path without control characters', + }), + ), +); + +/** + * How one role's image is built; each field left out falls back to the top-level field of the same + * name, then to its default. + */ +const RoleBuild = Schema.Struct({ + /** + * The Dockerfile stage to build; without it no `--target` flag is passed and the final stage is + * built. + */ + target: Schema.optionalKey(Target), + + /** + * The source directory for this role's Git checks and build, resolved like the top-level + * `workingDirectory`. + */ + workingDirectory: Schema.optionalKey(DirectoryPath), + + /** + * The Dockerfile, relative to the role's working directory; defaults to `Dockerfile` there. + */ + dockerfile: Schema.optionalKey(Schema.String), + + /** + * The build context, relative to the role's working directory; defaults to the root of the Git + * repository containing it. + */ + context: Schema.optionalKey(Schema.String), +}); + +/** + * What a destination or a local run may change about a role: its container environment and secrets. + * Destinations share the source, so they cannot add `build`. + */ +export const RoleSettings = Schema.Struct({ + /** + * Container environment of the service, merged over the role's `environment`. Values are literal. + */ + environment: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), + + /** + * Secret variable names from the stack's secret source, delivered to the service under the same + * names. Replaces the role's list. + */ + secrets: Schema.optionalKey(Schema.Array(Schema.String)), +}); + +/** + * The settings of one Compose service: what to build for it and which container environment and + * secrets it gets. + */ +export const Role = Schema.Struct({ + /** + * How to build the service's image; without it Compoza builds nothing for this service. + */ + build: Schema.optionalKey(RoleBuild), + + ...RoleSettings.fields, +}); + +/** + * Role settings by Compose service name. + */ +const RoleSettingsByName = Schema.Record(RoleName, RoleSettings); + /** * Settings that replace the deployment's for `--local` runs against this machine's Docker. */ @@ -78,6 +161,12 @@ export const LocalOverrides = Schema.Struct({ * Compose variables merged over `environment`, usually relative paths in place of host folders. */ environment: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), + + /** + * Role environment merged over the app's roles for a local run, and secrets replacing a role's + * list. + */ + roles: Schema.optionalKey(RoleSettingsByName), }); /** @@ -123,8 +212,8 @@ export const HealthConfig = Schema.Struct({ /** * A named deployment selection merged over the app defaults. * - * Destinations share source and image targets while selecting a host, Compose files and persistent - * host paths. + * Destinations share source and the roles' builds while selecting a host, Compose files and + * persistent host paths. */ export const Destination = Schema.Struct({ /** @@ -167,6 +256,11 @@ export const Destination = Schema.Struct({ */ environment: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), + /** + * Role environment merged over the app's roles, and secrets replacing a role's list. + */ + roles: Schema.optionalKey(RoleSettingsByName), + /** * The ingress replacing the app default, including `none` for private destinations. */ @@ -201,7 +295,7 @@ export const DeployConfig = Schema.Struct({ secrets: Schema.optionalKey(SecretsConfig), /** - * Compose project name and image name prefix (`-`). + * Compose project name and image name prefix (`-`). */ name: Name, @@ -211,36 +305,28 @@ export const DeployConfig = Schema.Struct({ host: Name, /** - * The source directory for Git and builds, relative to this file; absolute and `~/` paths are - * also accepted. Defaults to the deploy folder. Compose, assets, secrets and templates remain + * The default source directory for Git and builds, relative to this file; absolute and `~/` paths + * are also accepted. Defaults to the deploy folder. Compose, assets, secrets and templates remain * relative to this file. */ - workingDirectory: Schema.optionalKey( - Schema.String.pipe( - Schema.check( - Schema.isPattern(DIRECTORY_PATH_PATTERN, { - message: 'expected a nonempty directory path without control characters', - }), - ), - ), - ), + workingDirectory: Schema.optionalKey(DirectoryPath), /** - * The Dockerfile, relative to the working directory; defaults to `Dockerfile` there. + * The default Dockerfile, relative to the working directory; defaults to `Dockerfile` there. */ dockerfile: Schema.optionalKey(Schema.String), /** - * The build context, relative to the working directory; defaults to the root of the Git + * The default build context, relative to the working directory; defaults to the root of the Git * repository. */ context: Schema.optionalKey(Schema.String), /** - * Dockerfile targets to build; each becomes an `_IMAGE` Compose variable holding its - * image reference. + * Settings per Compose service. A role with `build` gets an image built by Compoza, and every + * role can set container environment and secrets for its service. */ - targets: Schema.Array(Target), + roles: Schema.optionalKey(Schema.Record(RoleName, Role)), /** * Compose files applied in order; relative paths use the deploy folder, and absolute paths select @@ -337,12 +423,22 @@ export interface App { readonly config: DeployConfig; } +/** + * The settings of one role after merging destinations and local runs. + */ +export type Role = typeof Role.Type; + +/** + * The container settings a destination or local run may set for a role. + */ +export type RoleSettings = typeof RoleSettings.Type; + /** * The source directory used by Git and builds, without requiring a checkout for source-free - * commands. + * commands. A role's own `workingDirectory` takes the place of the top-level one when given. */ -export function workingDirectoryOf(app: App): string { - return resolve(app.dir, expandHome(app.config.workingDirectory ?? '.')); +export function workingDirectoryOf(app: App, workingDirectory?: string): string { + return resolve(app.dir, expandHome(workingDirectory ?? app.config.workingDirectory ?? '.')); } /** @@ -399,6 +495,47 @@ export function environmentOf( return { ...config.environment, ...(local ? config.local?.environment : undefined) }; } +/** + * Merges role settings from a destination or local run over a base set of roles: per role, + * `environment` merges over the base's and `secrets` replaces the base's list when given. A role + * named only in `overrides` is kept as an environment-only role. + */ +export function mergeRoles( + base: Readonly> | undefined, + overrides: Readonly> | undefined, +): Record { + const merged: Record = { ...base }; + for (const [name, settings] of Object.entries(overrides ?? {})) { + const role = Object.hasOwn(merged, name) ? merged[name] : undefined; + const environment = { ...role?.environment, ...settings.environment }; + const secrets = settings.secrets ?? role?.secrets; + merged[name] = { + ...role, + ...(Object.keys(environment).length === 0 ? {} : { environment }), + ...(secrets === undefined ? {} : { secrets }), + }; + } + + return merged; +} + +/** + * The roles for a run: the app's, with the `local.roles` settings merged over them for a local run. + */ +export function rolesOf( + config: DeployConfig, + { local = false }: RunOptions = {}, +): Readonly> { + return local ? mergeRoles(config.roles, config.local?.roles) : (config.roles ?? {}); +} + +/** + * The variables the roles take from the secret source, without duplicates, in first-use order. + */ +export function roleSecretNames(roles: Readonly>): ReadonlyArray { + return [...new Set(Object.values(roles).flatMap((role) => role.secrets ?? []))]; +} + /** * Reads and validates the `deploy.yaml` at `target`, which may name the folder or the file itself. * @@ -415,7 +552,7 @@ export const loadApp = Effect.fn('loadApp')(function* ( file, DeployConfig, `No ${file}. Point compoza at a folder holding a ${DEPLOY_FILE}.`, - { privateInput: true }, + { privateInput: true, removedKeys: { targets: TARGETS_REMOVED_MESSAGE } }, ); return { dir: path.dirname(file), file, config }; }); @@ -460,10 +597,13 @@ export function selectDestination(app: App, name?: string): App { throw new Error(`Unknown destination "${name}"; choices: ${choices}`); } + const { roles: _roles, ...rest } = selected; + const roles = mergeRoles(app.config.roles, selected.roles); const config = { ...app.config, - ...selected, + ...rest, environment: { ...app.config.environment, ...selected.environment }, + ...(Object.keys(roles).length === 0 ? {} : { roles }), }; return { ...app, config }; } diff --git a/src/config/env-file.test.ts b/src/config/env-file.test.ts index b976a6d..e218a63 100644 --- a/src/config/env-file.test.ts +++ b/src/config/env-file.test.ts @@ -54,7 +54,7 @@ describe('readSecrets', () => { const config = yield* Schema.decodeUnknownEffect(DeployConfig)({ name: 'demo', host: 'atlas', - targets: ['web'], + roles: { web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml'], secrets: { 'env-file': { path: 'secrets.env' } }, }); @@ -71,7 +71,7 @@ describe('readSecrets', () => { const config = yield* Schema.decodeUnknownEffect(DeployConfig)({ name: 'demo', host: 'atlas', - targets: ['web'], + roles: { web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml'], }); assert.deepStrictEqual(yield* readSecrets({ dir: '/nonexistent', config }), {}); diff --git a/src/config/schema.ts b/src/config/schema.ts index a1fffff..494105c 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -13,6 +13,22 @@ export const Name = Schema.String.pipe( ), ); +/** + * A role name, which is a Compose service name that also becomes a component of an image name. + * Docker accepts lowercase letters and digits joined by single or double underscores or runs of + * dashes, so a name may not start or end with a separator or mix separators in a row (`web_`, + * `a_-b`). + */ +export const RoleName = Schema.String.pipe( + Schema.check( + Schema.isPattern(/^[a-z0-9]+(?:(?:_|__|-+)[a-z0-9]+)*$/, { + message: + 'expected lowercase letters and digits, joined by underscores or dashes, ' + + 'starting and ending with a letter or digit', + }), + ), +); + /** * A Dockerfile build target: like `Name`, but underscores are allowed because Dockerfile stage * names may use them. diff --git a/src/config/secrets.test.ts b/src/config/secrets.test.ts index 6f2d49e..e2be4b3 100644 --- a/src/config/secrets.test.ts +++ b/src/config/secrets.test.ts @@ -5,21 +5,21 @@ import { mkdtempDisposable, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { App } from './deploy.ts'; +import type { App, RoleSettings } from './deploy.ts'; import { OnePasswordReader } from '../extensions/1password/index.ts'; import { DryRun } from '../services/Shell.ts'; import { deploymentScript } from '../ssh-plan.ts'; -import { selectDestination } from './deploy.ts'; +import { mergeRoles, selectDestination } from './deploy.ts'; import { parseEnvFile } from './env-file.ts'; -import { readSecrets, secretSource, validateSecrets } from './secrets.ts'; +import { readSecrets, secretSource, validateSecrets, withRoleSecrets } from './secrets.ts'; const app: App = { dir: '/nonexistent', config: { name: 'app', host: 'atlas', - targets: ['web'], + roles: { web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml'], secrets: { '1password': { environmentId: 'fictional-environment' } }, }, @@ -35,6 +35,13 @@ const allowlisted = (variables: ReadonlyArray): App => ({ }, }); +/** + * The app with `roles` merged over its own, role settings only. + */ +function withRoles(base: App, roles: Readonly>): App { + return { ...base, config: { ...base.config, roles: mergeRoles(base.config.roles, roles) } }; +} + describe('secret sources', () => { it('replaces the entire inherited source at destinations', () => { const selected = selectDestination( @@ -87,7 +94,7 @@ describe('secret sources', () => { const plan = deploymentScript( selected, { ssh: 'fixture', platform: 'linux/arm64', transport: { mode: 'ssh-load' } }, - 'one', + { web: 'one' }, [], values, ); @@ -98,7 +105,7 @@ describe('secret sources', () => { .join('\n'); assert.include(uploaded, String.raw`TOKEN="$$HOME \"literal\""`); assert.include(uploaded, 'EMPTY=""'); - assert.include(uploaded, 'WEB_IMAGE="app-web:one"'); + assert.include(uploaded, 'image: "app-web:one"'); for (const excluded of ['OTHER', 'SENTINEL_EXCLUDED', 'PATH', 'WEB_IMAGE_SECRET', 'token']) { assert.notInclude(uploaded, excluded); } @@ -142,6 +149,96 @@ describe('secret sources', () => { ), ); + it.effect('adds the role secrets to a configured allowlist', () => + Effect.gen(function* () { + const selected = withRoles(allowlisted(['TOKEN']), { web: { secrets: ['ROLE_TOKEN'] } }); + assert.deepStrictEqual(yield* readSecrets(selected), { + TOKEN: 'first', + ROLE_TOKEN: 'second', + }); + }).pipe( + Effect.provide(NodeServices.layer), + Effect.provideService(OnePasswordReader, async () => [ + { name: 'TOKEN', value: 'first', masked: true }, + { name: 'ROLE_TOKEN', value: 'second', masked: true }, + { name: 'OTHER', value: 'SENTINEL_EXCLUDED', masked: true }, + ]), + ), + ); + + it.effect('imports every variable, role secrets included, without an allowlist', () => + Effect.gen(function* () { + const selected = withRoles(app, { web: { secrets: ['ROLE_TOKEN'] } }); + assert.deepStrictEqual(yield* readSecrets(selected), { ROLE_TOKEN: 'second', OTHER: 'x' }); + }).pipe( + Effect.provide(NodeServices.layer), + Effect.provideService(OnePasswordReader, async () => [ + { name: 'ROLE_TOKEN', value: 'second', masked: true }, + { name: 'OTHER', value: 'x', masked: false }, + ]), + ), + ); + + it('extends only a configured 1Password allowlist with role secrets', () => { + const configured = { provider: '1password', environmentId: 'e', variables: ['A'] } as const; + assert.deepStrictEqual(withRoleSecrets(configured, ['B', 'A']), { + ...configured, + variables: ['A', 'B'], + }); + + const open = { provider: '1password', environmentId: 'e' } as const; + assert.strictEqual(withRoleSecrets(open, ['B']), open); + + const file = { provider: 'env-file', path: 'x.env' } as const; + assert.strictEqual(withRoleSecrets(file, ['B']), file); + }); + + it.effect('fails when the source does not provide a role secret', () => + Effect.gen(function* () { + const selected = withRoles(app, { web: { secrets: ['ROLE_TOKEN'] } }); + const error = yield* readSecrets(selected).pipe(Effect.flip); + assert.include(error.message, 'web: ROLE_TOKEN'); + assert.notInclude(error.message, 'SENTINEL'); + }).pipe( + Effect.provide(NodeServices.layer), + Effect.provideService(OnePasswordReader, async () => [ + { name: 'OTHER', value: 'SENTINEL', masked: true }, + ]), + ), + ); + + it.effect.each([ + ['without a source', undefined], + ['with source none', { none: {} }], + ] as const)('rejects role secrets %s, even on a dry run', ([, secrets]) => + Effect.gen(function* () { + const { secrets: _secrets, ...rest } = app.config; + const selected = withRoles( + { ...app, config: { ...rest, ...(secrets === undefined ? {} : { secrets }) } }, + { web: { secrets: ['ROLE_TOKEN'] } }, + ); + const error = yield* readSecrets(selected).pipe( + Effect.provideService(DryRun, true), + Effect.flip, + ); + assert.include(error.message, 'Role "web" takes secrets, but the run has no secret source'); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect('reads local role secrets from the local source', () => + Effect.gen(function* () { + const selected: App = { + ...app, + config: { + ...app.config, + local: { secrets: { none: {} }, roles: { web: { secrets: ['ROLE_TOKEN'] } } }, + }, + }; + const error = yield* readSecrets(selected, { local: true }).pipe(Effect.flip); + assert.include(error.message, 'no secret source'); + }).pipe(Effect.provide(NodeServices.layer)), + ); + it.effect('still rejects duplicate selected variables and unsafe selected values', () => Effect.gen(function* () { const duplicate = yield* readSecrets(allowlisted(['TOKEN'])).pipe( @@ -242,7 +339,6 @@ describe('secret sources', () => { ); it.each([ - 'WEB_IMAGE', 'COMPOSE_PROJECT_NAME', 'DOCKER_HOST', 'BUILDX_BUILDER', diff --git a/src/config/secrets.ts b/src/config/secrets.ts index 16e5257..5ac28a8 100644 --- a/src/config/secrets.ts +++ b/src/config/secrets.ts @@ -1,13 +1,12 @@ import { Effect, type FileSystem, type Path, Result } from 'effect'; -import type { App, RunOptions } from './deploy.ts'; +import type { App, Role, RunOptions } from './deploy.ts'; import { DeployError } from '../errors.ts'; import { resolveSecrets, type ResolvedSecretsConfig } from '../extensions/config.ts'; import { credentialProvider, isCredentialEnvironment } from '../extensions/credentials.ts'; -import { imageVariable } from '../plan.ts'; import { DryRun } from '../services/Shell.ts'; -import { environmentOf } from './deploy.ts'; +import { environmentOf, roleSecretNames, rolesOf } from './deploy.ts'; import { VARIABLE_NAME_PATTERN } from './env-file.ts'; /** @@ -97,19 +96,14 @@ export function validateSecrets( values: Readonly>, { local = false }: RunOptions = {}, ): void { - const imageVariables = new Set(app.config.targets.map(imageVariable)); const environment = environmentOf(app.config, { local }); - if (imageVariables.size !== app.config.targets.length) { - throw new Error('Targets must map to distinct image variables'); - } - for (const [index, [key, value]] of Object.entries(values).entries()) { if (!VARIABLE_NAME_PATTERN.test(key) || typeof value !== 'string') { throw new Error(`Invalid secret variable at entry ${index + 1}`); } - if (isReservedVariable(key, imageVariables)) { + if (isReservedVariable(key)) { throw new Error(`Reserved secret variable: ${key}`); } @@ -146,11 +140,20 @@ export const readSecrets = Effect.fn('readSecrets')(function* ( app: App, options: RunOptions = {}, ): Effect.fn.Return, DeployError, FileSystem.FileSystem | Path.Path> { - const source = yield* Effect.fromResult(selectSecretSource(app, options)); - if (source === undefined || source.provider === 'none' || (yield* DryRun)) { + const selected = yield* Effect.fromResult(selectSecretSource(app, options)); + const roles = rolesOf(app.config, options); + const wanted = roleSecretNames(roles); + const sourceIssue = roleSecretSourceIssue(roles, selected); + if (sourceIssue !== undefined) { + return yield* new DeployError({ message: sourceIssue }); + } + + if (selected === undefined || selected.provider === 'none' || (yield* DryRun)) { return {} as Record; } + const source = withRoleSecrets(selected, wanted); + const provider = yield* Effect.try({ try: () => credentialProvider(source.provider), catch: () => new DeployError({ message: 'Unknown credential provider' }), @@ -161,19 +164,63 @@ export const readSecrets = Effect.fn('readSecrets')(function* ( catch: (cause) => new DeployError({ message: (cause as Error).message }), }); + const missing = Object.entries(roles).flatMap(([role, settings]) => + (settings.secrets ?? []) + .filter((name) => !Object.hasOwn(values, name)) + .map((name) => `${role}: ${name}`), + ); + if (missing.length > 0) { + return yield* new DeployError({ + message: `The secret source lacks variables that roles take (${missing.join(', ')})`, + }); + } + return { ...values }; }); /** - * Whether a secret named `key` would replace an image variable, a Docker or Compose setting, a - * secret provider's credential or a process variable the host's shell and runtimes read. + * The secret source as it is read for a run: a 1Password `variables` allowlist also lists every + * variable the roles take, so a role secret is imported without repeating it there. A source + * without an allowlist already imports every variable and is returned as is. + */ +export function withRoleSecrets( + source: ResolvedSecretsConfig, + names: ReadonlyArray, +): ResolvedSecretsConfig { + if (source.provider !== '1password' || source.variables === undefined) { + return source; + } + + return { ...source, variables: [...new Set([...source.variables, ...names])] }; +} + +/** + * Why the roles cannot get their secrets, or `undefined` when they can: some role takes secrets but + * the run has no secret source, or its provider is `none`. The message names the first such role. + * Reads no secret values. + */ +export function roleSecretSourceIssue( + roles: Readonly>, + source: ResolvedSecretsConfig | undefined, +): string | undefined { + if (source !== undefined && source.provider !== 'none') { + return undefined; + } + + const role = Object.entries(roles).find(([, settings]) => (settings.secrets ?? []).length > 0); + return role === undefined + ? undefined + : `Role "${role[0]}" takes secrets, but the run has no secret source; ` + + 'configure secrets (or local.secrets for --local) with a provider'; +} + +/** + * Whether a secret named `key` would replace a Docker or Compose setting, a secret provider's + * credential or a process variable the host's shell and runtimes read. */ -function isReservedVariable(key: string, imageVariables: ReadonlySet): boolean { +export function isReservedVariable(key: string): boolean { return ( - imageVariables.has(key) || - RESERVED_PREFIX_PATTERN.test(key) || - isCredentialEnvironment(key) || - RESERVED_VARIABLES.has(key) + RESERVED_PREFIX_PATTERN.test(key) || isCredentialEnvironment(key) || RESERVED_VARIABLES.has(key) ); } diff --git a/src/deploy.ts b/src/deploy.ts index abfaa75..6ccffa5 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -1,43 +1,48 @@ import type { HttpClient } from 'effect/http'; -import { Console, Effect, FileSystem, type Option, Path } from 'effect'; +import { Console, Effect, FileSystem, type Option, Path, type Scope } from 'effect'; +import { tmpdir } from 'node:os'; import type { Step } from './command.ts'; import type { IngressPlan } from './extensions/ingress.ts'; import type { Phase } from './progress.ts'; +import { ROLES_OVERRIDE_FILE } from './bundle.ts'; import { blocking, formatChecks, runChecks } from './checks.ts'; import { mkdirScript, printableStep } from './command.ts'; -import { - type App, - directoriesOf, - loadApp, - selectDestination, - workingDirectoryOf, -} from './config/deploy.ts'; +import { type App, directoriesOf, loadApp, selectDestination } from './config/deploy.ts'; import { dockerHost, type HostConfig, hostFor, loadHostsForRun } from './config/hosts.ts'; import { readSecrets } from './config/secrets.ts'; -import { DeployError, tryPlan } from './errors.ts'; +import { DeployError, platformFailure, tryPlan } from './errors.ts'; import { loadAppExtensionFiles } from './extensions/config.ts'; import { tryIngressFor, tryIngressPlan } from './extensions/ingress.ts'; import { planDeployment } from './plan.ts'; import { DRY_RUN_COMPLETE, formatDuration, phase, runPhases } from './progress.ts'; +import { + buildingRoles, + buildSettingsOf, + roleOverride, + type RoleSource, + type RoleSources, + validateRoles, +} from './roles.ts'; import { Git } from './services/Git.ts'; import { Remote } from './services/Remote.ts'; import { DryRun, Shell } from './services/Shell.ts'; import { deploySsh } from './ssh-deploy.ts'; +import { parseTagFlags, tagLabel, tagsOf } from './tags.ts'; import { WAIT_TIMEOUT_MILLIS, waitForUrl } from './wait.ts'; /** - * The tag of a deploy without targets: nothing is built, so there is no commit to name the images - * after. + * The message of a failed deployment plan when the planner throws something other than an `Error`. */ -const THIRD_PARTY_TAG = 'third-party'; +const INVALID_DEPLOYMENT_PLAN = 'Invalid deployment plan'; /** - * The message of a failed deployment plan when the planner throws something other than an `Error`. + * The permissions of the roles override file: readable by this user only, because it names the + * variables of role secrets. */ -const INVALID_DEPLOYMENT_PLAN = 'Invalid deployment plan'; +const PRIVATE_FILE_MODE = 0o600; /** * What the `deploy` command was asked to do. @@ -65,9 +70,11 @@ export interface DeployOptions { readonly allowDirty: boolean; /** - * Reuse images already built under this tag instead of building: a rollback to an earlier deploy. + * The values of the repeatable `--tag` flag: `` for every building role and `=` + * for one. Any value reuses images already built under those tags instead of building, a rollback + * to an earlier deploy. */ - readonly tag: Option.Option; + readonly tags: ReadonlyArray; /** * The hosts file, when not the default. @@ -115,70 +122,77 @@ export const resolve = Effect.fn('resolve')(function* ( }); /** - * The image tag of a deploy and the build context its images are built from. - */ -interface Source { - /** - * The tag the images are built and run under. - */ - readonly tag: string; - - /** - * The absolute build context: `context` from `deploy.yaml` resolved against the working - * directory, else the Git root. With `--tag` nothing is built, so it falls back to the working - * directory instead and is never read. - */ - readonly context: string; -} - -/** - * Chooses the image tag and build context: the commit, `-dirty` when the checkout has changes, or - * the tag given to reuse. + * Chooses the tag and build context of every building role: the commit of the role's repository, + * `-dirty` when its checkout has changes, or the tags given with `--tag` to reuse. * * A repository without commits cannot be deployed, because there is nothing to tag images with; a * dirty checkout is refused unless `--allow-dirty` or `--local`, so that a tag always names the - * exact source it was built from. With `--tag` Git is not consulted at all: a rollback reuses - * images, so neither the commit nor a clean checkout matters, and it has to work from a deploy - * folder that is no longer inside a checkout, where `git rev-parse` exits 128. The context is then - * only a placeholder for `PlanOptions`, which needs one even though no build step reads it. + * exact source it was built from. Roles whose working directories share a Git root share one check + * and one tag. With `--tag` Git is not consulted at all: a rollback reuses images, so neither the + * commit nor a clean checkout matters, and it has to work from a deploy folder that is no longer + * inside a checkout, where `git rev-parse` exits 128. The context is then only a placeholder for + * `PlanOptions`, which needs one even though no build step reads it. An app that builds nothing has + * no sources and does not consult Git, though `--tag` values are still parsed and validated. */ -const chooseSource = Effect.fn('chooseSource')(function* (app: App, options: DeployOptions) { - const path = yield* Path.Path; - const directory = workingDirectoryOf(app); - const configured = - app.config.context === undefined ? undefined : path.resolve(directory, app.config.context); +const chooseSource = Effect.fn('chooseSource')(function* ( + app: App, + options: DeployOptions, +): Effect.fn.Return { + const roles = buildingRoles(app.config).map((role) => buildSettingsOf(app, role)); + + if (options.tags.length > 0) { + const tags = yield* tryPlan(() => parseTagFlags(options.tags, app.config), 'Invalid --tag'); + const reused: Record = {}; + for (const settings of roles) { + const tag = Object.hasOwn(tags, settings.role) ? tags[settings.role] : undefined; + if (tag === undefined) { + return yield* new DeployError({ message: `No tag for role "${settings.role}"` }); + } + + reused[settings.role] = { tag, context: settings.context ?? settings.workingDirectory }; + } - if (options.tag._tag === 'Some') { - const reused: Source = { tag: options.tag.value, context: configured ?? directory }; return reused; } - if (app.config.targets.length === 0) { - return { tag: THIRD_PARTY_TAG, context: configured ?? directory }; + const git = yield* Git; + const tagsByRoot = new Map(); + const built: Record = {}; + for (const settings of roles) { + if (settings.workingDirectoryConfigured) { + yield* requireDirectory(settings.workingDirectory); + } + + const root = yield* git.root(settings.workingDirectory); + const known = tagsByRoot.get(root); + const tag = known ?? (yield* checkedTag(root, options)); + tagsByRoot.set(root, tag); + built[settings.role] = { tag, context: settings.context ?? root }; } - if (app.config.workingDirectory !== undefined) { - yield* requireDirectory(directory); - } + return built; +}); +/** + * The tag of one Git repository: its short commit, `-dirty` when the checkout has changes. Fails + * when it has no commits, or is dirty without `--allow-dirty` or `--local`. + */ +const checkedTag = Effect.fn('checkedTag')(function* (root: string, options: DeployOptions) { const git = yield* Git; - const root = yield* git.root(directory); if (!(yield* git.hasCommits(root))) { return yield* new DeployError({ - message: 'The repository has no commits yet; images are tagged with the commit', + message: `The repository ${root} has no commits yet; images are tagged with the commit`, }); } const dirty = yield* git.isDirty(root); if (dirty && !options.allowDirty && !options.local) { - return yield* new DeployError({ message: 'Commit your changes first, or pass --allow-dirty' }); + return yield* new DeployError({ + message: `Commit your changes in ${root} first, or pass --allow-dirty`, + }); } - const built: Source = { - tag: `${yield* git.shortSha(root)}${dirty ? '-dirty' : ''}`, - context: configured ?? root, - }; - return built; + return `${yield* git.shortSha(root)}${dirty ? '-dirty' : ''}`; }); /** @@ -242,13 +256,14 @@ const planRun = Effect.fn('planRun')(function* ( app: App, host: HostConfig, options: DeployOptions, - source: Source, + sources: RoleSources, + overrideFile: string | undefined, ) { const shell = yield* Shell; const dryRun = yield* DryRun; - const { tag } = source; - const steps = yield* planSteps(app, host, options, source); + const tag = tagLabel(sources); + const steps = yield* planSteps(app, host, options, sources, overrideFile); const where = options.local ? 'local Docker' : `${app.config.host} (${dockerHost(host)})`; const header = `Deploying ${app.config.name} ${tag} to ${where}`; @@ -273,7 +288,7 @@ const planRun = Effect.fn('planRun')(function* ( const ingress = options.local ? undefined : yield* tryIngressFor(app); const directories = directoriesOf(app.config); - const images = imageCount(app.config.targets.length); + const images = imageCount(Object.keys(sources).length); // `planDeployment` puts the Compose command last, after every build and transport step; with // `--tag` it is alone. @@ -351,7 +366,8 @@ const planSteps = Effect.fn('planSteps')(function* ( app: App, host: HostConfig, options: DeployOptions, - source: Source, + sources: RoleSources, + overrideFile: string | undefined, ) { const secrets = yield* readSecrets(app, { local: options.local }); const steps = yield* tryPlan( @@ -359,10 +375,10 @@ const planSteps = Effect.fn('planSteps')(function* ( planDeployment({ app, host, - tag: source.tag, + sources, local: options.local, - reuseImages: options.tag._tag === 'Some', - context: source.context, + reuseImages: options.tags.length > 0, + ...(overrideFile === undefined ? {} : { overrideFile }), secrets, }), INVALID_DEPLOYMENT_PLAN, @@ -472,24 +488,59 @@ export const deploy = Effect.fn('deploy')(function* ( ): Effect.fn.Return< void, DeployError, - FileSystem.FileSystem | Git | HttpClient.HttpClient | Path.Path | Remote | Shell + FileSystem.FileSystem | Git | HttpClient.HttpClient | Path.Path | Remote | Scope.Scope | Shell > { const { app, host } = yield* resolve(options.target, options.hosts, options.destination); if (!options.local && host.transport.mode === 'remote-build' && app.config.execution !== 'ssh') { return yield* new DeployError({ message: 'remote-build transport requires execution: ssh' }); } - const source = yield* chooseSource(app, options); + yield* validateRoles(app, { local: options.local }); + const sources = yield* chooseSource(app, options); if (!options.local && app.config.execution === 'ssh') { - return yield* deploySsh(app, host, options, source); + return yield* deploySsh(app, host, options, sources); } - yield* Effect.logDebug(`tag ${source.tag}, context ${source.context}`); + yield* Effect.logDebug(`tags ${tagLabel(sources)}`); - const run = yield* planRun(app, host, options, source); + const override = yield* tryPlan( + () => roleOverride({ app, host, tags: tagsOf(sources), local: options.local }), + INVALID_DEPLOYMENT_PLAN, + ); + const overrideFile = override === undefined ? undefined : yield* overridePath(override); + const run = yield* planRun(app, host, options, sources, overrideFile); yield* Console.log(run.header); + if (override !== undefined && (yield* DryRun)) { + yield* Console.log(`Roles override ${overrideFile}:\n${override.trimEnd()}`); + } + const total = yield* runPhases(run.phases); yield* run.summary(formatDuration(total)); +}, Effect.scoped); + +/** + * The path Compose reads the roles override from. Outside a dry run the override is written to a + * private temporary folder that is removed when the surrounding scope closes; a dry run writes + * nothing and only names where the file would go. + */ +const overridePath = Effect.fn('overridePath')(function* ( + override: string, +): Effect.fn.Return { + const path = yield* Path.Path; + if (yield* DryRun) { + return path.join(tmpdir(), 'compoza-roles-XXXXXX', ROLES_OVERRIDE_FILE); + } + + const fs = yield* FileSystem.FileSystem; + const dir = yield* fs + .makeTempDirectoryScoped({ prefix: 'compoza-roles-' }) + .pipe(Effect.mapError(platformFailure('Creating a temporary folder for the roles override'))); + const file = path.join(dir, ROLES_OVERRIDE_FILE); + yield* fs + .writeFileString(file, override, { mode: PRIVATE_FILE_MODE }) + .pipe(Effect.mapError(platformFailure('Writing the roles override'))); + + return file; }); diff --git a/src/extensions/caddy/templates.test.ts b/src/extensions/caddy/templates.test.ts index 32a811b..9e7b662 100644 --- a/src/extensions/caddy/templates.test.ts +++ b/src/extensions/caddy/templates.test.ts @@ -219,7 +219,6 @@ describe('Caddy templates', () => { yield* yaml(join(dir, 'repo', 'deploy.yaml'), { name: 'budget', host: 'server', - targets: [], composeFiles: ['compose.yaml'], ingress: { caddy: { diff --git a/src/extensions/cloudflare-tunnel/cloudflare.test.ts b/src/extensions/cloudflare-tunnel/cloudflare.test.ts index 838ee19..b15a57d 100644 --- a/src/extensions/cloudflare-tunnel/cloudflare.test.ts +++ b/src/extensions/cloudflare-tunnel/cloudflare.test.ts @@ -141,7 +141,6 @@ if (!url.endsWith('/configurations')) { name: 'app', projectName: 'app-staging', host: 'fixture', - targets: [], composeFiles: ['compose.yaml'], execution: 'ssh', remote: { stackDir: join(dir, 'stack') }, @@ -289,7 +288,7 @@ describe('Cloudflare Tunnel extension', { timeout: INTEGRATION_TIMEOUT_MILLIS }, succeeded(run(cloudflareIngress.setup(host).script)); const plan = cloudflareIngress.plan(app, host); - const deployment = deploymentScript(app, host, 'one', []); + const deployment = deploymentScript(app, host, {}, []); assert.notInclude(deployment.script + deployment.input, sentinel); const records = deployment.input diff --git a/src/extensions/cloudflare-tunnel/provision.test.ts b/src/extensions/cloudflare-tunnel/provision.test.ts index a1166d7..f4c565b 100644 --- a/src/extensions/cloudflare-tunnel/provision.test.ts +++ b/src/extensions/cloudflare-tunnel/provision.test.ts @@ -425,7 +425,6 @@ describe('Cloudflare connector provisioning', { timeout: INTEGRATION_TIMEOUT_MIL config: { name: 'app', host: 'fixture', - targets: [], composeFiles: ['compose.yaml'] as [string], ingress: { 'cloudflare-tunnel': { domain: 'app.example.com', service: 'http://app:80' } }, }, @@ -456,7 +455,6 @@ describe('Cloudflare connector provisioning', { timeout: INTEGRATION_TIMEOUT_MIL config: { name: 'app', host: 'fixture', - targets: [], composeFiles: ['compose.yaml'] as [string], ingress: { 'cloudflare-tunnel': { domain: 'app.example.com', service: 'https://app:443' }, diff --git a/src/extensions/cloudflare-tunnel/templates.test.ts b/src/extensions/cloudflare-tunnel/templates.test.ts index 90d6f07..ceeaeac 100644 --- a/src/extensions/cloudflare-tunnel/templates.test.ts +++ b/src/extensions/cloudflare-tunnel/templates.test.ts @@ -15,7 +15,7 @@ const proxy: CloudflareProxy = { const settings = { domain: 'app.example.com', service: 'http://app:80' }; const app: App = { dir: '/app', - config: { name: 'app', host: 'host', targets: [], composeFiles: ['compose.yaml'] }, + config: { name: 'app', host: 'host', composeFiles: ['compose.yaml'] }, }; const namedApp: App = { diff --git a/src/extensions/config.test.ts b/src/extensions/config.test.ts index 9453742..800fe8d 100644 --- a/src/extensions/config.test.ts +++ b/src/extensions/config.test.ts @@ -44,7 +44,6 @@ const app: App = { config: { name: 'app', host: 'server', - targets: [], composeFiles: ['compose.yaml'], execution: 'ssh', remote: { stackDir: '/srv/app' }, @@ -230,8 +229,8 @@ describe('named extension configuration', () => { extension.checks(selectedApp, singleHost, [], {}), ); assert.deepStrictEqual( - deploymentScript(selectedApp, combined, 'one', []), - deploymentScript(selectedApp, singleHost, 'one', []), + deploymentScript(selectedApp, combined, {}, []), + deploymentScript(selectedApp, singleHost, {}, []), ); } diff --git a/src/extensions/extensions.test.ts b/src/extensions/extensions.test.ts index 04e6319..ea6c9ba 100644 --- a/src/extensions/extensions.test.ts +++ b/src/extensions/extensions.test.ts @@ -22,7 +22,6 @@ const app: App = { name: 'demo', projectName: 'demo-staging', host: 'atlas', - targets: [], composeFiles: ['compose.yaml'], ingress: { caddy: { domain: 'demo.example.com', upstream: 'demo:80', healthPath: '/ready' }, diff --git a/src/ingress-setup.ts b/src/ingress-setup.ts index eac445e..5793ece 100644 --- a/src/ingress-setup.ts +++ b/src/ingress-setup.ts @@ -39,7 +39,6 @@ export const setupOperationInput = Effect.fn('setupOperationInput')(function* ( config: { name: 'ingress-setup', host: 'setup', - targets: [], composeFiles: ['compose.yaml'], secrets: operation.secrets, }, diff --git a/src/lifecycle.test.ts b/src/lifecycle.test.ts index a37b5cb..4ca2409 100644 --- a/src/lifecycle.test.ts +++ b/src/lifecycle.test.ts @@ -10,7 +10,12 @@ import { Shell } from './services/Shell.ts'; const resolved: Resolved = { app: { dir: '/repo/infra', - config: { name: 'demo', host: 'atlas', targets: ['web'], composeFiles: ['compose.yaml'] }, + config: { + name: 'demo', + host: 'atlas', + roles: { web: { build: { target: 'web' } } }, + composeFiles: ['compose.yaml'], + }, }, host: { ssh: 'deploy@atlas.example.com', diff --git a/src/multi-ingress.test.ts b/src/multi-ingress.test.ts index 8afb22a..10a833b 100644 --- a/src/multi-ingress.test.ts +++ b/src/multi-ingress.test.ts @@ -63,11 +63,11 @@ const loadedFixture = Effect.gen(function* () { (directory) => Effect.promise(() => directory.remove()), ); const directory = temporary.path; - const hostDirectory = join(directory, 'macmini-server'); + const hostDirectory = join(directory, 'linden'); mkdirSync(hostDirectory); writeFileSync( join(directory, 'hosts.yaml'), - YAML.stringify({ 'macmini-server': { $ref: './macmini-server/host.yaml' } }), + YAML.stringify({ linden: { $ref: './linden/host.yaml' } }), ); writeFileSync(join(hostDirectory, 'site.tmpl'), defaultSiteTemplate); writeFileSync(join(hostDirectory, 'Caddyfile.tmpl'), defaultConfigTemplate); @@ -88,8 +88,8 @@ const loadedFixture = Effect.gen(function* () { ); const hosts = yield* loadHosts(join(directory, 'hosts.yaml')); - const selected = hosts['macmini-server']!; - const caddySettings = yield* proxyOf(selected, 'macmini-server'); + const selected = hosts['linden']!; + const caddySettings = yield* proxyOf(selected, 'linden'); assert.deepStrictEqual(caddySettings.templates?.site, { inline: defaultSiteTemplate }); assert.deepStrictEqual(caddySettings.templates?.config, { inline: defaultConfigTemplate }); diff --git a/src/plan.test.ts b/src/plan.test.ts index 2ccae5d..8bd0b5c 100644 --- a/src/plan.test.ts +++ b/src/plan.test.ts @@ -4,20 +4,15 @@ import type { App } from './config/deploy.ts'; import type { HostConfig } from './config/hosts.ts'; import { isPipeline, printableStep, REMOTE_PATH, shellQuote } from './command.ts'; -import { - composePrefix, - dockerfileOf, - imageReference, - imageVariable, - planDeployment, -} from './plan.ts'; +import { composePrefix, planDeployment } from './plan.ts'; +import { buildSettingsOf, imageReference } from './roles.ts'; const app: App = { dir: '/repo/infra', config: { name: 'demo', host: 'atlas', - targets: ['builder', 'web'], + roles: { builder: { build: { target: 'builder' } }, web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml', 'compose.production.yaml'], environment: { DEMO_PORT: '8090' }, local: { composeFiles: ['compose.yaml'], environment: { DEMO_SOURCES_PATH: '../sources' } }, @@ -30,6 +25,13 @@ const registryHost: HostConfig = { transport: { mode: 'registry', registry: 'registry.example.com' }, }; +/** + * A role source under the fixture repository. + */ +function source(tag: string, context = '/repo') { + return { tag, context }; +} + const sshHost: HostConfig = { ...registryHost, transport: { mode: 'ssh-load' } }; describe('naming', () => { @@ -40,10 +42,10 @@ describe('naming', () => { ); assert.strictEqual(imageReference(app, registryHost, 'web', 'abc', true), 'demo-web:abc'); assert.strictEqual(imageReference(app, sshHost, 'web', 'abc', false), 'demo-web:abc'); - assert.strictEqual(imageVariable('scraper-api'), 'SCRAPER_API_IMAGE'); - assert.strictEqual(dockerfileOf(app), '/repo/infra/Dockerfile'); + assert.strictEqual(buildSettingsOf(app, 'web').dockerfile, '/repo/infra/Dockerfile'); assert.strictEqual( - dockerfileOf({ ...app, config: { ...app.config, dockerfile: '../Dockerfile' } }), + buildSettingsOf({ ...app, config: { ...app.config, dockerfile: '../Dockerfile' } }, 'web') + .dockerfile, '/repo/Dockerfile', ); }); @@ -53,15 +55,14 @@ describe('naming', () => { ...app, config: { ...app.config, dockerfile: '/abs/Dockerfile', composeFiles: ['/abs/compose.yaml'] }, }; - assert.strictEqual(dockerfileOf(absolute), '/abs/Dockerfile'); + assert.strictEqual(buildSettingsOf(absolute, 'web').dockerfile, '/abs/Dockerfile'); const steps = planDeployment({ app: absolute, host: registryHost, - tag: 'abc', + sources: { builder: source('abc'), web: source('abc') }, local: false, reuseImages: false, - context: '/repo', secrets: {}, }); assert.include(printableStep(steps[0]!), '--file /abs/Dockerfile '); @@ -91,10 +92,9 @@ describe('naming', () => { describe('planDeployment', () => { const base = { app, - tag: 'abc123', + sources: { builder: source('abc123'), web: source('abc123') }, local: false, reuseImages: false, - context: '/repo', secrets: { PG_PASSWORD: 'x' }, }; @@ -103,7 +103,7 @@ describe('planDeployment', () => { assert.deepStrictEqual(steps.map(printableStep), [ 'docker buildx build --platform linux/arm64 --push --file /repo/infra/Dockerfile --target builder --tag registry.example.com/demo-builder:abc123 /repo', 'docker buildx build --platform linux/arm64 --push --file /repo/infra/Dockerfile --target web --tag registry.example.com/demo-web:abc123 /repo', - 'DEMO_PORT=8090 PG_PASSWORD=*** BUILDER_IMAGE=registry.example.com/demo-builder:abc123 WEB_IMAGE=registry.example.com/demo-web:abc123 docker --host ssh://deploy@atlas.example.com compose --project-name demo --project-directory /repo/infra --env-file /dev/null --file /repo/infra/compose.yaml --file /repo/infra/compose.production.yaml up --detach --remove-orphans --no-build --pull always', + 'DEMO_PORT=8090 PG_PASSWORD=*** docker --host ssh://deploy@atlas.example.com compose --project-name demo --project-directory /repo/infra --env-file /dev/null --file /repo/infra/compose.yaml --file /repo/infra/compose.production.yaml up --detach --remove-orphans --no-build --pull always', ]); const compose = steps.at(-1); @@ -136,13 +136,70 @@ describe('planDeployment', () => { assert.deepStrictEqual(steps.map(printableStep), [ 'docker buildx build --load --file /repo/infra/Dockerfile --target builder --tag demo-builder:abc123 /repo', 'docker buildx build --load --file /repo/infra/Dockerfile --target web --tag demo-web:abc123 /repo', - 'DEMO_PORT=8090 DEMO_SOURCES_PATH=../sources PG_PASSWORD=*** BUILDER_IMAGE=demo-builder:abc123 WEB_IMAGE=demo-web:abc123 docker compose --project-name demo --project-directory /repo/infra --env-file /dev/null --file /repo/infra/compose.yaml up --detach --remove-orphans --no-build --pull missing', + 'DEMO_PORT=8090 DEMO_SOURCES_PATH=../sources PG_PASSWORD=*** docker compose --project-name demo --project-directory /repo/infra --env-file /dev/null --file /repo/infra/compose.yaml up --detach --remove-orphans --no-build --pull missing', ]); }); it('skips the build when reusing images', () => { const steps = planDeployment({ ...base, host: registryHost, reuseImages: true }); assert.strictEqual(steps.length, 1); - assert.include(printableStep(steps[0]!), 'WEB_IMAGE=registry.example.com/demo-web:abc123'); + assert.include(printableStep(steps[0]!), 'compose --project-name demo'); + }); + + it('builds a role without a target and with its own dockerfile, context and directory', () => { + const roles: App['config']['roles'] = { + api: { build: {} }, + worker: { + build: { + target: 'worker', + workingDirectory: '/other/worker', + dockerfile: 'infra/worker.Dockerfile', + context: 'src', + }, + }, + }; + const steps = planDeployment({ + ...base, + app: { ...app, config: { ...app.config, roles } }, + host: registryHost, + sources: { api: source('abc', '/repo'), worker: source('def', '/other/worker/src') }, + }); + const [api, worker] = steps; + assert.strictEqual( + printableStep(api!), + 'docker buildx build --platform linux/arm64 --push --file /repo/infra/Dockerfile --tag registry.example.com/demo-api:abc /repo', + ); + assert.strictEqual( + printableStep(worker!), + 'docker buildx build --platform linux/arm64 --push --file /other/worker/infra/worker.Dockerfile --target worker --tag registry.example.com/demo-worker:def /other/worker/src', + ); + assert.strictEqual(isPipeline(api!) ? '' : api!.cwd, '/repo/infra'); + assert.strictEqual(isPipeline(worker!) ? '' : worker!.cwd, '/other/worker'); + }); + + it('builds nothing for roles without a build', () => { + const roles = { postgres: { environment: { POSTGRES_USER: 'demo' } } }; + const steps = planDeployment({ + ...base, + app: { ...app, config: { ...app.config, roles } }, + host: registryHost, + sources: {}, + }); + assert.strictEqual(steps.length, 1); + }); + + it('passes the roles override to compose as the last file', () => { + const overrideFile = '/tmp/compoza-roles-x/compoza.roles.yaml'; + const client = planDeployment({ ...base, host: registryHost, overrideFile }); + assert.include( + printableStep(client.at(-1)!), + '--file /repo/infra/compose.production.yaml --file /tmp/compoza-roles-x/compoza.roles.yaml up', + ); + + const local = planDeployment({ ...base, host: registryHost, local: true, overrideFile }); + assert.include( + printableStep(local.at(-1)!), + '--file /repo/infra/compose.yaml --file /tmp/compoza-roles-x/compoza.roles.yaml up', + ); }); }); diff --git a/src/plan.ts b/src/plan.ts index 843e10b..ecd8b03 100644 --- a/src/plan.ts +++ b/src/plan.ts @@ -2,15 +2,16 @@ import { devNull } from 'node:os'; import { resolve } from 'node:path'; import { type Command, command, REMOTE_PATH, shellQuote, type Step } from './command.ts'; -import { - type App, - composeFilesOf, - environmentOf, - projectNameOf, - workingDirectoryOf, -} from './config/deploy.ts'; +import { type App, composeFilesOf, environmentOf, projectNameOf } from './config/deploy.ts'; import { dockerHost, type HostConfig } from './config/hosts.ts'; import { secretSource, validateSecrets } from './config/secrets.ts'; +import { + buildingRoles, + buildSettingsOf, + imageReference, + type RoleBuildSettings, + type RoleSources, +} from './roles.ts'; /** * Variables cleared for a build on the host's daemon, so a Docker context or Buildx builder @@ -34,8 +35,8 @@ const REMOTE_BUILD_ENVIRONMENT = { }; /** - * Everything a plan needs decided: the app and host, the tag, where the build context is, and the - * secrets. + * Everything a plan needs decided: the app and host, the tag and build context of every building + * role, and the secrets. */ export interface PlanOptions { /** @@ -49,9 +50,10 @@ export interface PlanOptions { readonly host: HostConfig; /** - * The image tag, normally the short commit. + * The tag and absolute build context of every building role; the tag is normally the short commit + * of the role's repository. */ - readonly tag: string; + readonly sources: RoleSources; /** * Whether the stack runs on this machine's Docker instead of the host. @@ -59,15 +61,16 @@ export interface PlanOptions { readonly local: boolean; /** - * Whether the build and transport steps are left out because the images already exist under - * `tag`. + * Whether the build and transport steps are left out because the images already exist under their + * tags. */ readonly reuseImages: boolean; /** - * The absolute build context. + * The absolute path of the generated roles override, passed to Compose as its last file. Without + * it Compose gets only the configured files. */ - readonly context: string; + readonly overrideFile?: string; /** * Secret Compose variables, passed to Compose and masked when printed. @@ -75,41 +78,6 @@ export interface PlanOptions { readonly secrets: Readonly>; } -/** - * The Compose variable that carries a target's image, e.g. `web` → `WEB_IMAGE`, `scraper-api` → - * `SCRAPER_API_IMAGE`. - */ -export function imageVariable(target: string): string { - return `${target.toUpperCase().replaceAll('-', '_')}_IMAGE`; -} - -/** - * The image reference for one target at one tag: prefixed with the registry when the host pulls - * from one, bare otherwise (local runs and SSH transport, where the image is loaded straight into a - * daemon). - */ -export function imageReference( - app: App, - host: HostConfig, - target: string, - tag: string, - local: boolean, -): string { - const bare = `${app.config.name}-${target}:${tag}`; - return !local && host.transport.mode === 'registry' ? `${host.transport.registry}/${bare}` : bare; -} - -/** - * The absolute Dockerfile path, defaulting to `Dockerfile` in the working directory. - * - * A relative `dockerfile` resolves against the working directory and an absolute one is kept as is, - * the same rule the build context follows in `deploy.ts`; `join` would instead append an absolute - * path to the folder. - */ -export function dockerfileOf(app: App): string { - return resolve(workingDirectoryOf(app), app.config.dockerfile ?? 'Dockerfile'); -} - /** * The script the SSH transport runs on the host to load an image from standard input: `gunzip | * docker load` after `REMOTE_PATH`, under `bash -c`. @@ -137,8 +105,8 @@ export function composePrefix(app: App, host?: HostConfig): ReadonlyArray { - const { app, host, tag, local } = options; + const { app, host, local } = options; validateSecrets(app, options.secrets, { local }); const steps: Array = []; - const images: Record = {}; - const dockerfile = dockerfileOf(app); - - for (const target of app.config.targets) { - const reference = imageReference(app, host, target, tag, local); - images[imageVariable(target)] = reference; - if (options.reuseImages) { - continue; - } - - steps.push(buildCommand(options, dockerfile, target, reference)); - - if (!local && host.transport.mode === 'ssh-load') { - steps.push(loadPipeline(host, reference)); + if (!options.reuseImages) { + for (const role of buildingRoles(app.config)) { + const source = Object.hasOwn(options.sources, role) ? options.sources[role] : undefined; + if (source === undefined) { + throw new Error(`No source for role "${role}"`); + } + + const reference = imageReference(app, host, role, source.tag, local); + steps.push(buildCommand(options, buildSettingsOf(app, role), source.context, reference)); + + if (!local && host.transport.mode === 'ssh-load') { + steps.push(loadPipeline(host, reference)); + } } } - steps.push(composeUpCommand(options, images)); + steps.push(composeUpCommand(options)); return steps; } /** - * The `docker buildx build` command for one target: pushed to the registry, built on the host's own + * The `docker buildx build` command for one role: pushed to the registry, built on the host's own * daemon, or loaded into this machine's daemon, depending on the transport and `local`. */ function buildCommand( options: PlanOptions, - dockerfile: string, - target: string, + settings: RoleBuildSettings, + context: string, reference: string, ): Command { - const { app, host, local } = options; + const { host, local } = options; const push = !local && host.transport.mode === 'registry'; const remoteBuild = !local && host.transport.mode === 'remote-build'; return command( @@ -199,15 +166,14 @@ function buildCommand( ...(local ? [] : ['--platform', host.platform]), push ? '--push' : '--load', '--file', - dockerfile, - '--target', - target, + settings.dockerfile, + ...(settings.target === undefined ? [] : ['--target', settings.target]), '--tag', reference, - options.context, + context, ], { - cwd: workingDirectoryOf(app), + cwd: settings.workingDirectory, ...(remoteBuild ? { env: REMOTE_BUILD_ENVIRONMENT } : {}), }, ); @@ -228,12 +194,13 @@ function loadPipeline(host: HostConfig, reference: string): Step { } /** - * The `docker compose up` command that starts the stack with the built image references. + * The `docker compose up` command that starts the stack, with the generated roles override last + * when the plan has one. * * A secret source or secret values replace Compose's automatic `.env` lookup with `/dev/null`, so a * stray dotenv file in the project directory cannot add or override variables. */ -function composeUpCommand(options: PlanOptions, images: Readonly>): Command { +function composeUpCommand(options: PlanOptions): Command { const { app, host, local } = options; const pull = !local && host.transport.mode === 'registry' ? 'always' : 'missing'; const replacesDotenv = @@ -243,10 +210,10 @@ function composeUpCommand(options: PlanOptions, images: Readonly [ - '--file', - resolve(app.dir, file), - ]), + ...[ + ...composeFilesOf(app.config, { local }).map((file) => resolve(app.dir, file)), + ...(options.overrideFile === undefined ? [] : [options.overrideFile]), + ].flatMap((file) => ['--file', file]), ...(app.config.profiles ?? []).flatMap((profile) => ['--profile', profile]), 'up', '--detach', @@ -257,7 +224,7 @@ function composeUpCommand(options: PlanOptions, images: Readonly { - const plan = deploymentScript(app, host, 'same', [ + const plan = deploymentScript(app, host, { web: 'same' }, [ source, { path: 'settings.txt', encoded: Buffer.from(value).toString('base64') }, ]); @@ -180,15 +180,15 @@ describe.skipIf(!composeAvailable)( composeFixture(({ app, root, run }) => { Object.assign(app.config, { composeFiles: ['normalized.yaml'] }); const source = Buffer.from('services: {web: {image: nginx:alpine}}').toString('base64'); - const first = deploymentScript(app, host, 'one', [ + const first = deploymentScript(app, host, { web: 'one' }, [ { path: 'normalized.yaml', encoded: source }, ]); const firstResult = run(first.script, first.input); assert.strictEqual(firstResult.status, 0, String(firstResult.stderr)); - assert.include(readFileSync(join(root, 'compose.yaml'), 'utf8'), 'nginx:alpine'); + assert.include(readFileSync(join(root, 'compose.yaml'), 'utf8'), 'fixture-web:one'); Object.assign(app.config, { composeFiles: ['compose.yaml'] }); - const second = deploymentScript(app, host, 'two', [ + const second = deploymentScript(app, host, { web: 'two' }, [ { path: 'compose.yaml', encoded: Buffer.from( @@ -210,7 +210,7 @@ describe.skipIf(!composeAvailable)( const compose = `services: web: - image: \${WEB_IMAGE:?required image} + image: nginx environment: TOKEN: \${TOKEN:?required token} LITERAL: $$literal @@ -227,16 +227,32 @@ describe.skipIf(!composeAvailable)( { path: 'production.yaml', encoded: Buffer.from(production).toString('base64') }, { path: 'settings.txt', encoded: Buffer.from('settings').toString('base64') }, ]; - const plan = deploymentScript(app, host, 'known', bundle, { + Object.assign(app.config, { + roles: { + web: { + build: { target: 'web' }, + environment: { + ROLE_LITERAL: 'cost $5 ${NOT_A_VARIABLE}', + ROLE_UNDERSCORE: '1_000', + ROLE_BINARY: '0b101', + ROLE_DATE: '2001-12-14', + }, + secrets: ['ROLE_SECRET'], + }, + }, + }); + const plan = deploymentScript(app, host, { web: 'known' }, bundle, { TOKEN: '$private "quoted" \\ trailing\\', + ROLE_SECRET: 'role "secret" $x', }); const result = run(plan.script, plan.input); assert.strictEqual(result.status, 0, String(result.stderr)); const installed = readFileSync(join(root, 'compose.yaml'), 'utf8'); - assert.include(installed, '${WEB_IMAGE:?required image}'); assert.include(installed, '${TOKEN:?required token}'); assert.notInclude(installed, '$private'); + assert.notInclude(installed, 'role "secret"'); + assert.include(installed, '${ROLE_SECRET}'); assert.notInclude(installed, '.deploy-lock'); const model = YAML.parse(installed); @@ -270,6 +286,18 @@ describe.skipIf(!composeAvailable)( ); assert.strictEqual(service.environment.RAW.replaceAll('$$', '$'), '$LITERAL "quoted"'); assert.strictEqual(service.environment.LITERAL.replaceAll('$$', '$'), '$literal'); + assert.strictEqual( + service.environment.ROLE_LITERAL.replaceAll('$$', '$'), + 'cost $5 ${NOT_A_VARIABLE}', + ); + assert.strictEqual(service.environment.ROLE_UNDERSCORE, '1_000'); + assert.strictEqual(service.environment.ROLE_BINARY, '0b101'); + assert.strictEqual(service.environment.ROLE_DATE, '2001-12-14'); + assert.strictEqual( + service.environment.ROLE_SECRET.replaceAll('$$', '$'), + 'role "secret" $x', + ); + assert.strictEqual(service.environment.MODE, 'production'); assert.strictEqual(service.volumes[0].source, join(root, 'settings.txt')); assert.deepStrictEqual(readdirSync(root).sort(), [ '.compoza.json', @@ -291,11 +319,11 @@ describe.skipIf(!composeAvailable)( remote: { stackDir: root, overlayFile }, }); - const plan = deploymentScript(app, host, 'known', [ + const plan = deploymentScript(app, host, { web: 'known' }, [ { path: 'compose.yaml', encoded: Buffer.from( - 'services: {web: {image: "${WEB_IMAGE}", environment: {TOKEN: "${TOKEN}"}}}', + 'services: {web: {image: nginx, environment: {TOKEN: "${TOKEN}"}}}', ).toString('base64'), }, ]); diff --git a/src/roles-deploy.test.ts b/src/roles-deploy.test.ts new file mode 100644 index 0000000..ea5615f --- /dev/null +++ b/src/roles-deploy.test.ts @@ -0,0 +1,337 @@ +import { NodeServices } from '@effect/platform-node'; +import { assert, describe, it } from '@effect/vitest'; +import { Effect, Layer, Option } from 'effect'; +import { FetchHttpClient } from 'effect/http'; +import { TestConsole } from 'effect/testing'; +import { existsSync, readFileSync } from 'node:fs'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as YAML from 'yaml'; + +import type { Step } from './command.ts'; + +import { deploy } from './deploy.ts'; +import { Git } from './services/Git.ts'; +import { Remote } from './services/Remote.ts'; +import { DryRun, Shell } from './services/Shell.ts'; + +/** + * Two Git repositories (`web-repo` and `worker-repo`) and a deploy folder, with Git and the shell + * replaced by recorders. + */ +const fixture = Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => mkdtemp(join(tmpdir(), 'compoza-roles-deploy-'))), + (directory) => Effect.promise(() => rm(directory, { recursive: true, force: true })), + ); + const deployDir = join(root, 'deploy'); + const webRepo = join(root, 'web-repo'); + const workerRepo = join(root, 'worker-repo'); + const hosts = join(root, 'hosts.yaml'); + yield* Effect.promise(async () => { + for (const directory of [deployDir, webRepo, workerRepo]) { + await mkdir(directory, { recursive: true }); + } + + await writeFile( + hosts, + YAML.stringify({ + host: { ssh: 'fixture', platform: 'linux/arm64', transport: { mode: 'ssh-load' } }, + }), + ); + await writeFile( + join(deployDir, 'compose.yaml'), + 'services: {web: {image: base}, worker: {image: base}, db: {image: postgres}}', + ); + }); + + const steps: Array = []; + const overrides: Array = []; + const overrideFiles: Array = []; + const gitCalls: Array = []; + const dirty = new Set(); + const shas: Record = { [webRepo]: 'aaaaaaaaaaaa', [workerRepo]: 'bbbbbbbbbbbb' }; + const shell = Layer.succeed( + Shell, + Shell.of({ + run: (step) => + Effect.sync(() => { + steps.push(step); + if ('program' in step && step.args.includes('up')) { + const file = step.args.filter((arg, index) => step.args[index - 1] === '--file').at(-1); + if (file !== undefined && file.endsWith('compoza.roles.yaml') && existsSync(file)) { + overrideFiles.push(file); + overrides.push(readFileSync(file, 'utf8')); + } + } + }), + capture: () => Effect.die('Git is replaced'), + }), + ); + const git = Layer.succeed( + Git, + Git.of({ + root: (dir) => + Effect.sync(() => { + gitCalls.push(`root ${dir}`); + return dir.startsWith(workerRepo) ? workerRepo : webRepo; + }), + hasCommits: (repository) => + Effect.sync(() => { + gitCalls.push(`hasCommits ${repository}`); + return true; + }), + isDirty: (repository) => + Effect.sync(() => { + gitCalls.push(`isDirty ${repository}`); + return dirty.has(repository); + }), + shortSha: (repository) => Effect.succeed(shas[repository] ?? ''), + }), + ); + const services = Layer.mergeAll( + NodeServices.layer, + FetchHttpClient.layer, + shell, + git, + Layer.succeed( + Remote, + Remote.of({ + run: () => Effect.die('No remote writes expected'), + probe: () => Effect.die('No remote probes expected'), + }), + ), + ); + const write = (config: unknown) => + Effect.promise(() => + writeFile( + join(deployDir, 'deploy.yaml'), + YAML.stringify({ + name: 'demo', + host: 'host', + composeFiles: ['compose.yaml'], + ...(config as object), + }), + ), + ); + const options = { + target: deployDir, + hosts: Option.some(hosts), + local: true, + dryRun: false, + allowDirty: false, + tags: [] as ReadonlyArray, + }; + return { + deployDir, + webRepo, + workerRepo, + steps, + overrides, + overrideFiles, + gitCalls, + dirty, + services, + write, + options, + }; +}); + +const roles = { + web: { build: { target: 'server', workingDirectory: '../web-repo' }, secrets: [] }, + worker: { + build: { workingDirectory: '../worker-repo', dockerfile: 'worker.Dockerfile' }, + environment: { MY_API_BASE_URL: 'https://api.example.com/v1', LOG_FORMAT: '$remote_addr' }, + }, + db: { environment: { POSTGRES_USER: 'demo' } }, +}; + +describe('deploying roles', () => { + it.effect('tags each role with the commit of its own repository and checks each once', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles }); + yield* deploy(f.options).pipe(Effect.provide(f.services)); + const lines = yield* TestConsole.logLines; + + const builds = f.steps.filter((step) => 'program' in step && step.args.includes('buildx')); + assert.lengthOf(builds, 2); + const [web, worker] = builds as ReadonlyArray>; + assert.include(web!.args, 'demo-web:aaaaaaaaaaaa'); + assert.strictEqual(web!.cwd, f.webRepo); + assert.strictEqual(web!.args.at(-1), f.webRepo); + assert.include(worker!.args, 'demo-worker:bbbbbbbbbbbb'); + assert.include(worker!.args, join(f.workerRepo, 'worker.Dockerfile')); + assert.notInclude(worker!.args, '--target'); + assert.strictEqual(worker!.cwd, f.workerRepo); + assert.deepStrictEqual( + f.gitCalls.filter((call) => call.startsWith('hasCommits')), + [`hasCommits ${f.webRepo}`, `hasCommits ${f.workerRepo}`], + ); + assert.strictEqual( + lines[0], + 'Deploying demo web=aaaaaaaaaaaa worker=bbbbbbbbbbbb to local Docker', + ); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer)), + ); + + it.effect('checks a repository shared by two roles once and shares its tag', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ + workingDirectory: '../web-repo', + roles: { web: { build: {} }, worker: { build: {} } }, + }); + yield* deploy(f.options).pipe(Effect.provide(f.services)); + const lines = yield* TestConsole.logLines; + + assert.lengthOf( + f.gitCalls.filter((call) => call.startsWith('isDirty')), + 1, + ); + assert.strictEqual(lines[0], 'Deploying demo aaaaaaaaaaaa to local Docker'); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer)), + ); + + it.effect('refuses a dirty repository of any role unless allowed', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles }); + f.dirty.add(f.workerRepo); + const error = yield* deploy({ ...f.options, local: false }).pipe( + Effect.provide(f.services), + Effect.flip, + ); + assert.include(error.message, `Commit your changes in ${f.workerRepo} first`); + assert.lengthOf(f.steps, 0); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer), Effect.provideService(DryRun, true)), + ); + + it.effect('writes the override as the last Compose file, then removes it', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles }); + yield* deploy(f.options).pipe(Effect.provide(f.services)); + + assert.lengthOf(f.overrides, 1); + assert.deepStrictEqual(YAML.parse(f.overrides[0]!), { + services: { + web: { image: 'demo-web:aaaaaaaaaaaa' }, + worker: { + image: 'demo-worker:bbbbbbbbbbbb', + environment: { + MY_API_BASE_URL: 'https://api.example.com/v1', + LOG_FORMAT: '$$remote_addr', + }, + }, + db: { environment: { POSTGRES_USER: 'demo' } }, + }, + }); + assert.isFalse(existsSync(f.overrideFiles[0]!)); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer)), + ); + + it.effect('reuses images with per-role tags without consulting Git', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles }); + yield* deploy({ ...f.options, tags: ['known', 'worker=older'] }).pipe( + Effect.provide(f.services), + ); + const lines = yield* TestConsole.logLines; + + assert.lengthOf(f.gitCalls, 0); + assert.lengthOf(f.steps, 1); + assert.lengthOf(f.overrides, 1); + const parsed = YAML.parse(f.overrides[0]!); + assert.strictEqual(parsed.services.web.image, 'demo-web:known'); + assert.strictEqual(parsed.services.worker.image, 'demo-worker:older'); + assert.strictEqual(lines[0], 'Deploying demo web=known worker=older to local Docker'); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer)), + ); + + it.effect.each([ + [['ghost=1'], 'Unknown role "ghost" in --tag'], + [['db=1', 'x'], 'Role "db" has no build'], + [['web=1'], 'No tag for role "worker"'], + [['a', 'b'], 'Only one --tag without a role is allowed'], + ] as const)('rejects --tag %j before building', ([tags, message]) => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles }); + const error = yield* deploy({ ...f.options, tags }).pipe( + Effect.provide(f.services), + Effect.flip, + ); + assert.include(error.message, message); + assert.lengthOf(f.steps, 0); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer)), + ); + + it.effect('builds nothing and uses the third-party tag without building roles', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles: { db: roles.db } }); + yield* deploy(f.options).pipe(Effect.provide(f.services)); + const lines = yield* TestConsole.logLines; + + assert.lengthOf(f.gitCalls, 0); + assert.lengthOf(f.steps, 1); + assert.lengthOf(f.overrides, 1); + assert.strictEqual(lines[0], 'Deploying demo third-party to local Docker'); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer)), + ); + + it.effect('fails before building when a role is not a Compose service', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles: { ...roles, wbe: { environment: { A: '1' } } } }); + const error = yield* deploy(f.options).pipe(Effect.provide(f.services), Effect.flip); + assert.strictEqual( + error.message, + 'Role "wbe" is not a service in the Compose files; services: web, worker, db', + ); + assert.lengthOf(f.steps, 0); + assert.lengthOf(f.gitCalls, 0); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer)), + ); + + it.effect('prints the override on a dry run without writing it', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles }); + yield* deploy(f.options).pipe(Effect.provide(f.services)); + const lines = yield* TestConsole.logLines; + + assert.lengthOf(f.overrideFiles, 0); + const printed = lines.find((line) => String(line).startsWith('Roles override ')); + assert.isDefined(printed); + assert.include(String(printed), 'compoza.roles.yaml:'); + assert.include(String(printed), 'image: "demo-web:aaaaaaaaaaaa"'); + assert.include(String(printed), 'LOG_FORMAT: "$$remote_addr"'); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer), Effect.provideService(DryRun, true)), + ); + + it.effect('prints the override on an SSH dry run', () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.write({ roles, execution: 'ssh', remote: { stackDir: '/srv/demo' } }); + const remote = Remote.of({ + run: () => Effect.succeed(''), + probe: () => + Effect.succeed('docker=yes\ncompose=yes\njq=yes\nstack_path=yes\nhost_overlay=yes\n'), + }); + yield* deploy({ ...f.options, local: false, dryRun: true }).pipe( + Effect.provideService(Remote, remote), + Effect.provide(f.services), + ); + const lines = yield* TestConsole.logLines; + + const printed = lines.find((line) => String(line).startsWith('Roles override ')); + assert.isDefined(printed); + assert.include(String(printed), 'compoza.roles.yaml:'); + assert.include(String(printed), 'LOG_FORMAT: "$$remote_addr"'); + }).pipe(Effect.scoped, Effect.provide(TestConsole.layer), Effect.provideService(DryRun, true)), + ); +}); diff --git a/src/roles.test.ts b/src/roles.test.ts new file mode 100644 index 0000000..f72c8ae --- /dev/null +++ b/src/roles.test.ts @@ -0,0 +1,597 @@ +import { NodeServices } from '@effect/platform-node'; +import { assert, describe, it } from '@effect/vitest'; +import { Effect, type FileSystem } from 'effect'; +import { mkdir, mkdtempDisposable, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { parse } from 'yaml'; + +import type { App, DeployConfig } from './config/deploy.ts'; +import type { HostConfig } from './config/hosts.ts'; + +import { + buildingRoles, + buildSettingsOf, + roleOverride, + validateRoles, + validateRoleSettings, +} from './roles.ts'; + +const host: HostConfig = { + ssh: 'deploy@atlas.example.com', + platform: 'linux/arm64', + transport: { mode: 'registry', registry: 'registry.example.com' }, +}; + +const config: DeployConfig = { + name: 'demo', + host: 'atlas', + composeFiles: ['compose.yaml'], + workingDirectory: '../repo', + dockerfile: 'infra/Dockerfile', + roles: { + web: { build: { target: 'server' }, secrets: ['READ_TOKEN'] }, + worker: { + build: { workingDirectory: '/srv/worker', dockerfile: 'worker.Dockerfile', context: 'src' }, + environment: { + MY_API_BASE_URL: 'https://api.example.com/v1', + LOG_FORMAT: '$remote_addr ${status}', + }, + }, + postgres: { environment: { POSTGRES_USER: 'demo' } }, + }, +}; + +const { roles: _roles, ...withoutRoles } = config; + +const app: App = { dir: '/deployments/demo', config }; + +describe('buildSettingsOf', () => { + it('inherits the top-level build fields and resolves them against the working directory', () => { + assert.deepStrictEqual(buildSettingsOf(app, 'web'), { + role: 'web', + target: 'server', + workingDirectory: '/deployments/repo', + workingDirectoryConfigured: true, + dockerfile: '/deployments/repo/infra/Dockerfile', + context: undefined, + }); + }); + + it('lets a role override the working directory, dockerfile and context', () => { + assert.deepStrictEqual(buildSettingsOf(app, 'worker'), { + role: 'worker', + target: undefined, + workingDirectory: '/srv/worker', + workingDirectoryConfigured: true, + dockerfile: '/srv/worker/worker.Dockerfile', + context: '/srv/worker/src', + }); + }); + + it('defaults to the deploy folder and a Dockerfile there', () => { + const { workingDirectory: _workingDirectory, dockerfile: _dockerfile, ...rest } = config; + const bare: App = { dir: '/d', config: { ...rest, roles: { api: { build: {} } } } }; + assert.deepStrictEqual(buildSettingsOf(bare, 'api'), { + role: 'api', + target: undefined, + workingDirectory: '/d', + workingDirectoryConfigured: false, + dockerfile: '/d/Dockerfile', + context: undefined, + }); + }); + + it('keeps absolute paths and rejects a role without a build', () => { + const absolute: App = { + ...app, + config: { + ...config, + roles: { api: { build: { dockerfile: '/abs/Dockerfile', context: '/abs' } } }, + }, + }; + assert.strictEqual(buildSettingsOf(absolute, 'api').dockerfile, '/abs/Dockerfile'); + assert.strictEqual(buildSettingsOf(absolute, 'api').context, '/abs'); + assert.throws(() => buildSettingsOf(app, 'postgres'), 'Role "postgres" has no build'); + }); + + it('lists the building roles in configuration order', () => { + assert.deepStrictEqual(buildingRoles(config), ['web', 'worker']); + assert.deepStrictEqual(buildingRoles(withoutRoles), []); + }); +}); + +describe('roleOverride', () => { + const tags = { web: 'abc', worker: 'def' }; + + it('sets the image, literal environment and secret references per role', () => { + const override = roleOverride({ app, host, tags }); + assert.deepStrictEqual(parse(override ?? ''), { + services: { + web: { + image: 'registry.example.com/demo-web:abc', + environment: { READ_TOKEN: '${READ_TOKEN}' }, + }, + worker: { + image: 'registry.example.com/demo-worker:def', + environment: { + MY_API_BASE_URL: 'https://api.example.com/v1', + LOG_FORMAT: '$$remote_addr $${status}', + }, + }, + postgres: { environment: { POSTGRES_USER: 'demo' } }, + }, + }); + }); + + it('reads a building role named like an Object.prototype member only from its own tag', () => { + const roles = { constructor: { build: {} } }; + const override = roleOverride({ + app: { ...app, config: { ...app.config, roles } }, + host, + tags: { constructor: 'abc' }, + }); + assert.include(override, 'registry.example.com/demo-constructor:abc'); + assert.throws( + () => roleOverride({ app: { ...app, config: { ...app.config, roles } }, host, tags: {} }), + 'No tag for role "constructor"', + ); + }); + + it('double-quotes values that YAML 1.1 readers would turn into numbers or dates', () => { + const odd = { A: '1_000', B: '0b101', C: '2001-12-14', D: '1_0.5', E: 'yes' }; + const override = + roleOverride({ + app: { ...app, config: { ...app.config, roles: { web: { environment: odd } } } }, + host, + tags: {}, + }) ?? ''; + + for (const [key, value] of Object.entries(odd)) { + assert.include(override, `${key}: "${value}"`); + } + }); + + it('names images without a registry for local runs and gives roles without a build no image', () => { + const local = parse(roleOverride({ app, host, tags, local: true }) ?? ''); + assert.strictEqual(local.services.web.image, 'demo-web:abc'); + assert.notProperty(local.services.postgres, 'image'); + }); + + it('merges the local roles for a local run', () => { + const local: App = { + ...app, + config: { + ...config, + local: { roles: { postgres: { environment: { EXTRA: '1' }, secrets: ['PG_PASSWORD'] } } }, + }, + }; + const merged = parse(roleOverride({ app: local, host, tags, local: true }) ?? ''); + assert.deepStrictEqual(merged.services.postgres.environment, { + POSTGRES_USER: 'demo', + EXTRA: '1', + PG_PASSWORD: '${PG_PASSWORD}', + }); + + const remote = parse(roleOverride({ app: local, host, tags }) ?? ''); + assert.deepStrictEqual(remote.services.postgres.environment, { POSTGRES_USER: 'demo' }); + }); + + it('never contains a secret value, only references', () => { + const override = roleOverride({ app, host, tags }) ?? ''; + assert.include(override, '${READ_TOKEN}'); + }); + + it('is absent without roles or settings', () => { + assert.isUndefined(roleOverride({ app: { ...app, config: withoutRoles }, host, tags })); + assert.isUndefined( + roleOverride({ app: { ...app, config: { ...config, roles: { idle: {} } } }, host, tags }), + ); + }); + + it('fails for a building role without a tag', () => { + assert.throws( + () => roleOverride({ app, host, tags: { web: 'abc' } }), + 'No tag for role "worker"', + ); + }); +}); + +describe('validateRoleSettings', () => { + it.each([ + [{ web: { environment: { '1BAD': 'x' } } }, 'Role "web" environment name 1BAD is not portable'], + [{ web: { environment: { 'A-B': 'x' } } }, 'environment name A-B is not portable'], + [ + { web: { environment: { A: 'one\ntwo' } } }, + 'Role "web" environment A has a multi-line value', + ], + [{ web: { environment: { A: 'one\rtwo' } } }, 'environment A has a multi-line value'], + [{ web: { environment: { A: 'one\0' } } }, 'environment A has a multi-line value'], + [{ web: { secrets: ['bad name'] } }, 'Role "web" secret name bad name is not portable'], + [ + { web: { environment: { TOKEN: 'x' }, secrets: ['TOKEN'] } }, + 'Role "web" lists TOKEN in both environment and secrets', + ], + ] as const)('rejects %j', (roles, message) => { + assert.throws(() => validateRoleSettings(roles), message); + }); + + it('never repeats an environment value in the message', () => { + try { + validateRoleSettings({ web: { environment: { TOKEN: 'SENTINEL\nSECOND' } } }); + assert.fail('Expected a failure'); + } catch (error) { + assert.notInclude((error as Error).message, 'SENTINEL'); + } + }); +}); + +describe('validateRoles', () => { + /** + * Runs `use` with an app whose deploy folder holds `files`. + */ + function withCompose( + files: Readonly>, + roles: DeployConfig['roles'], + use: (app: App) => Effect.Effect, + extra: Partial = {}, + ) { + return Effect.acquireUseRelease( + Effect.promise(() => mkdtempDisposable(join(tmpdir(), 'compoza-roles-'))), + ({ path: dir }) => + Effect.gen(function* () { + for (const [name, text] of Object.entries(files)) { + yield* Effect.promise(async () => { + await mkdir(dirname(join(dir, name)), { recursive: true }); + await writeFile(join(dir, name), text); + }); + } + + const { roles: _roles, ...base } = config; + const names = Object.keys(files); + return yield* use({ + dir, + config: { + ...base, + secrets: { 'env-file': { path: 'dev.env' } }, + composeFiles: [names[0] ?? 'compose.yaml', ...names.slice(1)], + ...(roles === undefined ? {} : { roles }), + ...extra, + }, + }); + }), + (directory) => Effect.promise(() => directory.remove()), + ).pipe(Effect.provide(NodeServices.layer)); + } + + it.effect('accepts roles that name services in any selected Compose file', () => + withCompose( + { 'compose.yaml': 'services: {web: {image: a}}', 'extra.yaml': 'services: {worker: {}}' }, + { web: { environment: { A: '1' } }, worker: { secrets: ['B'] } }, + (app) => validateRoles(app), + ), + ); + + it.effect('reads nothing for an app without roles', () => + withCompose({}, undefined, (app) => + validateRoles({ ...app, config: { ...app.config, composeFiles: ['missing.yaml'] } }), + ), + ); + + it.effect('names the known services when a role is not one of them', () => + withCompose( + { 'compose.yaml': 'services: {web: {image: a}, worker: {image: b}}' }, + { wbe: { environment: { A: '1' } } }, + (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.strictEqual( + error.message, + 'Role "wbe" is not a service in the Compose files; services: web, worker', + ); + }), + ), + ); + + it.effect('checks the local Compose files for a local run', () => + withCompose( + { 'compose.yaml': 'services: {web: {}}', 'local.yaml': 'services: {web: {}, db: {}}' }, + { db: { environment: { A: '1' } } }, + (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.include(error.message, 'Role "db" is not a service'); + yield* validateRoles(app, { local: true }); + }), + { composeFiles: ['compose.yaml'], local: { composeFiles: ['local.yaml'] } }, + ), + ); + + it.effect('rejects invalid role settings before reading files', () => + withCompose( + { 'compose.yaml': 'services: {web: {}}' }, + { web: { secrets: ['bad name'] } }, + (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.include(error.message, 'secret name bad name is not portable'); + }), + ), + ); + + it.effect.each([ + ['map form', 'services: {web: {environment: {MODE: x}}}', { environment: { MODE: 'y' } }], + [ + 'list form KEY=value', + 'services: {web: {environment: ["MODE=x"]}}', + { environment: { MODE: 'y' } }, + ], + ['list form KEY', 'services: {web: {environment: [MODE]}}', { environment: { MODE: 'y' } }], + ['a role secret', 'services: {web: {environment: {TOKEN: x}}}', { secrets: ['TOKEN'] }], + ] as const)('rejects a variable the service also sets in %s', ([, compose, role]) => + withCompose({ 'compose.yaml': compose }, { web: role }, (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.match( + error.message, + /^Role "web" sets (MODE|TOKEN), which service web also sets in environment in compose\.yaml; set it in only one place$/, + ); + }), + ), + ); + + it.effect('resolves YAML merge keys when finding the services own environment', () => + withCompose( + { + 'compose.yaml': + 'x-env: &env {MODE: x}\nservices:\n web:\n environment:\n <<: *env\n', + }, + { web: { environment: { MODE: 'y' } } }, + (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.include(error.message, 'sets MODE'); + }), + ), + ); + + it.effect('sees environment and extends inherited through a service-level merge key', () => + Effect.gen(function* () { + yield* withCompose( + { 'compose.yaml': 'x-base: &base {environment: {MODE: x}}\nservices: {web: {<<: *base}}' }, + { web: { environment: { MODE: 'y' } } }, + (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.include(error.message, 'sets MODE'); + }), + ); + yield* withCompose( + { + 'compose.yaml': + 'x-ext: &ext {extends: base}\nservices:\n base: {environment: {MODE: x}}\n web: {<<: *ext}', + }, + { web: { environment: { MODE: 'y' } } }, + (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.include(error.message, 'sets MODE'); + }), + ); + }), + ); + + it.effect('lets an explicit environment on the service win over a merged one', () => + withCompose( + { + 'compose.yaml': + 'x-base: &base {environment: {MODE: x}}\nservices: {web: {<<: *base, environment: {OTHER: z}}}', + }, + { web: { environment: { MODE: 'y' } } }, + (app) => validateRoles(app), + ), + ); + + it.effect('treats a whole-environment !reset as dropping its payload too', () => + withCompose( + { + 'compose.yaml': 'services: {web: {environment: {MODE: x}}}', + 'override.yaml': 'services: {web: {environment: !reset {MODE: x}}}', + }, + { web: { environment: { MODE: 'y' } } }, + (app) => validateRoles(app), + ), + ); + + it.effect('keeps the payload of a whole-environment !override', () => + withCompose( + { + 'compose.yaml': 'services: {web: {image: a}}', + 'override.yaml': 'services: {web: {environment: !override {MODE: x}}}', + }, + { web: { environment: { MODE: 'y' } } }, + (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.include(error.message, 'sets MODE'); + }), + ), + ); + + it.effect('follows include for a local run even when the app deploys over SSH', () => + withCompose( + { + 'compose.yaml': 'include: [inc.yaml]\nservices: {web: {image: a}}', + 'inc.yaml': 'services: {db: {image: b}}', + }, + { db: { environment: { A: '1' } } }, + (app) => + Effect.gen(function* () { + const error = yield* validateRoles(app).pipe(Effect.flip); + assert.include(error.message, 'Role "db" is not a service'); + yield* validateRoles(app, { local: true }); + }), + { execution: 'ssh', composeFiles: ['compose.yaml'] }, + ), + ); + + it.effect('fails when a role takes secrets but the run has no secret source', () => + withCompose( + { 'compose.yaml': 'services: {web: {}}' }, + { web: { secrets: ['TOKEN'] } }, + (app) => + Effect.gen(function* () { + const expected = + 'Role "web" takes secrets, but the run has no secret source; ' + + 'configure secrets (or local.secrets for --local) with a provider'; + const none = yield* validateRoles(app).pipe(Effect.flip); + assert.strictEqual(none.message, expected); + + const { secrets: _secrets, ...rest } = app.config; + const absent = yield* validateRoles({ ...app, config: rest }).pipe(Effect.flip); + assert.strictEqual(absent.message, expected); + }), + { secrets: { none: {} } }, + ), + ); + + it.effect('rejects reserved role secrets and names an environment variable already set', () => + withCompose( + { 'compose.yaml': 'services: {web: {}}' }, + { web: { secrets: ['COMPOSE_FILE'] } }, + (app) => + Effect.gen(function* () { + const reserved = yield* validateRoles(app).pipe(Effect.flip); + assert.strictEqual( + reserved.message, + 'Role "web" secret COMPOSE_FILE is a reserved variable; rename it', + ); + + const colliding = yield* validateRoles({ + ...app, + config: { ...app.config, roles: { web: { secrets: ['SHARED'] } } }, + }).pipe(Effect.flip); + assert.include( + colliding.message, + 'Role "web" secret SHARED has the name of an environment', + ); + }), + { environment: { SHARED: 'x' } }, + ), + ); + + it.effect('does not report a variable that a later file clears with !reset or !override', () => + withCompose( + { + 'compose.yaml': 'services: {web: {environment: {A: x, B: x, C: x}}}', + 'overlay.yaml': 'services:\n web:\n environment:\n A: !reset null\n D: y\n', + }, + { web: { environment: { A: '1' } } }, + (app) => + Effect.gen(function* () { + yield* validateRoles(app); + + const kept = yield* validateRoles({ + ...app, + config: { ...app.config, roles: { web: { environment: { B: '1' } } } }, + }).pipe(Effect.flip); + assert.include(kept.message, 'sets B'); + assert.include(kept.message, 'in compose.yaml'); + + const added = yield* validateRoles({ + ...app, + config: { ...app.config, roles: { web: { environment: { D: '1' } } } }, + }).pipe(Effect.flip); + assert.include(added.message, 'in overlay.yaml'); + }), + ), + ); + + it.effect.each([ + ['!reset', '!reset []'], + ['!override', '!override {Z: z}'], + ] as const)('follows %s on a whole environment in a later file', ([, tagged]) => + withCompose( + { + 'compose.yaml': 'services: {web: {environment: {A: x}}}', + 'overlay.yaml': `services: {web: {environment: ${tagged}}}`, + }, + { web: { environment: { A: '1' } } }, + (app) => validateRoles(app), + ), + ); + + it.effect('reports a variable inherited through extends, in the same or another file', () => + withCompose( + { + 'compose.yaml': + 'services:\n base: {environment: [A=1]}\n web: {extends: base}\n' + + ' api: {extends: {service: remote, file: lib/shared.yaml}}\n', + 'lib/shared.yaml': 'services: {remote: {environment: {B: x}}}', + }, + { web: { environment: { A: '1' } } }, + (app) => + Effect.gen(function* () { + const same = yield* validateRoles(app).pipe(Effect.flip); + assert.include(same.message, 'Role "web" sets A'); + + const other = yield* validateRoles({ + ...app, + config: { ...app.config, roles: { api: { secrets: ['B'] } } }, + }).pipe(Effect.flip); + assert.include(other.message, 'Role "api" sets B'); + assert.include(other.message, 'in lib/shared.yaml'); + + yield* validateRoles({ + ...app, + config: { ...app.config, roles: { api: { environment: { A: '1' } } } }, + }); + }), + { composeFiles: ['compose.yaml'] }, + ), + ); + + it.effect('counts services of included files under client execution only', () => + withCompose( + { + 'compose.yaml': + 'include:\n - sub/db.yaml\n - path: [sub/cache.yaml]\nservices: {web: {}}', + 'sub/db.yaml': 'services: {db: {environment: {PG: x}}}', + 'sub/cache.yaml': 'services: {cache: {}}', + }, + { db: { environment: { A: '1' } }, cache: { environment: { A: '1' } } }, + (app) => + Effect.gen(function* () { + yield* validateRoles(app); + + const clash = yield* validateRoles({ + ...app, + config: { ...app.config, roles: { db: { environment: { PG: '1' } } } }, + }).pipe(Effect.flip); + assert.include(clash.message, 'in sub/db.yaml'); + + const ssh = yield* validateRoles({ + ...app, + config: { ...app.config, execution: 'ssh' }, + }).pipe(Effect.flip); + assert.include(ssh.message, 'Role "db" is not a service'); + }), + { composeFiles: ['compose.yaml'] }, + ), + ); + + it.effect('reports an unreadable or unparsable Compose file', () => + withCompose({ 'compose.yaml': 'services: [' }, { web: {} }, (app) => + Effect.gen(function* () { + const bad = yield* validateRoles(app).pipe(Effect.flip); + assert.strictEqual(bad.message, 'Cannot parse compose.yaml; check its YAML syntax'); + + const missing = yield* validateRoles({ + ...app, + config: { ...app.config, composeFiles: ['nope.yaml'] }, + }).pipe(Effect.flip); + assert.strictEqual(missing.message, 'Cannot read Compose file nope.yaml'); + }), + ), + ); +}); diff --git a/src/roles.ts b/src/roles.ts new file mode 100644 index 0000000..f95798f --- /dev/null +++ b/src/roles.ts @@ -0,0 +1,339 @@ +import { Effect, FileSystem } from 'effect'; +import { resolve } from 'node:path'; +import { stringify } from 'yaml'; + +import type { App, DeployConfig, Role, RunOptions } from './config/deploy.ts'; +import type { HostConfig } from './config/hosts.ts'; + +import { readComposeEnvironment } from './compose-environment.ts'; +import { environmentOf, roleSecretNames, rolesOf, workingDirectoryOf } from './config/deploy.ts'; +import { VARIABLE_NAME_PATTERN } from './config/env-file.ts'; +import { isReservedVariable, roleSecretSourceIssue, selectSecretSource } from './config/secrets.ts'; +import { DeployError, tryPlan } from './errors.ts'; + +/** + * Characters a role environment value may not contain, matching the rule for secret values. + */ +const UNSUPPORTED_VALUE_PATTERN = /[\r\n\0]/; + +/** + * Why a variable name is rejected, shared by environment and secret names. + */ +const PORTABLE_NAME_HINT = 'use a letter or underscore, then letters, digits and underscores'; + +/** + * Where one building role's image comes from: the tag it runs under and the absolute build context. + */ +export interface RoleSource { + /** + * The tag the role's image is built and run under. + */ + readonly tag: string; + + /** + * The absolute build context: the role's `context` resolved against its working directory, else + * the Git root. With `--tag` nothing is built, so it falls back to the working directory and is + * never read. + */ + readonly context: string; +} + +/** + * Image sources by role name, for the roles that build an image. + */ +export type RoleSources = Readonly>; + +/** + * A role's build settings after falling back to the top-level fields and defaults. + */ +export interface RoleBuildSettings { + /** + * The role, which is also the Compose service. + */ + readonly role: string; + + /** + * The Dockerfile stage, or `undefined` to build the final stage. + */ + readonly target: string | undefined; + + /** + * The absolute working directory used for Git checks and as the build's `cwd`. + */ + readonly workingDirectory: string; + + /** + * Whether a working directory was configured for the role, at either level. + */ + readonly workingDirectoryConfigured: boolean; + + /** + * The absolute Dockerfile path. + */ + readonly dockerfile: string; + + /** + * The absolute build context when one is configured; the Git root is used otherwise. + */ + readonly context: string | undefined; +} + +/** + * The names of the roles that build an image, in configuration order. Only the app's roles can + * build: destinations and local runs share the source. + */ +export function buildingRoles(config: DeployConfig): ReadonlyArray { + return Object.entries(config.roles ?? {}) + .filter(([, role]) => role.build !== undefined) + .map(([name]) => name); +} + +/** + * The build settings of a role that has `build`: each field falls back to the top-level field of + * the same name, then to its default (`Dockerfile`, the Git root). Throws when the role has no + * `build`. + * + * A relative `dockerfile` and `context` resolve against the role's working directory and an + * absolute one is kept as is. + */ +export function buildSettingsOf(app: App, role: string): RoleBuildSettings { + const roles = app.config.roles ?? {}; + const build = Object.hasOwn(roles, role) ? roles[role]?.build : undefined; + if (build === undefined) { + throw new Error(`Role "${role}" has no build`); + } + + const configured = build.workingDirectory ?? app.config.workingDirectory; + const workingDirectory = workingDirectoryOf(app, configured); + const context = build.context ?? app.config.context; + return { + role, + target: build.target, + workingDirectory, + workingDirectoryConfigured: configured !== undefined, + dockerfile: resolve( + workingDirectory, + build.dockerfile ?? app.config.dockerfile ?? 'Dockerfile', + ), + context: context === undefined ? undefined : resolve(workingDirectory, context), + }; +} + +/** + * The image reference for one role at one tag: prefixed with the registry when the host pulls from + * one, bare otherwise (local runs and SSH transport, where the image is loaded straight into a + * daemon). + */ +export function imageReference( + app: App, + host: HostConfig, + role: string, + tag: string, + local: boolean, +): string { + const bare = `${app.config.name}-${role}:${tag}`; + return !local && host.transport.mode === 'registry' ? `${host.transport.registry}/${bare}` : bare; +} + +/** + * Checks the role settings that need no Compose file: environment names and values, secret names, + * and a name used as both environment and secret. Throws an `Error` naming the role and variable + * (never a value) for the first problem found. + */ +export function validateRoleSettings(roles: Readonly>): void { + for (const [name, role] of Object.entries(roles)) { + const environment = role.environment ?? {}; + const secrets = role.secrets ?? []; + + for (const [key, value] of Object.entries(environment)) { + if (!VARIABLE_NAME_PATTERN.test(key)) { + throw new Error( + `Role "${name}" environment name ${key} is not portable; ${PORTABLE_NAME_HINT}`, + ); + } + + if (UNSUPPORTED_VALUE_PATTERN.test(value)) { + throw new Error( + `Role "${name}" environment ${key} has a multi-line value; ` + + 'it must be a single line without line breaks or NUL', + ); + } + } + + for (const secret of secrets) { + if (!VARIABLE_NAME_PATTERN.test(secret)) { + throw new Error( + `Role "${name}" secret name ${secret} is not portable; ${PORTABLE_NAME_HINT}`, + ); + } + + if (Object.hasOwn(environment, secret)) { + throw new Error(`Role "${name}" lists ${secret} in both environment and secrets`); + } + } + } +} + +/** + * Options of `roleOverride`. + */ +export interface RoleOverrideOptions { + /** + * The app whose roles are rendered. + */ + readonly app: App; + + /** + * The host the images are named for. + */ + readonly host: HostConfig; + + /** + * The tag of every building role. + */ + readonly tags: Readonly>; + + /** + * Whether this is a local run, which merges `local.roles` and names images without a registry. + * `false` when left out. + */ + readonly local?: boolean; +} + +/** + * Renders the Compose override that sets each building role's image and each role's container + * environment, or `undefined` when the app has no roles. + * + * Environment values are literal, so `$` is written as `$$`. A role secret is only the reference + * `${NAME}`: the value stays in the interpolation pool (the stack `.env` or the process + * environment), so the file never holds a secret. Throws when the role settings are invalid or a + * building role has no tag. + */ +export function roleOverride({ + app, + host, + tags, + local = false, +}: RoleOverrideOptions): string | undefined { + const roles = rolesOf(app.config, { local }); + if (Object.keys(roles).length === 0) { + return undefined; + } + + validateRoleSettings(roles); + + const services: Record> = {}; + for (const [name, role] of Object.entries(roles)) { + const service: Record = {}; + if (role.build !== undefined) { + const tag = Object.hasOwn(tags, name) ? tags[name] : undefined; + if (tag === undefined) { + throw new Error(`No tag for role "${name}"`); + } + + service.image = imageReference(app, host, name, tag, local); + } + + const environment = { + ...Object.fromEntries( + Object.entries(role.environment ?? {}).map(([key, value]) => [ + key, + value.replaceAll('$', () => '$$'), + ]), + ), + ...Object.fromEntries((role.secrets ?? []).map((secret) => [secret, `\${${secret}}`])), + }; + if (Object.keys(environment).length > 0) { + service.environment = environment; + } + + if (Object.keys(service).length > 0) { + services[name] = service; + } + } + + if (Object.keys(services).length === 0) { + return undefined; + } + + // Double-quoting every string keeps Compose's YAML 1.1 reader from turning plain `1_000`, + // `0b101` or `2001-12-14` into a number or a date. + return stringify( + { services }, + { defaultStringType: 'QUOTE_DOUBLE', defaultKeyType: 'PLAIN', lineWidth: 0 }, + ); +} + +/** + * Checks the roles of a run against the run's secret source, environment and Compose files, before + * any build or host write. + * + * Fails with a `DeployError` when a role setting is invalid; a role takes secrets but the run has + * no secret source; a role secret is a reserved variable or has the name of an `environment` + * variable of the run; a role is not a service in any selected Compose file (the override would + * silently create an imageless service); or a service sets in its own `environment:` a variable its + * role also sets. The Compose rule follows how Compose merges the files: later files and + * `!reset`/`!override` replace or remove earlier variables, `extends` contributes the base + * service's variables, and under client execution `include:` files count. Files are parsed locally + * and no secret value is read; an app without roles reads nothing. + */ +export const validateRoles = Effect.fn('validateRoles')(function* ( + app: App, + options: RunOptions = {}, +): Effect.fn.Return { + const roles = rolesOf(app.config, options); + if (Object.keys(roles).length === 0) { + return; + } + + yield* tryPlan(() => validateRoleSettings(roles), 'Invalid roles'); + + if (roleSecretNames(roles).length > 0) { + const source = yield* Effect.fromResult(selectSecretSource(app, options)); + const sourceIssue = roleSecretSourceIssue(roles, source); + if (sourceIssue !== undefined) { + return yield* new DeployError({ message: sourceIssue }); + } + } + + const environment = environmentOf(app.config, options); + for (const [name, role] of Object.entries(roles)) { + for (const secret of role.secrets ?? []) { + if (isReservedVariable(secret)) { + return yield* new DeployError({ + message: `Role "${name}" secret ${secret} is a reserved variable; rename it`, + }); + } + + if (Object.hasOwn(environment, secret)) { + return yield* new DeployError({ + message: + `Role "${name}" secret ${secret} has the name of an environment variable; ` + + 'set it in only one place', + }); + } + } + } + + const services = yield* readComposeEnvironment(app, options); + for (const [name, role] of Object.entries(roles)) { + const own = services.get(name); + if (own === undefined) { + return yield* new DeployError({ + message: + `Role "${name}" is not a service in the Compose files; ` + + `services: ${[...services.keys()].join(', ') || 'none'}`, + }); + } + + const keys = [...Object.keys(role.environment ?? {}), ...(role.secrets ?? [])]; + const clash = keys.find((key) => own.has(key)); + if (clash !== undefined) { + return yield* new DeployError({ + message: + `Role "${name}" sets ${clash}, which service ${name} also sets in environment ` + + `in ${own.get(clash)}; set it in only one place`, + }); + } + } +}); diff --git a/src/ssh-deploy.test.ts b/src/ssh-deploy.test.ts index 96f53c4..ea02d7c 100644 --- a/src/ssh-deploy.test.ts +++ b/src/ssh-deploy.test.ts @@ -25,7 +25,7 @@ const app: App = { name: 'app', host: 'atlas', workingDirectory: '/source-app', - targets: ['web'], + roles: { web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml'], execution: 'ssh', remote: { stackDir: '/srv/app' }, @@ -75,10 +75,9 @@ describe('SSH deployment selection and transports', () => { const steps = planDeployment({ app, host: selectedHost, - tag: 'abc', + sources: { web: { tag: 'abc', context: '/source' } }, local: false, reuseImages: false, - context: '/source', secrets: {}, }); const build = steps[0]!; @@ -111,10 +110,9 @@ describe('SSH deployment selection and transports', () => { const reused = planDeployment({ app, host: selectedHost, - tag: 'abc', + sources: { web: { tag: 'abc', context: '/source' } }, local: false, reuseImages: true, - context: '/source', secrets: {}, }); assert.lengthOf(reused, 1); @@ -180,7 +178,6 @@ describe('SSH deployment selection and transports', () => { name: 'third-party', workingDirectory: '/missing/app-checkout', host: 'host', - targets: [], execution: 'ssh', remote: { stackDir: '/srv/third-party' }, composeFiles: ['compose.yaml'], @@ -199,7 +196,7 @@ describe('SSH deployment selection and transports', () => { dryRun: true, local: false, allowDirty: false, - tag: Option.none(), + tags: [], }); assert.lengthOf(writes, 0); diff --git a/src/ssh-deploy.ts b/src/ssh-deploy.ts index e9ced85..89550a4 100644 --- a/src/ssh-deploy.ts +++ b/src/ssh-deploy.ts @@ -4,16 +4,18 @@ import type { App } from './config/deploy.ts'; import type { HostConfig } from './config/hosts.ts'; import type { DeployOptions } from './deploy.ts'; -import { readBundle } from './bundle.ts'; +import { readBundle, ROLES_OVERRIDE_FILE } from './bundle.ts'; import { blocking, formatChecks, runChecks } from './checks.ts'; import { command, shellQuote } from './command.ts'; import { readSecrets, selectSecretSource } from './config/secrets.ts'; import { DeployError, tryPlan } from './errors.ts'; import { planDeployment } from './plan.ts'; import { DRY_RUN_COMPLETE, phase, runPhases } from './progress.ts'; +import { roleOverride, type RoleSources } from './roles.ts'; import { Remote } from './services/Remote.ts'; import { DryRun, Shell } from './services/Shell.ts'; import { deployedScript, deploymentScript } from './ssh-plan.ts'; +import { tagLabel, tagsOf } from './tags.ts'; /** * What the host stack transaction does after staging the bundle, shown in its progress label. @@ -47,7 +49,7 @@ export const deploySsh = Effect.fn('deploySsh')(function* ( app: App, host: HostConfig, options: DeployOptions, - source: { readonly tag: string; readonly context: string }, + sources: RoleSources, ): Effect.fn.Return { const shell = yield* Shell; const remote = yield* Remote; @@ -59,7 +61,7 @@ export const deploySsh = Effect.fn('deploySsh')(function* ( const managed = sourceConfig !== undefined && sourceConfig.provider !== 'none'; const deployment = yield* planned(() => - deploymentScript(app, host, source.tag, bundle, managed ? values : undefined), + deploymentScript(app, host, tagsOf(sources), bundle, managed ? values : undefined), ); // The plan's last step is the local Compose command; over SSH the host runs Compose itself. @@ -67,10 +69,9 @@ export const deploySsh = Effect.fn('deploySsh')(function* ( planDeployment({ app, host, - tag: source.tag, - context: source.context, + sources, local: false, - reuseImages: options.tag._tag === 'Some', + reuseImages: options.tags.length > 0, secrets: {}, }), ); @@ -93,7 +94,7 @@ export const deploySsh = Effect.fn('deploySsh')(function* ( const build = phase( 'build', - `Building and shipping ${app.config.targets.length} images (${host.transport.mode})`, + `Building and shipping ${Object.keys(sources).length} images (${host.transport.mode})`, Effect.forEach(builds, (step) => shell.run(step), { discard: true }), ); @@ -107,7 +108,13 @@ export const deploySsh = Effect.fn('deploySsh')(function* ( Effect.flatMap((output) => (output === '' ? Effect.void : Console.log(output.trimEnd()))), ); - yield* Console.log(`Deploying ${app.config.name} ${source.tag} to ${app.config.host} over SSH`); + const tag = tagLabel(sources); + yield* Console.log(`Deploying ${app.config.name} ${tag} to ${app.config.host} over SSH`); + const override = yield* planned(() => roleOverride({ app, host, tags: tagsOf(sources) })); + if (override !== undefined && (yield* DryRun)) { + yield* Console.log(`Roles override ${ROLES_OVERRIDE_FILE}:\n${override.trimEnd()}`); + } + yield* runPhases([ readiness, ...(builds.length === 0 ? [] : [build]), @@ -117,7 +124,7 @@ export const deploySsh = Effect.fn('deploySsh')(function* ( yield* Console.log( (yield* DryRun) ? SSH_DRY_RUN_SUMMARY - : `Deployed ${app.config.name} ${source.tag} to ${app.config.host}; stack folder updated.`, + : `Deployed ${app.config.name} ${tag} to ${app.config.host}; stack folder updated.`, ); }); diff --git a/src/ssh-plan.test.ts b/src/ssh-plan.test.ts index d684fdb..5ee1961 100644 --- a/src/ssh-plan.test.ts +++ b/src/ssh-plan.test.ts @@ -49,7 +49,7 @@ const host: HostConfig = { const source: BundleFile = { path: 'compose.yaml', - encoded: Buffer.from('services: {web: {image: "${WEB_IMAGE}"}}').toString('base64'), + encoded: Buffer.from('services: {web: {image: nginx}}').toString('base64'), }; /** @@ -105,7 +105,7 @@ esac config: { name: 'fixture', host: 'host', - targets: ['web'], + roles: { web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml'], execution: 'ssh', remote: { stackDir: root }, @@ -126,7 +126,7 @@ esac secrets?: Readonly>, bundle = [source] as ReadonlyArray, ) => { - const plan = deploymentScript(app, host, tag, bundle, secrets); + const plan = deploymentScript(app, host, { web: tag }, bundle, secrets); return run(plan.script, plan.input); }; @@ -148,7 +148,7 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { config: { name: 'fixture', host: 'host', - targets: ['web'], + roles: { web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml'], secrets: { '1password': { environmentId: 'fictional', variables: ['TOKEN'] } }, }, @@ -163,7 +163,7 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { .split('\n') .map((line) => line.slice(0, line.indexOf('='))) .sort((left, right) => left.localeCompare(right)), - ['TOKEN', 'WEB_IMAGE'], + ['TOKEN'], ); assert.notInclude(environment, 'excluded-secret'); assert.strictEqual(statSync(join(root, '.env')).mode & 0o777, 0o600); @@ -191,14 +191,18 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { assert.deepStrictEqual(readFileSync(join(root, 'settings.txt')), Buffer.from([0, 1, 255])); assert.strictEqual(statSync(join(root, '.env')).mode & 0o777, 0o600); assert.strictEqual(statSync(join(root, '.compoza.json')).mode & 0o777, 0o600); - assert.include(readFileSync(join(root, '.env'), 'utf8'), 'fixture-web:one'); + assert.deepStrictEqual(JSON.parse(readFileSync(join(root, '.compoza.json'), 'utf8')).tags, { + web: 'one', + }); assert.notInclude(readFileSync(join(root, 'compose.yaml'), 'utf8'), 'private-token'); assert.notInclude(readFileSync(join(root, '.compoza.json'), 'utf8'), 'private-token'); assert.include(readFileSync(join(dir, 'environment'), 'utf8'), 'CALLER_SECRET=\n'); succeeded(deploy('older', { TOKEN: 'rotated-token' }, [source, asset])); - assert.include(readFileSync(join(root, '.env'), 'utf8'), 'fixture-web:older'); + assert.deepStrictEqual(JSON.parse(readFileSync(join(root, '.compoza.json'), 'utf8')).tags, { + web: 'older', + }); assert.notInclude(readFileSync(join(root, '.env'), 'utf8'), 'private-token'); succeeded(run(deployedScript(app, ['stop'], true))); @@ -206,6 +210,59 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { assert.isFalse(existsSync(join(root, '.deploy-lock'))); })); + it('normalizes the generated roles override last and installs only the flat stack file', () => + fixture(({ app, dir, root, deploy }) => { + Object.assign(app.config, { + roles: { + web: { + build: { target: 'web' }, + environment: { MODE: 'a$b' }, + secrets: ['TOKEN'], + }, + }, + }); + succeeded(deploy('one', { TOKEN: 'private-token' })); + + const normalize = readFileSync(join(dir, 'calls'), 'utf8') + .split('\n') + .find((call) => call.includes('--no-interpolate')); + const files = [...(normalize ?? '').matchAll(/--file (\S+)/g)].map((match) => match[1]); + assert.lengthOf(files, 2); + assert.match(files[0]!, /\/upload\/compose\.yaml$/); + assert.match(files[1]!, /\/upload\/compoza\.roles\.yaml$/); + assert.deepStrictEqual(readdirSync(root).sort(), ['.compoza.json', '.env', 'compose.yaml']); + assert.isFalse( + JSON.parse(readFileSync(join(root, '.compoza.json'), 'utf8')).files.includes( + 'compoza.roles.yaml', + ), + ); + assert.notInclude(readFileSync(join(root, 'compose.yaml'), 'utf8'), 'private-token'); + assert.include(readFileSync(join(root, '.env'), 'utf8'), 'TOKEN="private-token"'); + })); + + it('rejects a Compose file or asset that uses the reserved override name', () => + fixture(({ app }) => { + Object.assign(app.config, { roles: { web: { build: {} } } }); + assert.throws( + () => + deploymentScript(app, host, { web: 'one' }, [ + source, + { ...source, path: 'compoza.roles.yaml' }, + ]), + 'compoza.roles.yaml is reserved for the generated roles override', + ); + })); + + it('sends no override without roles', () => + fixture(({ app, dir, run }) => { + const { roles: _roles, ...config } = app.config; + const plan = deploymentScript({ ...app, config }, host, {}, [source]); + succeeded(run(plan.script, plan.input)); + + const calls = readFileSync(join(dir, 'calls'), 'utf8'); + assert.notInclude(calls, 'compoza.roles.yaml'); + })); + it('validates before replacing stack files and never prints secret-bearing diagnostics', () => fixture(({ dir, root, deploy }) => { succeeded(deploy('good', { TOKEN: 'active-token' })); @@ -307,7 +364,7 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { succeeded(deploy()); const before = readFileSync(join(root, '.env'), 'utf8'); - const plan = deploymentScript(app, host, 'bad', [source]); + const plan = deploymentScript(app, host, { web: 'bad' }, [source]); const result = run(plan.script, plan.input.split('\n')[0] + '\n'); assert.notStrictEqual(result.status, 0); assert.include(String(result.stderr), 'Interrupted stack upload'); @@ -579,7 +636,9 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { assert.notStrictEqual(result.status, 0); assert.include(String(result.stderr), 'known image tag'); assert.notInclude(String(result.stdout) + String(result.stderr), 'PRIVATE_DIAGNOSTIC'); - assert.include(readFileSync(join(root, '.env'), 'utf8'), 'fixture-web:bad'); + assert.deepStrictEqual(JSON.parse(readFileSync(join(root, '.compoza.json'), 'utf8')).tags, { + web: 'bad', + }); assert.deepStrictEqual(readdirSync(root).sort(), ['.compoza.json', '.env', 'compose.yaml']); })); @@ -594,7 +653,9 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { it('keeps secrets out of script arguments even with inherited xtrace', () => fixture(({ app, run }) => { - const plan = deploymentScript(app, host, 'one', [source], { TOKEN: 'SENTINEL_PRIVATE' }); + const plan = deploymentScript(app, host, { web: 'one' }, [source], { + TOKEN: 'SENTINEL_PRIVATE', + }); assert.notInclude(plan.script, 'SENTINEL_PRIVATE'); const result = run('set -x\n' + plan.script, plan.input); @@ -607,14 +668,13 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { ); })); - it('encodes literal configured values and rejects reserved image variables', () => { + it('encodes literal configured values', () => { const environment = { DOLLAR: '$HOME ${OTHER}', QUOTE: 'a"b', BACK: 'ends\\', EMPTY: '' }; const app: App = { dir: '/app', config: { name: 'app', host: 'host', - targets: [], composeFiles: ['compose.yaml'], environment, }, @@ -629,15 +689,6 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { .join('='); assert.strictEqual(JSON.parse(encoded).replaceAll('$$', '$'), value); } - - assert.throws( - () => - generatedEnvironment({ - ...app, - config: { ...app.config, targets: ['web'], environment: { WEB_IMAGE: 'bad' } }, - }), - 'reserved', - ); }); it('names the first requirement a configured environment variable fails', () => { @@ -646,7 +697,6 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { config: { name: 'app', host: 'host', - targets: ['web'], composeFiles: ['compose.yaml'], environment, }, @@ -654,7 +704,6 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { const cases: ReadonlyArray, string]> = [ [{ '1TOKEN': 'value' }, 'Environment variable name 1TOKEN is not portable; use a letter'], [{ 'BAD-NAME': 'value' }, 'Environment variable name BAD-NAME is not portable'], - [{ WEB_IMAGE: 'value' }, "Environment variable WEB_IMAGE is reserved for a target's image"], [{ PATH: '/bin' }, 'Environment variable PATH is reserved by Compose, Docker or the shell'], [{ COMPOSE_FILE: 'x.yaml' }, 'COMPOSE_FILE is reserved by Compose, Docker or the shell'], [{ DOCKER_HOST: 'tcp://x' }, 'DOCKER_HOST is reserved by Compose, Docker or the shell'], @@ -687,7 +736,6 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { config: { name: 'app', host: 'host', - targets: [], composeFiles: ['compose.yaml'], remote: { stackDir }, }, @@ -711,7 +759,7 @@ describe('flat SSH stack transactions', { timeout: FIXTURE_TIMEOUT_MS }, () => { it('requires remote.stackDir for SSH execution', () => { const app: App = { dir: '/app', - config: { name: 'app', host: 'host', targets: [], composeFiles: ['compose.yaml'] }, + config: { name: 'app', host: 'host', composeFiles: ['compose.yaml'] }, }; assert.throws(() => stackRoot(app), 'SSH execution requires remote.stackDir; set it'); diff --git a/src/ssh-plan.ts b/src/ssh-plan.ts index 7ca7ee6..f15476b 100644 --- a/src/ssh-plan.ts +++ b/src/ssh-plan.ts @@ -5,12 +5,12 @@ import type { App } from './config/deploy.ts'; import type { HostConfig } from './config/hosts.ts'; import type { IngressPlan } from './extensions/ingress.ts'; -import { composeBundlePath } from './bundle.ts'; +import { composeBundlePath, ROLES_OVERRIDE_FILE, ROLES_OVERRIDE_RESERVED } from './bundle.ts'; import { REMOTE_PATH, shellQuote } from './command.ts'; import { directoriesOf, ingressOf, projectNameOf } from './config/deploy.ts'; import { secretEnvironment, validateSecrets } from './config/secrets.ts'; import { ingressFor, ingressOperationScript } from './extensions/ingress.ts'; -import { imageReference, imageVariable } from './plan.ts'; +import { buildingRoles, roleOverride } from './roles.ts'; import { runtimeSource } from './ssh-runtime.ts'; /** @@ -86,6 +86,12 @@ interface TransactionScriptParts { */ readonly bundle: ReadonlyArray; + /** + * Whether the generated roles override follows the bundle on standard input. It is staged as + * `ROLES_OVERRIDE_FILE` and normalized last, but never installed. + */ + readonly hasOverride: boolean; + /** * Host directories created before Compose starts the stack. */ @@ -138,19 +144,12 @@ export function stackRoot(app: App): string { /** * Builds the non-secret dotenv payload using Compose's double-quoted literal escaping. * - * Image variables are reserved and written into the generated dotenv, so a separate host dotenv or - * inherited shell variables cannot replace them. Configured variables must be explicit non-secret - * strings. + * Configured variables must be explicit non-secret strings. */ export function generatedEnvironment(app: App): string { - const imageVariables = new Set(app.config.targets.map(imageVariable)); - if (imageVariables.size !== app.config.targets.length) { - throw new Error('Targets must map to distinct image variables'); - } - const environment = app.config.environment ?? {}; for (const [key, value] of Object.entries(environment)) { - const issue = environmentVariableIssue(key, value, imageVariables); + const issue = environmentVariableIssue(key, value); if (issue !== undefined) { throw new Error(issue); } @@ -205,7 +204,7 @@ done export function deploymentScript( app: App, host: HostConfig, - tag: string, + tags: Readonly>, bundle: ReadonlyArray, secrets?: Readonly>, ): { readonly script: string; readonly input: string } { @@ -220,22 +219,27 @@ export function deploymentScript( throw new Error('compose.yaml is the generated stack file and cannot be declared as an asset'); } + if (bundle.some((file) => file.path === ROLES_OVERRIDE_FILE)) { + throw new Error(ROLES_OVERRIDE_RESERVED); + } + + const override = roleOverride({ app, host, tags }); + const files = ['compose.yaml', ...assets.map((file) => file.path)]; const outputs = [...files, '.env', '.compoza.json']; assertOverlaySeparate(app, root, outputs); const ingress = ingressFor(app)?.plan(app, host); const url = app.config.health?.url ?? ingress?.healthUrl; - const metadata = deploymentMetadata(app, tag, files, ingress, secrets !== undefined); - const environment = - generatedEnvironment(app) + - secretEnvironment({ ...secrets, ...imageEnvironment(app, host, tag) }); + const metadata = deploymentMetadata(app, tags, files, ingress, secrets !== undefined); + const environment = generatedEnvironment(app) + secretEnvironment({ ...secrets }); const script = transactionScript({ root, composeFiles, outputs, bundle, + hasOverride: override !== undefined, directories: directoriesOf(app.config), pull: host.transport.mode === 'registry' ? 'always' : 'missing', waitTimeoutSeconds: app.config.remote?.waitTimeoutSeconds ?? DEFAULT_WAIT_TIMEOUT_SECONDS, @@ -250,6 +254,7 @@ export function deploymentScript( const input = [ ...bundle.map((file) => file.encoded), + ...(override === undefined ? [] : [Buffer.from(override).toString('base64')]), Buffer.from(metadata).toString('base64'), Buffer.from(environment).toString('base64'), ].join('\n') + '\n'; @@ -318,7 +323,8 @@ export function sshReadinessScript(app: App, host: HostConfig): string { const overlayCheck = readableFileCheck(app.config.remote?.overlayFile); const probesHttp = app.config.health !== undefined || ingressOf(app.config).mode !== 'none'; const curlCheck = probesHttp ? 'command -v curl >/dev/null 2>&1' : 'true'; - const buildsOnHost = host.transport.mode === 'remote-build' && app.config.targets.length > 0; + const buildsOnHost = + host.transport.mode === 'remote-build' && buildingRoles(app.config).length > 0; const buildkitCheck = buildsOnHost ? 'docker buildx version >/dev/null 2>&1' : 'true'; return `set -eu ${REMOTE_PATH} @@ -363,17 +369,13 @@ function stackRootIssue(root: string): string | undefined { /** * Why a configured variable may not be written into the generated dotenv, or `undefined` when it - * may: its name must be portable, must not be an image variable, must not be reserved by Compose, - * Docker or the shell, and its value must be a single line without NUL. The requirements are - * checked in that order and the first one that fails is reported. + * may: its name must be portable, must not be reserved by Compose, Docker or the shell, and its + * value must be a single line without NUL. The requirements are checked in that order and the first + * one that fails is reported. * * Messages name the variable but never include its value. */ -function environmentVariableIssue( - key: string, - value: string, - imageVariables: ReadonlySet, -): string | undefined { +function environmentVariableIssue(key: string, value: string): string | undefined { if (!ENVIRONMENT_KEY.test(key)) { return ( `Environment variable name ${key} is not portable; ` + @@ -381,10 +383,6 @@ function environmentVariableIssue( ); } - if (imageVariables.has(key)) { - return `Environment variable ${key} is reserved for a target's image; rename it in environment`; - } - if ( RESERVED_ENVIRONMENT_KEYS.has(key) || RESERVED_ENVIRONMENT_PREFIXES.some((prefix) => key.startsWith(prefix)) @@ -439,12 +437,13 @@ function assertOverlaySeparate(app: App, root: string, outputs: ReadonlyArray>, files: ReadonlyArray, ingress: IngressPlan | undefined, managed: boolean, @@ -452,7 +451,7 @@ function deploymentMetadata( return JSON.stringify({ version: 1, project: projectNameOf(app), - tag, + tags, profiles: app.config.profiles ?? [], files, ownedFiles: files, @@ -462,18 +461,6 @@ function deploymentMetadata( }); } -/** - * The image variable of every target, set to the reference the host runs at `tag`. - */ -function imageEnvironment(app: App, host: HostConfig, tag: string): Record { - return Object.fromEntries( - app.config.targets.map((target) => [ - imageVariable(target), - imageReference(app, host, target, tag, false), - ]), - ); -} - /** * Renders the stack transaction: lock, receive the bundle, normalize and validate it in the lock * folder, install the stack files, then start the stack, route ingress and wait for health. @@ -487,7 +474,13 @@ function transactionScript(parts: TransactionScriptParts): string { `compoza_receive "$upload"/${shellQuote(file.path)}`, ].join('\n'), ); - const uploadedComposeFiles = composeFiles.map((file) => `"$upload"/${shellQuote(file)}`); + const stagedOverride = parts.hasOverride + ? [`compoza_receive "$upload"/${shellQuote(ROLES_OVERRIDE_FILE)}`] + : []; + const uploadedComposeFiles = [ + ...composeFiles, + ...(parts.hasOverride ? [ROLES_OVERRIDE_FILE] : []), + ].map((file) => `"$upload"/${shellQuote(file)}`); const installs = outputs.map((file) => [ `mkdir -p -- ${shellQuote(posix.dirname(`${root}/${file}`))}`, @@ -511,7 +504,7 @@ compoza_receive() { IFS= read -r encoded || { echo 'Interrupted stack upload' >&2; return 1; } printf '%s' "$encoded" | compoza_decode > "$1" || { echo 'Invalid stack upload' >&2; return 1; } } -${staging.join('\n')} +${[...staging, ...stagedOverride].join('\n')} compoza_receive "$upload/.compoza.json" compoza_receive "$upload/.env" metadata="$upload/.compoza.json" diff --git a/src/tags.test.ts b/src/tags.test.ts new file mode 100644 index 0000000..bebd865 --- /dev/null +++ b/src/tags.test.ts @@ -0,0 +1,88 @@ +import { assert, describe, it } from '@effect/vitest'; + +import type { DeployConfig } from './config/deploy.ts'; + +import { parseTagFlags, tagLabel, tagsOf } from './tags.ts'; + +const config: DeployConfig = { + name: 'demo', + host: 'atlas', + composeFiles: ['compose.yaml'], + roles: { + web: { build: { target: 'web' } }, + worker: { build: {} }, + postgres: { environment: { POSTGRES_USER: 'demo' } }, + }, +}; + +describe('parseTagFlags', () => { + it('handles a building role named like an Object.prototype member', () => { + const named: DeployConfig = { ...config, roles: { constructor: { build: {} } } }; + assert.deepStrictEqual(parseTagFlags(['abc'], named), { constructor: 'abc' }); + assert.deepStrictEqual(parseTagFlags(['constructor=def'], named), { constructor: 'def' }); + assert.throws(() => parseTagFlags(['constructor=a', 'constructor=b'], named), 'given twice'); + }); + + it('applies a bare tag to every building role', () => { + assert.deepStrictEqual(parseTagFlags(['abc'], config), { web: 'abc', worker: 'abc' }); + }); + + it('lets a role tag override the bare tag, in either order', () => { + const expected = { web: 'abc', worker: 'def' }; + assert.deepStrictEqual(parseTagFlags(['abc', 'worker=def'], config), expected); + assert.deepStrictEqual(parseTagFlags(['worker=def', 'abc'], config), expected); + }); + + it('accepts role tags alone when every building role is named', () => { + assert.deepStrictEqual(parseTagFlags(['web=1', 'worker=2'], config), { web: '1', worker: '2' }); + }); + + it('keeps everything after the first equals sign in the tag', () => { + assert.throws(() => parseTagFlags(['abc', 'web=a=b'], config), 'not a valid Docker tag'); + }); + + it('accepts Docker tags with dots, dashes and underscores', () => { + assert.strictEqual(parseTagFlags(['v1.2_3-rc'], config).web, 'v1.2_3-rc'); + assert.strictEqual(parseTagFlags(['_a'.padEnd(128, 'b')], config).web?.length, 128); + }); + + it.each([ + [['ghost=abc'], 'Unknown role "ghost" in --tag; roles: web, worker, postgres'], + [['postgres=abc', 'abc'], 'Role "postgres" has no build, so --tag cannot name it'], + [['abc', 'web=1', 'web=2'], 'Role "web" is given twice in --tag'], + [['one', 'two'], 'Only one --tag without a role is allowed'], + [['web=1'], 'No tag for role "worker"; pass --tag or --tag worker='], + [[''], '--tag needs a tag'], + [['web='], '--tag web= needs a tag after ='], + [['web:abc'], 'to name a role use --tag ='], + [['.abc'], 'not a valid Docker tag'], + [['-abc'], 'not a valid Docker tag'], + [['web=a b'], 'not a valid Docker tag'], + [['a'.repeat(129)], 'not a valid Docker tag'], + ] as const)('rejects %j', (values, message) => { + assert.throws(() => parseTagFlags(values, config), message); + }); + + it('names no roles when the app has none', () => { + const bare: DeployConfig = { name: 'demo', host: 'atlas', composeFiles: ['compose.yaml'] }; + assert.throws(() => parseTagFlags(['web=1'], bare), 'roles: none'); + assert.deepStrictEqual(parseTagFlags(['abc'], bare), {}); + }); +}); + +describe('tagLabel', () => { + it('shows one tag when every role shares it', () => { + const sources = { web: { tag: 'abc', context: '/a' }, worker: { tag: 'abc', context: '/b' } }; + assert.strictEqual(tagLabel(sources), 'abc'); + }); + + it('lists the tag of each role when they differ', () => { + const sources = { web: { tag: 'abc', context: '/a' }, worker: { tag: 'def', context: '/b' } }; + assert.strictEqual(tagLabel(sources), 'web=abc worker=def'); + assert.deepStrictEqual(tagsOf(sources), { web: 'abc', worker: 'def' }); + }); + + it('names the third-party tag when nothing is built', () => { + assert.strictEqual(tagLabel({}), 'third-party'); + }); +}); diff --git a/src/tags.ts b/src/tags.ts new file mode 100644 index 0000000..30f9fb3 --- /dev/null +++ b/src/tags.ts @@ -0,0 +1,115 @@ +import type { DeployConfig } from './config/deploy.ts'; +import type { RoleSources } from './roles.ts'; + +import { buildingRoles } from './roles.ts'; + +/** + * The tag shown for a deploy that builds nothing, so there is no commit to name the images after. + */ +const THIRD_PARTY_TAG = 'third-party'; + +/** + * What Docker accepts as an image tag: up to 128 letters, digits, underscores, dots and dashes, not + * starting with a dot or dash. + */ +const TAG_PATTERN = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/; + +/** + * Reads the values of the repeatable `--tag` flag into one tag per building role. + * + * `--tag ` is the default for every building role and `--tag =` overrides it for + * one role. Throws an `Error` naming the problem when a role is unknown or has no `build`, a role + * or the bare tag is given twice, a tag is empty or not a valid Docker tag, or a building role is + * left without a tag. + */ +export function parseTagFlags( + values: ReadonlyArray, + config: DeployConfig, +): Readonly> { + const roles = Object.keys(config.roles ?? {}); + const building = buildingRoles(config); + const named = new Map(); + const bare: Array = []; + + for (const value of values) { + const separator = value.indexOf('='); + const role = separator < 0 ? undefined : value.slice(0, separator); + const tag = separator < 0 ? value : value.slice(separator + 1); + if (tag === '') { + throw new Error( + role === undefined ? '--tag needs a tag' : `--tag ${role}= needs a tag after =`, + ); + } + + if (!TAG_PATTERN.test(tag)) { + throw new Error( + `--tag ${value} is not a valid Docker tag (letters, digits, underscores, dots and ` + + 'dashes, at most 128 characters, not starting with a dot or dash); ' + + 'to name a role use --tag =', + ); + } + + if (role === undefined) { + bare.push(tag); + continue; + } + + if (!roles.includes(role)) { + throw new Error(`Unknown role "${role}" in --tag; roles: ${roles.join(', ') || 'none'}`); + } + + if (!building.includes(role)) { + throw new Error( + `Role "${role}" has no build, so --tag cannot name it; ` + + `roles with build: ${building.join(', ') || 'none'}`, + ); + } + + if (named.has(role)) { + throw new Error(`Role "${role}" is given twice in --tag`); + } + + named.set(role, tag); + } + + if (bare.length > 1) { + throw new Error('Only one --tag without a role is allowed; use --tag = per role'); + } + + const tags: Record = {}; + for (const role of building) { + const tag = named.get(role) ?? bare[0]; + if (tag === undefined) { + throw new Error(`No tag for role "${role}"; pass --tag or --tag ${role}=`); + } + + tags[role] = tag; + } + + return tags; +} + +/** + * The tags of a deploy as one phrase for messages: the single tag when every role shares it, + * `web=abc worker=def` otherwise, and `third-party` when nothing is built. + */ +export function tagLabel(sources: RoleSources): string { + const entries = Object.entries(sources); + const tags = new Set(entries.map(([, source]) => source.tag)); + if (tags.size === 0) { + return THIRD_PARTY_TAG; + } + + if (tags.size === 1) { + return [...tags][0] ?? THIRD_PARTY_TAG; + } + + return entries.map(([role, source]) => `${role}=${source.tag}`).join(' '); +} + +/** + * The tag of every role in `sources`, as recorded in the deployed stack's metadata. + */ +export function tagsOf(sources: RoleSources): Record { + return Object.fromEntries(Object.entries(sources).map(([role, source]) => [role, source.tag])); +} diff --git a/src/working-directory.test.ts b/src/working-directory.test.ts index 83e6b4c..63e0522 100644 --- a/src/working-directory.test.ts +++ b/src/working-directory.test.ts @@ -83,7 +83,7 @@ const fixture = Effect.gen(function* () { const config: DeployConfig = { name: 'foo', host: 'host', - targets: ['web'], + roles: { web: { build: { target: 'web' } } }, composeFiles: ['compose.yaml'], workingDirectory: '../../../foo/app source', destinations: { development: { projectName: 'foo-dev' } }, @@ -97,7 +97,7 @@ const fixture = Effect.gen(function* () { local: true, dryRun: true, allowDirty: false, - tag: Option.none(), + tags: [] as ReadonlyArray, destination: 'development', }; return { @@ -183,7 +183,7 @@ describe('separate working directory', () => { Effect.provide(f.services), Effect.flip, ); - assert.include(error.message, 'Commit your changes first'); + assert.include(error.message, 'Commit your changes in'); assert.lengthOf(f.steps, 0); yield* deploy(f.options).pipe(Effect.provide(f.services)); @@ -246,11 +246,11 @@ describe('separate working directory', () => { const f = yield* fixture; yield* f.writeConfig({ workingDirectory: '/missing/app-checkout', - ...(mode === 'third-party' ? { targets: [] } : {}), + ...(mode === 'third-party' ? { roles: {} } : {}), }); yield* deploy({ ...f.options, - tag: mode === 'tag' ? Option.some('known') : Option.none(), + tags: mode === 'tag' ? ['known'] : [], }).pipe(Effect.provide(f.services)); const resolved = yield* resolve(f.file, Option.some(f.hosts), 'development').pipe( Effect.provide(f.services),