diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..36991bb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +**/.terraform +**/.bootstrap +local/.env +*.md +docs +adrs +runbooks +examples +aws +gcp +azure +.github diff --git a/.github/workflows/build-ami.yml b/.github/workflows/build-ami.yml new file mode 100644 index 0000000..1976a24 --- /dev/null +++ b/.github/workflows/build-ami.yml @@ -0,0 +1,61 @@ +# Bake the Vault node AMI once and copy it to every region in AMI_REGIONS. +# +# Repository configuration: +# secret AWS_ROLE_ARN IAM role assumed through GitHub OIDC; Packer's EC2 policy plus ec2:CopyImage +# var AWS_REGION region the bake runs in (default us-east-1) +# var AMI_REGIONS comma-separated regions to copy the AMI to (optional) +name: build-ami + +on: + workflow_dispatch: + inputs: + vault_version: + description: Vault CE version to install + default: "2.0.4" + push: + branches: [main] + paths: + - vault-node/** + - cmd/** + - internal/** + - go.mod + - go.sum + +permissions: + id-token: write + contents: read + +concurrency: build-ami + +jobs: + bake: + runs-on: ubuntu-latest + env: + AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.26.x" + + - name: Build vault-utils + run: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o vault-node/vault-utils ./cmd/vault-utils + + - uses: hashicorp/setup-packer@v3 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Bake + working-directory: vault-node + env: + AMI_REGIONS: ${{ vars.AMI_REGIONS }} + VAULT_VERSION: ${{ inputs.vault_version || '2.0.4' }} + run: | + regions=$(printf '%s\n' "$AMI_REGIONS" | jq -Rc 'split(",") | map(select(length > 0))') + packer init . + packer validate -var region="$AWS_REGION" -var vault_version="$VAULT_VERSION" -var "ami_regions=$regions" vault.pkr.hcl + packer build -color=false -var region="$AWS_REGION" -var vault_version="$VAULT_VERSION" -var "ami_regions=$regions" vault.pkr.hcl diff --git a/.github/workflows/test-local.yml b/.github/workflows/test-local.yml new file mode 100644 index 0000000..86bbf44 --- /dev/null +++ b/.github/workflows/test-local.yml @@ -0,0 +1,38 @@ +name: test-local + +on: + push: + branches: [main, master, develop] + pull_request: + +permissions: + contents: read + +jobs: + go: + name: go tests (docker) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.26.x" + + - name: Isolation and credentials + run: go test ./internal/vaultcluster -count=1 -timeout 15m + + stack: + name: compose runtime conformance + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.26.x" + + - name: Local compose runtime test + run: go test ./local -count=1 -timeout 15m -v diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..28c07e2 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,150 @@ +# Static validation. +name: validate + +on: + push: + branches: [main, master, develop] + pull_request: + +permissions: + contents: read + +jobs: + go: + name: go unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.26.x" + + - name: Unit tests (no Docker) + run: go test -short ./... + + architecture: + name: runtime separation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: HTTP-only trees must not reference any runtime + run: | + set -euo pipefail + fail=0 + check() { + local pattern="$1" label="$2" + if grep -rInE "$pattern" \ + internal/vaultcluster/policies \ + --include='*.sh' --include='*.tpl' --include='*.hcl' \ + | grep -vE '^\s*[^:]+:[0-9]+:\s*#' ; then + echo "FAIL: HTTP-only tree references $label" + fail=1 + fi + } + check '\bdocker\b' 'docker' + check '\bdocker[- ]compose\b' 'docker compose' + check 'vault-cluster-(vault|postgres)' 'container names' + check '/vault/(file|logs|config)' 'container filesystem paths' + exit "$fail" + + security: + name: secret scan and CE guard + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: No bootstrap material is committed + run: | + set -euo pipefail + fail=0 + while IFS= read -r f; do + echo "FAIL: bootstrap material committed: $f"; fail=1 + done < <(find . -path ./.git -prune -o \ + \( -name 'vault-init.json' -o -name '*.snap' \ + -o -name '*.token' -o -name 'unseal*' \) -print) + + if grep -rInE 'hvs\.[A-Za-z0-9_-]{20,}|"root_token"\s*:\s*"hvs' . \ + --exclude-dir=.git --exclude=CHANGELOG.md; then + echo "FAIL: what looks like a real Vault token is committed"; fail=1 + fi + exit "$fail" + + - name: Fake credentials are labelled as fake + run: | + set -euo pipefail + if grep -rInE '(password|secret|api_key)\s*[:=]\s*"[A-Za-z0-9]{16,}"' \ + local internal 2>/dev/null \ + | grep -viE 'FAKE-|local-dev-|\{\{|\$\{|not-a-real'; then + echo "FAIL: an unlabelled credential-shaped literal was found" + exit 1 + fi + + - name: No Vault Enterprise features are used + run: | + set -euo pipefail + fail=0 + if grep -rInE 'sys/namespaces|X-Vault-Namespace|sys/replication|sys/control-group|sentinel' \ + local/ cmd/ internal/ --include='*.sh' --include='*.hcl' --include='*.tpl' --include='*.go' \ + | grep -vE ':\s*#' | grep -v 'VAULT_NAMESPACE'; then + echo "FAIL: Vault Enterprise-only feature referenced (Community Edition only)" + fail=1 + fi + if grep -rIn 'hashicorp/vault-enterprise' local cmd internal --exclude-dir=.git; then + echo "FAIL: Enterprise image referenced"; fail=1 + fi + exit "$fail" + + - name: Images are pinned by digest + run: | + set -euo pipefail + fail=0 + if grep -nE '^\s*image:' local/compose.yml | grep -v '@sha256:'; then + echo "FAIL: an image is not digest-pinned"; fail=1 + fi + if grep -rn ':latest' local/ --include='*.yml' --include='.env.example'; then + echo "FAIL: ':latest' found - the environment could change without a commit"; fail=1 + fi + exit "$fail" + + compose: + name: compose config + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Compose file is valid with a one-shot bootstrap + run: | + set -euo pipefail + cd local + cp .env.example .env + + docker compose --env-file .env config --quiet + + default_svcs="$(docker compose --env-file .env config --services)" + printf 'default services:\n%s\n' "${default_svcs}" + if printf '%s\n' "${default_svcs}" | grep -qx 'unseal'; then + echo "FAIL: long-running unseal sidecar should not be a default service" + exit 1 + fi + if ! printf '%s\n' "${default_svcs}" | grep -qx 'bootstrap'; then + echo "FAIL: bootstrap one-shot missing from default compose services" + exit 1 + fi + if printf '%s\n' "${default_svcs}" | grep -qx 'postgres'; then + echo "FAIL: the local stack must not run PostgreSQL" + exit 1 + fi + + docs: + name: documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Required documents exist + run: | + set -euo pipefail + [ -f README.md ] || { echo "missing: README.md"; exit 1; } + [ -f CHANGELOG.md ] || { echo "missing: CHANGELOG.md"; exit 1; } diff --git a/.gitignore b/.gitignore index 78e7733..a6221d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,44 +1,76 @@ -# Local .terraform directories -.terraform/ +# Published module surface. +# +# Shipped: runtime code, CI, /README.md, /CHANGELOG.md +# Local-only (kept on disk, not published): ADRs, design docs, runbooks, examples, +# and any other markdown. -# .tfstate files -*.tfstate -*.tfstate.* +*.md +!/README.md +!/CHANGELOG.md +docs/ +runbooks/ +examples/ +adrs/ +adapters/ -# Crash log files -crash.log -crash.*.log +# Unseal keys, tokens, snapshots. Never commit. +.bootstrap/ +**/.bootstrap/ +*.unseal +*.unseal-keys +vault-init.json +.vault-init +.vault-token +*.token +*.snap +*.snapshot +audit.log +audit*.log +local/data/ +local/audit/ +local/backups/ -# Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject -# to change depending on the environment. -*.tfvars -*.tfvars.json +# Generated policies. +local/.rendered-policies/ +*.rendered.hcl -# Ignore override files as they are usually used to override resources locally and so -# are not checked in +# Secrets and local env. +.env +.env.* +!.env.example +*.pem +*.key +!**/testdata/**/*.key +*.crt +!**/testdata/**/*.crt +*.p12 +*.pfx +role-id +secret-id + +# OpenTofu / Terraform state (none yet). +**/.terraform/* +*.tfstate +*.tfstate.* +*.tfplan +crash.log override.tf override.tf.json *_override.tf *_override.tf.json -# Ignore transient lock info files created by terraform apply -.terraform.tfstate.lock.info - -# Include override files you do wish to add to version control using negated pattern -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - -# Ignore CLI configuration files -.terraformrc -terraform.rc - -# Optional: ignore graph output files generated by `terraform graph` -# *.dot +# Go build artifacts. +/vault-utils +coverage.out -# Optional: ignore plan files saved before destroying Terraform configuration -# Uncomment the line below if you want to ignore planout files. -# planout \ No newline at end of file +# Test/editor noise. +test-results/ +*.tap +.coverage +.DS_Store +**/.DS_Store +.idea/ +.vscode/ +.cursor/ +*.swp +*~ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e23ce31 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,2 @@ +# 0.1.0 (Unreleased) +* Initial release diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..031212a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.26-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd/ cmd/ +COPY internal/ internal/ +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /vault-utils ./cmd/vault-utils + +FROM alpine:3.21 +COPY --from=build /vault-utils /usr/local/bin/vault-utils +ENTRYPOINT ["/usr/local/bin/vault-utils"] diff --git a/README.md b/README.md index e0553a7..8094c86 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,385 @@ -# vault-cluster +# Secrets Vault Cluster + Multi-platform modules to configure a self-hosted Vault cluster on local, AWS, GCP, and Azure + +Applications share one Vault. They do not see each other's secrets. Isolation is a path prefix plus ACL policy plus AppRole. Vault CE has no namespaces. This is not Enterprise. + +**Dev machine:** `cd local && docker compose up -d --build`. You do not init or unseal by hand. + +| Target | Role | Status | +|---|---|---| +| `local/` | Docker Compose | Implemented | +| `aws/aws-ec2-vault-cluster/` | `aws-ec2-vault-cluster` | Connections, IAM, SM, security groups, AMI, user-data, launch template. `bootstrap aws`, health on 8210, S3 snapshots. ASG/NLB not built yet. | +| `gcp/` | GCP | Not implemented | +| `azure/` | Azure | Not implemented | + +There is no `module "vault_cluster" { source = "./${var.cloud}" }` switch. Shared trees talk to Vault only through `VAULT_ADDR` and a token. + +## Contents + +1. [Scope](#scope) +2. [Architecture](#architecture) +3. [Repository layout](#repository-layout) +4. [Prerequisites](#prerequisites) +5. [Quick start](#quick-start) +6. [Commands](#commands) +7. [Tenant isolation](#tenant-isolation) +8. [Identities](#identities) +9. [Health](#health) +10. [Backup, restore, and disaster recovery](#backup-restore-and-disaster-recovery) +11. [Break-glass](#break-glass) +12. [Testing](#testing) +13. [Troubleshooting](#troubleshooting) +14. [Security](#security) + +## Scope + +Implemented: + +- Vault CE 2.0 (never `-dev`, never Enterprise) +- Docker Compose, Raft on a named volume +- KV v2 at `kv/customers/{tenant}/*` +- Dynamic PostgreSQL credentials in the Go library only (`TestCredentialsMatrix`); the local stack runs no PostgreSQL and never mounts `database/` +- File audit on a volume separate from Raft +- Auto-init (first start) and one-shot Shamir unseal via `vault-utils` +- Isolation tests in Go (`go test`); credentials tests in Go (`TestCredentialsMatrix`) +- `bootstrap aws` (KMS auto-unseal, Secrets Manager tokens), health on 8210, S3 snapshots +- AWS AMI bake (Vault CE + vault-utils) and instance user-data (fail-closed bootstrap, health on 8210) + +Not implemented: + +- AWS ASG and NLB +- GCP, Azure, Kubernetes +- KMS auto-unseal proven on a running EC2 cluster +- TLS, multi-node Raft, DR replication + +Local unseal submits Shamir shares (5 shares, threshold 3) for laptop use. It is not AWS KMS, Cloud KMS, or Azure Key Vault auto-unseal. + +## Architecture + +``` +Developer + | +Docker Compose + | + +-- Vault + | Raft + | KV v2 + | ACL policies + | AppRole + | Audit + | + +-- vault-utils bootstrap (one-shot: init, unseal, configure) +``` + +Isolation and credentials tests are Go (`go test ./internal/vaultcluster`). Both run throwaway containers; the credentials suite brings its own PostgreSQL, so the local stack never runs one. + +## Repository layout + +``` +vault-cluster/ +├── README.md +├── CHANGELOG.md +├── Dockerfile vault-utils image +├── cmd/ Go app entrypoints (vault-utils CLI) +├── internal/vaultcluster/ shared Vault library +├── internal/aws/ AWS adapters (secretsmanager, s3) +├── local/ Compose target, snapshots +├── aws/aws-ec2-vault-cluster/ Nullstone module (IAM/SM/SG/AMI/user-data/launch template; no ASG yet) +├── gcp/ Nullstone Terraform module (not yet implemented) +└── azure/ Nullstone Terraform module (not yet implemented) +``` + +## Prerequisites + +Docker Desktop (Compose v2). Go 1.26 for `go test`. `curl` and `jq` for the manual examples below; Vault CLI is optional except break-glass decode. + +Images are pinned by tag and digest in `local/compose.yml` (Vault 2.0, PostgreSQL 18-alpine). Never `latest`. + +```bash +docker --version +docker compose version +``` + +## Quick start + +```bash +cd local +docker compose up -d +``` + +First run: + +1. Starts persistent Vault CE +2. Initializes Shamir 5/3 +3. Writes keys to `local/.bootstrap/` (gitignored, mode 600) +4. Unseals +5. Enables audit, KV v2, AppRole, policies +6. Revokes the root token + +Bootstrap only initializes the cluster. Tenants are created explicitly with `tenants create`. + +Later runs skip init if the provisioning token still works. A Vault process restart reseals. Unseal is a one-shot (`docker compose run --rm bootstrap`), not a long-running sidecar. `docker compose up -d` starts Vault and runs bootstrap once, then bootstrap exits. + +Then: + +```bash +export VAULT_ADDR=http://127.0.0.1:8200 +export VAULT_TOKEN=$(cat local/.bootstrap/provisioning.token) +``` + +Back up `local/.bootstrap/vault-init.json` immediately. Without it this volume cannot be unsealed. + +Do not commit `.bootstrap/` or `.env`. Do not run `vault operator unseal`. + +## Commands + +Run from `local/` unless noted. Destructive commands require `--yes`. + +| Command | Destructive | Purpose | +|---|---|---| +| `docker compose up -d --build` | no | Start, init (first time), unseal, configure | +| `docker compose down` | no | Stop containers. Keeps all data. | +| `docker compose down --volumes --remove-orphans && rm -rf .bootstrap` | yes | Destroys volumes and unseal keys | +| `docker compose run --rm -e VAULT_TOKEN=... bootstrap tenants create ` | no | Onboard a tenant | +| `docker compose run --rm -e VAULT_TOKEN=... bootstrap tenants destroy --yes` | yes (access) | Revoke access; secrets kept | +| `docker compose run --rm -e VAULT_TOKEN=... bootstrap tenants destroy --yes --purge-secrets` | yes | Also destroy secret versions | +| `docker compose run --rm bootstrap snapshot take` | no | Raft snapshot plus SHA-256 | +| `docker compose run --rm bootstrap snapshot restore --yes` | yes | Replaces all Vault state | +| `go test -short ./...` | no | Unit tests (tenant ID, policy lint, render, compose lint) — repo root | +| `go test ./internal/vaultcluster` | no | Isolation and credentials (needs Docker) — repo root | +| `go test ./local` | no | Compose runtime conformance (needs Docker) — repo root | + +## Tenant isolation + +Paths (KV v2 requires `data/` and `metadata/`): + +``` +kv/data/customers/{tenant_id}/* +kv/metadata/customers/{tenant_id}/* +``` + +`kv/customers/...` matches nothing. + +Tenant IDs: `^[a-z0-9]([a-z0-9-]{1,30}[a-z0-9])$` (3-32 characters). Rejected: `/`, `..`, `*`, `sys`, `data`, `root`, and similar reserved names. + +```bash +export VAULT_ADDR=http://127.0.0.1:8200 +export VAULT_TOKEN=$(cat local/.bootstrap/provisioning.token) +cd local +docker compose run --rm -e VAULT_TOKEN bootstrap tenants create acme-corp +``` + +`role_id` and `secret_id` print once and are not stored. Re-issue a secret_id if lost. + +Write as the tenant writer: + +```bash +curl -s -H "X-Vault-Token: ${TENANT_TOKEN}" \ + -X POST --data '{"data":{"api_key":"FAKE-value"}}' \ + "${VAULT_ADDR}/v1/kv/data/customers/acme-corp/app-config" +``` + +Offboard (revoke access, keep secrets): + +```bash +cd local +docker compose run --rm -e VAULT_TOKEN bootstrap tenants destroy acme-corp --yes +``` + +Offboard and destroy data (needs break-glass; provisioning cannot read or purge KV): + +```bash +cd local +docker compose run --rm -e VAULT_TOKEN bootstrap tenants destroy acme-corp --yes --purge-secrets +``` + +Cross-tenant, wildcard, and traversal reads return HTTP 403. That is the isolation contract. A 200 on those paths is a breach. + +## Identities + +| Identity | Role | +|---|---| +| Root | Bootstrap only. Revoked when setup finishes. | +| Provisioning | Create and offboard tenants. Cannot read tenant secrets. | +| Tenant AppRole | One reader and one writer per tenant. | +| Operator | Health, mounts, snapshots. Not a tenant secret reader. | + +## Health + +```bash +curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8200/v1/sys/health +``` + +| Code | Meaning | +|---|---| +| 200 | Unsealed and active | +| 501 | Uninitialized | +| 503 | Sealed | + +After a Vault process restart, expect 503 (sealed). Re-run the one-shot: + +```bash +cd local && docker compose run --rm bootstrap +``` + +Vault fails closed when audit cannot write. If every request is denied: + +```bash +cd local && docker compose exec vault sh -c 'ls -la /vault/logs && df -h /vault/logs' +``` + +## Backup, restore, and disaster recovery + +A snapshot is the whole cluster (secrets, policies, tokens). Treat it like Vault itself. A backup that has never been restored is not a backup. + +Keep the matching `vault-init.json` with each snapshot. After restore, Vault unseals only with the shares that were current when the snapshot was taken. + +Snapshots are written to `local/.bootstrap/backups/` on the host, which the bootstrap container sees as `/bootstrap/backups/`. + +### Backup + +```bash +cd local +docker compose run --rm bootstrap snapshot take +docker compose run --rm bootstrap snapshot list +docker compose run --rm bootstrap snapshot verify /bootstrap/backups/vault-.snap +``` + +### Restore (destructive) + +Restore needs a token with `sys/storage/raft/snapshot-force`. The operator token cannot restore; generate a break-glass root first (see [Break-glass](#break-glass)). + +```bash +cd local +docker compose run --rm -e VAULT_TOKEN= bootstrap snapshot restore /bootstrap/backups/vault-.snap --yes +docker compose run --rm bootstrap +``` + +### Restore drill + +```bash +cd local +docker compose up -d --build +export VAULT_ADDR=http://127.0.0.1:8200 +export VAULT_TOKEN=$(cat .bootstrap/provisioning.token) +docker compose run --rm -e VAULT_TOKEN bootstrap tenants create tenant-a + +docker compose run --rm bootstrap snapshot take +cp .bootstrap/vault-init.json /tmp/keys-at-snapshot.json + +docker compose run --rm -e VAULT_TOKEN bootstrap tenants create tenant-drill +docker compose down --volumes --remove-orphans && rm -rf .bootstrap + +docker compose up -d +cp /tmp/keys-at-snapshot.json .bootstrap/vault-init.json +# generate a break-glass root (see Break-glass), then: +docker compose run --rm -e VAULT_TOKEN= bootstrap snapshot restore /bootstrap/backups/vault-.snap --yes +docker compose run --rm bootstrap +go test ./internal/vaultcluster -run TestIsolationMatrix +``` + +Expected: `tenant-a` exists, `tenant-drill` does not, isolation suite passes. + +### If unseal keys are lost + +The Raft volume cannot be unsealed. Data is gone. That is Shamir working. Restore from a snapshot that still has its matching keys, or destroy the volumes (`docker compose down --volumes --remove-orphans && rm -rf .bootstrap` in `local/`) and start empty. + +## Break-glass + +Use only when no routine identity can do the job (read tenant data in an incident, purge secrets, repair audit). Two people. Record why first. + +```bash +export VAULT_ADDR=http://127.0.0.1:8200 +curl -s -X PUT "${VAULT_ADDR}/v1/sys/generate-root/attempt" | jq +# each of 3 share holders: +curl -s -X PUT --data '{"key":"","nonce":""}' \ + "${VAULT_ADDR}/v1/sys/generate-root/update" | jq +vault operator generate-root -decode= -otp= +# one recorded action, then: +curl -s -X POST -H "X-Vault-Token: ${ROOT_TOKEN}" \ + "${VAULT_ADDR}/v1/auth/token/revoke-self" +``` + +Cancel an in-flight attempt: + +```bash +curl -s -X DELETE "${VAULT_ADDR}/v1/sys/generate-root/attempt" +``` + +## Testing + +```bash +go test -short ./... + +go test ./internal/vaultcluster + +go test ./local +``` + +In `local/`, `TestLocalComposeStatic` lints `compose.yml` (digest pins, no dev mode, loopback-only ports). `TestLocalComposeRuntime` brings up an isolated copy of the Compose stack and verifies the one-shot bootstrap, Shamir over Raft, audit/Raft volume separation, and persistence across a restart. It never touches your dev stack or `local/.bootstrap/`. + +Denials must be HTTP 403. A 404 is a different failure. + +### AWS module (`aws/aws-ec2-vault-cluster/`) + +OpenTofu in this directory is connections, IAM, Secrets Manager, security groups, AMI lookup, and user-data. `go test ./internal/aws/...` covers the SM KeyStore and S3 snapshot helpers. `go test ./internal/vaultcluster` covers Raft health, leader check, and cron parse. There is no live-AWS test in CI. + +From `aws/aws-ec2-vault-cluster/`: + +```bash +tofu fmt -check +tofu init -backend=false +tofu validate +``` + +`tofu plan` and `tofu apply` need a Nullstone workspace plus AWS credentials. Without them, plan fails with `no nullstone workspace 0/0/0` and missing AWS credentials. That is expected. Do not `tofu apply` from this repo unless you intend to create IAM, Secrets Manager, and security groups. + +To plan against real connections: + +1. Install the Nullstone CLI (`ns`). +2. In an AWS Nullstone stack, attach this module and connect: + - `network` → `network/aws/vpc` (same VPC pattern as Nullstone EC2 apps) + - `snapshots_bucket` → `datastore/aws/s3` (snapshot bucket) + - `unseal_key` → `datastore/aws/kms` (dedicated unseal key, not the bucket SSE key) +3. Run workspace preview/plan in Nullstone so `ns_connection` outputs resolve. +4. In the plan, expect an IAM role + instance profile, SSM attach, inline IAM policy, three Secrets Manager secrets (`init` / `provisioning` / `operator`), two security groups (NLB + nodes) with the 8200/8201/8210 rules, a launch template (baked AMI + user-data), and no ASG or NLB yet. + +Bake the node AMI (x86_64, matches default `t3.micro`) from `vault-node/`: + +```bash +GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o vault-node/vault-utils ./cmd/vault-utils +cd vault-node +packer init . +packer build -var region="$AWS_REGION" vault.pkr.hcl +``` + +`.github/workflows/build-ami.yml` runs the same bake on demand or on a push to `main` that touches the image inputs, and copies the result to every region in the `AMI_REGIONS` repository variable. It assumes the `AWS_ROLE_ARN` secret through GitHub OIDC and bakes in `AWS_REGION` (default `us-east-1`). + +`vault-node/files/` holds the cloud-neutral image content: base `vault.hcl` and the systemd units. `vault-node/aws/vault-node-configure` is the only AWS-specific piece, and other clouds add a sibling directory. The bake installs Vault CE 2.0, `vault-utils`, and that content, then enables every unit. + +On boot, `vault-configure.service` runs after cloud-init, writes `/etc/vault.d/cloud.hcl` and `/etc/vault.d/node.env`, and exits. Systemd ordering then starts Vault, bootstrap, health, and snapshots. User-data only writes `/etc/vault.d/vault-utils.env`. Override with `ami` when using a different architecture. + +## Troubleshooting + +| Symptom | Action | +|---|---| +| Docker daemon is not running | Start Docker Desktop | +| Port already allocated | Change `VAULT_HOST_PORT` in `local/.env` | +| Initialized but `.bootstrap` missing | Restore `vault-init.json`, or destroy volumes and start empty | +| Health 503 | Vault is sealed. Run `docker compose run --rm bootstrap` in `local/` | +| Permission denied on tenant create | Use the provisioning token, not operator | +| Permission denied on tenant secrets | Expected for provisioning | +| Everything denied | Audit volume full or unwritable | + +## Security + +- Host ports bind to `127.0.0.1` only +- No Vault `-dev` mode +- Root token revoked after bootstrap +- Unseal keys, tokens, and `.env` are gitignored (mode 600). Never printed to logs +- Audit values are HMAC'd. Raw secrets must not appear in the audit log +- Provisioning cannot read tenant KV +- Operator cannot read tenant KV +- Local Compose sets `disable_mlock = true` because Docker and GitHub Actions cannot mlock. Production hosts should use `IPC_LOCK` +- Dynamic credentials (Go library and tests only): bounded TTL, revoke drops the Postgres role, residue scan expects zero leftover `v-*` roles diff --git a/aws/aws-ec2-vault-cluster/.nullstone/module.yml b/aws/aws-ec2-vault-cluster/.nullstone/module.yml new file mode 100644 index 0000000..a9985e7 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/.nullstone/module.yml @@ -0,0 +1,14 @@ +org_name: nullstone +name: aws-ec2-vault-cluster +friendly_name: Vault Cluster (AWS EC2) +description: Self-hosted Vault cluster running on an EC2 Auto Scaling Group +category: datastore +subcategory: "" +provider_types: + - aws +platform: vault +subplatform: ec2 +type: "" +appCategories: [] +is_public: true +tool_name: opentofu diff --git a/aws/aws-ec2-vault-cluster/.terraform.lock.hcl b/aws/aws-ec2-vault-cluster/.terraform.lock.hcl new file mode 100644 index 0000000..c518e6b --- /dev/null +++ b/aws/aws-ec2-vault-cluster/.terraform.lock.hcl @@ -0,0 +1,107 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "6.62.0" + hashes = [ + "h1:5x/4mMlIqSyeMJQ8tD0A+FTNynpdHW8IA7F2zqrgpwU=", + "h1:Etlx1tUvlB7sqFWPciSn9Yz1g50ctM+dGlyVMpu720I=", + "h1:FBnaqFZDf3DdVb4SVpCPoT+TZQUSChNmmDViEGPV1rw=", + "h1:MRIBAtFWiQAyo4kpbBUtIvxlSmpQ9Eel0nLMYCb6MH4=", + "h1:MeMP80kzq1meAJQ0l+kU6KVW1S+4EhIRSy6q3L++LPU=", + "h1:N8W8KgcjlG1kb9mPapfieH/vYzyNrJBsb4RS0axwg5Y=", + "h1:OB5obEZKuaX4gQ7DYxYvAMaf4I6v2keeKOPfOpFMkgI=", + "h1:WnhkO4yQc0QvVgx/xJFI+UbM1y8BC8yvApoDUlhb5XY=", + "h1:fN9PvxrT2/rAiT86Hen0vicyMbKXAZG6VC/TEP9ZpD8=", + "h1:kRrdLje5ab/tmObt9lOKOi2KCyIB8B3xZCKhcfdS2K0=", + "h1:l5VASLhVAOCr2Q+7ywGqWb+JSEIJ5UjilOMBjFMdQ7U=", + "h1:mugvZRK3/kSysUmpt664tMNpOZtGbQd6nGKaM4p3kwo=", + "h1:nY4ct0BTUQ7se8gbnzXsPCN2tiP4mZsGnEw2juSpXNs=", + "h1:qRFHk1ksyfMj2KOrs9pEntzagM+Tzqm+qznJfxVrJj0=", + "h1:szVx4GPJr2hJt9t82VilK2Gu/N9WR/KbdeLZ9wb0Nek=", + "zh:072d542e40ca0b8c5e081c9f834e3e41aa2ad31c01219522f314685d5c482823", + "zh:0b2643473f5bb154d724e64e75632814d74acda5fd81b6488d0ed13f413152b2", + "zh:108b3e175886e45c4e955f1ef323aa849dfd82fc450108238be2ab7c0d1f0c2e", + "zh:3586ca03a5d07e40a67b01b7fdebc6f7636a42c2a2f6ab88635ffca8f6928dfe", + "zh:47bd626153ee94e6b45533c7c02f579dc6d586d6d9580e03a040643f647be50b", + "zh:49aafbe6f665b3c4c97522905fd09229f2758a782c7cfc508280607030134ef8", + "zh:6e3397227c3f8f3becc04f95ebcb977d7012f025c9921f9ef5e603fe6d5da665", + "zh:70f4478658a13eeb57725e3b424329494a0ffbd78ac2734effea4e60d802cfa2", + "zh:774d4357447f94a5d4981c7569f85cd140ac577974174c5e7b477d834c6bfb43", + "zh:7971ef7bc532ab0edb3ee739ac3010b46321ec8541f873956a20dabbd6a0f189", + "zh:8ec18fba906468a9e6ce15329aa098dfb45ab1b9dca446a8fc2079a0d6ef590c", + "zh:98328661edd4dfc4e4782fd03a7010854218d3fbf35435837ac33e7ee8aafe7e", + "zh:9cbbf5033116f72749ebd4cf9add6aadf7e9e4bc7704eec86a8f4c139c16c120", + "zh:9faf944474f9233ddbb3606bc495570d4b952fea21f09c28676f9847d619e50f", + "zh:d0636d6de8a7b0b0bcc2add80556ee7118981d738998d4772558b9de0df99af1", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.9.0" + hashes = [ + "h1:8EQU5KSxezcjo/phRSe69rDOI0lk4pSaggj7FsskYp8=", + "h1:Lw9im2VBBJQ3RyAbHPQ0rcvcmmcZWm3x+kIOpN+Tv9s=", + "h1:U8KXqGCoNI9/guYbTvzgdtVk3fRthoG0UXwm1JoEpIs=", + "h1:YXaVd4p6qXPPVaxIBaIDNXmBwT02ZqDn0qD+tYpw8sA=", + "h1:cOpc03fphEt/G9Rfc4jLL/fW0D7tgvlXqiDKPF4vuww=", + "h1:g09RR7T1xWkeGrZwWvWMT9ncJrFGr1k3CBD585UmO7w=", + "h1:gGDdPPibmw2EWROx+sh1RGLjR5+nPwZyrf6/N9jXfeM=", + "h1:haE7/nXCOhXKP4oXeEnER3t5CaVQWqujz4nBnpeTUv4=", + "h1:ieSVpfZS2lKuMr05ph0QsOVpCzg7uk3cgKBaXR+Ikug=", + "h1:ig2s1IS9IzehorRjvVAnKIsUUj8fkgyxct1L/kswcc4=", + "h1:j3lS+ZEERFnoab8t1ppDrScGVP/cgWbzlCrEYKTCXYw=", + "h1:lxezrKmOiQIySHAM+os8qLVq7hqufDr8h3Hpzvsk+78=", + "h1:lzRqBJAG+NETxHbEZUJ/YP3RMEjZBinTX7VmgH3lw60=", + "h1:tdSNWK5ApqUsgbdYieyeYLTu6nIZUV3hR1oFqUfAuGo=", + "h1:xedet8yH/zI2CfdxsGlK0nlFWc/Bp61yrWsEa3fHB8g=", + "zh:03f1114cc20b8913523735ab76e0f0a2b16ce13c92923a53304bf85f07fc0dbc", + "zh:105b678ee72322a3067f105d7e05e940f6143238f377f6e87ff4ec909246ac2a", + "zh:55f3bbf13ea18cbace61a706566a80f25f33fe2b1780b6f3d7b582af2a05b6d2", + "zh:63adf996db48f082f7a6351eb485e219cd88795fc71e6ec60a837263ab0d2cb1", + "zh:7e99550738a4e3cc68b8a467714b0d69371025fe95e3326d5323d026d55653e9", + "zh:8342b54af3a18a37e075eeae61be57f4de2ba71b35d95c5075d402dd2c1f289d", + "zh:83ee18e32ac9dd5fc91298554b7c4cfa4c3a1db50f4c797945637cc93c0844ae", + "zh:993ecc0adbf6bd535a59fbc9b735d8c33950e6f6eb5e621d750da9b71d65d80a", + "zh:ad722bc59d4edbf1415e827fc007c0efe6e0e9462d5568bae20b34be1058a261", + "zh:ae9448e1f87b2f9a6c5197a0e9862162ec6b137cb3a3835e11522995d8939e7c", + "zh:bc9cdd3aac784f759125c6627f6f6416e8726a1c184eb9cf3e55b9edbc94c627", + "zh:c8e35b89572ba1c40a9b20022e033a3395fb8d42e7604d50c900f193ba10382e", + "zh:e2deaa8a9975ef81d9f62baed12c41286918b0a10908e0e031f13f69a3b730a1", + "zh:ee39707557210a0ab1098aa357d2cdfe502e5a312d0dbdffb09d08facc4d3fc5", + "zh:f81afe4eb63e8aa9e0ea71be6c990f0dc69cb360e7191c0742a991f4a5081b64", + ] +} + +provider "registry.opentofu.org/nullstone-io/ns" { + version = "0.11.1" + constraints = "~> 0.11.0" + hashes = [ + "h1:6BxUpd3+1TLtEsj54HBS4nOkR9FicIuuXEVDPCByF7Q=", + "h1:71TqF9V72ZFWQf/RFzarQgxgKglL4VvPaOuwb/xBc7k=", + "h1:EYaDXEvkg6WmXCw47d4Tijcm9rlOw3DYkGF05wx0H3o=", + "h1:HaGl+RgDBjpxvxrxbuUhDiuwMkmc+IFf/jDQW8tQY1c=", + "h1:PRRORu4Y+QfYbNvjIEiaQXlQ9veksGKAWHm0Por7ZHk=", + "h1:Q45Qrs0APQjOr+HhvnxB49f4wPE04/Pq5ukoq8uUZp4=", + "h1:ShGG1d+/n+cwFvvncUjjzh64Hqc3TQdJxuy7YKfP1yo=", + "h1:TRhYX38Zlx/32PbScJyI3mKmIPp8xXDkUSCGZCRQNNQ=", + "h1:YrKu51K7Bae6Mj/CFN9MX08ITrgjcdH7MchfL0tqv9M=", + "h1:cTiiEMOG2BDPmWv4VtpGV+nRnGTgs7GYMUS4gR9slvk=", + "h1:f1sN6fJ4vDYh0AiPl3TKT+0qBxvdxzmX+ZbGiNg4h3U=", + "h1:mR6frDhMROvg82IrNOXeZUywhMRLHByK1am8G4SOBxg=", + "h1:o/Hd6E8+H1/+Haa96SjdI2JGN1jN0yPi79JxBZplIY0=", + "zh:2e7aa2baf793e68155ce2e3d1bf17f2fcbb1d33b0986edf8ac37f35c09e0b217", + "zh:50e49ef17f8edc2c65a434a46994326740a1975ebf28aeea0964cd2dcfbaeed8", + "zh:56c1d3ad611599e1946992bc78f7172f4f5774b38e08c9f3384988cfc844374a", + "zh:7fe50367ad319f9e5b07aa925cfa337f98c079d119a48e87f6df2c9772297a68", + "zh:8947b5f6360b414f3f9f4dc56d2efe6bea981b24ae8c0d76a7b3191d47512897", + "zh:95f5581d67edbf0923332eaa95805a8388bd48b34d8f8a714c66985b7dc66449", + "zh:a666ee46523be51cb9445d0fece4441776bbcb7c161cfa374a6195448ed005d8", + "zh:ad17d9f33d1f69c025de5b803a85fe0d937113d4cac22c74fb814a862e3fb750", + "zh:b3541c9fee4b362f30ba50cc1daf74d910fae1942e71252f306711ec22d33415", + "zh:bf68295cbb945321453f51537f0ed1dbeedeb274e600672d53f6f43f953b51b0", + "zh:ce56a788e0cd05cfb094e9834b64bbf51fb570d2626b19e60540b8d56842db70", + "zh:d577d8aed3d4389231a21fa3fbca2433a98930f05e54c82ff030279e9871090f", + "zh:ede865f1859ffe2788bc0f15ec45de337fece8247e752a0e817a0ad5c85b8031", + ] +} diff --git a/aws/aws-ec2-vault-cluster/ami.tf b/aws/aws-ec2-vault-cluster/ami.tf new file mode 100644 index 0000000..6976b7c --- /dev/null +++ b/aws/aws-ec2-vault-cluster/ami.tf @@ -0,0 +1,24 @@ +data "aws_ami" "vault" { + count = var.ami == "" ? 1 : 0 + most_recent = true + owners = ["self"] + + filter { + name = "tag:Name" + values = ["nullstone-vault"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "state" + values = ["available"] + } +} + +locals { + ami = var.ami != "" ? var.ami : data.aws_ami.vault[0].id +} diff --git a/aws/aws-ec2-vault-cluster/aws.tf b/aws/aws-ec2-vault-cluster/aws.tf new file mode 100644 index 0000000..8e86a87 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/aws.tf @@ -0,0 +1,7 @@ +provider "aws" { + default_tags { + tags = local.tags + } +} + +data "aws_region" "this" {} diff --git a/aws/aws-ec2-vault-cluster/connections.tf b/aws/aws-ec2-vault-cluster/connections.tf new file mode 100644 index 0000000..3275892 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/connections.tf @@ -0,0 +1,33 @@ +data "ns_connection" "network" { + name = "network" + contract = "network/aws/vpc" +} + +data "ns_connection" "snapshots_bucket" { + name = "snapshots_bucket" + contract = "datastore/aws/s3" +} + +data "ns_connection" "unseal_key" { + name = "unseal_key" + contract = "datastore/aws/kms" +} + +locals { + vpc_id = data.ns_connection.network.outputs.vpc_id + vpc_cidr = data.ns_connection.network.outputs.vpc_cidr + + snapshot_bucket_arn = data.ns_connection.snapshots_bucket.outputs.db_arn + snapshot_bucket_name = trimprefix(local.snapshot_bucket_arn, "arn:aws:s3:::") + snapshot_kms_key_arn = try(data.ns_connection.snapshots_bucket.outputs.kms_key_arn, "") + + unseal_kms_key_arn = data.ns_connection.unseal_key.outputs.kms_key_arn + + vault_api_port = 8200 + vault_cluster_port = 8201 + vault_health_port = 8210 + snapshot_prefix = "vault-snapshots" + + vault_cluster_tag_key = "vault-cluster" + vault_cluster_tag_value = local.resource_name +} diff --git a/aws/aws-ec2-vault-cluster/iam.tf b/aws/aws-ec2-vault-cluster/iam.tf new file mode 100644 index 0000000..15b3a4e --- /dev/null +++ b/aws/aws-ec2-vault-cluster/iam.tf @@ -0,0 +1,102 @@ +data "aws_iam_policy_document" "assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "this" { + name = local.resource_name + assume_role_policy = data.aws_iam_policy_document.assume.json + tags = local.tags +} + +resource "aws_iam_instance_profile" "this" { + name = local.resource_name + role = aws_iam_role.this.name + tags = local.tags +} + +resource "aws_iam_role_policy_attachment" "ssm" { + role = aws_iam_role.this.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +data "aws_iam_policy_document" "this" { + statement { + sid = "UnsealKey" + effect = "Allow" + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:DescribeKey", + ] + resources = [local.unseal_kms_key_arn] + } + + dynamic "statement" { + for_each = local.snapshot_kms_key_arn == "" ? [] : [local.snapshot_kms_key_arn] + content { + sid = "SnapshotKey" + effect = "Allow" + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKey", + ] + resources = [statement.value] + } + } + + statement { + sid = "SnapshotList" + effect = "Allow" + actions = [ + "s3:ListBucket", + ] + resources = [local.snapshot_bucket_arn] + condition { + test = "StringLike" + variable = "s3:prefix" + values = [local.snapshot_prefix, "${local.snapshot_prefix}/*"] + } + } + + statement { + sid = "SnapshotObjects" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + resources = ["${local.snapshot_bucket_arn}/${local.snapshot_prefix}/*"] + } + + statement { + sid = "PlatformTokens" + effect = "Allow" + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:PutSecretValue", + ] + resources = [for s in aws_secretsmanager_secret.platform : s.arn] + } + + statement { + sid = "RaftJoin" + effect = "Allow" + actions = ["ec2:DescribeInstances"] + resources = ["*"] + } +} + +resource "aws_iam_role_policy" "this" { + name = local.resource_name + role = aws_iam_role.this.id + policy = data.aws_iam_policy_document.this.json +} diff --git a/aws/aws-ec2-vault-cluster/launch-template.tf b/aws/aws-ec2-vault-cluster/launch-template.tf new file mode 100644 index 0000000..4e5a243 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/launch-template.tf @@ -0,0 +1,30 @@ +resource "aws_launch_template" "this" { + name_prefix = "${local.resource_name}-" + image_id = local.ami + instance_type = var.instance_type + user_data = base64encode(local.user_data) + + iam_instance_profile { + name = aws_iam_instance_profile.this.name + } + + vpc_security_group_ids = [aws_security_group.nodes.id] + + metadata_options { + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + http_tokens = "required" + } + + tag_specifications { + resource_type = "instance" + tags = merge(local.tags, { + Name = local.resource_name + (local.vault_cluster_tag_key) = local.vault_cluster_tag_value + }) + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/aws/aws-ec2-vault-cluster/nullstone.tf b/aws/aws-ec2-vault-cluster/nullstone.tf new file mode 100644 index 0000000..a66bc88 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/nullstone.tf @@ -0,0 +1,30 @@ +terraform { + required_providers { + ns = { + source = "nullstone-io/ns" + version = "~> 0.11.0" + } + aws = { + source = "hashicorp/aws" + } + random = { + source = "hashicorp/random" + } + } +} + +data "ns_workspace" "this" {} + +resource "random_string" "resource_suffix" { + length = 5 + lower = true + upper = false + numeric = false + special = false +} + +locals { + tags = data.ns_workspace.this.aws_tags + block_name = data.ns_workspace.this.block_name + resource_name = "${data.ns_workspace.this.block_ref}-${random_string.resource_suffix.result}" +} diff --git a/aws/aws-ec2-vault-cluster/outputs.tf b/aws/aws-ec2-vault-cluster/outputs.tf new file mode 100644 index 0000000..d1733ae --- /dev/null +++ b/aws/aws-ec2-vault-cluster/outputs.tf @@ -0,0 +1,34 @@ +output "ami_id" { + value = local.ami + description = "string ||| AMI ID for Vault nodes." +} + +output "role_name" { + value = aws_iam_role.this.name + description = "string ||| IAM role name for Vault EC2 instances." +} + +output "instance_profile_name" { + value = aws_iam_instance_profile.this.name + description = "string ||| Instance profile name for the launch template." +} + +output "security_group_id" { + value = aws_security_group.nodes.id + description = "string ||| Security group attached to Vault nodes." +} + +output "nlb_security_group_id" { + value = aws_security_group.nlb.id + description = "string ||| Security group attached to the internal NLB." +} + +output "operator_secret_arn" { + value = aws_secretsmanager_secret.platform["operator"].arn + description = "string ||| Secrets Manager ARN for the operator token." +} + +output "provisioning_secret_arn" { + value = aws_secretsmanager_secret.platform["provisioning"].arn + description = "string ||| Secrets Manager ARN for the provisioning token." +} diff --git a/aws/aws-ec2-vault-cluster/secrets.tf b/aws/aws-ec2-vault-cluster/secrets.tf new file mode 100644 index 0000000..1eca7c8 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/secrets.tf @@ -0,0 +1,11 @@ +locals { + platform_secret_names = toset(["init", "provisioning", "operator"]) +} + +resource "aws_secretsmanager_secret" "platform" { + for_each = local.platform_secret_names + + name_prefix = "${local.block_name}/vault/${each.key}/" + recovery_window_in_days = 0 + tags = local.tags +} diff --git a/aws/aws-ec2-vault-cluster/security.tf b/aws/aws-ec2-vault-cluster/security.tf new file mode 100644 index 0000000..afb2341 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/security.tf @@ -0,0 +1,102 @@ +resource "aws_security_group" "nlb" { + name = "${local.resource_name}/nlb" + vpc_id = local.vpc_id + tags = merge(local.tags, { Name = "${local.resource_name}/nlb" }) +} + +resource "aws_security_group" "nodes" { + name = "${local.resource_name}/nodes" + vpc_id = local.vpc_id + tags = merge(local.tags, { Name = "${local.resource_name}/nodes" }) +} + +resource "aws_security_group_rule" "nlb_api_from_vpc" { + security_group_id = aws_security_group.nlb.id + type = "ingress" + protocol = "tcp" + from_port = local.vault_api_port + to_port = local.vault_api_port + cidr_blocks = [local.vpc_cidr] +} + +resource "aws_security_group_rule" "nlb_to_api" { + security_group_id = aws_security_group.nlb.id + type = "egress" + protocol = "tcp" + from_port = local.vault_api_port + to_port = local.vault_api_port + source_security_group_id = aws_security_group.nodes.id +} + +resource "aws_security_group_rule" "nlb_to_health" { + security_group_id = aws_security_group.nlb.id + type = "egress" + protocol = "tcp" + from_port = local.vault_health_port + to_port = local.vault_health_port + source_security_group_id = aws_security_group.nodes.id +} + +resource "aws_security_group_rule" "nodes_api_from_nlb" { + security_group_id = aws_security_group.nodes.id + type = "ingress" + protocol = "tcp" + from_port = local.vault_api_port + to_port = local.vault_api_port + source_security_group_id = aws_security_group.nlb.id +} + +resource "aws_security_group_rule" "nodes_health_from_nlb" { + security_group_id = aws_security_group.nodes.id + type = "ingress" + protocol = "tcp" + from_port = local.vault_health_port + to_port = local.vault_health_port + source_security_group_id = aws_security_group.nlb.id +} + +resource "aws_security_group_rule" "nodes_raft" { + security_group_id = aws_security_group.nodes.id + type = "ingress" + protocol = "tcp" + from_port = local.vault_cluster_port + to_port = local.vault_cluster_port + self = true +} + +# Raft auto-join calls the leader API port before moving to the cluster port. +resource "aws_security_group_rule" "nodes_api_from_nodes" { + security_group_id = aws_security_group.nodes.id + type = "ingress" + protocol = "tcp" + from_port = local.vault_api_port + to_port = local.vault_api_port + self = true +} + +resource "aws_security_group_rule" "nodes_api_egress" { + security_group_id = aws_security_group.nodes.id + type = "egress" + protocol = "tcp" + from_port = local.vault_api_port + to_port = local.vault_api_port + self = true +} + +resource "aws_security_group_rule" "nodes_https" { + security_group_id = aws_security_group.nodes.id + type = "egress" + protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "nodes_raft_egress" { + security_group_id = aws_security_group.nodes.id + type = "egress" + protocol = "tcp" + from_port = local.vault_cluster_port + to_port = local.vault_cluster_port + self = true +} diff --git a/aws/aws-ec2-vault-cluster/templates/user-data.sh.tpl b/aws/aws-ec2-vault-cluster/templates/user-data.sh.tpl new file mode 100644 index 0000000..33c0ca5 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/templates/user-data.sh.tpl @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +install -m 0600 -o vault -g vault /dev/null /etc/vault.d/vault-utils.env +cat >/etc/vault.d/vault-utils.env <= 1 && var.cluster_size % 2 == 1 + error_message = "cluster_size must be an odd number of instances (1, 3, 5, 7, ...)." + } +} + +variable "instance_type" { + type = string + default = "t3.micro" + description = < [args] + +Commands: + bootstrap local|aws|azure|gcp Init once, unseal, configure + tenants create + tenants destroy --yes [--purge-secrets] + snapshot take Write a Raft snapshot + snapshot list + snapshot verify + snapshot restore --yes + snapshot schedule Cron loop (BACKUP_SCHEDULE; empty disables) + health Print seal status + health serve HTTP on :8210 (200 only if this node is a Raft voter and caught up) + +Local key material: BOOTSTRAP_DIR (default .bootstrap). +AWS: VAULT_INIT_SECRET_ARN, VAULT_PROVISIONING_SECRET_ARN, VAULT_OPERATOR_SECRET_ARN. +Optional: SNAPSHOT_BUCKET, SNAPSHOT_PREFIX (default vault-snapshots). +`) +} + +func run(cmd string, args []string) error { + cfg := vaultcluster.ConfigFromEnv() + c, err := vaultcluster.New(cfg) + if err != nil { + return err + } + switch cmd { + case "bootstrap": + return runBootstrap(c, args) + case "tenants": + return runTenants(c, args) + case "snapshot": + return runSnapshot(c, args) + case "health": + if len(args) > 0 && args[0] == "serve" { + return runHealthServe(c) + } + return c.Health() + default: + usage() + return fmt.Errorf("unknown command %q", cmd) + } +} + +func runBootstrap(c *vaultcluster.Client, args []string) error { + platform := "" + if len(args) > 0 { + platform = args[0] + } + if platform == "" { + platform = os.Getenv("VAULT_PLATFORM") + } + if platform == "" { + return fmt.Errorf("usage: vault-utils bootstrap local|aws|azure|gcp") + } + switch platform { + case "local": + shares, _ := strconv.Atoi(getenv("VAULT_INIT_KEY_SHARES", "5")) + threshold, _ := strconv.Atoi(getenv("VAULT_INIT_KEY_THRESHOLD", "3")) + return c.RunBootstrap(fileKeyStore(), vaultcluster.BootstrapOptions{ + Shares: shares, + Threshold: threshold, + KeepRoot: getenv("KEEP_ROOT", "false") == "true", + }) + case "aws": + store, err := awsKeyStore() + if err != nil { + return err + } + shares, _ := strconv.Atoi(getenv("VAULT_INIT_RECOVERY_SHARES", "1")) + threshold, _ := strconv.Atoi(getenv("VAULT_INIT_RECOVERY_THRESHOLD", "1")) + return c.RunBootstrap(store, vaultcluster.BootstrapOptions{ + Shares: shares, + Threshold: threshold, + KeepRoot: getenv("KEEP_ROOT", "false") == "true", + AutoUnseal: true, + }) + case "azure", "gcp": + return fmt.Errorf("bootstrap %s is not implemented yet", platform) + default: + return fmt.Errorf("unknown platform %q (local, aws, azure, gcp)", platform) + } +} + +func runTenants(c *vaultcluster.Client, args []string) error { + if len(args) < 1 { + return fmt.Errorf("usage: vault-utils tenants create|destroy ") + } + sub, rest := args[0], args[1:] + switch sub { + case "create": + id := "" + for _, a := range rest { + if !strings.HasPrefix(a, "-") { + id = a + } + } + if id == "" { + return fmt.Errorf("usage: vault-utils tenants create ") + } + return c.CreateTenant(id, true) + case "destroy": + yes, purge := false, false + id := "" + for _, a := range rest { + switch a { + case "--yes": + yes = true + case "--purge-secrets": + purge = true + default: + if !strings.HasPrefix(a, "-") { + id = a + } + } + } + if id == "" || !yes { + return fmt.Errorf("usage: vault-utils tenants destroy --yes [--purge-secrets]") + } + return c.OffboardTenant(id, purge) + default: + return fmt.Errorf("unknown subcommand %q (create, destroy)", sub) + } +} + +func runSnapshot(c *vaultcluster.Client, args []string) error { + if len(args) < 1 { + return fmt.Errorf("usage: vault-utils snapshot take | list | verify | restore --yes | schedule") + } + backupDir := filepath.Join(bootstrapDir(), "backups") + switch args[0] { + case "take": + if err := useOperatorToken(c); err != nil { + return err + } + file, err := takeSnapshot(c, backupDir) + if err != nil { + return err + } + log.Printf("snapshot written: %s", file) + log.Printf("this file contains every secret in the cluster; treat it as one") + return nil + case "list": + files, err := listSnapshots(backupDir) + if err != nil { + return err + } + if len(files) == 0 { + log.Printf("no snapshots") + return nil + } + for _, f := range files { + fmt.Println(f) + } + return nil + case "verify": + if len(args) < 2 { + return fmt.Errorf("usage: vault-utils snapshot verify ") + } + if err := vaultcluster.SnapshotVerify(args[1]); err != nil { + return err + } + log.Printf("checksum OK: %s", args[1]) + return nil + case "restore": + if len(args) < 3 || args[2] != "--yes" { + return fmt.Errorf("restore replaces the entire cluster; re-run with: vault-utils snapshot restore --yes") + } + if c.Cfg.Token == "" { + return fmt.Errorf("restore requires VAULT_TOKEN with sys/storage/raft/snapshot-force (break-glass root); the operator token cannot restore") + } + if err := c.SnapshotRestore(args[1]); err != nil { + return err + } + log.Printf("restore submitted; Vault will seal") + log.Printf("unseal with the key shares that were current when this snapshot was taken") + return nil + case "schedule": + return runSnapshotSchedule(c, backupDir) + default: + return fmt.Errorf("unknown subcommand %q (take, list, verify, restore, schedule)", args[0]) + } +} + +func runSnapshotSchedule(c *vaultcluster.Client, backupDir string) error { + sched, err := vaultcluster.ParseBackupSchedule(os.Getenv("BACKUP_SCHEDULE")) + if err != nil { + return err + } + if sched == nil { + log.Printf("scheduled snapshots disabled") + return nil + } + if err := useOperatorToken(c); err != nil { + return err + } + go c.RenewToken(tokenRenewInterval, nil) + nodeID := os.Getenv("VAULT_RAFT_NODE_ID") + for { + wait := time.Until(sched.Next(time.Now())) + if wait > 0 { + time.Sleep(wait) + } + if nodeID != "" { + data, err := c.RaftAutopilot() + if err != nil { + log.Printf("snapshot skipped: %v", err) + continue + } + if !vaultcluster.NodeIsLeader(nodeID, data) { + log.Printf("snapshot skipped: not raft leader") + continue + } + } + file, err := takeSnapshot(c, backupDir) + if err != nil { + log.Printf("snapshot failed: %v", err) + continue + } + log.Printf("snapshot written: %s", file) + } +} + +func runHealthServe(c *vaultcluster.Client) error { + nodeID := os.Getenv("VAULT_RAFT_NODE_ID") + if nodeID == "" { + return fmt.Errorf("VAULT_RAFT_NODE_ID is required for health serve") + } + if err := useOperatorToken(c); err != nil { + return err + } + go c.RenewToken(tokenRenewInterval, nil) + addr := getenv("VAULT_HEALTH_ADDR", ":8210") + log.Printf("health listening on %s", addr) + return c.ServeHealth(addr, nodeID) +} + +func takeSnapshot(c *vaultcluster.Client, backupDir string) (string, error) { + if bucket := os.Getenv("SNAPSHOT_BUCKET"); bucket != "" { + store, err := s3.New() + if err != nil { + return "", err + } + b, err := c.RaftSnapshot() + if err != nil { + return "", err + } + return s3.PutSnapshot(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots"), b) + } + return c.SnapshotTake(backupDir) +} + +func listSnapshots(backupDir string) ([]string, error) { + if bucket := os.Getenv("SNAPSHOT_BUCKET"); bucket != "" { + store, err := s3.New() + if err != nil { + return nil, err + } + return s3.ListSnapshots(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots")) + } + return vaultcluster.SnapshotList(backupDir) +} + +func useOperatorToken(c *vaultcluster.Client) error { + if c.Cfg.Token != "" { + return nil + } + store, err := keyStore() + if err != nil { + return err + } + tok, err := store.LoadToken("operator") + if err != nil { + return fmt.Errorf("set VAULT_TOKEN or bootstrap first (operator token not found): %w", err) + } + c.API.SetToken(tok) + c.Cfg.Token = tok + return nil +} + +func keyStore() (vaultcluster.KeyStore, error) { + if os.Getenv("VAULT_OPERATOR_SECRET_ARN") != "" || os.Getenv("VAULT_INIT_SECRET_ARN") != "" { + return awsKeyStore() + } + return fileKeyStore(), nil +} + +func fileKeyStore() vaultcluster.FileKeyStore { + return vaultcluster.FileKeyStore{Dir: bootstrapDir()} +} + +func awsKeyStore() (*secretsmanager.KeyStore, error) { + return secretsmanager.New( + os.Getenv("VAULT_INIT_SECRET_ARN"), + os.Getenv("VAULT_PROVISIONING_SECRET_ARN"), + os.Getenv("VAULT_OPERATOR_SECRET_ARN"), + ) +} + +func bootstrapDir() string { + return getenv("BOOTSTRAP_DIR", ".bootstrap") +} + +func getenv(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} diff --git a/gcp/.gitkeep b/gcp/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..38107f0 --- /dev/null +++ b/go.mod @@ -0,0 +1,48 @@ +module github.com/nullstone-modules/vault-cluster + +go 1.26.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.2 + github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6 + github.com/hashicorp/vault/api v1.16.0 + github.com/robfig/cron/v3 v3.0.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect + github.com/aws/smithy-go v1.24.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9df16b5 --- /dev/null +++ b/go.sum @@ -0,0 +1,119 @@ +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= +github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6 h1:l4mxH8imZoflVEWWa8VT8skwObm+t0KEveqEskyiKEo= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6/go.mod h1:1qwmvfRBGTQ5shUxu+eQO/S2+O6o6SxbvcvtN62kmc0= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11/go.mod h1:0DO9B5EUJQlIDif+XJRWCljZRKsAFKh3gpFz7UnDtOo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWAXLGFIizeqkdkKgRlJwWc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= +github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0= +github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= +github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= +github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/vault/api v1.16.0 h1:nbEYGJiAPGzT9U4oWgaaB0g+Rj8E59QuHKyA5LhwQN4= +github.com/hashicorp/vault/api v1.16.0/go.mod h1:KhuUhzOD8lDSk29AtzNjgAu2kxRA9jL9NAbkFlqvkBA= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/aws/s3/snapshot.go b/internal/aws/s3/snapshot.go new file mode 100644 index 0000000..107cad1 --- /dev/null +++ b/internal/aws/s3/snapshot.go @@ -0,0 +1,98 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type ObjectStore interface { + Put(ctx context.Context, bucket, key string, body []byte) error + List(ctx context.Context, bucket, prefix string) ([]string, error) +} + +type Store struct { + inner *awss3.Client +} + +func New() (Store, error) { + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return Store{}, fmt.Errorf("AWS credentials: %w", err) + } + return Store{inner: awss3.NewFromConfig(cfg)}, nil +} + +func (s Store) Put(ctx context.Context, bucket, key string, body []byte) error { + _, err := s.inner.PutObject(ctx, &awss3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(body), + }) + if err != nil { + return fmt.Errorf("s3 put s3://%s/%s: %w", bucket, key, err) + } + return nil +} + +func (s Store) List(ctx context.Context, bucket, prefix string) ([]string, error) { + out, err := s.inner.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + }) + if err != nil { + return nil, fmt.Errorf("s3 list s3://%s/%s: %w", bucket, prefix, err) + } + var keys []string + for _, obj := range out.Contents { + if obj.Key == nil || !strings.HasSuffix(*obj.Key, ".snap") { + continue + } + keys = append(keys, "s3://"+bucket+"/"+*obj.Key) + } + return keys, nil +} + +func normalizePrefix(prefix string) string { + if p := strings.Trim(prefix, "/"); p != "" { + return p + } + return "vault-snapshots" +} + +func ObjectKey(prefix, stamp string) string { + return normalizePrefix(prefix) + "/vault-" + stamp + ".snap" +} + +func PutSnapshot(store ObjectStore, bucket, prefix string, data []byte) (string, error) { + if bucket == "" { + return "", fmt.Errorf("SNAPSHOT_BUCKET is not set") + } + if len(data) == 0 { + return "", fmt.Errorf("snapshot is empty; refusing to keep it") + } + key := ObjectKey(prefix, time.Now().UTC().Format("20060102T150405Z")) + if err := store.Put(context.Background(), bucket, key, data); err != nil { + return "", err + } + sum := sha256.Sum256(data) + if err := store.Put(context.Background(), bucket, key+".sha256", []byte(hex.EncodeToString(sum[:])+"\n")); err != nil { + return "", err + } + return "s3://" + bucket + "/" + key, nil +} + +func ListSnapshots(store ObjectStore, bucket, prefix string) ([]string, error) { + if bucket == "" { + return nil, fmt.Errorf("SNAPSHOT_BUCKET is not set") + } + return store.List(context.Background(), bucket, normalizePrefix(prefix)+"/") +} diff --git a/internal/aws/s3/snapshot_test.go b/internal/aws/s3/snapshot_test.go new file mode 100644 index 0000000..f67c4ad --- /dev/null +++ b/internal/aws/s3/snapshot_test.go @@ -0,0 +1,81 @@ +package s3 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +type memObjects map[string][]byte + +func (m memObjects) Put(_ context.Context, bucket, key string, body []byte) error { + m[bucket+"/"+key] = append([]byte(nil), body...) + return nil +} + +func (m memObjects) List(_ context.Context, bucket, prefix string) ([]string, error) { + var out []string + root := bucket + "/" + for k := range m { + if !strings.HasPrefix(k, root+prefix) { + continue + } + key := strings.TrimPrefix(k, root) + if strings.HasSuffix(key, ".snap") { + out = append(out, "s3://"+bucket+"/"+key) + } + } + return out, nil +} + +func TestObjectKey(t *testing.T) { + got := ObjectKey("vault-snapshots", "20260101T000000Z") + if got != "vault-snapshots/vault-20260101T000000Z.snap" { + t.Fatalf("key %q", got) + } +} + +func TestListSnapshots(t *testing.T) { + store := memObjects{ + "b/vault-snapshots/vault-1.snap": []byte("a"), + "b/vault-snapshots/vault-1.snap.sha256": []byte("x"), + "b/other/vault-2.snap": []byte("c"), + } + got, err := ListSnapshots(store, "b", "vault-snapshots") + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0] != "s3://b/vault-snapshots/vault-1.snap" { + t.Fatalf("list %v", got) + } +} + +func TestPutSnapshotWritesChecksum(t *testing.T) { + store := memObjects{} + uri, err := PutSnapshot(store, "b", "", []byte("snap")) + if err != nil { + t.Fatal(err) + } + key := strings.TrimPrefix(uri, "s3://b/") + if !strings.HasPrefix(key, "vault-snapshots/vault-") { + t.Fatalf("empty prefix should default: %q", uri) + } + if string(store["b/"+key]) != "snap" { + t.Fatalf("snapshot body %q", store["b/"+key]) + } + sum := sha256.Sum256([]byte("snap")) + if got := string(store["b/"+key+".sha256"]); got != hex.EncodeToString(sum[:])+"\n" { + t.Fatalf("checksum %q", got) + } +} + +func TestPutSnapshotRejectsEmpty(t *testing.T) { + if _, err := PutSnapshot(memObjects{}, "b", "vault-snapshots", nil); err == nil { + t.Fatal("expected empty snapshot to fail") + } + if _, err := PutSnapshot(memObjects{}, "", "vault-snapshots", []byte("x")); err == nil { + t.Fatal("expected missing bucket to fail") + } +} diff --git a/internal/aws/secretsmanager/keystore.go b/internal/aws/secretsmanager/keystore.go new file mode 100644 index 0000000..5c0ed72 --- /dev/null +++ b/internal/aws/secretsmanager/keystore.go @@ -0,0 +1,123 @@ +package secretsmanager + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/hashicorp/vault/api" +) + +type SecretStore interface { + Get(ctx context.Context, arn string) ([]byte, error) + Put(ctx context.Context, arn string, val []byte) error +} + +type KeyStore struct { + Secrets SecretStore + InitARN string + ProvisioningARN string + OperatorARN string +} + +func New(initARN, provisioningARN, operatorARN string) (*KeyStore, error) { + if initARN == "" || provisioningARN == "" || operatorARN == "" { + return nil, fmt.Errorf("VAULT_INIT_SECRET_ARN, VAULT_PROVISIONING_SECRET_ARN, and VAULT_OPERATOR_SECRET_ARN are required") + } + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return nil, fmt.Errorf("AWS credentials: %w", err) + } + return &KeyStore{ + Secrets: smClient{inner: secretsmanager.NewFromConfig(cfg)}, + InitARN: initARN, + ProvisioningARN: provisioningARN, + OperatorARN: operatorARN, + }, nil +} + +func (s KeyStore) tokenARN(name string) (string, error) { + switch name { + case "provisioning": + return s.ProvisioningARN, nil + case "operator": + return s.OperatorARN, nil + default: + return "", fmt.Errorf("unknown token %q", name) + } +} + +func (s KeyStore) SaveInit(resp *api.InitResponse) error { + b, err := json.Marshal(resp) + if err != nil { + return err + } + return s.Secrets.Put(context.Background(), s.InitARN, b) +} + +func (s KeyStore) LoadInit() (*api.InitResponse, error) { + raw, err := s.Secrets.Get(context.Background(), s.InitARN) + if err != nil { + return nil, err + } + var resp api.InitResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (s KeyStore) SaveToken(name, token string) error { + arn, err := s.tokenARN(name) + if err != nil { + return err + } + return s.Secrets.Put(context.Background(), arn, []byte(token)) +} + +func (s KeyStore) LoadToken(name string) (string, error) { + arn, err := s.tokenARN(name) + if err != nil { + return "", err + } + b, err := s.Secrets.Get(context.Background(), arn) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +type smClient struct { + inner *secretsmanager.Client +} + +func (c smClient) Get(ctx context.Context, arn string) ([]byte, error) { + out, err := c.inner.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(arn), + }) + if err != nil { + return nil, fmt.Errorf("secrets manager get %s: %w", arn, err) + } + if out.SecretString != nil { + return []byte(*out.SecretString), nil + } + if len(out.SecretBinary) > 0 { + return out.SecretBinary, nil + } + return nil, fmt.Errorf("secrets manager get %s: empty secret", arn) +} + +func (c smClient) Put(ctx context.Context, arn string, val []byte) error { + _, err := c.inner.PutSecretValue(ctx, &secretsmanager.PutSecretValueInput{ + SecretId: aws.String(arn), + SecretString: aws.String(string(val)), + }) + if err != nil { + return fmt.Errorf("secrets manager put %s: %w", arn, err) + } + return nil +} diff --git a/internal/aws/secretsmanager/keystore_test.go b/internal/aws/secretsmanager/keystore_test.go new file mode 100644 index 0000000..67c180b --- /dev/null +++ b/internal/aws/secretsmanager/keystore_test.go @@ -0,0 +1,84 @@ +package secretsmanager + +import ( + "context" + "errors" + "testing" + + "github.com/hashicorp/vault/api" +) + +type memSecrets map[string][]byte + +func (m memSecrets) Get(_ context.Context, arn string) ([]byte, error) { + b, ok := m[arn] + if !ok { + return nil, errors.New("missing") + } + return b, nil +} + +func (m memSecrets) Put(_ context.Context, arn string, val []byte) error { + if arn == "fail" { + return errors.New("denied") + } + m[arn] = append([]byte(nil), val...) + return nil +} + +func testStore(m memSecrets) KeyStore { + return KeyStore{ + Secrets: m, + InitARN: "arn:init", + ProvisioningARN: "arn:provisioning", + OperatorARN: "arn:operator", + } +} + +func TestKeyStoreRoundTrip(t *testing.T) { + store := testStore(memSecrets{}) + init := &api.InitResponse{RootToken: "hvs.root", RecoveryKeysB64: []string{"abc"}} + if err := store.SaveInit(init); err != nil { + t.Fatal(err) + } + got, err := store.LoadInit() + if err != nil { + t.Fatal(err) + } + if got.RootToken != "hvs.root" || len(got.RecoveryKeysB64) != 1 { + t.Fatalf("init mismatch: %+v", got) + } + if err := store.SaveToken("operator", "hvs.op"); err != nil { + t.Fatal(err) + } + tok, err := store.LoadToken("operator") + if err != nil { + t.Fatal(err) + } + if tok != "hvs.op" { + t.Fatalf("token %q", tok) + } +} + +func TestKeyStoreFailClosed(t *testing.T) { + store := testStore(memSecrets{}) + if _, err := store.LoadInit(); err == nil { + t.Fatal("expected missing init to fail") + } + if _, err := store.LoadToken("operator"); err == nil { + t.Fatal("expected missing token to fail") + } + if err := store.SaveToken("root", "x"); err == nil { + t.Fatal("expected unknown token name to fail") + } + store.InitARN = "fail" + if err := store.SaveInit(&api.InitResponse{RootToken: "x"}); err == nil { + t.Fatal("expected put failure") + } +} + +func TestNewRequiresARNs(t *testing.T) { + if _, err := New("", "a", "b"); err == nil { + t.Fatal("expected error") + } +} diff --git a/internal/vaultcluster/bootstrap.go b/internal/vaultcluster/bootstrap.go new file mode 100644 index 0000000..2ad1dc2 --- /dev/null +++ b/internal/vaultcluster/bootstrap.go @@ -0,0 +1,236 @@ +package vaultcluster + +import ( + "fmt" + "log" + "time" + + "github.com/hashicorp/vault/api" +) + +type BootstrapOptions struct { + Shares int + Threshold int + KeepRoot bool + AutoUnseal bool +} + +const revokedRootMarker = "revoked-at-bootstrap" + +// On a first boot the KMS seal waits for instance credentials before Vault opens its +// listener; about a minute was measured on a freshly created instance profile. +const readyTimeout = 3 * time.Minute + +func (c *Client) RunBootstrap(store KeyStore, opts BootstrapOptions) error { + if err := c.WaitReady(readyTimeout); err != nil { + return err + } + + st, err := c.API.Sys().SealStatus() + if err != nil { + return err + } + if !st.Initialized { + if err := c.initVault(store, opts); err != nil { + return err + } + } else if _, err := store.LoadInit(); err != nil { + return fmt.Errorf("Vault is initialized but key material is missing: %w", err) + } + + if opts.AutoUnseal { + if err := c.waitUnsealed(); err != nil { + return err + } + } else if err := c.unseal(store, opts.Threshold); err != nil { + return err + } + + if tok, err := store.LoadToken("provisioning"); err == nil { + c.API.SetToken(tok) + if _, err := c.API.Auth().Token().LookupSelf(); err == nil { + c.Cfg.Token = tok + if err := c.enableCredentialsIfNeeded(store); err != nil { + return err + } + log.Printf("platform already bootstrapped") + return nil + } + } + + initResp, err := store.LoadInit() + if err != nil { + return err + } + root := initResp.RootToken + if root == "" || root == revokedRootMarker { + return fmt.Errorf("root token is missing or already revoked; restore key material or bootstrap with keep-root") + } + c.API.SetToken(root) + c.Cfg.Token = root + + if err := c.Configure(); err != nil { + return err + } + + for _, name := range []string{"provisioning", "operator"} { + tok, err := c.issueOrphanToken(name) + if err != nil { + return err + } + if err := store.SaveToken(name, tok); err != nil { + return err + } + } + + if opts.KeepRoot { + log.Printf("keeping the root token active") + return nil + } + if err := c.API.Auth().Token().RevokeSelf(""); err != nil { + log.Printf("root token revocation failed: %v", err) + } else { + initResp.RootToken = revokedRootMarker + _ = store.SaveInit(initResp) + log.Printf("revoked the initial root token") + } + return nil +} + +func initRequest(opts BootstrapOptions) *api.InitRequest { + if opts.AutoUnseal { + return &api.InitRequest{ + RecoveryShares: opts.Shares, + RecoveryThreshold: opts.Threshold, + } + } + return &api.InitRequest{ + SecretShares: opts.Shares, + SecretThreshold: opts.Threshold, + } +} + +func (c *Client) initVault(store KeyStore, opts BootstrapOptions) error { + if opts.AutoUnseal { + log.Printf("initializing Vault (recovery %d/%d, auto-unseal)", opts.Shares, opts.Threshold) + } else { + log.Printf("initializing Vault (%d/%d Shamir)", opts.Shares, opts.Threshold) + } + resp, err := c.API.Sys().Init(initRequest(opts)) + if err != nil { + st, stErr := c.API.Sys().SealStatus() + if stErr == nil && st.Initialized { + return nil + } + return err + } + if err := store.SaveInit(resp); err != nil { + return err + } + log.Printf("initialized; key material saved (not logged)") + return nil +} + +func (c *Client) waitUnsealed() error { + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + st, err := c.API.Sys().SealStatus() + if err != nil { + time.Sleep(time.Second) + continue + } + if !st.Sealed { + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("Vault remained sealed (auto-unseal failed)") +} + +func (c *Client) WaitReady(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + st, err := c.API.Sys().SealStatus() + if err == nil && st != nil { + return nil + } + time.Sleep(time.Second) + } + return fmt.Errorf("Vault did not respond within %s", timeout) +} + +func (c *Client) unseal(store KeyStore, threshold int) error { + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + st, err := c.API.Sys().SealStatus() + if err != nil { + time.Sleep(time.Second) + continue + } + if !st.Sealed { + return nil + } + initResp, err := store.LoadInit() + if err != nil { + return err + } + keys := initResp.KeysB64 + if len(keys) == 0 { + keys = initResp.Keys + } + n := threshold + if n > len(keys) { + n = len(keys) + } + for i := 0; i < n; i++ { + if _, err := c.API.Sys().Unseal(keys[i]); err != nil { + log.Printf("unseal share %d: %v", i, err) + } + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("Vault remained sealed") +} + +func (c *Client) enableCredentialsIfNeeded(store KeyStore) error { + if !c.Cfg.EnableCredentials || c.databaseMounted() { + return nil + } + tok, err := store.LoadToken("operator") + if err != nil { + return fmt.Errorf("database engine is not mounted and the operator token is missing") + } + if err := c.WithToken(tok).mountDatabase(); err != nil { + return fmt.Errorf("enable database engine: %w", err) + } + log.Printf("enabled database engine") + return nil +} + +func (c *Client) databaseMounted() bool { + mounts, err := c.API.Sys().ListMounts() + if err != nil { + return false + } + _, ok := mounts[c.Cfg.DatabaseMount+"/"] + return ok +} + +func (c *Client) issueOrphanToken(policy string) (string, error) { + sec, err := c.API.Auth().Token().Create(&api.TokenCreateRequest{ + Policies: []string{policy}, + Period: "24h", + Renewable: boolPtr(true), + DisplayName: policy, + NoParent: true, + }) + if err != nil { + return "", err + } + if sec == nil || sec.Auth == nil { + return "", fmt.Errorf("failed to issue %s token", policy) + } + return sec.Auth.ClientToken, nil +} + +func boolPtr(b bool) *bool { return &b } diff --git a/internal/vaultcluster/bootstrap_test.go b/internal/vaultcluster/bootstrap_test.go new file mode 100644 index 0000000..6a39c1c --- /dev/null +++ b/internal/vaultcluster/bootstrap_test.go @@ -0,0 +1,23 @@ +package vaultcluster + +import "testing" + +func TestInitRequestAutoUnsealUsesRecovery(t *testing.T) { + req := initRequest(BootstrapOptions{Shares: 1, Threshold: 1, AutoUnseal: true}) + if req.RecoveryShares != 1 || req.RecoveryThreshold != 1 { + t.Fatalf("recovery: %+v", req) + } + if req.SecretShares != 0 || req.SecretThreshold != 0 { + t.Fatalf("shamir should be unset: %+v", req) + } +} + +func TestInitRequestLocalUsesShamir(t *testing.T) { + req := initRequest(BootstrapOptions{Shares: 5, Threshold: 3}) + if req.SecretShares != 5 || req.SecretThreshold != 3 { + t.Fatalf("shamir: %+v", req) + } + if req.RecoveryShares != 0 { + t.Fatalf("recovery should be unset: %+v", req) + } +} diff --git a/internal/vaultcluster/client.go b/internal/vaultcluster/client.go new file mode 100644 index 0000000..13ad194 --- /dev/null +++ b/internal/vaultcluster/client.go @@ -0,0 +1,95 @@ +package vaultcluster + +import ( + "fmt" + "io" + "net/url" + "strings" + + "github.com/hashicorp/vault/api" +) + +type Client struct { + API *api.Client + Cfg Config +} + +func New(cfg Config) (*Client, error) { + if cfg.Addr == "" { + return nil, fmt.Errorf("VAULT_ADDR is not set") + } + ac := api.DefaultConfig() + ac.Address = cfg.Addr + if cfg.HTTPTimeout > 0 { + ac.Timeout = cfg.HTTPTimeout + } + vc, err := api.NewClient(ac) + if err != nil { + return nil, err + } + if cfg.Token != "" { + vc.SetToken(cfg.Token) + } + return &Client{API: vc, Cfg: cfg}, nil +} + +func (c *Client) WithToken(token string) *Client { + clone, err := c.API.Clone() + if err != nil { + cp := *c + return &cp + } + clone.SetToken(token) + cp := *c + cp.API = clone + cp.Cfg.Token = token + return &cp +} + +type HTTPResult struct { + Status int + Body []byte +} + +func (c *Client) Do(method, path string, body any) (HTTPResult, error) { + path = strings.TrimPrefix(path, "/") + query := "" + if i := strings.Index(path, "?"); i >= 0 { + query = path[i+1:] + path = path[:i] + } + req := c.API.NewRequest(method, "/v1/"+path) + if query != "" { + vals, err := url.ParseQuery(query) + if err != nil { + return HTTPResult{}, err + } + req.Params = vals + } + if body != nil { + if err := req.SetJSONBody(body); err != nil { + return HTTPResult{}, err + } + } + resp, err := c.API.RawRequest(req) + if err != nil && resp == nil { + return HTTPResult{Status: 0}, err + } + if resp == nil { + return HTTPResult{}, err + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return HTTPResult{Status: resp.StatusCode, Body: b}, nil +} + +func (c *Client) Must(method, path string, body any) (HTTPResult, error) { + r, err := c.Do(method, path, body) + if err != nil { + return r, err + } + if r.Status < 200 || r.Status >= 300 { + return r, fmt.Errorf("%s %s failed (HTTP %d): %s", method, path, r.Status, strings.TrimSpace(string(r.Body))) + } + return r, nil +} diff --git a/internal/vaultcluster/config.go b/internal/vaultcluster/config.go new file mode 100644 index 0000000..530115b --- /dev/null +++ b/internal/vaultcluster/config.go @@ -0,0 +1,85 @@ +package vaultcluster + +import ( + "fmt" + "os" + "strings" + "time" +) + +type Config struct { + Addr string + Token string + KVMount string + TenantPrefix string + AuthMount string + DatabaseMount string + AuditPath string + AuditDevice string + EnableAudit bool + EnableCredentials bool + DatabaseURL string + DatabaseUsername string + DatabasePassword string + DatabaseConnName string + DatabaseTTL string + DatabaseMaxTTL string + TokenTTL string + TokenMaxTTL string + HTTPTimeout time.Duration +} + +func ConfigFromEnv() Config { + c := Config{ + Addr: getenv("VAULT_ADDR", ""), + Token: getenv("VAULT_TOKEN", ""), + KVMount: getenv("KV_MOUNT", "kv"), + TenantPrefix: getenv("TENANT_PREFIX", "customers"), + AuthMount: getenv("AUTH_MOUNT", "approle"), + DatabaseMount: getenv("DATABASE_MOUNT", "database"), + AuditPath: getenv("AUDIT_LOG_PATH", "/vault/logs/audit.log"), + AuditDevice: getenv("AUDIT_DEVICE_NAME", "file"), + EnableAudit: getenv("ENABLE_AUDIT", "true") == "true", + // The dynamic-credentials fields (EnableCredentials, DatabaseURL, + // DatabaseUsername, ...) are never set from the environment: the + // database engine is configured only by callers that opt in + // programmatically. DatabaseMount stays because the tenant and + // platform policies reference its paths even when the engine is + // not mounted. + TokenTTL: getenv("DEFAULT_TOKEN_TTL", "1h"), + TokenMaxTTL: getenv("MAX_TOKEN_TTL", "24h"), + HTTPTimeout: 15 * time.Second, + } + return c +} + +func getenv(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func (c Config) TenantPolicy(kind, tenantID string) string { + return fmt.Sprintf("tenant-%s-%s", tenantID, kind) +} + +func (c Config) TenantRole(kind, tenantID string) string { + return fmt.Sprintf("tenant-%s-%s", tenantID, kind) +} + +func (c Config) KVDataPath(tenantID, secret string) string { + p := fmt.Sprintf("%s/data/%s/%s", c.KVMount, c.TenantPrefix, tenantID) + if secret != "" { + p += "/" + strings.TrimPrefix(secret, "/") + } + return p +} + +func (c Config) KVMetaPath(tenantID, secret string) string { + p := fmt.Sprintf("%s/metadata/%s/%s", c.KVMount, c.TenantPrefix, tenantID) + if secret != "" { + p += "/" + strings.TrimPrefix(secret, "/") + } + return p +} diff --git a/internal/vaultcluster/configure.go b/internal/vaultcluster/configure.go new file mode 100644 index 0000000..d474672 --- /dev/null +++ b/internal/vaultcluster/configure.go @@ -0,0 +1,166 @@ +package vaultcluster + +import ( + "fmt" + "log" + + "github.com/hashicorp/vault/api" +) + +func (c *Client) Configure() error { + if c.Cfg.Token == "" { + return fmt.Errorf("VAULT_TOKEN is not set") + } + log.Printf("configuring platform at %s", c.Cfg.Addr) + if err := c.enableAudit(); err != nil { + return err + } + if err := c.mountKV(); err != nil { + return err + } + if err := c.enableAppRole(); err != nil { + return err + } + for _, name := range []string{"provisioning", "operator"} { + hcl, err := RenderPolicy(name, "", c.Cfg) + if err != nil { + return err + } + if err := LintOrError(name, hcl, c.Cfg); err != nil { + return err + } + if err := c.API.Sys().PutPolicy(name, hcl); err != nil { + return fmt.Errorf("apply policy %s: %w", name, err) + } + log.Printf("applied policy %s", name) + } + if err := c.mountDatabase(); err != nil { + return err + } + log.Printf("platform configuration complete") + return nil +} + +func (c *Client) enableAudit() error { + if !c.Cfg.EnableAudit { + log.Printf("ENABLE_AUDIT is false; skipping") + return nil + } + if c.Cfg.AuditPath == "" { + return fmt.Errorf("AUDIT_LOG_PATH is required") + } + audits, err := c.API.Sys().ListAudit() + if err != nil { + return err + } + if _, ok := audits[c.Cfg.AuditDevice+"/"]; ok { + log.Printf("audit device %s already enabled", c.Cfg.AuditDevice) + return nil + } + return c.API.Sys().EnableAuditWithOptions(c.Cfg.AuditDevice, &api.EnableAuditOptions{ + Type: "file", + Description: "Platform audit device", + Options: map[string]string{ + "file_path": c.Cfg.AuditPath, + "log_raw": "false", + "hmac_accessor": "true", + "mode": "0600", + "format": "json", + }, + }) +} + +func (c *Client) mountKV() error { + mounts, err := c.API.Sys().ListMounts() + if err != nil { + return err + } + if _, ok := mounts[c.Cfg.KVMount+"/"]; !ok { + if err := c.API.Sys().Mount(c.Cfg.KVMount, &api.MountInput{ + Type: "kv", + Description: "Multi-tenant secret store", + Options: map[string]string{"version": "2"}, + }); err != nil { + return err + } + } + tune, err := c.API.Logical().Read("sys/mounts/" + c.Cfg.KVMount + "/tune") + if err != nil { + return err + } + ver := "" + if tune != nil { + if opts, ok := tune.Data["options"].(map[string]any); ok { + if v, ok := opts["version"].(string); ok { + ver = v + } + } + } + if ver != "2" { + return fmt.Errorf("%s/ is not KV v2", c.Cfg.KVMount) + } + _, err = c.API.Logical().Write(c.Cfg.KVMount+"/config", map[string]any{ + "max_versions": 10, + "cas_required": false, + "delete_version_after": "0s", + }) + return err +} + +func (c *Client) enableAppRole() error { + auths, err := c.API.Sys().ListAuth() + if err != nil { + return err + } + if _, ok := auths[c.Cfg.AuthMount+"/"]; !ok { + if err := c.API.Sys().EnableAuthWithOptions(c.Cfg.AuthMount, &api.EnableAuthOptions{ + Type: "approle", + Description: "Tenant workload identities", + }); err != nil { + return err + } + } + return c.API.Sys().TuneMount("auth/"+c.Cfg.AuthMount, api.MountConfigInput{ + DefaultLeaseTTL: c.Cfg.TokenTTL, + MaxLeaseTTL: c.Cfg.TokenMaxTTL, + }) +} + +func (c *Client) mountDatabase() error { + if !c.Cfg.EnableCredentials { + log.Printf("dynamic credentials disabled") + return nil + } + if c.Cfg.DatabaseURL == "" || c.Cfg.DatabaseUsername == "" || c.Cfg.DatabasePassword == "" { + return fmt.Errorf("database connection settings are required when credentials are enabled") + } + mounts, err := c.API.Sys().ListMounts() + if err != nil { + return err + } + if _, ok := mounts[c.Cfg.DatabaseMount+"/"]; !ok { + if err := c.API.Sys().Mount(c.Cfg.DatabaseMount, &api.MountInput{ + Type: "database", + Description: "Dynamic database credentials", + Config: api.MountConfigInput{ + DefaultLeaseTTL: c.Cfg.DatabaseTTL, + MaxLeaseTTL: c.Cfg.DatabaseMaxTTL, + }, + }); err != nil { + return err + } + } + _, err = c.API.Logical().Write(c.Cfg.DatabaseMount+"/config/"+c.Cfg.DatabaseConnName, map[string]any{ + "plugin_name": "postgresql-database-plugin", + "connection_url": c.Cfg.DatabaseURL, + "username": c.Cfg.DatabaseUsername, + "password": c.Cfg.DatabasePassword, + "allowed_roles": []string{"tenant-*"}, + "max_open_connections": 8, + "max_idle_connections": 2, + "max_connection_lifetime": "5m", + "verify_connection": true, + "password_authentication": "password", + }) + return err +} diff --git a/internal/vaultcluster/credentials_test.go b/internal/vaultcluster/credentials_test.go new file mode 100644 index 0000000..dc9d259 --- /dev/null +++ b/internal/vaultcluster/credentials_test.go @@ -0,0 +1,222 @@ +package vaultcluster + +import ( + "fmt" + "os/exec" + "strings" + "testing" + "time" +) + +func TestCredentialsMatrix(t *testing.T) { + pg, c := credentialsStack(t) + c.Cfg.EnableCredentials = true + if err := c.Configure(); err != nil { + t.Fatal(err) + } + if err := c.CreateTenant("tenant-a", false); err != nil { + t.Fatal(err) + } + if err := c.CreateTenant("tenant-b", false); err != nil { + t.Fatal(err) + } + + tokA, err := c.LoginAppRole(c.Cfg.TenantRole("writer", "tenant-a")) + if err != nil { + t.Fatal(err) + } + tokB, err := c.LoginAppRole(c.Cfg.TenantRole("writer", "tenant-b")) + if err != nil { + t.Fatal(err) + } + tokAR, err := c.LoginAppRole(c.Cfg.TenantRole("reader", "tenant-a")) + if err != nil { + t.Fatal(err) + } + + a := c.WithToken(tokA) + ro, err := a.API.Logical().Read(c.Cfg.DatabaseMount + "/creds/tenant-tenant-a-readonly") + if err != nil || ro == nil || ro.Data == nil { + t.Fatalf("issue readonly: %v %#v", err, ro) + } + roUser := fmt.Sprint(ro.Data["username"]) + roPass := fmt.Sprint(ro.Data["password"]) + + out, err := pgQuery(pg, roUser, roPass, "SELECT count(*) FROM app.items;") + if err != nil { + t.Fatalf("readonly SELECT: %v %s", err, out) + } + if strings.TrimSpace(out) != "1" { + t.Fatalf("readonly SELECT count=%q", out) + } + if _, err := pgQuery(pg, roUser, roPass, "INSERT INTO app.items (name) VALUES ('nope');"); err == nil { + t.Fatal("readonly INSERT succeeded") + } + + rw, err := a.API.Logical().Read(c.Cfg.DatabaseMount + "/creds/tenant-tenant-a-readwrite") + if err != nil || rw == nil || rw.Data == nil { + t.Fatalf("issue readwrite: %v", err) + } + rwUser := fmt.Sprint(rw.Data["username"]) + rwPass := fmt.Sprint(rw.Data["password"]) + if _, err := pgQuery(pg, rwUser, rwPass, "INSERT INTO app.items (name) VALUES ('ok');"); err != nil { + t.Fatalf("readwrite INSERT: %v", err) + } + + b := c.WithToken(tokB) + r, err := b.Do("GET", c.Cfg.DatabaseMount+"/creds/tenant-tenant-a-readonly", nil) + if err != nil && r.Status == 0 { + t.Fatal(err) + } + if r.Status != 403 { + t.Fatalf("cross-tenant creds: HTTP %d want 403", r.Status) + } + r, err = c.WithToken(tokAR).Do("GET", c.Cfg.DatabaseMount+"/creds/tenant-tenant-a-readonly", nil) + if err != nil && r.Status == 0 { + t.Fatal(err) + } + if r.Status != 403 { + t.Fatalf("reader creds: HTTP %d want 403", r.Status) + } + + if err := c.API.Sys().Revoke(ro.LeaseID); err != nil { + t.Fatal(err) + } + if _, err := pgQuery(pg, roUser, roPass, "SELECT 1;"); err == nil { + t.Fatal("revoked readonly still connects") + } +} + +func TestCredentialsAfterIsolation(t *testing.T) { + pg, c := credentialsStack(t) + c.Cfg.EnableCredentials = false + if err := c.Configure(); err != nil { + t.Fatal(err) + } + if err := c.CreateTenant("tenant-a", false); err != nil { + t.Fatal(err) + } + + store := FileKeyStore{Dir: t.TempDir()} + for _, name := range []string{"operator", "provisioning"} { + tok, err := c.issueOrphanToken(name) + if err != nil { + t.Fatal(err) + } + if err := store.SaveToken(name, tok); err != nil { + t.Fatal(err) + } + } + provTok, err := store.LoadToken("provisioning") + if err != nil { + t.Fatal(err) + } + prov := c.WithToken(provTok) + prov.Cfg.EnableCredentials = true + if err := prov.enableCredentialsIfNeeded(store); err != nil { + t.Fatal(err) + } + if err := prov.CreateTenant("tenant-a", false); err != nil { + t.Fatal(err) + } + + tokA, err := c.LoginAppRole(c.Cfg.TenantRole("writer", "tenant-a")) + if err != nil { + t.Fatal(err) + } + ro, err := c.WithToken(tokA).API.Logical().Read(c.Cfg.DatabaseMount + "/creds/tenant-tenant-a-readonly") + if err != nil || ro == nil || ro.Data == nil { + t.Fatalf("issue readonly after isolation: %v %#v", err, ro) + } + roUser := fmt.Sprint(ro.Data["username"]) + roPass := fmt.Sprint(ro.Data["password"]) + out, err := pgQuery(pg, roUser, roPass, "SELECT count(*) FROM app.items;") + if err != nil { + t.Fatalf("readonly SELECT: %v %s", err, out) + } + if strings.TrimSpace(out) != "1" { + t.Fatalf("readonly SELECT count=%q", out) + } +} + +func credentialsStack(t *testing.T) (string, *Client) { + t.Helper() + requireDocker(t) + + net := fmt.Sprintf("vctest-%d", time.Now().UnixNano()) + if err := exec.Command("docker", "network", "create", net).Run(); err != nil { + t.Fatalf("network create: %v", err) + } + t.Cleanup(func() { _ = exec.Command("docker", "network", "rm", net).Run() }) + + pgPass := "local-dev-only-not-a-real-secret" + pg := dockerRun(t, + "--name", fmt.Sprintf("vctest-pg-%d", time.Now().UnixNano()), + "--network", net, "--network-alias", "pg", + "-e", "POSTGRES_PASSWORD="+pgPass, + "-e", "POSTGRES_DB=appdb", + postgresTestImage, + ) + deadline := time.Now().Add(45 * time.Second) + var last string + ready := false + for time.Now().Before(deadline) { + out, err := pgQuery(pg, "postgres", pgPass, "SELECT 1;") + if err == nil { + ready = true + break + } + last = strings.TrimSpace(fmt.Sprintf("%v %s", err, out)) + time.Sleep(250 * time.Millisecond) + } + if !ready { + logs, _ := exec.Command("docker", "logs", pg).CombinedOutput() + t.Fatalf("postgres did not become ready: %s\n%s", last, logs) + } + + mustPG(t, pg, "postgres", pgPass, ` +CREATE ROLE app_readonly NOLOGIN; +CREATE ROLE app_readwrite NOLOGIN; +CREATE ROLE vault_admin LOGIN CREATEROLE PASSWORD 'local-dev-only-vault-admin-not-a-real-secret'; +GRANT app_readonly TO vault_admin WITH ADMIN OPTION; +GRANT app_readwrite TO vault_admin WITH ADMIN OPTION; +CREATE SCHEMA app; +CREATE TABLE app.items (id serial PRIMARY KEY, name text NOT NULL); +INSERT INTO app.items (name) VALUES ('row-a'); +GRANT USAGE ON SCHEMA app TO app_readonly, app_readwrite; +GRANT SELECT ON ALL TABLES IN SCHEMA app TO app_readonly; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA app TO app_readwrite; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA app TO app_readwrite; +`) + + c := startVaultInmem(t, "--network", net) + c.Cfg.KVMount = "kv" + c.Cfg.TenantPrefix = "customers" + c.Cfg.AuthMount = "approle" + c.Cfg.DatabaseMount = "database" + c.Cfg.EnableAudit = false + c.Cfg.DatabaseURL = "postgresql://{{username}}:{{password}}@pg:5432/appdb?sslmode=disable" + c.Cfg.DatabaseUsername = "vault_admin" + c.Cfg.DatabasePassword = "local-dev-only-vault-admin-not-a-real-secret" + c.Cfg.DatabaseConnName = "app" + c.Cfg.DatabaseTTL = "1h" + c.Cfg.DatabaseMaxTTL = "24h" + c.Cfg.TokenTTL = "1h" + c.Cfg.TokenMaxTTL = "24h" + return pg, c +} + +func mustPG(t *testing.T, id, user, pass, sql string) { + t.Helper() + out, err := pgQuery(id, user, pass, sql) + if err != nil { + t.Fatalf("psql: %v\n%s", err, out) + } +} + +func pgQuery(id, user, pass, sql string) (string, error) { + cmd := exec.Command("docker", "exec", "-e", "PGPASSWORD="+pass, id, + "psql", "-U", user, "-d", "appdb", "-v", "ON_ERROR_STOP=1", "-tAc", sql) + out, err := cmd.CombinedOutput() + return string(out), err +} diff --git a/internal/vaultcluster/docker_test.go b/internal/vaultcluster/docker_test.go new file mode 100644 index 0000000..cd91a00 --- /dev/null +++ b/internal/vaultcluster/docker_test.go @@ -0,0 +1,119 @@ +package vaultcluster + +import ( + "os/exec" + "strings" + "testing" + "time" + + "github.com/hashicorp/vault/api" +) + +const ( + vaultTestImage = "hashicorp/vault:1.21.4@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569" + postgresTestImage = "postgres:16.15-alpine@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685" +) + +func requireDocker(t *testing.T) { + t.Helper() + if testing.Short() { + t.Skip("skipping docker") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available") + } + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skip("docker daemon is not running") + } +} + +func dockerOutput(t *testing.T, args ...string) string { + t.Helper() + out, err := exec.Command("docker", args...).Output() + if err != nil { + stderr := []byte(nil) + if ee, ok := err.(*exec.ExitError); ok { + stderr = ee.Stderr + } + t.Fatalf("docker %v: %v\n%s\n%s", args, err, out, stderr) + } + return strings.TrimSpace(string(out)) +} + +func dockerRun(t *testing.T, args ...string) string { + t.Helper() + id := lastLine(dockerOutput(t, append([]string{"run", "-d"}, args...)...)) + t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", id).Run() }) + return id +} + +func lastLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.LastIndex(s, "\n"); i >= 0 { + return strings.TrimSpace(s[i+1:]) + } + return s +} + +func vaultListenAddr(t *testing.T, id string) string { + t.Helper() + var portOut []byte + var portErr error + for i := 0; i < 20; i++ { + portOut, portErr = exec.Command("docker", "port", id, "8200").CombinedOutput() + if portErr == nil && strings.Contains(string(portOut), ":") { + break + } + time.Sleep(250 * time.Millisecond) + } + if portErr != nil || !strings.Contains(string(portOut), ":") { + logs, _ := exec.Command("docker", "logs", id).CombinedOutput() + t.Fatalf("docker port %s: %v\n%s\nlogs:\n%s", id, portErr, portOut, logs) + } + line := strings.TrimSpace(strings.Split(string(portOut), "\n")[0]) + hostPort := line + if i := strings.LastIndex(line, ":"); i >= 0 { + hostPort = line[i+1:] + } + return "http://127.0.0.1:" + hostPort +} + +func startVaultInmem(t *testing.T, extraRunArgs ...string) *Client { + t.Helper() + requireDocker(t) + cfgJSON := `{"disable_mlock":true,"listener":{"tcp":{"address":"0.0.0.0:8200","tls_disable":true}},"storage":{"inmem":{}}}` + args := append([]string{ + "-p", "127.0.0.1::8200", + "-e", "VAULT_LOCAL_CONFIG=" + cfgJSON, + "--cap-add", "IPC_LOCK", + }, extraRunArgs...) + args = append(args, vaultTestImage, "server") + id := dockerRun(t, args...) + c, err := New(Config{Addr: vaultListenAddr(t, id), HTTPTimeout: 10 * time.Second}) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + if st, err := c.API.Sys().SealStatus(); err == nil && st != nil { + break + } + time.Sleep(200 * time.Millisecond) + } + resp, err := c.API.Sys().Init(&api.InitRequest{SecretShares: 1, SecretThreshold: 1}) + if err != nil { + t.Fatal(err) + } + key := "" + if len(resp.KeysB64) > 0 { + key = resp.KeysB64[0] + } else if len(resp.Keys) > 0 { + key = resp.Keys[0] + } + if _, err := c.API.Sys().Unseal(key); err != nil { + t.Fatal(err) + } + c.API.SetToken(resp.RootToken) + c.Cfg.Token = resp.RootToken + return c +} diff --git a/internal/vaultcluster/health.go b/internal/vaultcluster/health.go new file mode 100644 index 0000000..47268ff --- /dev/null +++ b/internal/vaultcluster/health.go @@ -0,0 +1,27 @@ +package vaultcluster + +import ( + "fmt" + "os" +) + +func (c *Client) Health() error { + st, err := c.API.Sys().SealStatus() + if err != nil { + return err + } + fmt.Printf("initialized %v\n", st.Initialized) + fmt.Printf("sealed %v\n", st.Sealed) + if st.Sealed { + return fmt.Errorf("vault is sealed") + } + if c.Cfg.Token != "" { + if _, err := c.API.Logical().Read("sys/mounts/" + c.Cfg.KVMount + "/tune"); err != nil { + fmt.Fprintf(os.Stderr, "kv mount: %v\n", err) + return err + } + fmt.Printf("kv %s/ (v2)\n", c.Cfg.KVMount) + } + fmt.Println("healthy") + return nil +} diff --git a/internal/vaultcluster/health_http.go b/internal/vaultcluster/health_http.go new file mode 100644 index 0000000..2d57e94 --- /dev/null +++ b/internal/vaultcluster/health_http.go @@ -0,0 +1,34 @@ +package vaultcluster + +import ( + "net/http" + "time" +) + +// The NLB marks a probe failed after 6s. Answers slower than that are useless, and a +// stalled Vault must not pin probe connections open, so both sides are bounded below it. +const healthProbeTimeout = 5 * time.Second + +func (c *Client) HealthHandler(nodeID string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if err := c.NodeHealthOK(nodeID); err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) + }) +} + +func (c *Client) ServeHealth(addr, nodeID string) error { + c.API.SetClientTimeout(healthProbeTimeout) + srv := &http.Server{ + Addr: addr, + Handler: c.HealthHandler(nodeID), + ReadHeaderTimeout: healthProbeTimeout, + ReadTimeout: healthProbeTimeout, + WriteTimeout: healthProbeTimeout, + IdleTimeout: time.Minute, + } + return srv.ListenAndServe() +} diff --git a/internal/vaultcluster/health_http_test.go b/internal/vaultcluster/health_http_test.go new file mode 100644 index 0000000..7bb2827 --- /dev/null +++ b/internal/vaultcluster/health_http_test.go @@ -0,0 +1,20 @@ +package vaultcluster + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestHealthHandlerUnavailable(t *testing.T) { + c, err := New(Config{Addr: "http://127.0.0.1:1", HTTPTimeout: 50 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + c.HealthHandler("n1").ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status %d", rec.Code) + } +} diff --git a/internal/vaultcluster/health_raft.go b/internal/vaultcluster/health_raft.go new file mode 100644 index 0000000..79ceb3e --- /dev/null +++ b/internal/vaultcluster/health_raft.go @@ -0,0 +1,107 @@ +package vaultcluster + +import ( + "encoding/json" + "fmt" + "strings" +) + +type raftServer struct { + ID string + Status string + Healthy bool + Voter bool +} + +func parseAutopilot(data map[string]any) []raftServer { + raw, _ := json.Marshal(data) + var parsed struct { + Voters []string `json:"voters"` + Servers map[string]struct { + ID string `json:"id"` + Status string `json:"status"` + Healthy bool `json:"healthy"` + } `json:"servers"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil + } + voters := map[string]bool{} + for _, id := range parsed.Voters { + voters[id] = true + } + var out []raftServer + for id, s := range parsed.Servers { + if s.ID == "" { + s.ID = id + } + status := strings.ToLower(s.Status) + out = append(out, raftServer{ + ID: s.ID, + Status: status, + Healthy: s.Healthy, + Voter: voters[s.ID] || status == "leader" || status == "voter", + }) + } + return out +} + +func NodeIsLeader(nodeID string, data map[string]any) bool { + if nodeID == "" || data == nil { + return false + } + for _, s := range parseAutopilot(data) { + if s.ID == nodeID && s.Status == "leader" { + return true + } + } + return false +} + +func NodeRaftReady(nodeID string, data map[string]any) error { + if nodeID == "" { + return fmt.Errorf("VAULT_RAFT_NODE_ID is not set") + } + if data == nil { + return fmt.Errorf("raft autopilot state is missing") + } + for _, s := range parseAutopilot(data) { + if s.ID != nodeID { + continue + } + if !s.Voter { + return fmt.Errorf("node %s is not a raft voter", nodeID) + } + if !s.Healthy { + return fmt.Errorf("node %s is not caught up", nodeID) + } + return nil + } + return fmt.Errorf("node %s is not in the raft cluster", nodeID) +} + +func (c *Client) RaftAutopilot() (map[string]any, error) { + sec, err := c.API.Logical().Read("sys/storage/raft/autopilot/state") + if err != nil { + return nil, err + } + if sec == nil || sec.Data == nil { + return nil, fmt.Errorf("raft autopilot state is empty") + } + return sec.Data, nil +} + +func (c *Client) NodeHealthOK(nodeID string) error { + st, err := c.API.Sys().SealStatus() + if err != nil { + return err + } + if !st.Initialized || st.Sealed { + return fmt.Errorf("vault is not ready") + } + data, err := c.RaftAutopilot() + if err != nil { + return err + } + return NodeRaftReady(nodeID, data) +} diff --git a/internal/vaultcluster/health_raft_test.go b/internal/vaultcluster/health_raft_test.go new file mode 100644 index 0000000..2feefb2 --- /dev/null +++ b/internal/vaultcluster/health_raft_test.go @@ -0,0 +1,69 @@ +package vaultcluster + +import "testing" + +func autopilot(nodeID, status string, healthy bool, voters []string) map[string]any { + return map[string]any{ + "voters": voters, + "servers": map[string]any{ + nodeID: map[string]any{ + "id": nodeID, + "status": status, + "healthy": healthy, + }, + }, + } +} + +func TestNodeRaftReadyVoterCaughtUp(t *testing.T) { + data := autopilot("n1", "voter", true, []string{"n1", "n2"}) + if err := NodeRaftReady("n1", data); err != nil { + t.Fatal(err) + } +} + +func TestNodeRaftReadyLeader(t *testing.T) { + data := autopilot("n1", "leader", true, []string{"n1"}) + if err := NodeRaftReady("n1", data); err != nil { + t.Fatal(err) + } +} + +func TestNodeRaftReadyNonVoter(t *testing.T) { + data := autopilot("n3", "non-voter", true, []string{"n1", "n2"}) + if err := NodeRaftReady("n3", data); err == nil { + t.Fatal("expected non-voter to fail") + } +} + +func TestNodeRaftReadyNotCaughtUp(t *testing.T) { + data := autopilot("n1", "voter", false, []string{"n1"}) + if err := NodeRaftReady("n1", data); err == nil { + t.Fatal("expected unhealthy voter to fail") + } +} + +func TestNodeRaftReadyMissingNode(t *testing.T) { + data := autopilot("n1", "leader", true, []string{"n1"}) + if err := NodeRaftReady("n2", data); err == nil { + t.Fatal("expected missing node to fail") + } +} + +func TestNodeRaftReadyRequiresNodeID(t *testing.T) { + if err := NodeRaftReady("", autopilot("n1", "leader", true, []string{"n1"})); err == nil { + t.Fatal("expected empty node id to fail") + } +} + +func TestNodeIsLeader(t *testing.T) { + if !NodeIsLeader("n1", autopilot("n1", "leader", true, []string{"n1"})) { + t.Fatal("expected leader") + } + if NodeIsLeader("n1", autopilot("n1", "voter", true, []string{"n1"})) { + t.Fatal("expected follower to be false") + } + if NodeIsLeader("n2", autopilot("n1", "leader", true, []string{"n1"})) { + t.Fatal("expected other node to be false") + } +} diff --git a/internal/vaultcluster/isolation_test.go b/internal/vaultcluster/isolation_test.go new file mode 100644 index 0000000..5386435 --- /dev/null +++ b/internal/vaultcluster/isolation_test.go @@ -0,0 +1,167 @@ +package vaultcluster + +import ( + "fmt" + "testing" + + "github.com/hashicorp/vault/api" +) + +func TestIsolationMatrix(t *testing.T) { + c := startVaultInmem(t) + c.Cfg.KVMount = "kv" + c.Cfg.TenantPrefix = "customers" + c.Cfg.AuthMount = "approle" + c.Cfg.DatabaseMount = "database" + c.Cfg.EnableAudit = false + if err := c.Configure(); err != nil { + t.Fatal(err) + } + if err := c.CreateTenant("tenant-a", false); err != nil { + t.Fatal(err) + } + if err := c.CreateTenant("tenant-b", false); err != nil { + t.Fatal(err) + } + + tokA, err := c.LoginAppRole(c.Cfg.TenantRole("reader", "tenant-a")) + if err != nil { + t.Fatal(err) + } + tokB, err := c.LoginAppRole(c.Cfg.TenantRole("reader", "tenant-b")) + if err != nil { + t.Fatal(err) + } + tokAW, err := c.LoginAppRole(c.Cfg.TenantRole("writer", "tenant-a")) + if err != nil { + t.Fatal(err) + } + tokBW, err := c.LoginAppRole(c.Cfg.TenantRole("writer", "tenant-b")) + if err != nil { + t.Fatal(err) + } + + provSec, err := c.API.Auth().Token().Create(&api.TokenCreateRequest{ + Policies: []string{"provisioning"}, + Period: "1h", + NoParent: true, + DisplayName: "provisioning", + }) + if err != nil || provSec == nil || provSec.Auth == nil { + t.Fatalf("provisioning token: %v", err) + } + tokProv := provSec.Auth.ClientToken + + mustWrite := func(token, path, val string) { + t.Helper() + cl := c.WithToken(token) + if _, err := cl.API.Logical().Write(path, map[string]any{"data": map[string]any{"value": val}}); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + mustWrite(tokAW, c.Cfg.KVDataPath("tenant-a", "fixture"), "secret-a") + mustWrite(tokBW, c.Cfg.KVDataPath("tenant-b", "fixture"), "secret-b") + + assertStatus := func(token, method, path string, body any, want int) { + t.Helper() + cl := c.WithToken(token) + r, err := cl.Do(method, path, body) + if err != nil && r.Status == 0 { + t.Fatalf("%s %s: %v", method, path, err) + } + if r.Status != want { + t.Fatalf("%s %s: got HTTP %d want %d body %s", method, path, r.Status, want, r.Body) + } + } + deny := func(token, method, path string, body any) { + t.Helper() + assertStatus(token, method, path, body, 403) + } + + aData := c.Cfg.KVDataPath("tenant-a", "fixture") + bData := c.Cfg.KVDataPath("tenant-b", "fixture") + aMeta := c.Cfg.KVMetaPath("tenant-a", "fixture") + bMeta := c.Cfg.KVMetaPath("tenant-b", "fixture") + crossWrite := map[string]any{"data": map[string]any{"value": "FAKE-cross-tenant-write"}} + + assertStatus(tokA, "GET", aData, nil, 200) + assertStatus(tokB, "GET", bData, nil, 200) + assertStatus(tokA, "GET", "auth/token/lookup-self", nil, 200) + + deny(tokA, "GET", bData, nil) + deny(tokB, "GET", aData, nil) + deny(tokAW, "GET", bData, nil) + deny(tokBW, "GET", aData, nil) + deny(tokAW, "POST", bData, crossWrite) + deny(tokBW, "POST", aData, crossWrite) + deny(tokAW, "POST", c.Cfg.KVDataPath("tenant-b", "newly-planted-secret"), map[string]any{ + "data": map[string]any{"value": "FAKE-planted"}, + }) + deny(tokAW, "DELETE", bData, nil) + deny(tokAW, "POST", c.Cfg.KVMount+"/destroy/"+c.Cfg.TenantPrefix+"/tenant-b/fixture", map[string]any{"versions": []int{1}}) + deny(tokAW, "DELETE", bMeta, nil) + deny(tokA, "GET", bMeta, nil) + deny(tokB, "GET", aMeta, nil) + deny(tokA, "GET", c.Cfg.KVMount+"/metadata/"+c.Cfg.TenantPrefix+"?list=true", nil) + deny(tokB, "GET", c.Cfg.KVMount+"/metadata/"+c.Cfg.TenantPrefix+"?list=true", nil) + deny(tokA, "GET", c.Cfg.KVMetaPath("tenant-b", "")+"?list=true", nil) + + deny(tokA, "GET", c.Cfg.KVMount+"/data/"+c.Cfg.TenantPrefix, nil) + deny(tokA, "GET", c.Cfg.KVMount+"/data", nil) + deny(tokA, "GET", c.Cfg.KVMount+"/metadata/"+c.Cfg.TenantPrefix, nil) + deny(tokAW, "POST", c.Cfg.KVMount+"/data/"+c.Cfg.TenantPrefix+"/shared-secret", map[string]any{ + "data": map[string]any{"value": "FAKE-planted-at-parent"}, + }) + deny(tokA, "GET", c.Cfg.KVMount+"/data/platform/root-credentials", nil) + deny(tokAW, "POST", c.Cfg.KVMount+"/data/platform/root-credentials", map[string]any{ + "data": map[string]any{"value": "FAKE-planted-sibling"}, + }) + deny(tokA, "GET", c.Cfg.KVDataPath("tenant-a-extended", "secret"), nil) + deny(tokA, "GET", c.Cfg.KVDataPath("tenant-ax", "secret"), nil) + deny(tokA, "GET", c.Cfg.KVMount+"/data/"+c.Cfg.TenantPrefix+"/*", nil) + + deny(tokA, "GET", "sys/mounts", nil) + deny(tokA, "GET", "sys/auth", nil) + deny(tokAW, "POST", "sys/mounts/rogue", map[string]any{"type": "kv", "options": map[string]any{"version": "2"}}) + deny(tokAW, "DELETE", "sys/mounts/"+c.Cfg.KVMount, nil) + deny(tokA, "GET", "sys/audit", nil) + deny(tokA, "GET", "sys/policies/acl?list=true", nil) + deny(tokA, "GET", "sys/policies/acl/"+c.Cfg.TenantPolicy("reader", "tenant-b"), nil) + deny(tokAW, "PUT", "sys/policies/acl/"+c.Cfg.TenantPolicy("reader", "tenant-a"), map[string]any{ + "policy": `path "kv/data/*" { capabilities = ["read", "list"] }`, + }) + deny(tokA, "GET", "auth/"+c.Cfg.AuthMount+"/role?list=true", nil) + deny(tokA, "GET", "auth/"+c.Cfg.AuthMount+"/role/"+c.Cfg.TenantRole("reader", "tenant-b")+"/role-id", nil) + deny(tokAW, "POST", "auth/"+c.Cfg.AuthMount+"/role/"+c.Cfg.TenantRole("reader", "tenant-b")+"/secret-id", map[string]any{}) + deny(tokAW, "POST", "auth/token/create", map[string]any{"policies": []string{"operator"}}) + deny(tokA, "GET", "auth/token/accessors?list=true", nil) + + deny(tokProv, "GET", aData, nil) + deny(tokProv, "GET", aMeta, nil) + deny(tokProv, "POST", aData, map[string]any{"data": map[string]any{"value": "FAKE-written-by-provisioning"}}) + deny(tokProv, "PUT", "sys/policies/acl/provisioning", map[string]any{ + "policy": `path "kv/data/*" { capabilities = ["read"] }`, + }) + assertStatus(tokProv, "GET", "auth/"+c.Cfg.AuthMount+"/role/"+c.Cfg.TenantRole("reader", "tenant-a")+"/role-id", nil, 200) + + for _, traversal := range []string{ + fmt.Sprintf("%s/data/%s/tenant-a/../tenant-b/fixture", c.Cfg.KVMount, c.Cfg.TenantPrefix), + fmt.Sprintf("%s/data/%s/tenant-a/..%%2ftenant-b/fixture", c.Cfg.KVMount, c.Cfg.TenantPrefix), + } { + cl := c.WithToken(tokA) + r, err := cl.Do("GET", traversal, nil) + if err != nil && r.Status == 0 { + t.Fatalf("traversal %s: %v", traversal, err) + } + if r.Status >= 200 && r.Status < 300 { + t.Fatalf("traversal granted HTTP %d: %s", r.Status, traversal) + } + } + + if err := c.OffboardTenant("tenant-a", false); err != nil { + t.Fatal(err) + } + if _, err := c.LoginAppRole(c.Cfg.TenantRole("reader", "tenant-a")); err == nil { + t.Fatal("expected AppRole login to fail after offboard") + } +} diff --git a/internal/vaultcluster/keystore.go b/internal/vaultcluster/keystore.go new file mode 100644 index 0000000..2fbddf3 --- /dev/null +++ b/internal/vaultcluster/keystore.go @@ -0,0 +1,63 @@ +package vaultcluster + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/vault/api" +) + +type KeyStore interface { + SaveInit(resp *api.InitResponse) error + LoadInit() (*api.InitResponse, error) + SaveToken(name, token string) error + LoadToken(name string) (string, error) +} + +type FileKeyStore struct { + Dir string +} + +func (s FileKeyStore) SaveInit(resp *api.InitResponse) error { + b, err := json.Marshal(resp) + if err != nil { + return err + } + return s.write(filepath.Join(s.Dir, "vault-init.json"), b) +} + +func (s FileKeyStore) LoadInit() (*api.InitResponse, error) { + raw, err := os.ReadFile(filepath.Join(s.Dir, "vault-init.json")) + if err != nil { + return nil, err + } + var resp api.InitResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (s FileKeyStore) SaveToken(name, token string) error { + return s.write(filepath.Join(s.Dir, name+".token"), []byte(token)) +} + +func (s FileKeyStore) LoadToken(name string) (string, error) { + b, err := os.ReadFile(filepath.Join(s.Dir, name+".token")) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +func (s FileKeyStore) write(path string, b []byte) error { + if err := os.MkdirAll(s.Dir, 0o700); err != nil { + return err + } + if err := os.WriteFile(path, b, 0o600); err != nil { + return err + } + return chownToDirOwner(s.Dir, path) +} diff --git a/internal/vaultcluster/keystore_unix.go b/internal/vaultcluster/keystore_unix.go new file mode 100644 index 0000000..a0c70d4 --- /dev/null +++ b/internal/vaultcluster/keystore_unix.go @@ -0,0 +1,34 @@ +//go:build !windows + +package vaultcluster + +import ( + "os" + "strconv" + "syscall" +) + +func chownToDirOwner(dir, path string) error { + if os.Getuid() != 0 { + return nil + } + uid, gid := 0, 0 + if fi, err := os.Stat(dir); err == nil { + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + uid = int(st.Uid) + gid = int(st.Gid) + } + } + if v := os.Getenv("BOOTSTRAP_UID"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + uid = n + } + } + if v := os.Getenv("BOOTSTRAP_GID"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + gid = n + } + } + _ = os.Chown(dir, uid, gid) + return os.Chown(path, uid, gid) +} diff --git a/internal/vaultcluster/keystore_windows.go b/internal/vaultcluster/keystore_windows.go new file mode 100644 index 0000000..faf9d71 --- /dev/null +++ b/internal/vaultcluster/keystore_windows.go @@ -0,0 +1,8 @@ +//go:build windows + +package vaultcluster + +// Ownership fixups only apply inside the Linux bootstrap container. +func chownToDirOwner(dir, path string) error { + return nil +} diff --git a/internal/vaultcluster/lint.go b/internal/vaultcluster/lint.go new file mode 100644 index 0000000..64560dc --- /dev/null +++ b/internal/vaultcluster/lint.go @@ -0,0 +1,140 @@ +package vaultcluster + +import ( + "fmt" + "regexp" + "strings" +) + +var ( + pathRE = regexp.MustCompile(`path\s+"([^"]+)"`) + capsRE = regexp.MustCompile(`capabilities\s*=\s*\[([^\]]*)\]`) +) + +var kvV2Segments = map[string]struct{}{ + "data": {}, "metadata": {}, "delete": {}, "undelete": {}, + "destroy": {}, "config": {}, "subkeys": {}, +} + +var validCaps = map[string]struct{}{ + "create": {}, "read": {}, "update": {}, "delete": {}, "list": {}, + "patch": {}, "sudo": {}, "deny": {}, "recover": {}, "subscribe": {}, +} + +type Finding struct { + Policy string + Msg string +} + +func (f Finding) String() string { + return f.Policy + ": " + f.Msg +} + +func LintPolicy(policyName, src string, cfg Config) []Finding { + type pair struct{ path, caps string } + var pairs []pair + lines := strings.Split(src, "\n") + current := "" + for _, line := range lines { + trim := strings.TrimSpace(line) + if strings.HasPrefix(trim, "#") { + continue + } + if m := pathRE.FindStringSubmatch(line); m != nil { + current = m[1] + continue + } + if m := capsRE.FindStringSubmatch(line); m != nil && current != "" { + caps := strings.ReplaceAll(m[1], `"`, "") + caps = strings.ReplaceAll(caps, " ", "") + pairs = append(pairs, pair{current, caps}) + current = "" + } + } + + var findings []Finding + add := func(msg string) { + findings = append(findings, Finding{Policy: policyName, Msg: msg}) + } + if len(pairs) == 0 { + add("no path rules found") + return findings + } + + seen := map[string]struct{}{} + for _, p := range pairs { + if p.path == "*" || p.path == "/*" { + if p.caps != "deny" { + add(fmt.Sprintf("path %q grants [%s] over the entire Vault API", p.path, p.caps)) + } + } + if p.caps == "" { + add(fmt.Sprintf("path %q has an empty capabilities list", p.path)) + } + for _, c := range strings.Split(p.caps, ",") { + if c == "" { + continue + } + if _, ok := validCaps[c]; !ok { + add(fmt.Sprintf("path %q has unknown capability %q", p.path, c)) + } + } + if strings.Contains(","+p.caps+",", ",deny,") && p.caps != "deny" { + add(fmt.Sprintf("path %q mixes deny with [%s]", p.path, p.caps)) + } + if strings.Contains(","+p.caps+",", ",sudo,") { + if policyName != "operator" && policyName != "admin" { + add(fmt.Sprintf("path %q grants sudo", p.path)) + } + } + + if strings.HasPrefix(p.path, cfg.KVMount+"/") { + rest := strings.TrimPrefix(p.path, cfg.KVMount+"/") + second, _, _ := strings.Cut(rest, "/") + ok := p.caps == "deny" || second == "*" + if _, hit := kvV2Segments[second]; hit { + ok = true + } + if !ok { + add(fmt.Sprintf("path %q is not a valid KV v2 path", p.path)) + } + } + + if strings.HasSuffix(p.path, "/"+cfg.TenantPrefix+"/*") || strings.HasSuffix(p.path, "/"+cfg.TenantPrefix+"/") { + if strings.HasPrefix(p.path, cfg.KVMount+"/") && p.caps != "deny" { + segs := strings.Split(p.path, "/") + if len(segs) == 4 && segs[2] == cfg.TenantPrefix && segs[3] == "*" && p.caps != "deny" { + add(fmt.Sprintf("path %q grants [%s] across all tenants", p.path, p.caps)) + } + } + } + + switch p.path { + case cfg.KVMount + "/*", cfg.KVMount + "/data/*", cfg.KVMount + "/metadata/*": + if p.caps != "deny" { + add(fmt.Sprintf("path %q grants [%s] over the entire KV mount", p.path, p.caps)) + } + } + if p.path == cfg.DatabaseMount+"/creds/*" && p.caps != "deny" { + add(fmt.Sprintf("path %q grants [%s] on every tenant's database credentials", p.path, p.caps)) + } + if _, ok := seen[p.path]; ok { + add(fmt.Sprintf("path %q is declared more than once", p.path)) + } + seen[p.path] = struct{}{} + } + return findings +} + +func LintOrError(policyName, src string, cfg Config) error { + fs := LintPolicy(policyName, src, cfg) + if len(fs) == 0 { + return nil + } + var b strings.Builder + fmt.Fprintf(&b, "policy lint failed (%d):", len(fs)) + for _, f := range fs { + fmt.Fprintf(&b, "\n %s", f) + } + return fmt.Errorf("%s", b.String()) +} diff --git a/internal/vaultcluster/lint_test.go b/internal/vaultcluster/lint_test.go new file mode 100644 index 0000000..eadb514 --- /dev/null +++ b/internal/vaultcluster/lint_test.go @@ -0,0 +1,58 @@ +package vaultcluster + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLintFixtures(t *testing.T) { + cfg := Config{KVMount: "kv", TenantPrefix: "customers", DatabaseMount: "database", AuthMount: "approle"} + root := filepath.Join("testdata", "lint") + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".hcl") { + continue + } + b, err := os.ReadFile(filepath.Join(root, e.Name())) + if err != nil { + t.Fatal(err) + } + name := strings.TrimSuffix(e.Name(), ".hcl") + findings := LintPolicy(name, string(b), cfg) + wantBad := strings.HasPrefix(e.Name(), "BAD-") + if wantBad && len(findings) == 0 { + t.Errorf("%s: expected findings, got none", e.Name()) + } + if !wantBad && len(findings) > 0 { + t.Errorf("%s: unexpected findings: %v", e.Name(), findings) + } + } +} + +func TestRenderAndLintPlatformPolicies(t *testing.T) { + cfg := Config{KVMount: "kv", TenantPrefix: "customers", DatabaseMount: "database", AuthMount: "approle"} + for _, name := range []string{"provisioning", "operator"} { + hcl, err := RenderPolicy(name, "", cfg) + if err != nil { + t.Fatal(err) + } + if err := LintOrError(name, hcl, cfg); err != nil { + t.Fatal(err) + } + } + for _, tmpl := range []string{"tenant-reader", "tenant-writer", "tenant-database"} { + hcl, err := RenderPolicy(tmpl, "tenant-a", cfg) + if err != nil { + t.Fatal(err) + } + name := "tenant-tenant-a-" + strings.TrimPrefix(tmpl, "tenant-") + if err := LintOrError(name, hcl, cfg); err != nil { + t.Fatal(err) + } + } +} diff --git a/internal/vaultcluster/policies/embed.go b/internal/vaultcluster/policies/embed.go new file mode 100644 index 0000000..ebb3fc0 --- /dev/null +++ b/internal/vaultcluster/policies/embed.go @@ -0,0 +1,6 @@ +package policies + +import "embed" + +//go:embed templates/*.hcl.tpl +var Templates embed.FS diff --git a/internal/vaultcluster/policies/templates/operator.hcl.tpl b/internal/vaultcluster/policies/templates/operator.hcl.tpl new file mode 100644 index 0000000..a7187db --- /dev/null +++ b/internal/vaultcluster/policies/templates/operator.hcl.tpl @@ -0,0 +1,124 @@ +# Local operator: health and config read. Not a tenant secret reader. No restore. +path "sys/health" { + capabilities = ["read", "sudo"] +} + +path "sys/seal-status" { + capabilities = ["read"] +} + +path "sys/leader" { + capabilities = ["read"] +} + +path "sys/mounts" { + capabilities = ["read", "sudo"] +} + +path "sys/mounts/{{.DatabaseMount}}" { + capabilities = ["create", "update", "sudo"] +} + +path "sys/mounts/*" { + capabilities = ["read"] +} + +path "sys/auth" { + capabilities = ["read"] +} + +path "sys/policies/acl" { + capabilities = ["list"] +} + +path "sys/policies/acl/*" { + capabilities = ["read"] +} + +path "sys/audit" { + capabilities = ["read", "sudo"] +} + +path "auth/{{.AuthMount}}/role" { + capabilities = ["list"] +} + +path "auth/{{.AuthMount}}/role/*" { + capabilities = ["read"] +} + +path "{{.DatabaseMount}}/roles" { + capabilities = ["list"] +} + +path "{{.DatabaseMount}}/roles/*" { + capabilities = ["read"] +} + +path "{{.DatabaseMount}}/config/*" { + capabilities = ["create", "update", "read"] +} + +path "sys/leases/lookup" { + capabilities = ["update"] +} + +path "sys/leases/lookup/*" { + capabilities = ["list", "read"] +} + +path "sys/leases/revoke" { + capabilities = ["update"] +} + +path "sys/leases/revoke-prefix/*" { + capabilities = ["update", "sudo"] +} + +path "sys/storage/raft/snapshot" { + capabilities = ["read"] +} + +path "sys/storage/raft/configuration" { + capabilities = ["read"] +} + +path "sys/storage/raft/autopilot/state" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} + +path "{{.KVMount}}/data/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/metadata/*" { + capabilities = ["deny"] +} + +path "{{.DatabaseMount}}/creds/*" { + capabilities = ["deny"] +} + +path "sys/policies/acl/operator" { + capabilities = ["deny"] +} + +path "sys/audit/*" { + capabilities = ["deny"] +} + +path "sys/storage/raft/snapshot-force" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/policies/templates/provisioning.hcl.tpl b/internal/vaultcluster/policies/templates/provisioning.hcl.tpl new file mode 100644 index 0000000..de620cb --- /dev/null +++ b/internal/vaultcluster/policies/templates/provisioning.hcl.tpl @@ -0,0 +1,72 @@ +# Onboard/offboard tenants. Cannot read tenant secrets. +path "sys/policies/acl/tenant-*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "sys/policies/acl" { + capabilities = ["list"] +} + +path "auth/{{.AuthMount}}/role/tenant-*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "auth/{{.AuthMount}}/role" { + capabilities = ["list"] +} + +path "{{.DatabaseMount}}/roles/tenant-*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "sys/mounts" { + capabilities = ["read"] +} + +path "sys/auth" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "{{.KVMount}}/*" { + capabilities = ["deny"] +} + +path "sys/audit" { + capabilities = ["deny"] +} + +path "sys/audit/*" { + capabilities = ["deny"] +} + +path "sys/audit-hash/*" { + capabilities = ["deny"] +} + +path "{{.DatabaseMount}}/creds/*" { + capabilities = ["deny"] +} + +path "sys/mounts/*" { + capabilities = ["deny"] +} + +path "sys/auth/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/policies/templates/tenant-database.hcl.tpl b/internal/vaultcluster/policies/templates/tenant-database.hcl.tpl new file mode 100644 index 0000000..e668a56 --- /dev/null +++ b/internal/vaultcluster/policies/templates/tenant-database.hcl.tpl @@ -0,0 +1,36 @@ +# Dynamic DB creds for this tenant only. Additive; does not widen KV access. +path "{{.DatabaseMount}}/creds/tenant-{{.TenantID}}-*" { + capabilities = ["read"] +} + +path "{{.DatabaseMount}}/roles/tenant-{{.TenantID}}-*" { + capabilities = ["read"] +} + +path "sys/leases/renew" { + capabilities = ["update"] +} + +path "sys/leases/revoke" { + capabilities = ["update"] +} + +path "{{.DatabaseMount}}/creds/*" { + capabilities = ["deny"] +} + +path "{{.DatabaseMount}}/roles/*" { + capabilities = ["deny"] +} + +path "{{.DatabaseMount}}/config/*" { + capabilities = ["deny"] +} + +path "{{.DatabaseMount}}/static-creds/*" { + capabilities = ["deny"] +} + +path "{{.DatabaseMount}}/rotate-root/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/policies/templates/tenant-reader.hcl.tpl b/internal/vaultcluster/policies/templates/tenant-reader.hcl.tpl new file mode 100644 index 0000000..b204317 --- /dev/null +++ b/internal/vaultcluster/policies/templates/tenant-reader.hcl.tpl @@ -0,0 +1,56 @@ +# Read-only on one tenant. KV v2 needs data/ and metadata/; kv/{prefix}/{id} matches nothing. +path "{{.KVMount}}/data/{{.TenantPrefix}}/{{.TenantID}}/*" { + capabilities = ["read"] +} + +path "{{.KVMount}}/metadata/{{.TenantPrefix}}/{{.TenantID}}/*" { + capabilities = ["read", "list"] +} + +path "{{.KVMount}}/data/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/metadata/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/delete/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/undelete/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/destroy/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "auth/{{.AuthMount}}/role/*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/config" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/policies/templates/tenant-writer.hcl.tpl b/internal/vaultcluster/policies/templates/tenant-writer.hcl.tpl new file mode 100644 index 0000000..c39c5ab --- /dev/null +++ b/internal/vaultcluster/policies/templates/tenant-writer.hcl.tpl @@ -0,0 +1,68 @@ +# Full KV lifecycle on one tenant. Name all five KV v2 path families. +path "{{.KVMount}}/data/{{.TenantPrefix}}/{{.TenantID}}/*" { + capabilities = ["create", "read", "update", "patch", "delete", "list"] +} + +path "{{.KVMount}}/metadata/{{.TenantPrefix}}/{{.TenantID}}/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "{{.KVMount}}/delete/{{.TenantPrefix}}/{{.TenantID}}/*" { + capabilities = ["update"] +} + +path "{{.KVMount}}/undelete/{{.TenantPrefix}}/{{.TenantID}}/*" { + capabilities = ["update"] +} + +path "{{.KVMount}}/destroy/{{.TenantPrefix}}/{{.TenantID}}/*" { + capabilities = ["update"] +} + +path "{{.KVMount}}/data/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/metadata/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/delete/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/undelete/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/destroy/{{.TenantPrefix}}/*" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "auth/{{.AuthMount}}/role/*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} + +path "{{.KVMount}}/config" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/policy.go b/internal/vaultcluster/policy.go new file mode 100644 index 0000000..9f22377 --- /dev/null +++ b/internal/vaultcluster/policy.go @@ -0,0 +1,38 @@ +package vaultcluster + +import ( + "fmt" + "strings" + "text/template" + + "github.com/nullstone-modules/vault-cluster/internal/vaultcluster/policies" +) + +func RenderPolicy(templateName, tenantID string, cfg Config) (string, error) { + if tenantID != "" { + if err := ValidateTenantID(tenantID); err != nil { + return "", err + } + } + name := templateName + ".hcl.tpl" + raw, err := policies.Templates.ReadFile("templates/" + name) + if err != nil { + return "", fmt.Errorf("no such template %q: %w", templateName, err) + } + tpl, err := template.New(name).Option("missingkey=error").Parse(string(raw)) + if err != nil { + return "", fmt.Errorf("parse template %q: %w", templateName, err) + } + data := struct { + TenantID string + KVMount string + TenantPrefix string + DatabaseMount string + AuthMount string + }{tenantID, cfg.KVMount, cfg.TenantPrefix, cfg.DatabaseMount, cfg.AuthMount} + var b strings.Builder + if err := tpl.Execute(&b, data); err != nil { + return "", fmt.Errorf("render template %q: %w", templateName, err) + } + return b.String(), nil +} diff --git a/internal/vaultcluster/snapshot.go b/internal/vaultcluster/snapshot.go new file mode 100644 index 0000000..73fffce --- /dev/null +++ b/internal/vaultcluster/snapshot.go @@ -0,0 +1,114 @@ +package vaultcluster + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +func (c *Client) RaftSnapshot() ([]byte, error) { + req := c.API.NewRequest("GET", "/v1/sys/storage/raft/snapshot") + resp, err := c.API.RawRequest(req) + if err != nil { + return nil, fmt.Errorf("snapshot failed: %w", err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if len(b) == 0 { + return nil, fmt.Errorf("snapshot is empty; refusing to keep it") + } + return b, nil +} + +func snapshotStamp() string { + return time.Now().UTC().Format("20060102T150405Z") +} + +func (c *Client) SnapshotTake(dir string) (string, error) { + b, err := c.RaftSnapshot() + if err != nil { + return "", err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", err + } + file := filepath.Join(dir, "vault-"+snapshotStamp()+".snap") + if err := os.WriteFile(file, b, 0o600); err != nil { + return "", err + } + sum := sha256.Sum256(b) + if err := os.WriteFile(file+".sha256", []byte(hex.EncodeToString(sum[:])+"\n"), 0o600); err != nil { + return "", err + } + _ = chownToDirOwner(dir, file) + _ = chownToDirOwner(dir, file+".sha256") + return file, nil +} + +func SnapshotVerify(file string) error { + b, err := os.ReadFile(file) + if err != nil { + return err + } + want, err := os.ReadFile(file + ".sha256") + if err != nil { + return fmt.Errorf("no checksum beside %s; integrity cannot be established", file) + } + sum := sha256.Sum256(b) + got := hex.EncodeToString(sum[:]) + if got != strings.TrimSpace(string(want)) { + return fmt.Errorf("checksum mismatch for %s; this snapshot is corrupt and must not be restored", file) + } + return nil +} + +func SnapshotList(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var files []string + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".snap") { + files = append(files, filepath.Join(dir, e.Name())) + } + } + sort.Sort(sort.Reverse(sort.StringSlice(files))) + return files, nil +} + +func (c *Client) SnapshotRestore(file string) error { + if _, err := os.Stat(file + ".sha256"); err == nil { + if err := SnapshotVerify(file); err != nil { + return err + } + } + b, err := os.ReadFile(file) + if err != nil { + return err + } + req := c.API.NewRequest("POST", "/v1/sys/storage/raft/snapshot-force") + req.BodyBytes = b + resp, err := c.API.RawRequest(req) + if err != nil { + return fmt.Errorf("restore failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("restore failed (HTTP %d): %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + return nil +} diff --git a/internal/vaultcluster/snapshot_schedule.go b/internal/vaultcluster/snapshot_schedule.go new file mode 100644 index 0000000..9dc2675 --- /dev/null +++ b/internal/vaultcluster/snapshot_schedule.go @@ -0,0 +1,20 @@ +package vaultcluster + +import ( + "fmt" + "strings" + + "github.com/robfig/cron/v3" +) + +func ParseBackupSchedule(expr string) (cron.Schedule, error) { + expr = strings.TrimSpace(expr) + if expr == "" { + return nil, nil + } + sched, err := cron.ParseStandard(expr) + if err != nil { + return nil, fmt.Errorf("backup_schedule: %w", err) + } + return sched, nil +} diff --git a/internal/vaultcluster/snapshot_schedule_test.go b/internal/vaultcluster/snapshot_schedule_test.go new file mode 100644 index 0000000..cfddd31 --- /dev/null +++ b/internal/vaultcluster/snapshot_schedule_test.go @@ -0,0 +1,17 @@ +package vaultcluster + +import "testing" + +func TestParseBackupSchedule(t *testing.T) { + off, err := ParseBackupSchedule("") + if err != nil || off != nil { + t.Fatalf("empty: %v %v", off, err) + } + on, err := ParseBackupSchedule("0 3 * * *") + if err != nil || on == nil { + t.Fatalf("cron: %v %v", on, err) + } + if _, err := ParseBackupSchedule("not-a-cron"); err == nil { + t.Fatal("expected invalid cron to fail") + } +} diff --git a/internal/vaultcluster/tenant.go b/internal/vaultcluster/tenant.go new file mode 100644 index 0000000..1c72697 --- /dev/null +++ b/internal/vaultcluster/tenant.go @@ -0,0 +1,227 @@ +package vaultcluster + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "strings" + + "github.com/hashicorp/vault/api" +) + +func (c *Client) CreateTenant(tenantID string, issueCreds bool) error { + if err := ValidateTenantID(tenantID); err != nil { + return err + } + readerP := c.Cfg.TenantPolicy("reader", tenantID) + writerP := c.Cfg.TenantPolicy("writer", tenantID) + dbP := c.Cfg.TenantPolicy("database", tenantID) + readerR := c.Cfg.TenantRole("reader", tenantID) + writerR := c.Cfg.TenantRole("writer", tenantID) + + apply := func(tmpl, name string) error { + hcl, err := RenderPolicy(tmpl, tenantID, c.Cfg) + if err != nil { + return err + } + if err := LintOrError(name, hcl, c.Cfg); err != nil { + return err + } + return c.API.Sys().PutPolicy(name, hcl) + } + if err := apply("tenant-reader", readerP); err != nil { + return err + } + if err := apply("tenant-writer", writerP); err != nil { + return err + } + if c.Cfg.EnableCredentials { + if err := apply("tenant-database", dbP); err != nil { + return err + } + } + + if err := c.writeAppRole(readerR, []string{readerP}); err != nil { + return err + } + writerPolicies := []string{writerP} + if c.Cfg.EnableCredentials { + writerPolicies = append(writerPolicies, dbP) + } + if err := c.writeAppRole(writerR, writerPolicies); err != nil { + return err + } + + if c.Cfg.EnableCredentials { + if err := c.writeDBRole(tenantID, "readonly", "app_readonly"); err != nil { + return err + } + if err := c.writeDBRole(tenantID, "readwrite", "app_readwrite"); err != nil { + return err + } + } + + if !issueCreds { + log.Printf("tenant %s onboarded (no credentials issued)", tenantID) + return nil + } + if err := c.printAppRoleCreds(readerR); err != nil { + return err + } + return c.printAppRoleCreds(writerR) +} + +func (c *Client) writeAppRole(role string, policies []string) error { + _, err := c.API.Logical().Write("auth/"+c.Cfg.AuthMount+"/role/"+role, map[string]any{ + "token_policies": policies, + "token_ttl": c.Cfg.TokenTTL, + "token_max_ttl": c.Cfg.TokenMaxTTL, + "token_type": "service", + "secret_id_ttl": "24h", + "secret_id_num_uses": 0, + "bind_secret_id": true, + }) + return err +} + +func (c *Client) writeDBRole(tenantID, suffix, group string) error { + role := fmt.Sprintf("tenant-%s-%s", tenantID, suffix) + stmt1 := `CREATE ROLE "{{name}}" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';` + stmt2 := fmt.Sprintf(`GRANT %s TO "{{name}}";`, group) + _, err := c.API.Logical().Write(c.Cfg.DatabaseMount+"/roles/"+role, map[string]any{ + "db_name": c.Cfg.DatabaseConnName, + "creation_statements": []string{stmt1, stmt2}, + "default_ttl": c.Cfg.DatabaseTTL, + "max_ttl": c.Cfg.DatabaseMaxTTL, + }) + return err +} + +func (c *Client) printAppRoleCreds(role string) error { + s, err := c.API.Logical().Read("auth/" + c.Cfg.AuthMount + "/role/" + role + "/role-id") + if err != nil { + return err + } + sec, err := c.API.Logical().Write("auth/"+c.Cfg.AuthMount+"/role/"+role+"/secret-id", map[string]any{}) + if err != nil { + return err + } + fmt.Printf("----------------------------------------------------------\n") + fmt.Printf("role %s\n", role) + fmt.Printf("role_id %s\n", s.Data["role_id"]) + fmt.Printf("secret_id %s\n", sec.Data["secret_id"]) + return nil +} + +func (c *Client) LoginAppRole(role string) (string, error) { + s, err := c.API.Logical().Read("auth/" + c.Cfg.AuthMount + "/role/" + role + "/role-id") + if err != nil { + return "", err + } + sec, err := c.API.Logical().Write("auth/"+c.Cfg.AuthMount+"/role/"+role+"/secret-id", map[string]any{}) + if err != nil { + return "", err + } + login, err := c.API.Logical().Write("auth/"+c.Cfg.AuthMount+"/login", map[string]any{ + "role_id": s.Data["role_id"], + "secret_id": sec.Data["secret_id"], + }) + if err != nil { + return "", err + } + if login == nil || login.Auth == nil { + return "", fmt.Errorf("approle login returned no auth") + } + return login.Auth.ClientToken, nil +} + +func (c *Client) OffboardTenant(tenantID string, purge bool) error { + if err := ValidateTenantID(tenantID); err != nil { + return err + } + for _, role := range []string{c.Cfg.TenantRole("reader", tenantID), c.Cfg.TenantRole("writer", tenantID)} { + if err := c.deleteMissingOK("auth/" + c.Cfg.AuthMount + "/role/" + role); err != nil { + return fmt.Errorf("delete role %s: %w", role, err) + } + } + if c.Cfg.EnableCredentials { + for _, suffix := range []string{"readonly", "readwrite"} { + dbRole := fmt.Sprintf("tenant-%s-%s", tenantID, suffix) + _ = c.API.Sys().RevokePrefix(c.Cfg.DatabaseMount + "/creds/" + dbRole) + if err := c.deleteMissingOK(c.Cfg.DatabaseMount + "/roles/" + dbRole); err != nil { + return fmt.Errorf("delete database role %s: %w", dbRole, err) + } + } + } + for _, name := range []string{ + c.Cfg.TenantPolicy("reader", tenantID), + c.Cfg.TenantPolicy("writer", tenantID), + c.Cfg.TenantPolicy("database", tenantID), + } { + if err := c.API.Sys().DeletePolicy(name); err != nil && !isNotFound(err) { + return fmt.Errorf("delete policy %s: %w", name, err) + } + } + if purge { + if err := c.purgeTenantSecrets(tenantID); err != nil { + return err + } + } + log.Printf("tenant %s offboarded", tenantID) + return nil +} + +func (c *Client) purgeTenantSecrets(tenantID string) error { + meta := c.Cfg.KVMetaPath(tenantID, "") + r, err := c.Do("GET", meta+"?list=true", nil) + if err != nil && r.Status == 0 { + return err + } + if r.Status == 403 { + return fmt.Errorf("permission denied listing %s (provisioning cannot purge tenant data)", meta) + } + if r.Status == 404 { + return nil + } + if r.Status < 200 || r.Status >= 300 { + return fmt.Errorf("list %s failed (HTTP %d)", meta, r.Status) + } + var wrap struct { + Data struct { + Keys []string `json:"keys"` + } `json:"data"` + } + if err := json.Unmarshal(r.Body, &wrap); err != nil { + return err + } + for _, key := range wrap.Data.Keys { + path := strings.TrimSuffix(meta+"/"+strings.TrimSuffix(key, "/"), "/") + if err := c.deleteMissingOK(path); err != nil { + return fmt.Errorf("destroy %s: %w", path, err) + } + } + return nil +} + +func (c *Client) deleteMissingOK(path string) error { + r, err := c.Do("DELETE", path, nil) + if err != nil && r.Status == 0 { + return err + } + if r.Status == 404 || (r.Status >= 200 && r.Status < 300) { + return nil + } + return fmt.Errorf("DELETE %s failed (HTTP %d): %s", path, r.Status, strings.TrimSpace(string(r.Body))) +} + +func isNotFound(err error) bool { + if err == nil { + return false + } + var re *api.ResponseError + if errors.As(err, &re) && re.StatusCode == 404 { + return true + } + return strings.Contains(err.Error(), "404") +} diff --git a/internal/vaultcluster/tenantid.go b/internal/vaultcluster/tenantid.go new file mode 100644 index 0000000..c5474e2 --- /dev/null +++ b/internal/vaultcluster/tenantid.go @@ -0,0 +1,58 @@ +package vaultcluster + +import ( + "fmt" + "regexp" + "strings" + "unicode" +) + +var tenantIDPattern = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{1,30}[a-z0-9])$`) + +var reservedTenantIDs = map[string]struct{}{ + "sys": {}, "auth": {}, "identity": {}, "cubbyhole": {}, "root": {}, + "default": {}, "admin": {}, "data": {}, "metadata": {}, "delete": {}, + "undelete": {}, "destroy": {}, "config": {}, "subkeys": {}, "tenant": {}, + "customers": {}, +} + +func ValidateTenantID(id string) error { + if id == "" { + return fmt.Errorf("tenant ID is empty") + } + for _, r := range id { + if unicode.IsSpace(r) { + return fmt.Errorf("tenant ID contains whitespace") + } + if r > unicode.MaxASCII { + return fmt.Errorf("tenant ID contains non-ASCII characters") + } + } + switch { + case strings.Contains(id, "/"): + return fmt.Errorf("tenant ID contains a path separator") + case strings.Contains(id, ".."): + return fmt.Errorf("tenant ID contains '..'") + case strings.Contains(id, "*"): + return fmt.Errorf("tenant ID contains the ACL wildcard '*'") + case strings.Contains(id, "+"): + return fmt.Errorf("tenant ID contains '+'") + case strings.ContainsAny(id, "{}"): + return fmt.Errorf("tenant ID contains a brace") + case strings.Contains(id, `\`): + return fmt.Errorf("tenant ID contains a backslash") + case strings.ContainsAny(id, `"'`): + return fmt.Errorf("tenant ID contains a quote") + case strings.ContainsAny(id, "$`"): + return fmt.Errorf("tenant ID contains a shell metacharacter") + case strings.Contains(id, "%"): + return fmt.Errorf("tenant ID contains '%%'") + } + if !tenantIDPattern.MatchString(id) { + return fmt.Errorf("tenant ID %q does not match %s", id, tenantIDPattern) + } + if _, ok := reservedTenantIDs[id]; ok { + return fmt.Errorf("tenant ID %q is reserved", id) + } + return nil +} diff --git a/internal/vaultcluster/tenantid_test.go b/internal/vaultcluster/tenantid_test.go new file mode 100644 index 0000000..8b0eba4 --- /dev/null +++ b/internal/vaultcluster/tenantid_test.go @@ -0,0 +1,22 @@ +package vaultcluster + +import "testing" + +func TestValidateTenantID_accepts(t *testing.T) { + for _, id := range []string{"tenant-a", "tenant-b", "acme-corp"} { + if err := ValidateTenantID(id); err != nil { + t.Errorf("%s: %v", id, err) + } + } +} + +func TestValidateTenantID_rejects(t *testing.T) { + for _, id := range []string{ + "a/b", "../etc", "tenant-*", "TENANT", "ab", "sys", "data", + "a b", "a_b", "-x", "x-", "", + } { + if err := ValidateTenantID(id); err == nil { + t.Errorf("%q should have been rejected", id) + } + } +} diff --git a/internal/vaultcluster/testdata/lint/BAD-cross-tenant-wildcard.hcl b/internal/vaultcluster/testdata/lint/BAD-cross-tenant-wildcard.hcl new file mode 100644 index 0000000..7fd09c2 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/BAD-cross-tenant-wildcard.hcl @@ -0,0 +1,9 @@ +# MUST BE REJECTED - the wildcard sits directly under the tenant prefix, so it +# covers every tenant. This is the "temporary broad allow" that gets added when +# a correctly-written policy appears not to work. +path "kv/data/customers/*" { + capabilities = ["read", "list"] +} +path "kv/metadata/customers/*" { + capabilities = ["read", "list"] +} diff --git a/internal/vaultcluster/testdata/lint/BAD-duplicate-path.hcl b/internal/vaultcluster/testdata/lint/BAD-duplicate-path.hcl new file mode 100644 index 0000000..b2dcaf2 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/BAD-duplicate-path.hcl @@ -0,0 +1,7 @@ +# MUST BE REJECTED - only one stanza takes effect and which one is unspecified. +path "kv/data/customers/tenant-a/*" { + capabilities = ["read"] +} +path "kv/data/customers/tenant-a/*" { + capabilities = ["create", "read", "update", "delete"] +} diff --git a/internal/vaultcluster/testdata/lint/BAD-empty.hcl b/internal/vaultcluster/testdata/lint/BAD-empty.hcl new file mode 100644 index 0000000..eec9bb2 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/BAD-empty.hcl @@ -0,0 +1,2 @@ +# MUST BE REJECTED - no rules at all, which is what a failed template render +# produces. diff --git a/internal/vaultcluster/testdata/lint/BAD-kv-v1-path.hcl b/internal/vaultcluster/testdata/lint/BAD-kv-v1-path.hcl new file mode 100644 index 0000000..4b15d40 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/BAD-kv-v1-path.hcl @@ -0,0 +1,5 @@ +# MUST BE REJECTED - the KV v2 path split. Looks correct, parses correctly, +# applies without error, and matches nothing at all. +path "kv/customers/tenant-a/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} diff --git a/internal/vaultcluster/testdata/lint/BAD-mixed-deny.hcl b/internal/vaultcluster/testdata/lint/BAD-mixed-deny.hcl new file mode 100644 index 0000000..e8fe409 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/BAD-mixed-deny.hcl @@ -0,0 +1,5 @@ +# MUST BE REJECTED - deny overrides the other capabilities, so this rule denies +# everything while reading as though it allows reads. +path "kv/data/customers/tenant-a/*" { + capabilities = ["read", "list", "deny"] +} diff --git a/internal/vaultcluster/testdata/lint/BAD-sudo-in-tenant-policy.hcl b/internal/vaultcluster/testdata/lint/BAD-sudo-in-tenant-policy.hcl new file mode 100644 index 0000000..5e01410 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/BAD-sudo-in-tenant-policy.hcl @@ -0,0 +1,5 @@ +# MUST BE REJECTED - sudo is root-equivalent on the path and has no place in a +# tenant policy. +path "kv/data/customers/tenant-a/*" { + capabilities = ["read", "sudo"] +} diff --git a/internal/vaultcluster/testdata/lint/BAD-universal-wildcard.hcl b/internal/vaultcluster/testdata/lint/BAD-universal-wildcard.hcl new file mode 100644 index 0000000..2813e91 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/BAD-universal-wildcard.hcl @@ -0,0 +1,5 @@ +# MUST BE REJECTED - grants read over the entire Vault API, including sys/, +# auth/, and every tenant's secrets. +path "*" { + capabilities = ["read", "list"] +} diff --git a/internal/vaultcluster/testdata/lint/GOOD-tenant-scoped.hcl b/internal/vaultcluster/testdata/lint/GOOD-tenant-scoped.hcl new file mode 100644 index 0000000..2971bc6 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/GOOD-tenant-scoped.hcl @@ -0,0 +1,17 @@ +# MUST PASS - correctly scoped to one tenant, covers both KV v2 path families, +# and denies the rest of the tenant prefix. +path "kv/data/customers/tenant-a/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} +path "kv/metadata/customers/tenant-a/*" { + capabilities = ["read", "list"] +} +path "kv/data/customers/*" { + capabilities = ["deny"] +} +path "kv/metadata/customers/*" { + capabilities = ["deny"] +} +path "sys/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/token.go b/internal/vaultcluster/token.go new file mode 100644 index 0000000..3c50c5f --- /dev/null +++ b/internal/vaultcluster/token.go @@ -0,0 +1,24 @@ +package vaultcluster + +import ( + "log" + "time" +) + +// RenewToken keeps the client's periodic token alive for long-running commands. +// Bootstrap issues 24h periodic tokens, which expire unless renewed within the period. +// A failed renewal is logged and retried on the next tick; the caller decides the interval. +func (c *Client) RenewToken(every time.Duration, stop <-chan struct{}) { + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-stop: + return + case <-t.C: + if _, err := c.API.Auth().Token().RenewSelf(0); err != nil { + log.Printf("token renewal failed: %v", err) + } + } + } +} diff --git a/internal/vaultcluster/token_test.go b/internal/vaultcluster/token_test.go new file mode 100644 index 0000000..c945400 --- /dev/null +++ b/internal/vaultcluster/token_test.go @@ -0,0 +1,43 @@ +package vaultcluster + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func TestRenewTokenCallsRenewSelf(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/auth/token/renew-self" || r.Header.Get("X-Vault-Token") != "tok" { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + } + calls.Add(1) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"auth":{"client_token":"tok","renewable":true,"lease_duration":86400}}`)) + })) + defer srv.Close() + + c, err := New(Config{Addr: srv.URL, Token: "tok"}) + if err != nil { + t.Fatal(err) + } + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + c.RenewToken(10*time.Millisecond, stop) + close(done) + }() + + deadline := time.Now().Add(2 * time.Second) + for calls.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + close(stop) + <-done + if calls.Load() < 2 { + t.Fatalf("expected at least 2 renewals, got %d", calls.Load()) + } +} diff --git a/local/.env.example b/local/.env.example new file mode 100644 index 0000000..7e402c6 --- /dev/null +++ b/local/.env.example @@ -0,0 +1,14 @@ +# Optional overrides for the local Compose stack. +# +# Every variable has a working default baked into compose.yml; no .env is +# required. To override, copy this file: cp .env.example .env +# Compose reads .env automatically. .env is gitignored. This configures a local +# test environment only; never put real credentials here. + +# Host port Vault publishes on. Always bound to loopback in compose.yml. +VAULT_HOST_PORT=8200 + +# Unseal shares generated at init. 5/3 mirrors a realistic quorum so the unseal +# flow exercised locally matches the flow used elsewhere. +VAULT_INIT_KEY_SHARES=5 +VAULT_INIT_KEY_THRESHOLD=3 diff --git a/local/compose.yml b/local/compose.yml new file mode 100644 index 0000000..b0b2e23 --- /dev/null +++ b/local/compose.yml @@ -0,0 +1,60 @@ +name: vault-cluster + +services: + vault: + image: hashicorp/vault:2.0@sha256:5be49781ecf78bfe775c5309c6a4d9f4e9e040b6c885c99eb2b12fb69855e1a2 + restart: unless-stopped + command: ["vault", "server", "-config=/vault/config/config.hcl"] + cap_add: + - IPC_LOCK + environment: + SKIP_CHOWN: "true" + SKIP_SETCAP: "true" + ports: + # Loopback only. Binding 0.0.0.0 would expose an unsealed plaintext Vault + # to the LAN. + - "127.0.0.1:${VAULT_HOST_PORT:-8200}:8200" + volumes: + - ./vault/config.hcl:/vault/config/config.hcl:ro + - vault-data:/vault/file + - vault-audit:/vault/logs + healthcheck: + # Sealed and uninitialized still count as healthy so the one-shot + # bootstrap can start and perform init/unseal. + test: >- + wget --quiet --tries=1 --spider + 'http://127.0.0.1:8200/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200' + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + networks: + - vault-cluster + + bootstrap: + build: + context: .. + dockerfile: Dockerfile + restart: "no" + user: "0:0" + depends_on: + vault: + condition: service_healthy + environment: + VAULT_ADDR: "http://vault:8200" + BOOTSTRAP_DIR: /bootstrap + AUDIT_LOG_PATH: /vault/logs/audit.log + VAULT_INIT_KEY_SHARES: "${VAULT_INIT_KEY_SHARES:-5}" + VAULT_INIT_KEY_THRESHOLD: "${VAULT_INIT_KEY_THRESHOLD:-3}" + volumes: + - "${BOOTSTRAP_HOST_DIR:-./.bootstrap}:/bootstrap" + command: ["bootstrap", "local"] + networks: + - vault-cluster + +volumes: + vault-data: + vault-audit: + +networks: + vault-cluster: diff --git a/local/localcompose_test.go b/local/localcompose_test.go new file mode 100644 index 0000000..f01f70f --- /dev/null +++ b/local/localcompose_test.go @@ -0,0 +1,249 @@ +package local + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/nullstone-modules/vault-cluster/internal/vaultcluster" +) + +const localComposeFile = "compose.yml" + +func requireDocker(t *testing.T) { + t.Helper() + if testing.Short() { + t.Skip("skipping docker") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available") + } + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skip("docker daemon is not running") + } +} + +func dockerOutput(t *testing.T, args ...string) string { + t.Helper() + out, err := exec.Command("docker", args...).Output() + if err != nil { + stderr := []byte(nil) + if ee, ok := err.(*exec.ExitError); ok { + stderr = ee.Stderr + } + t.Fatalf("docker %v: %v\n%s\n%s", args, err, out, stderr) + } + return strings.TrimSpace(string(out)) +} + +// TestLocalComposeStatic lints compose.yml without Docker: images are +// digest-pinned and never latest, no Vault dev mode, and every published port +// binds to loopback only. +func TestLocalComposeStatic(t *testing.T) { + b, err := os.ReadFile(localComposeFile) + if err != nil { + t.Fatal(err) + } + src := string(b) + + for _, line := range strings.Split(src, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "image:") { + if !strings.Contains(trimmed, "@sha256:") { + t.Errorf("image is not digest-pinned: %s", trimmed) + } + if strings.Contains(trimmed, ":latest") { + t.Errorf("image uses the latest tag: %s", trimmed) + } + } + } + + if strings.Contains(src, "VAULT_DEV_") || regexp.MustCompile(`["'\s]-dev["'\s,\]]`).MatchString(src) { + t.Error("dev-mode configuration found in compose.yml") + } + + // Port publications look like - "host:container"; volume mounts contain a + // path separator in the host part. + portRe := regexp.MustCompile(`-\s+"([^"/]+):\d+"`) + found := false + for _, m := range portRe.FindAllStringSubmatch(src, -1) { + found = true + if !strings.HasPrefix(m[1], "127.0.0.1:") { + t.Errorf("published port is not bound to loopback: %s", m[0]) + } + } + if !found { + t.Error("no published ports found; the loopback check matched nothing") + } +} + +// TestLocalComposeRuntime runs the real local stack in an isolated Compose +// project: health-gated one-shot bootstrap, Shamir over Raft, separated audit +// and Raft volumes, loopback-only exposure, and persistence across a restart +// (Vault reseals, the one-shot unseals, data and tokens survive). +func TestLocalComposeRuntime(t *testing.T) { + requireDocker(t) + composeFile, err := filepath.Abs(localComposeFile) + if err != nil { + t.Fatal(err) + } + + project := fmt.Sprintf("vault-rt-%d", time.Now().UnixNano()) + bootstrapDir := t.TempDir() + env := append(os.Environ(), + "BOOTSTRAP_HOST_DIR="+bootstrapDir, + "VAULT_HOST_PORT=0", // ephemeral host port; never collides with a dev stack + ) + compose := func(args ...string) *exec.Cmd { + cmd := exec.Command("docker", append([]string{"compose", "-p", project, "-f", composeFile}, args...)...) + cmd.Env = env + return cmd + } + composeOut := func(args ...string) string { + t.Helper() + out, err := compose(args...).Output() + if err != nil { + stderr := []byte(nil) + if ee, ok := err.(*exec.ExitError); ok { + stderr = ee.Stderr + } + t.Fatalf("docker compose %v: %v\n%s\n%s", args, err, out, stderr) + } + return strings.TrimSpace(string(out)) + } + t.Cleanup(func() { + if t.Failed() { + logs, _ := compose("logs").CombinedOutput() + t.Logf("compose logs:\n%s", logs) + } + _ = compose("down", "--volumes", "--remove-orphans", "--timeout", "5").Run() + }) + + composeOut("up", "-d", "--wait", "vault") + composeOut("run", "--build", "--rm", "--no-deps", "bootstrap") + + running := composeOut("ps", "--status", "running", "--format", "{{.Service}}") + if running != "vault" { + t.Fatalf("expected only vault to stay running (bootstrap is one-shot), got: %q", running) + } + + addr := "http://" + composeOut("port", "vault", "8200") + c, err := vaultcluster.New(vaultcluster.Config{Addr: addr, HTTPTimeout: 10 * time.Second}) + if err != nil { + t.Fatal(err) + } + + if res, err := c.Do("GET", "sys/health?standbyok=true", nil); err != nil || res.Status != 200 { + t.Fatalf("expected health 200 (initialized, unsealed, active): status=%d err=%v", res.Status, err) + } + st, err := c.API.Sys().SealStatus() + if err != nil { + t.Fatal(err) + } + if st.Type != "shamir" { + t.Errorf("seal type = %q, want shamir (real unseal flow must be exercised)", st.Type) + } + if st.StorageType != "raft" { + t.Errorf("storage type = %q, want raft", st.StorageType) + } + + cid := composeOut("ps", "-q", "vault") + mountName := func(dest string) string { + return dockerOutput(t, "inspect", "--format", + `{{range .Mounts}}{{if eq .Destination "`+dest+`"}}{{.Name}}{{end}}{{end}}`, cid) + } + raftVol, auditVol := mountName("/vault/file"), mountName("/vault/logs") + if raftVol == "" || auditVol == "" { + t.Errorf("raft and audit volumes must both be mounted: raft=%q audit=%q", raftVol, auditVol) + } else if raftVol == auditVol { + t.Errorf("audit shares volume %q with raft; audit growth can fill or corrupt Raft storage", raftVol) + } + dockerOutput(t, "exec", cid, "sh", "-c", "test -s /vault/logs/audit.log && test -e /vault/file/vault.db") + + hostIPs := dockerOutput(t, "inspect", "--format", + `{{range $p, $conf := .NetworkSettings.Ports}}{{range $conf}}{{.HostIp}} {{end}}{{end}}`, cid) + if strings.Contains(hostIPs, "0.0.0.0") || !strings.Contains(hostIPs, "127.0.0.1") { + t.Errorf("Vault port must be published to 127.0.0.1 only, got bindings: %q", hostIPs) + } + + // Persistence probe: onboard a tenant, write as its AppRole writer, + // restart Vault (reseals under Shamir), unseal via the one-shot, and + // verify the secret and the pre-restart token both survived. + tokenBytes, err := os.ReadFile(filepath.Join(bootstrapDir, "provisioning.token")) + if err != nil { + t.Fatalf("bootstrap did not write provisioning.token: %v", err) + } + provToken := strings.TrimSpace(string(tokenBytes)) + composeOut("run", "--rm", "--no-deps", "-e", "VAULT_TOKEN="+provToken, + "bootstrap", "tenants", "create", "tenant-a") + + prov := c.WithToken(provToken) + roleResp, err := prov.API.Logical().Read("auth/approle/role/tenant-tenant-a-writer/role-id") + if err != nil || roleResp == nil { + t.Fatalf("reading writer role-id: %v", err) + } + secResp, err := prov.API.Logical().Write("auth/approle/role/tenant-tenant-a-writer/secret-id", nil) + if err != nil || secResp == nil { + t.Fatalf("issuing writer secret-id: %v", err) + } + loginResp, err := c.API.Logical().Write("auth/approle/login", map[string]any{ + "role_id": roleResp.Data["role_id"], + "secret_id": secResp.Data["secret_id"], + }) + if err != nil || loginResp == nil || loginResp.Auth == nil { + t.Fatalf("AppRole login as tenant writer: %v", err) + } + writer := c.WithToken(loginResp.Auth.ClientToken) + + probePath := "kv/data/customers/tenant-a/runtime-persistence-probe" + probeValue := fmt.Sprintf("FAKE-persistence-probe-%d", time.Now().UnixNano()) + if _, err := writer.API.Logical().Write(probePath, map[string]any{ + "data": map[string]any{"probe": probeValue}, + }); err != nil { + t.Fatalf("writing probe secret: %v", err) + } + + composeOut("restart", "vault") + // The ephemeral host port changes on restart; re-resolve and rebuild clients. + addr = "http://" + composeOut("port", "vault", "8200") + c, err = vaultcluster.New(vaultcluster.Config{Addr: addr, HTTPTimeout: 10 * time.Second}) + if err != nil { + t.Fatal(err) + } + writer = c.WithToken(loginResp.Auth.ClientToken) + + sealedAfterRestart := false + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + if st, err := c.API.Sys().SealStatus(); err == nil && st != nil { + sealedAfterRestart = st.Sealed + break + } + time.Sleep(500 * time.Millisecond) + } + if !sealedAfterRestart { + t.Error("Vault should reseal after a process restart (expected for Shamir)") + } + + composeOut("run", "--rm", "--no-deps", "bootstrap") + if st, err := c.API.Sys().SealStatus(); err != nil || st.Sealed { + t.Fatalf("one-shot bootstrap should leave Vault unsealed: sealed=%v err=%v", st != nil && st.Sealed, err) + } + + read, err := writer.API.Logical().Read(probePath) + if err != nil || read == nil { + t.Fatalf("reading probe secret after restart: %v", err) + } + data, _ := read.Data["data"].(map[string]any) + if got, _ := data["probe"].(string); got != probeValue { + t.Errorf("probe secret did not survive the restart: got %q, want %q", got, probeValue) + } + if _, err := writer.API.Auth().Token().LookupSelf(); err != nil { + t.Errorf("pre-restart writer token no longer valid (token storage did not persist): %v", err) + } +} diff --git a/local/vault/config.hcl b/local/vault/config.hcl new file mode 100644 index 0000000..1456ab4 --- /dev/null +++ b/local/vault/config.hcl @@ -0,0 +1,21 @@ +cluster_name = "vault-local" +ui = true + +storage "raft" { + path = "/vault/file" + node_id = "vault-local-1" +} + +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = 1 +} + +api_addr = "http://vault:8200" +cluster_addr = "http://vault:8201" + +log_level = "info" +log_format = "json" + +# Compose and GitHub Actions cannot mlock. Production hosts use IPC_LOCK instead. +disable_mlock = true diff --git a/vault-node/.gitignore b/vault-node/.gitignore new file mode 100644 index 0000000..b8c84d5 --- /dev/null +++ b/vault-node/.gitignore @@ -0,0 +1 @@ +vault-utils diff --git a/vault-node/aws/vault-node-configure b/vault-node/aws/vault-node-configure new file mode 100755 index 0000000..27a807a --- /dev/null +++ b/vault-node/aws/vault-node-configure @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +ENV_FILE=/etc/vault.d/vault-utils.env +NODE_ENV_FILE=/etc/vault.d/node.env +CLOUD_HCL=/etc/vault.d/cloud.hcl + +if [ ! -f "$ENV_FILE" ]; then + echo "missing $ENV_FILE" >&2 + exit 1 +fi + +# Read values individually. This file is systemd EnvironmentFile format and +# holds cron expressions, so sourcing it in bash is not safe. +env_value() { + local value + value="$(sed -n "s/^$1=//p" "$ENV_FILE" | tail -n 1)" + if [ -z "$value" ]; then + echo "missing $1 in $ENV_FILE" >&2 + exit 1 + fi + printf '%s' "$value" +} + +AWS_REGION="$(env_value AWS_REGION)" +UNSEAL_KMS_KEY_ARN="$(env_value UNSEAL_KMS_KEY_ARN)" +VAULT_CLUSTER_TAG_KEY="$(env_value VAULT_CLUSTER_TAG_KEY)" +VAULT_CLUSTER_TAG_VALUE="$(env_value VAULT_CLUSTER_TAG_VALUE)" + +imds_token() { + curl -fsS -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" +} + +imds() { + curl -fsS -H "X-aws-ec2-metadata-token: $1" "http://169.254.169.254/latest/meta-data/$2" +} + +TOKEN="$(imds_token)" +INSTANCE_ID="$(imds "$TOKEN" instance-id)" +PRIVATE_IP="$(imds "$TOKEN" local-ipv4)" + +printf 'VAULT_RAFT_NODE_ID=%s\n' "$INSTANCE_ID" >"$NODE_ENV_FILE" +chown vault:vault "$NODE_ENV_FILE" +chmod 0640 "$NODE_ENV_FILE" + +cat >"$CLOUD_HCL" </dev/null 2>&1; then + useradd --system --home /opt/vault --shell /sbin/nologin vault +fi + +install -d -m 0750 -o vault -g vault /opt/vault /opt/vault/data /opt/vault/audit /etc/vault.d + +dnf install -y unzip +# /tmp is a small tmpfs on AL2023; the Vault zip and binary do not fit on a 1 GB instance. +tmp="$(mktemp -d /var/tmp/vault-install.XXXXXX)" +trap 'rm -rf "$tmp"' EXIT +curl -fsSL "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip" -o "$tmp/vault.zip" +unzip -o "$tmp/vault.zip" -d "$tmp" +install -m 0755 "$tmp/vault" /usr/local/bin/vault +install -m 0755 /tmp/vault-utils /usr/local/bin/vault-utils +install -m 0640 -o vault -g vault /tmp/vault-image/vault.hcl /etc/vault.d/vault.hcl +install -m 0755 /tmp/vault-image/vault-node-configure /usr/local/bin/vault-node-configure +install -m 0644 /tmp/vault-image/vault.service /etc/systemd/system/vault.service +install -m 0644 /tmp/vault-image/vault-configure.service /etc/systemd/system/vault-configure.service +install -m 0644 /tmp/vault-image/vault-bootstrap.service /etc/systemd/system/vault-bootstrap.service +install -m 0644 /tmp/vault-image/vault-health.service /etc/systemd/system/vault-health.service +install -m 0644 /tmp/vault-image/vault-snapshot.service /etc/systemd/system/vault-snapshot.service + +bash -n /usr/local/bin/vault-node-configure +/usr/local/bin/vault version +/usr/local/bin/vault-utils >/dev/null || true + +systemctl daemon-reload +systemctl enable \ + amazon-ssm-agent \ + vault-configure.service \ + vault.service \ + vault-bootstrap.service \ + vault-health.service \ + vault-snapshot.service diff --git a/vault-node/vault.pkr.hcl b/vault-node/vault.pkr.hcl new file mode 100644 index 0000000..5ea2141 --- /dev/null +++ b/vault-node/vault.pkr.hcl @@ -0,0 +1,78 @@ +packer { + required_plugins { + amazon = { + source = "github.com/hashicorp/amazon" + version = ">= 1.3.0" + } + } +} + +variable "region" { + type = string +} + +variable "vault_version" { + type = string + default = "2.0.4" +} + +# Regions the finished AMI is copied to. One bake, then copies, instead of a build per region. +variable "ami_regions" { + type = list(string) + default = [] +} + +source "amazon-ebs" "vault" { + ami_name = "nullstone-vault-{{timestamp}}" + ami_description = "Vault node for self-hosting a vault cluster with nullstone vault-utils" + instance_type = "t3.micro" + region = var.region + ami_regions = var.ami_regions + ssh_username = "ec2-user" + + source_ami_filter { + filters = { + name = "al2023-ami-*-kernel-6.1-x86_64" + architecture = "x86_64" + root-device-type = "ebs" + virtualization-type = "hvm" + } + owners = ["amazon"] + most_recent = true + } + + tags = { + Name = "nullstone-vault" + } +} + +build { + sources = ["source.amazon-ebs.vault"] + + provisioner "shell" { + inline = ["mkdir -p /tmp/vault-image"] + } + + provisioner "file" { + source = "vault-utils" + destination = "/tmp/vault-utils" + } + + provisioner "file" { + source = "files/" + destination = "/tmp/vault-image" + } + + provisioner "file" { + source = "aws/vault-node-configure" + destination = "/tmp/vault-image/vault-node-configure" + } + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; sudo -E sh -c '{{ .Vars }} {{ .Path }}'" + environment_vars = [ + "VAULT_VERSION=${var.vault_version}", + ] + script = "${path.root}/provision.sh" + } +}