Add AWS vault-utils: Secrets Manager, KMS bootstrap, and Raft health

.github/workflows/test-local.yml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.23.x"
+          go-version: "1.26.x"
 
       - name: Isolation and credentials
         run: go test ./internal/vaultcluster -count=1 -timeout 15m
@@ -32,7 +32,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.23.x"
+          go-version: "1.26.x"
 
       - name: Local compose runtime test
         run: go test ./local -count=1 -timeout 15m -v

.github/workflows/validate.yml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.23.x"
+          go-version: "1.26.x"
 
       - name: Unit tests (no Docker)
         run: go test -short ./...

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23-alpine AS build
+FROM golang:1.26-alpine AS build
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download

README.md

@@ -9,7 +9,7 @@ Applications share one Vault. They do not see each other's secrets. Isolation is
 | Target | Role | Status |
 |---|---|---|
 | `local/` | Docker Compose | Implemented |
-| `aws/aws-ec2-vault-cluster/` | `aws-ec2-vault-cluster` | Connections, IAM, Secrets Manager tokens, security groups. ASG/NLB not built yet. |
+| `aws/aws-ec2-vault-cluster/` | `aws-ec2-vault-cluster` | Connections, IAM, SM, security groups. `bootstrap aws`, health on 8210, S3 snapshots. ASG/NLB not built yet. |
 | `gcp/` | GCP | Not implemented |
 | `azure/` | Azure | Not implemented |
 
@@ -43,12 +43,13 @@ Implemented:
 - File audit on a volume separate from Raft
 - Auto-init (first start) and one-shot Shamir unseal via `vault-utils`
 - Isolation tests in Go (`go test`); credentials tests in Go (`TestCredentialsMatrix`)
+- `bootstrap aws` (KMS auto-unseal, Secrets Manager tokens), health on 8210, S3 snapshots
 
 Not implemented:
 
-- AWS ASG, NLB, AMI, user-data, and `bootstrap aws`
+- AWS ASG, NLB, AMI, and user-data
 - GCP, Azure, Kubernetes
-- Production KMS auto-unseal on a running cluster
+- KMS auto-unseal proven on a running EC2 cluster
 - TLS, multi-node Raft, DR replication
 
 Local unseal submits Shamir shares (5 shares, threshold 3) for laptop use. It is not AWS KMS, Cloud KMS, or Azure Key Vault auto-unseal.
@@ -80,16 +81,17 @@ vault-cluster/
 ├── CHANGELOG.md
 ├── Dockerfile            vault-utils image
 ├── cmd/                  Go app entrypoints (vault-utils CLI)
-├── internal/             Go libraries, policy templates, lint fixtures
+├── internal/vaultcluster/ shared Vault library
+├── internal/aws/         AWS adapters (secretsmanager, s3)
 ├── local/                Compose target, snapshots
-├── aws/aws-ec2-vault-cluster/   Nullstone module (IAM/SM/SG; no ASG yet)
+├── aws/aws-ec2-vault-cluster/   Nullstone module (IAM/SM/SG; vault-utils AWS; no ASG yet)
 ├── gcp/                  Nullstone Terraform module (not yet implemented)
 └── azure/                Nullstone Terraform module (not yet implemented)
 ```
 
 ## Prerequisites
 
-Docker Desktop (Compose v2). Go 1.23 for `go test`. `curl` and `jq` for the manual examples below; Vault CLI is optional except break-glass decode.
+Docker Desktop (Compose v2). Go 1.26 for `go test`. `curl` and `jq` for the manual examples below; Vault CLI is optional except break-glass decode.
 
 Images are pinned by tag and digest in `local/compose.yml` (Vault 2.0, PostgreSQL 18-alpine). Never `latest`.
 
@@ -320,7 +322,7 @@ Denials must be HTTP 403. A 404 is a different failure.
 
 ### AWS module (`aws/aws-ec2-vault-cluster/`)
 
-`go test` does not cover this directory. The current slice is OpenTofu only (connections, IAM, Secrets Manager, security groups). There is no Docker or live-AWS test in CI.
+OpenTofu in this directory is connections, IAM, Secrets Manager, and security groups. `go test ./internal/aws/...` covers the SM KeyStore and S3 snapshot helpers. `go test ./internal/vaultcluster` covers Raft health and cron parse. There is no live-AWS test in CI.
 
 From `aws/aws-ec2-vault-cluster/`:
 

cmd/vault-utils/main.go

@@ -7,7 +7,10 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/nullstone-modules/vault-cluster/internal/aws/s3"
+	"github.com/nullstone-modules/vault-cluster/internal/aws/secretsmanager"
 	"github.com/nullstone-modules/vault-cluster/internal/vaultcluster"
 )
 
@@ -23,16 +26,23 @@ func main() {
 }
 
 func usage() {
-	fmt.Fprintf(os.Stderr, `vault-utils <command> <subcommand>
+	fmt.Fprintf(os.Stderr, `vault-utils <command> [args]
 
 Commands:
-  bootstrap local|aws|azure|gcp    Initialize a cluster: init (once), unseal, configure
+  bootstrap local|aws|azure|gcp     Init once, unseal, configure
   tenants create <id>
   tenants destroy <id> --yes [--purge-secrets]
-  snapshot take|list|verify <file>|restore <file> --yes
-  health
+  snapshot take                     Write a Raft snapshot
+  snapshot list
+  snapshot verify <file>
+  snapshot restore <file> --yes
+  snapshot schedule                 Cron loop (BACKUP_SCHEDULE; empty disables)
+  health                            Print seal status
+  health serve                      HTTP on :8210 (200 only if this node is a Raft voter and caught up)
 
-Key material for bootstrap local is stored under BOOTSTRAP_DIR (default .bootstrap).
+Local key material: BOOTSTRAP_DIR (default .bootstrap).
+AWS: VAULT_INIT_SECRET_ARN, VAULT_PROVISIONING_SECRET_ARN, VAULT_OPERATOR_SECRET_ARN.
+Optional: SNAPSHOT_BUCKET, SNAPSHOT_PREFIX (default vault-snapshots).
 `)
 }
 
@@ -50,6 +60,9 @@ func run(cmd string, args []string) error {
 	case "snapshot":
 		return runSnapshot(c, args)
 	case "health":
+		if len(args) > 0 && args[0] == "serve" {
+			return runHealthServe(c)
+		}
 		return c.Health()
 	default:
 		usage()
@@ -65,12 +78,25 @@ func runBootstrap(c *vaultcluster.Client, args []string) error {
 	case "local":
 		shares, _ := strconv.Atoi(getenv("VAULT_INIT_KEY_SHARES", "5"))
 		threshold, _ := strconv.Atoi(getenv("VAULT_INIT_KEY_THRESHOLD", "3"))
-		return c.RunBootstrap(keyStore(), vaultcluster.BootstrapOptions{
+		return c.RunBootstrap(fileKeyStore(), vaultcluster.BootstrapOptions{
 			Shares:    shares,
 			Threshold: threshold,
 			KeepRoot:  getenv("KEEP_ROOT", "false") == "true",
 		})
-	case "aws", "azure", "gcp":
+	case "aws":
+		store, err := awsKeyStore()
+		if err != nil {
+			return err
+		}
+		shares, _ := strconv.Atoi(getenv("VAULT_INIT_RECOVERY_SHARES", "1"))
+		threshold, _ := strconv.Atoi(getenv("VAULT_INIT_RECOVERY_THRESHOLD", "1"))
+		return c.RunBootstrap(store, vaultcluster.BootstrapOptions{
+			Shares:     shares,
+			Threshold:  threshold,
+			KeepRoot:   getenv("KEEP_ROOT", "false") == "true",
+			AutoUnseal: true,
+		})
+	case "azure", "gcp":
 		return fmt.Errorf("bootstrap %s is not implemented yet", args[0])
 	default:
 		return fmt.Errorf("unknown platform %q (local, aws, azure, gcp)", args[0])
@@ -120,28 +146,28 @@ func runTenants(c *vaultcluster.Client, args []string) error {
 
 func runSnapshot(c *vaultcluster.Client, args []string) error {
 	if len(args) < 1 {
-		return fmt.Errorf("usage: vault-utils snapshot take|list|verify <file>|restore <file> --yes")
+		return fmt.Errorf("usage: vault-utils snapshot take | list | verify <file> | restore <file> --yes | schedule")
 	}
 	backupDir := filepath.Join(bootstrapDir(), "backups")
 	switch args[0] {
 	case "take":
 		if err := useOperatorToken(c); err != nil {
 			return err
 		}
-		file, err := c.SnapshotTake(backupDir)
+		file, err := takeSnapshot(c, backupDir)
 		if err != nil {
 			return err
 		}
 		log.Printf("snapshot written: %s", file)
 		log.Printf("this file contains every secret in the cluster; treat it as one")
 		return nil
 	case "list":
-		files, err := vaultcluster.SnapshotList(backupDir)
+		files, err := listSnapshots(backupDir)
 		if err != nil {
 			return err
 		}
 		if len(files) == 0 {
-			log.Printf("no snapshots under %s", backupDir)
+			log.Printf("no snapshots")
 			return nil
 		}
 		for _, f := range files {
@@ -170,16 +196,87 @@ func runSnapshot(c *vaultcluster.Client, args []string) error {
 		log.Printf("restore submitted; Vault will seal")
 		log.Printf("unseal with the key shares that were current when this snapshot was taken")
 		return nil
+	case "schedule":
+		return runSnapshotSchedule(c, backupDir)
 	default:
-		return fmt.Errorf("unknown subcommand %q (take, list, verify, restore)", args[0])
+		return fmt.Errorf("unknown subcommand %q (take, list, verify, restore, schedule)", args[0])
+	}
+}
+
+func runSnapshotSchedule(c *vaultcluster.Client, backupDir string) error {
+	sched, err := vaultcluster.ParseBackupSchedule(os.Getenv("BACKUP_SCHEDULE"))
+	if err != nil {
+		return err
+	}
+	if sched == nil {
+		log.Printf("scheduled snapshots disabled")
+		return nil
+	}
+	if err := useOperatorToken(c); err != nil {
+		return err
+	}
+	for {
+		wait := time.Until(sched.Next(time.Now()))
+		if wait > 0 {
+			time.Sleep(wait)
+		}
+		file, err := takeSnapshot(c, backupDir)
+		if err != nil {
+			log.Printf("snapshot failed: %v", err)
+			continue
+		}
+		log.Printf("snapshot written: %s", file)
+	}
+}
+
+func runHealthServe(c *vaultcluster.Client) error {
+	nodeID := os.Getenv("VAULT_RAFT_NODE_ID")
+	if nodeID == "" {
+		return fmt.Errorf("VAULT_RAFT_NODE_ID is required for health serve")
 	}
+	if err := useOperatorToken(c); err != nil {
+		return err
+	}
+	addr := getenv("VAULT_HEALTH_ADDR", ":8210")
+	log.Printf("health listening on %s", addr)
+	return c.ServeHealth(addr, nodeID)
+}
+
+func takeSnapshot(c *vaultcluster.Client, backupDir string) (string, error) {
+	if bucket := os.Getenv("SNAPSHOT_BUCKET"); bucket != "" {
+		store, err := s3.New()
+		if err != nil {
+			return "", err
+		}
+		b, err := c.RaftSnapshot()
+		if err != nil {
+			return "", err
+		}
+		return s3.PutSnapshot(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots"), b)
+	}
+	return c.SnapshotTake(backupDir)
+}
+
+func listSnapshots(backupDir string) ([]string, error) {
+	if bucket := os.Getenv("SNAPSHOT_BUCKET"); bucket != "" {
+		store, err := s3.New()
+		if err != nil {
+			return nil, err
+		}
+		return s3.ListSnapshots(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots"))
+	}
+	return vaultcluster.SnapshotList(backupDir)
 }
 
 func useOperatorToken(c *vaultcluster.Client) error {
 	if c.Cfg.Token != "" {
 		return nil
 	}
-	tok, err := keyStore().LoadToken("operator")
+	store, err := keyStore()
+	if err != nil {
+		return err
+	}
+	tok, err := store.LoadToken("operator")
 	if err != nil {
 		return fmt.Errorf("set VAULT_TOKEN or bootstrap first (operator token not found): %w", err)
 	}
@@ -188,10 +285,25 @@ func useOperatorToken(c *vaultcluster.Client) error {
 	return nil
 }
 
-func keyStore() vaultcluster.FileKeyStore {
+func keyStore() (vaultcluster.KeyStore, error) {
+	if os.Getenv("VAULT_OPERATOR_SECRET_ARN") != "" || os.Getenv("VAULT_INIT_SECRET_ARN") != "" {
+		return awsKeyStore()
+	}
+	return fileKeyStore(), nil
+}
+
+func fileKeyStore() vaultcluster.FileKeyStore {
 	return vaultcluster.FileKeyStore{Dir: bootstrapDir()}
 }
 
+func awsKeyStore() (*secretsmanager.KeyStore, error) {
+	return secretsmanager.New(
+		os.Getenv("VAULT_INIT_SECRET_ARN"),
+		os.Getenv("VAULT_PROVISIONING_SECRET_ARN"),
+		os.Getenv("VAULT_OPERATOR_SECRET_ARN"),
+	)
+}
+
 func bootstrapDir() string {
 	return getenv("BOOTSTRAP_DIR", ".bootstrap")
 }

go.mod

@@ -1,10 +1,32 @@
 module github.com/nullstone-modules/vault-cluster
 
-go 1.23.0
+go 1.26.0
 
-require github.com/hashicorp/vault/api v1.16.0
+require (
+	github.com/aws/aws-sdk-go-v2 v1.41.2
+	github.com/aws/aws-sdk-go-v2/config v1.29.14
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6
+	github.com/hashicorp/vault/api v1.16.0
+	github.com/robfig/cron/v3 v3.0.1
+)
 
 require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect
+	github.com/aws/smithy-go v1.24.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect