From 9f91eaeb23374fedba2ed971877fabd82d8f87e6 Mon Sep 17 00:00:00 2001 From: Amit Biswas Date: Sat, 5 Sep 2026 14:04:21 -0400 Subject: [PATCH 1/2] Add AWS vault-utils: Secrets Manager, KMS bootstrap, and Raft health. Give EC2 a fail-closed path to init with KMS auto-unseal, store tokens in Secrets Manager, expose voter health on 8210, and write Raft snapshots to S3. --- README.md | 11 +- cmd/vault-utils/main.go | 122 ++++++++++++++++++-- go.mod | 24 +++- go.sum | 40 +++++++ internal/vaultcluster/bootstrap.go | 70 +++++++++--- internal/vaultcluster/health_http.go | 20 ++++ internal/vaultcluster/health_http_test.go | 20 ++++ internal/vaultcluster/health_raft.go | 95 ++++++++++++++++ internal/vaultcluster/health_raft_test.go | 57 ++++++++++ internal/vaultcluster/keystore_aws.go | 123 +++++++++++++++++++++ internal/vaultcluster/keystore_aws_test.go | 104 +++++++++++++++++ internal/vaultcluster/snapshot.go | 23 +++- internal/vaultcluster/snapshot_aws.go | 99 +++++++++++++++++ internal/vaultcluster/snapshot_aws_test.go | 65 +++++++++++ internal/vaultcluster/snapshot_schedule.go | 20 ++++ 15 files changed, 856 insertions(+), 37 deletions(-) create mode 100644 internal/vaultcluster/health_http.go create mode 100644 internal/vaultcluster/health_http_test.go create mode 100644 internal/vaultcluster/health_raft.go create mode 100644 internal/vaultcluster/health_raft_test.go create mode 100644 internal/vaultcluster/keystore_aws.go create mode 100644 internal/vaultcluster/keystore_aws_test.go create mode 100644 internal/vaultcluster/snapshot_aws.go create mode 100644 internal/vaultcluster/snapshot_aws_test.go create mode 100644 internal/vaultcluster/snapshot_schedule.go diff --git a/README.md b/README.md index 053f698..d76ad22 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Applications share one Vault. They do not see each other's secrets. Isolation is | Target | Role | Status | |---|---|---| | `local/` | Docker Compose | Implemented | -| `aws/aws-ec2-vault-cluster/` | `aws-ec2-vault-cluster` | Connections, IAM, Secrets Manager tokens, security groups. ASG/NLB not built yet. | +| `aws/aws-ec2-vault-cluster/` | `aws-ec2-vault-cluster` | Connections, IAM, SM, security groups. `bootstrap aws`, health on 8210, S3 snapshots. ASG/NLB not built yet. | | `gcp/` | GCP | Not implemented | | `azure/` | Azure | Not implemented | @@ -43,12 +43,13 @@ Implemented: - File audit on a volume separate from Raft - Auto-init (first start) and one-shot Shamir unseal via `vault-utils` - Isolation tests in Go (`go test`); credentials tests in Go (`TestCredentialsMatrix`) +- `bootstrap aws` (KMS auto-unseal, Secrets Manager tokens), health on 8210, S3 snapshots Not implemented: -- AWS ASG, NLB, AMI, user-data, and `bootstrap aws` +- AWS ASG, NLB, AMI, and user-data - GCP, Azure, Kubernetes -- Production KMS auto-unseal on a running cluster +- KMS auto-unseal proven on a running EC2 cluster - TLS, multi-node Raft, DR replication Local unseal submits Shamir shares (5 shares, threshold 3) for laptop use. It is not AWS KMS, Cloud KMS, or Azure Key Vault auto-unseal. @@ -82,7 +83,7 @@ vault-cluster/ ├── cmd/ Go app entrypoints (vault-utils CLI) ├── internal/ Go libraries, policy templates, lint fixtures ├── local/ Compose target, snapshots -├── aws/aws-ec2-vault-cluster/ Nullstone module (IAM/SM/SG; no ASG yet) +├── aws/aws-ec2-vault-cluster/ Nullstone module (IAM/SM/SG; vault-utils AWS; no ASG yet) ├── gcp/ Nullstone Terraform module (not yet implemented) └── azure/ Nullstone Terraform module (not yet implemented) ``` @@ -320,7 +321,7 @@ Denials must be HTTP 403. A 404 is a different failure. ### AWS module (`aws/aws-ec2-vault-cluster/`) -`go test` does not cover this directory. The current slice is OpenTofu only (connections, IAM, Secrets Manager, security groups). There is no Docker or live-AWS test in CI. +OpenTofu in this directory is connections, IAM, Secrets Manager, and security groups. `go test ./internal/vaultcluster` covers the AWS vault-utils logic (SM KeyStore, Raft health rule, S3 key/list, cron parse) with fakes. There is no live-AWS test in CI. From `aws/aws-ec2-vault-cluster/`: diff --git a/cmd/vault-utils/main.go b/cmd/vault-utils/main.go index 0b7620e..e66d77a 100644 --- a/cmd/vault-utils/main.go +++ b/cmd/vault-utils/main.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/nullstone-modules/vault-cluster/internal/vaultcluster" ) @@ -29,10 +30,11 @@ Commands: bootstrap local|aws|azure|gcp Initialize a cluster: init (once), unseal, configure tenants create tenants destroy --yes [--purge-secrets] - snapshot take|list|verify |restore --yes - health + snapshot take|list|verify |restore --yes|schedule + health [serve] Key material for bootstrap local is stored under BOOTSTRAP_DIR (default .bootstrap). +AWS uses Secrets Manager ARNs and optional SNAPSHOT_BUCKET. `) } @@ -50,6 +52,9 @@ func run(cmd string, args []string) error { case "snapshot": return runSnapshot(c, args) case "health": + if len(args) > 0 && args[0] == "serve" { + return runHealthServe(c) + } return c.Health() default: usage() @@ -65,12 +70,25 @@ func runBootstrap(c *vaultcluster.Client, args []string) error { case "local": shares, _ := strconv.Atoi(getenv("VAULT_INIT_KEY_SHARES", "5")) threshold, _ := strconv.Atoi(getenv("VAULT_INIT_KEY_THRESHOLD", "3")) - return c.RunBootstrap(keyStore(), vaultcluster.BootstrapOptions{ + return c.RunBootstrap(fileKeyStore(), vaultcluster.BootstrapOptions{ Shares: shares, Threshold: threshold, KeepRoot: getenv("KEEP_ROOT", "false") == "true", }) - case "aws", "azure", "gcp": + case "aws": + store, err := awsKeyStore() + if err != nil { + return err + } + shares, _ := strconv.Atoi(getenv("VAULT_INIT_RECOVERY_SHARES", "1")) + threshold, _ := strconv.Atoi(getenv("VAULT_INIT_RECOVERY_THRESHOLD", "1")) + return c.RunBootstrap(store, vaultcluster.BootstrapOptions{ + Shares: shares, + Threshold: threshold, + KeepRoot: getenv("KEEP_ROOT", "false") == "true", + AutoUnseal: true, + }) + case "azure", "gcp": return fmt.Errorf("bootstrap %s is not implemented yet", args[0]) default: return fmt.Errorf("unknown platform %q (local, aws, azure, gcp)", args[0]) @@ -120,7 +138,7 @@ func runTenants(c *vaultcluster.Client, args []string) error { func runSnapshot(c *vaultcluster.Client, args []string) error { if len(args) < 1 { - return fmt.Errorf("usage: vault-utils snapshot take|list|verify |restore --yes") + return fmt.Errorf("usage: vault-utils snapshot take|list|verify |restore --yes|schedule") } backupDir := filepath.Join(bootstrapDir(), "backups") switch args[0] { @@ -128,7 +146,7 @@ func runSnapshot(c *vaultcluster.Client, args []string) error { if err := useOperatorToken(c); err != nil { return err } - file, err := c.SnapshotTake(backupDir) + file, err := takeSnapshot(c, backupDir) if err != nil { return err } @@ -136,12 +154,12 @@ func runSnapshot(c *vaultcluster.Client, args []string) error { log.Printf("this file contains every secret in the cluster; treat it as one") return nil case "list": - files, err := vaultcluster.SnapshotList(backupDir) + files, err := listSnapshots(backupDir) if err != nil { return err } if len(files) == 0 { - log.Printf("no snapshots under %s", backupDir) + log.Printf("no snapshots") return nil } for _, f := range files { @@ -170,16 +188,83 @@ func runSnapshot(c *vaultcluster.Client, args []string) error { log.Printf("restore submitted; Vault will seal") log.Printf("unseal with the key shares that were current when this snapshot was taken") return nil + case "schedule": + return runSnapshotSchedule(c, backupDir) default: - return fmt.Errorf("unknown subcommand %q (take, list, verify, restore)", args[0]) + return fmt.Errorf("unknown subcommand %q (take, list, verify, restore, schedule)", args[0]) + } +} + +func runSnapshotSchedule(c *vaultcluster.Client, backupDir string) error { + sched, err := vaultcluster.ParseBackupSchedule(os.Getenv("BACKUP_SCHEDULE")) + if err != nil { + return err + } + if sched == nil { + log.Printf("scheduled snapshots disabled") + return nil + } + if err := useOperatorToken(c); err != nil { + return err + } + for { + wait := time.Until(sched.Next(time.Now())) + if wait > 0 { + time.Sleep(wait) + } + file, err := takeSnapshot(c, backupDir) + if err != nil { + log.Printf("snapshot failed: %v", err) + continue + } + log.Printf("snapshot written: %s", file) + } +} + +func runHealthServe(c *vaultcluster.Client) error { + nodeID := os.Getenv("VAULT_RAFT_NODE_ID") + if nodeID == "" { + return fmt.Errorf("VAULT_RAFT_NODE_ID is required for health serve") + } + if err := useOperatorToken(c); err != nil { + return err } + addr := getenv("VAULT_HEALTH_ADDR", ":8210") + log.Printf("health listening on %s", addr) + return c.ServeHealth(addr, nodeID) +} + +func takeSnapshot(c *vaultcluster.Client, backupDir string) (string, error) { + if bucket := os.Getenv("SNAPSHOT_BUCKET"); bucket != "" { + store, err := vaultcluster.NewS3ObjectStore() + if err != nil { + return "", err + } + return c.SnapshotTakeS3(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots")) + } + return c.SnapshotTake(backupDir) +} + +func listSnapshots(backupDir string) ([]string, error) { + if bucket := os.Getenv("SNAPSHOT_BUCKET"); bucket != "" { + store, err := vaultcluster.NewS3ObjectStore() + if err != nil { + return nil, err + } + return vaultcluster.SnapshotListS3(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots")) + } + return vaultcluster.SnapshotList(backupDir) } func useOperatorToken(c *vaultcluster.Client) error { if c.Cfg.Token != "" { return nil } - tok, err := keyStore().LoadToken("operator") + store, err := keyStore() + if err != nil { + return err + } + tok, err := store.LoadToken("operator") if err != nil { return fmt.Errorf("set VAULT_TOKEN or bootstrap first (operator token not found): %w", err) } @@ -188,10 +273,25 @@ func useOperatorToken(c *vaultcluster.Client) error { return nil } -func keyStore() vaultcluster.FileKeyStore { +func keyStore() (vaultcluster.KeyStore, error) { + if os.Getenv("VAULT_OPERATOR_SECRET_ARN") != "" || os.Getenv("VAULT_INIT_SECRET_ARN") != "" { + return awsKeyStore() + } + return fileKeyStore(), nil +} + +func fileKeyStore() vaultcluster.FileKeyStore { return vaultcluster.FileKeyStore{Dir: bootstrapDir()} } +func awsKeyStore() (*vaultcluster.SecretsManagerKeyStore, error) { + return vaultcluster.NewSecretsManagerKeyStore( + os.Getenv("VAULT_INIT_SECRET_ARN"), + os.Getenv("VAULT_PROVISIONING_SECRET_ARN"), + os.Getenv("VAULT_OPERATOR_SECRET_ARN"), + ) +} + func bootstrapDir() string { return getenv("BOOTSTRAP_DIR", ".bootstrap") } diff --git a/go.mod b/go.mod index 78c2fe4..fa9c244 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,31 @@ module github.com/nullstone-modules/vault-cluster go 1.23.0 -require github.com/hashicorp/vault/api v1.16.0 +require ( + github.com/aws/aws-sdk-go-v2 v1.41.2 + github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6 + github.com/hashicorp/vault/api v1.16.0 + github.com/robfig/cron/v3 v3.0.1 +) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect + github.com/aws/smithy-go v1.24.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 4c87563..9df16b5 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,42 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= +github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6 h1:l4mxH8imZoflVEWWa8VT8skwObm+t0KEveqEskyiKEo= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6/go.mod h1:1qwmvfRBGTQ5shUxu+eQO/S2+O6o6SxbvcvtN62kmc0= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11/go.mod h1:0DO9B5EUJQlIDif+XJRWCljZRKsAFKh3gpFz7UnDtOo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWAXLGFIizeqkdkKgRlJwWc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= +github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0= +github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -55,6 +93,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= diff --git a/internal/vaultcluster/bootstrap.go b/internal/vaultcluster/bootstrap.go index bbccef7..8f4ad3e 100644 --- a/internal/vaultcluster/bootstrap.go +++ b/internal/vaultcluster/bootstrap.go @@ -9,9 +9,10 @@ import ( ) type BootstrapOptions struct { - Shares int - Threshold int - KeepRoot bool + Shares int + Threshold int + KeepRoot bool + AutoUnseal bool } const revokedRootMarker = "revoked-at-bootstrap" @@ -26,23 +27,18 @@ func (c *Client) RunBootstrap(store KeyStore, opts BootstrapOptions) error { return err } if !st.Initialized { - log.Printf("initializing Vault (%d/%d Shamir)", opts.Shares, opts.Threshold) - resp, err := c.API.Sys().Init(&api.InitRequest{ - SecretShares: opts.Shares, - SecretThreshold: opts.Threshold, - }) - if err != nil { + if err := c.initVault(store, opts); err != nil { return err } - if err := store.SaveInit(resp); err != nil { - return err - } - log.Printf("initialized; key material saved (not logged)") } else if _, err := store.LoadInit(); err != nil { return fmt.Errorf("Vault is initialized but key material is missing: %w", err) } - if err := c.unseal(store, opts.Threshold); err != nil { + if opts.AutoUnseal { + if err := c.waitUnsealed(); err != nil { + return err + } + } else if err := c.unseal(store, opts.Threshold); err != nil { return err } @@ -97,6 +93,52 @@ func (c *Client) RunBootstrap(store KeyStore, opts BootstrapOptions) error { return nil } +func initRequest(opts BootstrapOptions) *api.InitRequest { + if opts.AutoUnseal { + return &api.InitRequest{ + RecoveryShares: opts.Shares, + RecoveryThreshold: opts.Threshold, + } + } + return &api.InitRequest{ + SecretShares: opts.Shares, + SecretThreshold: opts.Threshold, + } +} + +func (c *Client) initVault(store KeyStore, opts BootstrapOptions) error { + if opts.AutoUnseal { + log.Printf("initializing Vault (recovery %d/%d, auto-unseal)", opts.Shares, opts.Threshold) + } else { + log.Printf("initializing Vault (%d/%d Shamir)", opts.Shares, opts.Threshold) + } + resp, err := c.API.Sys().Init(initRequest(opts)) + if err != nil { + return err + } + if err := store.SaveInit(resp); err != nil { + return err + } + log.Printf("initialized; key material saved (not logged)") + return nil +} + +func (c *Client) waitUnsealed() error { + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + st, err := c.API.Sys().SealStatus() + if err != nil { + time.Sleep(time.Second) + continue + } + if !st.Sealed { + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("Vault remained sealed (auto-unseal failed)") +} + func (c *Client) WaitReady(timeout time.Duration) error { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { diff --git a/internal/vaultcluster/health_http.go b/internal/vaultcluster/health_http.go new file mode 100644 index 0000000..85d8e8e --- /dev/null +++ b/internal/vaultcluster/health_http.go @@ -0,0 +1,20 @@ +package vaultcluster + +import ( + "net/http" +) + +func (c *Client) HealthHandler(nodeID string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if err := c.NodeHealthOK(nodeID); err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) + }) +} + +func (c *Client) ServeHealth(addr, nodeID string) error { + return http.ListenAndServe(addr, c.HealthHandler(nodeID)) +} diff --git a/internal/vaultcluster/health_http_test.go b/internal/vaultcluster/health_http_test.go new file mode 100644 index 0000000..7bb2827 --- /dev/null +++ b/internal/vaultcluster/health_http_test.go @@ -0,0 +1,20 @@ +package vaultcluster + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestHealthHandlerUnavailable(t *testing.T) { + c, err := New(Config{Addr: "http://127.0.0.1:1", HTTPTimeout: 50 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + c.HealthHandler("n1").ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status %d", rec.Code) + } +} diff --git a/internal/vaultcluster/health_raft.go b/internal/vaultcluster/health_raft.go new file mode 100644 index 0000000..9ee6144 --- /dev/null +++ b/internal/vaultcluster/health_raft.go @@ -0,0 +1,95 @@ +package vaultcluster + +import ( + "encoding/json" + "fmt" + "strings" +) + +type raftServer struct { + ID string + Status string + Healthy bool + Voter bool +} + +func parseAutopilot(data map[string]any) []raftServer { + raw, _ := json.Marshal(data) + var parsed struct { + Voters []string `json:"voters"` + Servers map[string]struct { + ID string `json:"id"` + Status string `json:"status"` + Healthy bool `json:"healthy"` + } `json:"servers"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil + } + voters := map[string]bool{} + for _, id := range parsed.Voters { + voters[id] = true + } + var out []raftServer + for id, s := range parsed.Servers { + if s.ID == "" { + s.ID = id + } + status := strings.ToLower(s.Status) + out = append(out, raftServer{ + ID: s.ID, + Status: status, + Healthy: s.Healthy, + Voter: voters[s.ID] || status == "leader" || status == "voter", + }) + } + return out +} + +func NodeRaftReady(nodeID string, data map[string]any) error { + if nodeID == "" { + return fmt.Errorf("VAULT_RAFT_NODE_ID is not set") + } + if data == nil { + return fmt.Errorf("raft autopilot state is missing") + } + for _, s := range parseAutopilot(data) { + if s.ID != nodeID { + continue + } + if !s.Voter { + return fmt.Errorf("node %s is not a raft voter", nodeID) + } + if !s.Healthy { + return fmt.Errorf("node %s is not caught up", nodeID) + } + return nil + } + return fmt.Errorf("node %s is not in the raft cluster", nodeID) +} + +func (c *Client) RaftAutopilot() (map[string]any, error) { + sec, err := c.API.Logical().Read("sys/storage/raft/autopilot/state") + if err != nil { + return nil, err + } + if sec == nil || sec.Data == nil { + return nil, fmt.Errorf("raft autopilot state is empty") + } + return sec.Data, nil +} + +func (c *Client) NodeHealthOK(nodeID string) error { + st, err := c.API.Sys().SealStatus() + if err != nil { + return err + } + if !st.Initialized || st.Sealed { + return fmt.Errorf("vault is not ready") + } + data, err := c.RaftAutopilot() + if err != nil { + return err + } + return NodeRaftReady(nodeID, data) +} diff --git a/internal/vaultcluster/health_raft_test.go b/internal/vaultcluster/health_raft_test.go new file mode 100644 index 0000000..3b879e0 --- /dev/null +++ b/internal/vaultcluster/health_raft_test.go @@ -0,0 +1,57 @@ +package vaultcluster + +import "testing" + +func autopilot(nodeID, status string, healthy bool, voters []string) map[string]any { + return map[string]any{ + "voters": voters, + "servers": map[string]any{ + nodeID: map[string]any{ + "id": nodeID, + "status": status, + "healthy": healthy, + }, + }, + } +} + +func TestNodeRaftReadyVoterCaughtUp(t *testing.T) { + data := autopilot("n1", "voter", true, []string{"n1", "n2"}) + if err := NodeRaftReady("n1", data); err != nil { + t.Fatal(err) + } +} + +func TestNodeRaftReadyLeader(t *testing.T) { + data := autopilot("n1", "leader", true, []string{"n1"}) + if err := NodeRaftReady("n1", data); err != nil { + t.Fatal(err) + } +} + +func TestNodeRaftReadyNonVoter(t *testing.T) { + data := autopilot("n3", "non-voter", true, []string{"n1", "n2"}) + if err := NodeRaftReady("n3", data); err == nil { + t.Fatal("expected non-voter to fail") + } +} + +func TestNodeRaftReadyNotCaughtUp(t *testing.T) { + data := autopilot("n1", "voter", false, []string{"n1"}) + if err := NodeRaftReady("n1", data); err == nil { + t.Fatal("expected unhealthy voter to fail") + } +} + +func TestNodeRaftReadyMissingNode(t *testing.T) { + data := autopilot("n1", "leader", true, []string{"n1"}) + if err := NodeRaftReady("n2", data); err == nil { + t.Fatal("expected missing node to fail") + } +} + +func TestNodeRaftReadyRequiresNodeID(t *testing.T) { + if err := NodeRaftReady("", autopilot("n1", "leader", true, []string{"n1"})); err == nil { + t.Fatal("expected empty node id to fail") + } +} diff --git a/internal/vaultcluster/keystore_aws.go b/internal/vaultcluster/keystore_aws.go new file mode 100644 index 0000000..9dfba26 --- /dev/null +++ b/internal/vaultcluster/keystore_aws.go @@ -0,0 +1,123 @@ +package vaultcluster + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/hashicorp/vault/api" +) + +type secretKV interface { + Get(ctx context.Context, arn string) ([]byte, error) + Put(ctx context.Context, arn string, val []byte) error +} + +type SecretsManagerKeyStore struct { + Secrets secretKV + InitARN string + ProvisioningARN string + OperatorARN string +} + +func NewSecretsManagerKeyStore(initARN, provisioningARN, operatorARN string) (*SecretsManagerKeyStore, error) { + if initARN == "" || provisioningARN == "" || operatorARN == "" { + return nil, fmt.Errorf("VAULT_INIT_SECRET_ARN, VAULT_PROVISIONING_SECRET_ARN, and VAULT_OPERATOR_SECRET_ARN are required") + } + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return nil, fmt.Errorf("AWS credentials: %w", err) + } + return &SecretsManagerKeyStore{ + Secrets: smClient{inner: secretsmanager.NewFromConfig(cfg)}, + InitARN: initARN, + ProvisioningARN: provisioningARN, + OperatorARN: operatorARN, + }, nil +} + +func (s SecretsManagerKeyStore) tokenARN(name string) (string, error) { + switch name { + case "provisioning": + return s.ProvisioningARN, nil + case "operator": + return s.OperatorARN, nil + default: + return "", fmt.Errorf("unknown token %q", name) + } +} + +func (s SecretsManagerKeyStore) SaveInit(resp *api.InitResponse) error { + b, err := json.Marshal(resp) + if err != nil { + return err + } + return s.Secrets.Put(context.Background(), s.InitARN, b) +} + +func (s SecretsManagerKeyStore) LoadInit() (*api.InitResponse, error) { + raw, err := s.Secrets.Get(context.Background(), s.InitARN) + if err != nil { + return nil, err + } + var resp api.InitResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (s SecretsManagerKeyStore) SaveToken(name, token string) error { + arn, err := s.tokenARN(name) + if err != nil { + return err + } + return s.Secrets.Put(context.Background(), arn, []byte(token)) +} + +func (s SecretsManagerKeyStore) LoadToken(name string) (string, error) { + arn, err := s.tokenARN(name) + if err != nil { + return "", err + } + b, err := s.Secrets.Get(context.Background(), arn) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +type smClient struct { + inner *secretsmanager.Client +} + +func (c smClient) Get(ctx context.Context, arn string) ([]byte, error) { + out, err := c.inner.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(arn), + }) + if err != nil { + return nil, fmt.Errorf("secrets manager get %s: %w", arn, err) + } + if out.SecretString != nil { + return []byte(*out.SecretString), nil + } + if len(out.SecretBinary) > 0 { + return out.SecretBinary, nil + } + return nil, fmt.Errorf("secrets manager get %s: empty secret", arn) +} + +func (c smClient) Put(ctx context.Context, arn string, val []byte) error { + _, err := c.inner.PutSecretValue(ctx, &secretsmanager.PutSecretValueInput{ + SecretId: aws.String(arn), + SecretString: aws.String(string(val)), + }) + if err != nil { + return fmt.Errorf("secrets manager put %s: %w", arn, err) + } + return nil +} diff --git a/internal/vaultcluster/keystore_aws_test.go b/internal/vaultcluster/keystore_aws_test.go new file mode 100644 index 0000000..20dde03 --- /dev/null +++ b/internal/vaultcluster/keystore_aws_test.go @@ -0,0 +1,104 @@ +package vaultcluster + +import ( + "context" + "errors" + "testing" + + "github.com/hashicorp/vault/api" +) + +type memSecrets map[string][]byte + +func (m memSecrets) Get(_ context.Context, arn string) ([]byte, error) { + b, ok := m[arn] + if !ok { + return nil, errors.New("missing") + } + return b, nil +} + +func (m memSecrets) Put(_ context.Context, arn string, val []byte) error { + if arn == "fail" { + return errors.New("denied") + } + m[arn] = append([]byte(nil), val...) + return nil +} + +func testSMStore(m memSecrets) SecretsManagerKeyStore { + return SecretsManagerKeyStore{ + Secrets: m, + InitARN: "arn:init", + ProvisioningARN: "arn:provisioning", + OperatorARN: "arn:operator", + } +} + +func TestSecretsManagerKeyStoreRoundTrip(t *testing.T) { + store := testSMStore(memSecrets{}) + init := &api.InitResponse{RootToken: "hvs.root", RecoveryKeysB64: []string{"abc"}} + if err := store.SaveInit(init); err != nil { + t.Fatal(err) + } + got, err := store.LoadInit() + if err != nil { + t.Fatal(err) + } + if got.RootToken != "hvs.root" || len(got.RecoveryKeysB64) != 1 { + t.Fatalf("init mismatch: %+v", got) + } + if err := store.SaveToken("operator", "hvs.op"); err != nil { + t.Fatal(err) + } + tok, err := store.LoadToken("operator") + if err != nil { + t.Fatal(err) + } + if tok != "hvs.op" { + t.Fatalf("token %q", tok) + } +} + +func TestSecretsManagerKeyStoreFailClosed(t *testing.T) { + store := testSMStore(memSecrets{}) + if _, err := store.LoadInit(); err == nil { + t.Fatal("expected missing init to fail") + } + if _, err := store.LoadToken("operator"); err == nil { + t.Fatal("expected missing token to fail") + } + if err := store.SaveToken("root", "x"); err == nil { + t.Fatal("expected unknown token name to fail") + } + store.InitARN = "fail" + if err := store.SaveInit(&api.InitResponse{RootToken: "x"}); err == nil { + t.Fatal("expected put failure") + } +} + +func TestNewSecretsManagerKeyStoreRequiresARNs(t *testing.T) { + if _, err := NewSecretsManagerKeyStore("", "a", "b"); err == nil { + t.Fatal("expected error") + } +} + +func TestInitRequestAutoUnsealUsesRecovery(t *testing.T) { + req := initRequest(BootstrapOptions{Shares: 1, Threshold: 1, AutoUnseal: true}) + if req.RecoveryShares != 1 || req.RecoveryThreshold != 1 { + t.Fatalf("recovery: %+v", req) + } + if req.SecretShares != 0 || req.SecretThreshold != 0 { + t.Fatalf("shamir should be unset: %+v", req) + } +} + +func TestInitRequestLocalUsesShamir(t *testing.T) { + req := initRequest(BootstrapOptions{Shares: 5, Threshold: 3}) + if req.SecretShares != 5 || req.SecretThreshold != 3 { + t.Fatalf("shamir: %+v", req) + } + if req.RecoveryShares != 0 { + t.Fatalf("recovery should be unset: %+v", req) + } +} diff --git a/internal/vaultcluster/snapshot.go b/internal/vaultcluster/snapshot.go index 29b986e..946c2ff 100644 --- a/internal/vaultcluster/snapshot.go +++ b/internal/vaultcluster/snapshot.go @@ -12,25 +12,36 @@ import ( "time" ) -func (c *Client) SnapshotTake(dir string) (string, error) { +func (c *Client) raftSnapshot() ([]byte, error) { req := c.API.NewRequest("GET", "/v1/sys/storage/raft/snapshot") resp, err := c.API.RawRequest(req) if err != nil { - return "", fmt.Errorf("snapshot failed: %w", err) + return nil, fmt.Errorf("snapshot failed: %w", err) } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { - return "", err + return nil, err } if len(b) == 0 { - return "", fmt.Errorf("snapshot is empty; refusing to keep it") + return nil, fmt.Errorf("snapshot is empty; refusing to keep it") + } + return b, nil +} + +func snapshotStamp() string { + return time.Now().UTC().Format("20060102T150405Z") +} + +func (c *Client) SnapshotTake(dir string) (string, error) { + b, err := c.raftSnapshot() + if err != nil { + return "", err } if err := os.MkdirAll(dir, 0o700); err != nil { return "", err } - stamp := time.Now().UTC().Format("20060102T150405Z") - file := filepath.Join(dir, "vault-"+stamp+".snap") + file := filepath.Join(dir, "vault-"+snapshotStamp()+".snap") if err := os.WriteFile(file, b, 0o600); err != nil { return "", err } diff --git a/internal/vaultcluster/snapshot_aws.go b/internal/vaultcluster/snapshot_aws.go new file mode 100644 index 0000000..556365f --- /dev/null +++ b/internal/vaultcluster/snapshot_aws.go @@ -0,0 +1,99 @@ +package vaultcluster + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type objectStore interface { + Put(ctx context.Context, bucket, key string, body []byte) error + List(ctx context.Context, bucket, prefix string) ([]string, error) +} + +type s3Store struct { + inner *s3.Client +} + +func NewS3ObjectStore() (objectStore, error) { + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return nil, fmt.Errorf("AWS credentials: %w", err) + } + return s3Store{inner: s3.NewFromConfig(cfg)}, nil +} + +func (s s3Store) Put(ctx context.Context, bucket, key string, body []byte) error { + _, err := s.inner.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(body), + }) + if err != nil { + return fmt.Errorf("s3 put s3://%s/%s: %w", bucket, key, err) + } + return nil +} + +func (s s3Store) List(ctx context.Context, bucket, prefix string) ([]string, error) { + out, err := s.inner.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + }) + if err != nil { + return nil, fmt.Errorf("s3 list s3://%s/%s: %w", bucket, prefix, err) + } + var keys []string + for _, obj := range out.Contents { + if obj.Key == nil || !strings.HasSuffix(*obj.Key, ".snap") { + continue + } + keys = append(keys, "s3://"+bucket+"/"+*obj.Key) + } + return keys, nil +} + +func snapshotObjectKey(prefix, stamp string) string { + p := strings.Trim(prefix, "/") + if p == "" { + p = "vault-snapshots" + } + return p + "/vault-" + stamp + ".snap" +} + +func (c *Client) SnapshotTakeS3(store objectStore, bucket, prefix string) (string, error) { + if bucket == "" { + return "", fmt.Errorf("SNAPSHOT_BUCKET is not set") + } + b, err := c.raftSnapshot() + if err != nil { + return "", err + } + key := snapshotObjectKey(prefix, snapshotStamp()) + if err := store.Put(context.Background(), bucket, key, b); err != nil { + return "", err + } + sum := sha256.Sum256(b) + if err := store.Put(context.Background(), bucket, key+".sha256", []byte(hex.EncodeToString(sum[:])+"\n")); err != nil { + return "", err + } + return "s3://" + bucket + "/" + key, nil +} + +func SnapshotListS3(store objectStore, bucket, prefix string) ([]string, error) { + if bucket == "" { + return nil, fmt.Errorf("SNAPSHOT_BUCKET is not set") + } + p := strings.Trim(prefix, "/") + if p == "" { + p = "vault-snapshots" + } + return store.List(context.Background(), bucket, p+"/") +} diff --git a/internal/vaultcluster/snapshot_aws_test.go b/internal/vaultcluster/snapshot_aws_test.go new file mode 100644 index 0000000..3915bea --- /dev/null +++ b/internal/vaultcluster/snapshot_aws_test.go @@ -0,0 +1,65 @@ +package vaultcluster + +import ( + "context" + "strings" + "testing" +) + +type memObjects map[string][]byte + +func (m memObjects) Put(_ context.Context, bucket, key string, body []byte) error { + m[bucket+"/"+key] = append([]byte(nil), body...) + return nil +} + +func (m memObjects) List(_ context.Context, bucket, prefix string) ([]string, error) { + var out []string + root := bucket + "/" + for k := range m { + if !strings.HasPrefix(k, root+prefix) { + continue + } + key := strings.TrimPrefix(k, root) + if strings.HasSuffix(key, ".snap") { + out = append(out, "s3://"+bucket+"/"+key) + } + } + return out, nil +} + +func TestSnapshotObjectKey(t *testing.T) { + got := snapshotObjectKey("vault-snapshots", "20260101T000000Z") + if got != "vault-snapshots/vault-20260101T000000Z.snap" { + t.Fatalf("key %q", got) + } +} + +func TestSnapshotListS3(t *testing.T) { + store := memObjects{ + "b/vault-snapshots/vault-1.snap": []byte("a"), + "b/vault-snapshots/vault-1.snap.sha256": []byte("x"), + "b/other/vault-2.snap": []byte("c"), + } + got, err := SnapshotListS3(store, "b", "vault-snapshots") + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0] != "s3://b/vault-snapshots/vault-1.snap" { + t.Fatalf("list %v", got) + } +} + +func TestParseBackupSchedule(t *testing.T) { + off, err := ParseBackupSchedule("") + if err != nil || off != nil { + t.Fatalf("empty: %v %v", off, err) + } + on, err := ParseBackupSchedule("0 3 * * *") + if err != nil || on == nil { + t.Fatalf("cron: %v %v", on, err) + } + if _, err := ParseBackupSchedule("not-a-cron"); err == nil { + t.Fatal("expected invalid cron to fail") + } +} diff --git a/internal/vaultcluster/snapshot_schedule.go b/internal/vaultcluster/snapshot_schedule.go new file mode 100644 index 0000000..9dc2675 --- /dev/null +++ b/internal/vaultcluster/snapshot_schedule.go @@ -0,0 +1,20 @@ +package vaultcluster + +import ( + "fmt" + "strings" + + "github.com/robfig/cron/v3" +) + +func ParseBackupSchedule(expr string) (cron.Schedule, error) { + expr = strings.TrimSpace(expr) + if expr == "" { + return nil, nil + } + sched, err := cron.ParseStandard(expr) + if err != nil { + return nil, fmt.Errorf("backup_schedule: %w", err) + } + return sched, nil +} From 5b50be046f50c14f87415b940c46f4455450365f Mon Sep 17 00:00:00 2001 From: Amit Biswas Date: Mon, 7 Sep 2026 11:27:20 -0400 Subject: [PATCH 2/2] Split AWS adapters into internal/aws, clarify CLI help, and bump Go to 1.26. --- .github/workflows/test-local.yml | 4 +- .github/workflows/validate.yml | 2 +- Dockerfile | 2 +- README.md | 7 +- cmd/vault-utils/main.go | 38 ++++--- go.mod | 2 +- internal/aws/s3/snapshot.go | 98 ++++++++++++++++++ internal/aws/s3/snapshot_test.go | 81 +++++++++++++++ .../secretsmanager/keystore.go} | 22 ++--- .../secretsmanager/keystore_test.go} | 38 ++----- internal/vaultcluster/bootstrap_test.go | 23 +++++ internal/vaultcluster/snapshot.go | 4 +- internal/vaultcluster/snapshot_aws.go | 99 ------------------- internal/vaultcluster/snapshot_aws_test.go | 65 ------------ .../vaultcluster/snapshot_schedule_test.go | 17 ++++ 15 files changed, 275 insertions(+), 227 deletions(-) create mode 100644 internal/aws/s3/snapshot.go create mode 100644 internal/aws/s3/snapshot_test.go rename internal/{vaultcluster/keystore_aws.go => aws/secretsmanager/keystore.go} (81%) rename internal/{vaultcluster/keystore_aws_test.go => aws/secretsmanager/keystore_test.go} (59%) create mode 100644 internal/vaultcluster/bootstrap_test.go delete mode 100644 internal/vaultcluster/snapshot_aws.go delete mode 100644 internal/vaultcluster/snapshot_aws_test.go create mode 100644 internal/vaultcluster/snapshot_schedule_test.go diff --git a/.github/workflows/test-local.yml b/.github/workflows/test-local.yml index 36f88b1..86bbf44 100644 --- a/.github/workflows/test-local.yml +++ b/.github/workflows/test-local.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: "1.23.x" + go-version: "1.26.x" - name: Isolation and credentials run: go test ./internal/vaultcluster -count=1 -timeout 15m @@ -32,7 +32,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: "1.23.x" + go-version: "1.26.x" - name: Local compose runtime test run: go test ./local -count=1 -timeout 15m -v diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 06718ae..28c07e2 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: "1.23.x" + go-version: "1.26.x" - name: Unit tests (no Docker) run: go test -short ./... diff --git a/Dockerfile b/Dockerfile index 3fefa82..031212a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine AS build +FROM golang:1.26-alpine AS build WORKDIR /src COPY go.mod go.sum ./ RUN go mod download diff --git a/README.md b/README.md index d76ad22..0ff286e 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,8 @@ vault-cluster/ ├── CHANGELOG.md ├── Dockerfile vault-utils image ├── cmd/ Go app entrypoints (vault-utils CLI) -├── internal/ Go libraries, policy templates, lint fixtures +├── internal/vaultcluster/ shared Vault library +├── internal/aws/ AWS adapters (secretsmanager, s3) ├── local/ Compose target, snapshots ├── aws/aws-ec2-vault-cluster/ Nullstone module (IAM/SM/SG; vault-utils AWS; no ASG yet) ├── gcp/ Nullstone Terraform module (not yet implemented) @@ -90,7 +91,7 @@ vault-cluster/ ## Prerequisites -Docker Desktop (Compose v2). Go 1.23 for `go test`. `curl` and `jq` for the manual examples below; Vault CLI is optional except break-glass decode. +Docker Desktop (Compose v2). Go 1.26 for `go test`. `curl` and `jq` for the manual examples below; Vault CLI is optional except break-glass decode. Images are pinned by tag and digest in `local/compose.yml` (Vault 2.0, PostgreSQL 18-alpine). Never `latest`. @@ -321,7 +322,7 @@ Denials must be HTTP 403. A 404 is a different failure. ### AWS module (`aws/aws-ec2-vault-cluster/`) -OpenTofu in this directory is connections, IAM, Secrets Manager, and security groups. `go test ./internal/vaultcluster` covers the AWS vault-utils logic (SM KeyStore, Raft health rule, S3 key/list, cron parse) with fakes. There is no live-AWS test in CI. +OpenTofu in this directory is connections, IAM, Secrets Manager, and security groups. `go test ./internal/aws/...` covers the SM KeyStore and S3 snapshot helpers. `go test ./internal/vaultcluster` covers Raft health and cron parse. There is no live-AWS test in CI. From `aws/aws-ec2-vault-cluster/`: diff --git a/cmd/vault-utils/main.go b/cmd/vault-utils/main.go index e66d77a..141c110 100644 --- a/cmd/vault-utils/main.go +++ b/cmd/vault-utils/main.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/nullstone-modules/vault-cluster/internal/aws/s3" + "github.com/nullstone-modules/vault-cluster/internal/aws/secretsmanager" "github.com/nullstone-modules/vault-cluster/internal/vaultcluster" ) @@ -24,17 +26,23 @@ func main() { } func usage() { - fmt.Fprintf(os.Stderr, `vault-utils + fmt.Fprintf(os.Stderr, `vault-utils [args] Commands: - bootstrap local|aws|azure|gcp Initialize a cluster: init (once), unseal, configure + bootstrap local|aws|azure|gcp Init once, unseal, configure tenants create tenants destroy --yes [--purge-secrets] - snapshot take|list|verify |restore --yes|schedule - health [serve] + snapshot take Write a Raft snapshot + snapshot list + snapshot verify + snapshot restore --yes + snapshot schedule Cron loop (BACKUP_SCHEDULE; empty disables) + health Print seal status + health serve HTTP on :8210 (200 only if this node is a Raft voter and caught up) -Key material for bootstrap local is stored under BOOTSTRAP_DIR (default .bootstrap). -AWS uses Secrets Manager ARNs and optional SNAPSHOT_BUCKET. +Local key material: BOOTSTRAP_DIR (default .bootstrap). +AWS: VAULT_INIT_SECRET_ARN, VAULT_PROVISIONING_SECRET_ARN, VAULT_OPERATOR_SECRET_ARN. +Optional: SNAPSHOT_BUCKET, SNAPSHOT_PREFIX (default vault-snapshots). `) } @@ -138,7 +146,7 @@ func runTenants(c *vaultcluster.Client, args []string) error { func runSnapshot(c *vaultcluster.Client, args []string) error { if len(args) < 1 { - return fmt.Errorf("usage: vault-utils snapshot take|list|verify |restore --yes|schedule") + return fmt.Errorf("usage: vault-utils snapshot take | list | verify | restore --yes | schedule") } backupDir := filepath.Join(bootstrapDir(), "backups") switch args[0] { @@ -236,22 +244,26 @@ func runHealthServe(c *vaultcluster.Client) error { func takeSnapshot(c *vaultcluster.Client, backupDir string) (string, error) { if bucket := os.Getenv("SNAPSHOT_BUCKET"); bucket != "" { - store, err := vaultcluster.NewS3ObjectStore() + store, err := s3.New() if err != nil { return "", err } - return c.SnapshotTakeS3(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots")) + b, err := c.RaftSnapshot() + if err != nil { + return "", err + } + return s3.PutSnapshot(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots"), b) } return c.SnapshotTake(backupDir) } func listSnapshots(backupDir string) ([]string, error) { if bucket := os.Getenv("SNAPSHOT_BUCKET"); bucket != "" { - store, err := vaultcluster.NewS3ObjectStore() + store, err := s3.New() if err != nil { return nil, err } - return vaultcluster.SnapshotListS3(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots")) + return s3.ListSnapshots(store, bucket, getenv("SNAPSHOT_PREFIX", "vault-snapshots")) } return vaultcluster.SnapshotList(backupDir) } @@ -284,8 +296,8 @@ func fileKeyStore() vaultcluster.FileKeyStore { return vaultcluster.FileKeyStore{Dir: bootstrapDir()} } -func awsKeyStore() (*vaultcluster.SecretsManagerKeyStore, error) { - return vaultcluster.NewSecretsManagerKeyStore( +func awsKeyStore() (*secretsmanager.KeyStore, error) { + return secretsmanager.New( os.Getenv("VAULT_INIT_SECRET_ARN"), os.Getenv("VAULT_PROVISIONING_SECRET_ARN"), os.Getenv("VAULT_OPERATOR_SECRET_ARN"), diff --git a/go.mod b/go.mod index fa9c244..38107f0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/nullstone-modules/vault-cluster -go 1.23.0 +go 1.26.0 require ( github.com/aws/aws-sdk-go-v2 v1.41.2 diff --git a/internal/aws/s3/snapshot.go b/internal/aws/s3/snapshot.go new file mode 100644 index 0000000..107cad1 --- /dev/null +++ b/internal/aws/s3/snapshot.go @@ -0,0 +1,98 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type ObjectStore interface { + Put(ctx context.Context, bucket, key string, body []byte) error + List(ctx context.Context, bucket, prefix string) ([]string, error) +} + +type Store struct { + inner *awss3.Client +} + +func New() (Store, error) { + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return Store{}, fmt.Errorf("AWS credentials: %w", err) + } + return Store{inner: awss3.NewFromConfig(cfg)}, nil +} + +func (s Store) Put(ctx context.Context, bucket, key string, body []byte) error { + _, err := s.inner.PutObject(ctx, &awss3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(body), + }) + if err != nil { + return fmt.Errorf("s3 put s3://%s/%s: %w", bucket, key, err) + } + return nil +} + +func (s Store) List(ctx context.Context, bucket, prefix string) ([]string, error) { + out, err := s.inner.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + }) + if err != nil { + return nil, fmt.Errorf("s3 list s3://%s/%s: %w", bucket, prefix, err) + } + var keys []string + for _, obj := range out.Contents { + if obj.Key == nil || !strings.HasSuffix(*obj.Key, ".snap") { + continue + } + keys = append(keys, "s3://"+bucket+"/"+*obj.Key) + } + return keys, nil +} + +func normalizePrefix(prefix string) string { + if p := strings.Trim(prefix, "/"); p != "" { + return p + } + return "vault-snapshots" +} + +func ObjectKey(prefix, stamp string) string { + return normalizePrefix(prefix) + "/vault-" + stamp + ".snap" +} + +func PutSnapshot(store ObjectStore, bucket, prefix string, data []byte) (string, error) { + if bucket == "" { + return "", fmt.Errorf("SNAPSHOT_BUCKET is not set") + } + if len(data) == 0 { + return "", fmt.Errorf("snapshot is empty; refusing to keep it") + } + key := ObjectKey(prefix, time.Now().UTC().Format("20060102T150405Z")) + if err := store.Put(context.Background(), bucket, key, data); err != nil { + return "", err + } + sum := sha256.Sum256(data) + if err := store.Put(context.Background(), bucket, key+".sha256", []byte(hex.EncodeToString(sum[:])+"\n")); err != nil { + return "", err + } + return "s3://" + bucket + "/" + key, nil +} + +func ListSnapshots(store ObjectStore, bucket, prefix string) ([]string, error) { + if bucket == "" { + return nil, fmt.Errorf("SNAPSHOT_BUCKET is not set") + } + return store.List(context.Background(), bucket, normalizePrefix(prefix)+"/") +} diff --git a/internal/aws/s3/snapshot_test.go b/internal/aws/s3/snapshot_test.go new file mode 100644 index 0000000..f67c4ad --- /dev/null +++ b/internal/aws/s3/snapshot_test.go @@ -0,0 +1,81 @@ +package s3 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +type memObjects map[string][]byte + +func (m memObjects) Put(_ context.Context, bucket, key string, body []byte) error { + m[bucket+"/"+key] = append([]byte(nil), body...) + return nil +} + +func (m memObjects) List(_ context.Context, bucket, prefix string) ([]string, error) { + var out []string + root := bucket + "/" + for k := range m { + if !strings.HasPrefix(k, root+prefix) { + continue + } + key := strings.TrimPrefix(k, root) + if strings.HasSuffix(key, ".snap") { + out = append(out, "s3://"+bucket+"/"+key) + } + } + return out, nil +} + +func TestObjectKey(t *testing.T) { + got := ObjectKey("vault-snapshots", "20260101T000000Z") + if got != "vault-snapshots/vault-20260101T000000Z.snap" { + t.Fatalf("key %q", got) + } +} + +func TestListSnapshots(t *testing.T) { + store := memObjects{ + "b/vault-snapshots/vault-1.snap": []byte("a"), + "b/vault-snapshots/vault-1.snap.sha256": []byte("x"), + "b/other/vault-2.snap": []byte("c"), + } + got, err := ListSnapshots(store, "b", "vault-snapshots") + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0] != "s3://b/vault-snapshots/vault-1.snap" { + t.Fatalf("list %v", got) + } +} + +func TestPutSnapshotWritesChecksum(t *testing.T) { + store := memObjects{} + uri, err := PutSnapshot(store, "b", "", []byte("snap")) + if err != nil { + t.Fatal(err) + } + key := strings.TrimPrefix(uri, "s3://b/") + if !strings.HasPrefix(key, "vault-snapshots/vault-") { + t.Fatalf("empty prefix should default: %q", uri) + } + if string(store["b/"+key]) != "snap" { + t.Fatalf("snapshot body %q", store["b/"+key]) + } + sum := sha256.Sum256([]byte("snap")) + if got := string(store["b/"+key+".sha256"]); got != hex.EncodeToString(sum[:])+"\n" { + t.Fatalf("checksum %q", got) + } +} + +func TestPutSnapshotRejectsEmpty(t *testing.T) { + if _, err := PutSnapshot(memObjects{}, "b", "vault-snapshots", nil); err == nil { + t.Fatal("expected empty snapshot to fail") + } + if _, err := PutSnapshot(memObjects{}, "", "vault-snapshots", []byte("x")); err == nil { + t.Fatal("expected missing bucket to fail") + } +} diff --git a/internal/vaultcluster/keystore_aws.go b/internal/aws/secretsmanager/keystore.go similarity index 81% rename from internal/vaultcluster/keystore_aws.go rename to internal/aws/secretsmanager/keystore.go index 9dfba26..5c0ed72 100644 --- a/internal/vaultcluster/keystore_aws.go +++ b/internal/aws/secretsmanager/keystore.go @@ -1,4 +1,4 @@ -package vaultcluster +package secretsmanager import ( "context" @@ -12,19 +12,19 @@ import ( "github.com/hashicorp/vault/api" ) -type secretKV interface { +type SecretStore interface { Get(ctx context.Context, arn string) ([]byte, error) Put(ctx context.Context, arn string, val []byte) error } -type SecretsManagerKeyStore struct { - Secrets secretKV +type KeyStore struct { + Secrets SecretStore InitARN string ProvisioningARN string OperatorARN string } -func NewSecretsManagerKeyStore(initARN, provisioningARN, operatorARN string) (*SecretsManagerKeyStore, error) { +func New(initARN, provisioningARN, operatorARN string) (*KeyStore, error) { if initARN == "" || provisioningARN == "" || operatorARN == "" { return nil, fmt.Errorf("VAULT_INIT_SECRET_ARN, VAULT_PROVISIONING_SECRET_ARN, and VAULT_OPERATOR_SECRET_ARN are required") } @@ -32,7 +32,7 @@ func NewSecretsManagerKeyStore(initARN, provisioningARN, operatorARN string) (*S if err != nil { return nil, fmt.Errorf("AWS credentials: %w", err) } - return &SecretsManagerKeyStore{ + return &KeyStore{ Secrets: smClient{inner: secretsmanager.NewFromConfig(cfg)}, InitARN: initARN, ProvisioningARN: provisioningARN, @@ -40,7 +40,7 @@ func NewSecretsManagerKeyStore(initARN, provisioningARN, operatorARN string) (*S }, nil } -func (s SecretsManagerKeyStore) tokenARN(name string) (string, error) { +func (s KeyStore) tokenARN(name string) (string, error) { switch name { case "provisioning": return s.ProvisioningARN, nil @@ -51,7 +51,7 @@ func (s SecretsManagerKeyStore) tokenARN(name string) (string, error) { } } -func (s SecretsManagerKeyStore) SaveInit(resp *api.InitResponse) error { +func (s KeyStore) SaveInit(resp *api.InitResponse) error { b, err := json.Marshal(resp) if err != nil { return err @@ -59,7 +59,7 @@ func (s SecretsManagerKeyStore) SaveInit(resp *api.InitResponse) error { return s.Secrets.Put(context.Background(), s.InitARN, b) } -func (s SecretsManagerKeyStore) LoadInit() (*api.InitResponse, error) { +func (s KeyStore) LoadInit() (*api.InitResponse, error) { raw, err := s.Secrets.Get(context.Background(), s.InitARN) if err != nil { return nil, err @@ -71,7 +71,7 @@ func (s SecretsManagerKeyStore) LoadInit() (*api.InitResponse, error) { return &resp, nil } -func (s SecretsManagerKeyStore) SaveToken(name, token string) error { +func (s KeyStore) SaveToken(name, token string) error { arn, err := s.tokenARN(name) if err != nil { return err @@ -79,7 +79,7 @@ func (s SecretsManagerKeyStore) SaveToken(name, token string) error { return s.Secrets.Put(context.Background(), arn, []byte(token)) } -func (s SecretsManagerKeyStore) LoadToken(name string) (string, error) { +func (s KeyStore) LoadToken(name string) (string, error) { arn, err := s.tokenARN(name) if err != nil { return "", err diff --git a/internal/vaultcluster/keystore_aws_test.go b/internal/aws/secretsmanager/keystore_test.go similarity index 59% rename from internal/vaultcluster/keystore_aws_test.go rename to internal/aws/secretsmanager/keystore_test.go index 20dde03..67c180b 100644 --- a/internal/vaultcluster/keystore_aws_test.go +++ b/internal/aws/secretsmanager/keystore_test.go @@ -1,4 +1,4 @@ -package vaultcluster +package secretsmanager import ( "context" @@ -26,8 +26,8 @@ func (m memSecrets) Put(_ context.Context, arn string, val []byte) error { return nil } -func testSMStore(m memSecrets) SecretsManagerKeyStore { - return SecretsManagerKeyStore{ +func testStore(m memSecrets) KeyStore { + return KeyStore{ Secrets: m, InitARN: "arn:init", ProvisioningARN: "arn:provisioning", @@ -35,8 +35,8 @@ func testSMStore(m memSecrets) SecretsManagerKeyStore { } } -func TestSecretsManagerKeyStoreRoundTrip(t *testing.T) { - store := testSMStore(memSecrets{}) +func TestKeyStoreRoundTrip(t *testing.T) { + store := testStore(memSecrets{}) init := &api.InitResponse{RootToken: "hvs.root", RecoveryKeysB64: []string{"abc"}} if err := store.SaveInit(init); err != nil { t.Fatal(err) @@ -60,8 +60,8 @@ func TestSecretsManagerKeyStoreRoundTrip(t *testing.T) { } } -func TestSecretsManagerKeyStoreFailClosed(t *testing.T) { - store := testSMStore(memSecrets{}) +func TestKeyStoreFailClosed(t *testing.T) { + store := testStore(memSecrets{}) if _, err := store.LoadInit(); err == nil { t.Fatal("expected missing init to fail") } @@ -77,28 +77,8 @@ func TestSecretsManagerKeyStoreFailClosed(t *testing.T) { } } -func TestNewSecretsManagerKeyStoreRequiresARNs(t *testing.T) { - if _, err := NewSecretsManagerKeyStore("", "a", "b"); err == nil { +func TestNewRequiresARNs(t *testing.T) { + if _, err := New("", "a", "b"); err == nil { t.Fatal("expected error") } } - -func TestInitRequestAutoUnsealUsesRecovery(t *testing.T) { - req := initRequest(BootstrapOptions{Shares: 1, Threshold: 1, AutoUnseal: true}) - if req.RecoveryShares != 1 || req.RecoveryThreshold != 1 { - t.Fatalf("recovery: %+v", req) - } - if req.SecretShares != 0 || req.SecretThreshold != 0 { - t.Fatalf("shamir should be unset: %+v", req) - } -} - -func TestInitRequestLocalUsesShamir(t *testing.T) { - req := initRequest(BootstrapOptions{Shares: 5, Threshold: 3}) - if req.SecretShares != 5 || req.SecretThreshold != 3 { - t.Fatalf("shamir: %+v", req) - } - if req.RecoveryShares != 0 { - t.Fatalf("recovery should be unset: %+v", req) - } -} diff --git a/internal/vaultcluster/bootstrap_test.go b/internal/vaultcluster/bootstrap_test.go new file mode 100644 index 0000000..6a39c1c --- /dev/null +++ b/internal/vaultcluster/bootstrap_test.go @@ -0,0 +1,23 @@ +package vaultcluster + +import "testing" + +func TestInitRequestAutoUnsealUsesRecovery(t *testing.T) { + req := initRequest(BootstrapOptions{Shares: 1, Threshold: 1, AutoUnseal: true}) + if req.RecoveryShares != 1 || req.RecoveryThreshold != 1 { + t.Fatalf("recovery: %+v", req) + } + if req.SecretShares != 0 || req.SecretThreshold != 0 { + t.Fatalf("shamir should be unset: %+v", req) + } +} + +func TestInitRequestLocalUsesShamir(t *testing.T) { + req := initRequest(BootstrapOptions{Shares: 5, Threshold: 3}) + if req.SecretShares != 5 || req.SecretThreshold != 3 { + t.Fatalf("shamir: %+v", req) + } + if req.RecoveryShares != 0 { + t.Fatalf("recovery should be unset: %+v", req) + } +} diff --git a/internal/vaultcluster/snapshot.go b/internal/vaultcluster/snapshot.go index 946c2ff..73fffce 100644 --- a/internal/vaultcluster/snapshot.go +++ b/internal/vaultcluster/snapshot.go @@ -12,7 +12,7 @@ import ( "time" ) -func (c *Client) raftSnapshot() ([]byte, error) { +func (c *Client) RaftSnapshot() ([]byte, error) { req := c.API.NewRequest("GET", "/v1/sys/storage/raft/snapshot") resp, err := c.API.RawRequest(req) if err != nil { @@ -34,7 +34,7 @@ func snapshotStamp() string { } func (c *Client) SnapshotTake(dir string) (string, error) { - b, err := c.raftSnapshot() + b, err := c.RaftSnapshot() if err != nil { return "", err } diff --git a/internal/vaultcluster/snapshot_aws.go b/internal/vaultcluster/snapshot_aws.go deleted file mode 100644 index 556365f..0000000 --- a/internal/vaultcluster/snapshot_aws.go +++ /dev/null @@ -1,99 +0,0 @@ -package vaultcluster - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "strings" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" -) - -type objectStore interface { - Put(ctx context.Context, bucket, key string, body []byte) error - List(ctx context.Context, bucket, prefix string) ([]string, error) -} - -type s3Store struct { - inner *s3.Client -} - -func NewS3ObjectStore() (objectStore, error) { - cfg, err := config.LoadDefaultConfig(context.Background()) - if err != nil { - return nil, fmt.Errorf("AWS credentials: %w", err) - } - return s3Store{inner: s3.NewFromConfig(cfg)}, nil -} - -func (s s3Store) Put(ctx context.Context, bucket, key string, body []byte) error { - _, err := s.inner.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - Body: bytes.NewReader(body), - }) - if err != nil { - return fmt.Errorf("s3 put s3://%s/%s: %w", bucket, key, err) - } - return nil -} - -func (s s3Store) List(ctx context.Context, bucket, prefix string) ([]string, error) { - out, err := s.inner.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(bucket), - Prefix: aws.String(prefix), - }) - if err != nil { - return nil, fmt.Errorf("s3 list s3://%s/%s: %w", bucket, prefix, err) - } - var keys []string - for _, obj := range out.Contents { - if obj.Key == nil || !strings.HasSuffix(*obj.Key, ".snap") { - continue - } - keys = append(keys, "s3://"+bucket+"/"+*obj.Key) - } - return keys, nil -} - -func snapshotObjectKey(prefix, stamp string) string { - p := strings.Trim(prefix, "/") - if p == "" { - p = "vault-snapshots" - } - return p + "/vault-" + stamp + ".snap" -} - -func (c *Client) SnapshotTakeS3(store objectStore, bucket, prefix string) (string, error) { - if bucket == "" { - return "", fmt.Errorf("SNAPSHOT_BUCKET is not set") - } - b, err := c.raftSnapshot() - if err != nil { - return "", err - } - key := snapshotObjectKey(prefix, snapshotStamp()) - if err := store.Put(context.Background(), bucket, key, b); err != nil { - return "", err - } - sum := sha256.Sum256(b) - if err := store.Put(context.Background(), bucket, key+".sha256", []byte(hex.EncodeToString(sum[:])+"\n")); err != nil { - return "", err - } - return "s3://" + bucket + "/" + key, nil -} - -func SnapshotListS3(store objectStore, bucket, prefix string) ([]string, error) { - if bucket == "" { - return nil, fmt.Errorf("SNAPSHOT_BUCKET is not set") - } - p := strings.Trim(prefix, "/") - if p == "" { - p = "vault-snapshots" - } - return store.List(context.Background(), bucket, p+"/") -} diff --git a/internal/vaultcluster/snapshot_aws_test.go b/internal/vaultcluster/snapshot_aws_test.go deleted file mode 100644 index 3915bea..0000000 --- a/internal/vaultcluster/snapshot_aws_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package vaultcluster - -import ( - "context" - "strings" - "testing" -) - -type memObjects map[string][]byte - -func (m memObjects) Put(_ context.Context, bucket, key string, body []byte) error { - m[bucket+"/"+key] = append([]byte(nil), body...) - return nil -} - -func (m memObjects) List(_ context.Context, bucket, prefix string) ([]string, error) { - var out []string - root := bucket + "/" - for k := range m { - if !strings.HasPrefix(k, root+prefix) { - continue - } - key := strings.TrimPrefix(k, root) - if strings.HasSuffix(key, ".snap") { - out = append(out, "s3://"+bucket+"/"+key) - } - } - return out, nil -} - -func TestSnapshotObjectKey(t *testing.T) { - got := snapshotObjectKey("vault-snapshots", "20260101T000000Z") - if got != "vault-snapshots/vault-20260101T000000Z.snap" { - t.Fatalf("key %q", got) - } -} - -func TestSnapshotListS3(t *testing.T) { - store := memObjects{ - "b/vault-snapshots/vault-1.snap": []byte("a"), - "b/vault-snapshots/vault-1.snap.sha256": []byte("x"), - "b/other/vault-2.snap": []byte("c"), - } - got, err := SnapshotListS3(store, "b", "vault-snapshots") - if err != nil { - t.Fatal(err) - } - if len(got) != 1 || got[0] != "s3://b/vault-snapshots/vault-1.snap" { - t.Fatalf("list %v", got) - } -} - -func TestParseBackupSchedule(t *testing.T) { - off, err := ParseBackupSchedule("") - if err != nil || off != nil { - t.Fatalf("empty: %v %v", off, err) - } - on, err := ParseBackupSchedule("0 3 * * *") - if err != nil || on == nil { - t.Fatalf("cron: %v %v", on, err) - } - if _, err := ParseBackupSchedule("not-a-cron"); err == nil { - t.Fatal("expected invalid cron to fail") - } -} diff --git a/internal/vaultcluster/snapshot_schedule_test.go b/internal/vaultcluster/snapshot_schedule_test.go new file mode 100644 index 0000000..cfddd31 --- /dev/null +++ b/internal/vaultcluster/snapshot_schedule_test.go @@ -0,0 +1,17 @@ +package vaultcluster + +import "testing" + +func TestParseBackupSchedule(t *testing.T) { + off, err := ParseBackupSchedule("") + if err != nil || off != nil { + t.Fatalf("empty: %v %v", off, err) + } + on, err := ParseBackupSchedule("0 3 * * *") + if err != nil || on == nil { + t.Fatalf("cron: %v %v", on, err) + } + if _, err := ParseBackupSchedule("not-a-cron"); err == nil { + t.Fatal("expected invalid cron to fail") + } +}