From f29722920792f0d9cb4fb98ebc3263e5f0acf3ac Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Mon, 26 Oct 2026 06:24:40 -0700 Subject: [PATCH 1/2] Harden autoupdate deploy and recover the RustFS bucket Follow-ups to #338: - file-service health checks the bucket and recreates it when missing (RustFS uses an emptyDir, so a RustFS restart drops the bucket) - readiness probe timeout raised to fit the storage check - autoupdate: generate manifests before pulling images, source the env file only in that subshell, delete workers before applying manifests, and time-box the health check requests Co-Authored-By: Claude Opus 5.5 --- file-service/main.go | 69 +++++++++++++++++++++++-------------- k8/file-deployment.yaml.tpl | 5 +-- upgrade_k3s_containers.sh | 37 +++++++++++--------- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/file-service/main.go b/file-service/main.go index 85d1dc2..87f3fa6 100644 --- a/file-service/main.go +++ b/file-service/main.go @@ -68,34 +68,12 @@ func newServer() (*server, error) { if err != nil { return nil, fmt.Errorf("could not init S3 client: %w", err) } - err = s3Client.MakeBucket(context.Background(), BUCKET_NAME, minio.MakeBucketOptions{}) - if err != nil { - // Check to see if we already own this bucket (which happens if you run this twice) - exists, errBucketExists := s3Client.BucketExists(context.Background(), BUCKET_NAME) - if errBucketExists == nil && exists { - log.Printf("We already own %s\n", BUCKET_NAME) - } else { - return nil, fmt.Errorf("could not check if bucket exists: %w", err) - } - } else { - log.Printf("Successfully created bucket %s\n", BUCKET_NAME) - config := lifecycle.NewConfiguration() - config.Rules = []lifecycle.Rule{ - { - ID: "expire-bucket", - Status: "Enabled", - Expiration: lifecycle.Expiration{ - Days: 1, - }, - }, - } - if err := s3Client.SetBucketLifecycle(context.Background(), BUCKET_NAME, config); err != nil { - return nil, fmt.Errorf("could not set bucket lifecycle rule: %w", err) - } - } s := &server{ s3Client: s3Client, } + if err := s.ensureBucket(context.Background()); err != nil { + return nil, err + } s.echo = echo.New() s.echo.HTTPErrorHandler = echoutils.HTTPErrorHandler(s.echo) @@ -229,7 +207,48 @@ func (s *server) handleDownloadFile(c *echo.Context) error { return c.Stream(http.StatusOK, stat.ContentType, obj) } +// ensureBucket creates the upload bucket with its expiry rule if it does not exist yet. +func (s *server) ensureBucket(ctx context.Context) error { + err := s.s3Client.MakeBucket(ctx, BUCKET_NAME, minio.MakeBucketOptions{}) + if err != nil { + // Check to see if we already own this bucket (which happens if you run this twice) + exists, errBucketExists := s.s3Client.BucketExists(ctx, BUCKET_NAME) + if errBucketExists == nil && exists { + log.Printf("We already own %s\n", BUCKET_NAME) + return nil + } + return fmt.Errorf("could not check if bucket exists: %w", err) + } + log.Printf("Successfully created bucket %s\n", BUCKET_NAME) + config := lifecycle.NewConfiguration() + config.Rules = []lifecycle.Rule{ + { + ID: "expire-bucket", + Status: "Enabled", + Expiration: lifecycle.Expiration{ + Days: 1, + }, + }, + } + if err := s.s3Client.SetBucketLifecycle(ctx, BUCKET_NAME, config); err != nil { + return fmt.Errorf("could not set bucket lifecycle rule: %w", err) + } + return nil +} + +// handleHealth reports unhealthy while object storage is unreachable. RustFS +// keeps its data in an emptyDir, so the bucket is recreated after a restart. func (s *server) handleHealth(c *echo.Context) error { + ctx, cancel := context.WithTimeout(c.Request().Context(), 3*time.Second) + defer cancel() + exists, err := s.s3Client.BucketExists(ctx, BUCKET_NAME) + if err == nil && !exists { + err = s.ensureBucket(ctx) + } + if err != nil { + log.Printf("health check failed: %v", err) + return c.String(http.StatusServiceUnavailable, "storage unavailable") + } return c.String(http.StatusOK, "OK") } diff --git a/k8/file-deployment.yaml.tpl b/k8/file-deployment.yaml.tpl index 76e9d43..70e9800 100644 --- a/k8/file-deployment.yaml.tpl +++ b/k8/file-deployment.yaml.tpl @@ -37,12 +37,13 @@ spec: ports: - name: http containerPort: 8080 - # The HTTP server only starts after the storage bucket check passed, so - # a pod that cannot reach RustFS never becomes ready. + # /api/v1/health checks (and recreates) the RustFS bucket, so the pod + # is not ready while object storage is unavailable. readinessProbe: httpGet: path: /api/v1/health port: http initialDelaySeconds: 2 periodSeconds: 5 + timeoutSeconds: 5 restartPolicy: Always diff --git a/upgrade_k3s_containers.sh b/upgrade_k3s_containers.sh index e9f4106..9c0fadd 100755 --- a/upgrade_k3s_containers.sh +++ b/upgrade_k3s_containers.sh @@ -177,6 +177,20 @@ if [[ -z "$CONTINUE_SHA" ]]; then exec "$REPO/upgrade_k3s_containers.sh" --continue "$AFTER" fi +# Without this, manifest changes (new services, env vars, ingress routes) never +# reach the cluster and only image updates would be deployed. Generate before +# pulling images so a missing or broken env file fails fast. The env file is +# only sourced in the subshell so it cannot override this script's variables. +[[ -f "$ENV_FILE" ]] || die "env file ${ENV_FILE} not found; see README.md for the required variables" +log "generating k8/ manifests with ${ENV_FILE}" +( + set -a + # shellcheck source=/dev/null + . "$ENV_FILE" + set +a + cd "$REPO" && bash k8/generate.sh +) || die "k8/generate.sh failed" + CRICTL=(k3s crictl) if ! command -v k3s >/dev/null; then CRICTL=(crictl) @@ -187,15 +201,11 @@ for image in "${IMAGES[@]}"; do "${CRICTL[@]}" pull "$image" done -# Without this, manifest changes (new services, env vars, ingress routes) never -# reach the cluster and only image updates would be deployed. -[[ -f "$ENV_FILE" ]] || die "env file ${ENV_FILE} not found; see README.md for the required variables" -log "generating k8/ manifests with ${ENV_FILE}" -set -a -# shellcheck source=/dev/null -. "$ENV_FILE" -set +a -(cd "$REPO" && bash k8/generate.sh) || die "k8/generate.sh failed" +# Delete workers before bouncing control (including a rollout triggered by the +# apply below) so the new control process creates a fresh pool. Do not delete +# workers after control is ready. +log "removing worker pods before control restart" +kubectl delete pod -l role=worker --wait=false || true log "applying k8/ manifests" kubectl apply -f "$REPO/k8/" @@ -203,11 +213,6 @@ log "deleting retired resources: ${RETIRED_RESOURCES[*]}" kubectl delete "${RETIRED_RESOURCES[@]}" --ignore-not-found kubectl rollout status deployment/rustfs --timeout=180s -# Delete workers before bouncing control so the new control process creates a -# fresh pool. Do not delete workers after control is ready. -log "removing worker pods before control restart" -kubectl delete pod -l role=worker --wait=false || true - log "restarting deployments: ${APP_DEPLOYMENTS[*]}" restart_args=() for deploy in "${APP_DEPLOYMENTS[@]}"; do @@ -231,7 +236,7 @@ fi HEALTH_URL="${BASE_URL}/service/control/health" log "checking health at ${HEALTH_URL}" -curl -fsS --retry 12 --retry-all-errors --retry-delay 5 "$HEALTH_URL" >/dev/null \ +curl -fsS --max-time 10 --retry 12 --retry-all-errors --retry-delay 5 "$HEALTH_URL" >/dev/null \ || die "health check failed" # A missing object returns 404 only when file-service can reach object storage; @@ -240,7 +245,7 @@ STORAGE_URL="${BASE_URL}/file-uploads/00000000-0000-0000-0000-000000000000.png" log "checking object storage via ${STORAGE_URL}" storage_status="" for _ in $(seq 1 12); do - storage_status="$(curl -s -o /dev/null -w '%{http_code}' -I "$STORAGE_URL" || true)" + storage_status="$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' -I "$STORAGE_URL" || true)" [[ "$storage_status" == "404" ]] && break sleep 5 done From 25279a605a48f2eba0b9d0900b9a69b149c1489d Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Mon, 26 Oct 2026 06:27:25 -0700 Subject: [PATCH 2/2] Address review: keep worker deletion after apply, retry expiry rule - Revert deleting workers before apply: a failing apply or rustfs wait would leave control with only deleted workers - Retry the bucket lifecycle rule until it is set - Don't exit on startup when storage is not reachable yet; stay up and let the readiness probe report it - Keep both errors when bucket creation and the existence check fail Co-Authored-By: Claude Opus 5.5 --- file-service/main.go | 30 ++++++++++++++++++++++-------- k8/file-deployment.yaml.tpl | 2 +- upgrade_k3s_containers.sh | 13 +++++++------ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/file-service/main.go b/file-service/main.go index 87f3fa6..97929e3 100644 --- a/file-service/main.go +++ b/file-service/main.go @@ -13,6 +13,7 @@ import ( "regexp" "slices" "strconv" + "sync/atomic" "syscall" "time" @@ -35,6 +36,8 @@ type server struct { echo *echo.Echo httpServer *http.Server s3Client *minio.Client + // lifecycleSet is false until the bucket expiry rule has been applied. + lifecycleSet atomic.Bool } const BUCKET_NAME = "file-uploads" @@ -71,8 +74,10 @@ func newServer() (*server, error) { s := &server{ s3Client: s3Client, } + // Storage may not be up yet (e.g. RustFS starting in parallel). Keep running; + // the health check reports not ready and retries until the bucket exists. if err := s.ensureBucket(context.Background()); err != nil { - return nil, err + log.Printf("storage not ready yet: %v", err) } s.echo = echo.New() @@ -207,19 +212,27 @@ func (s *server) handleDownloadFile(c *echo.Context) error { return c.Stream(http.StatusOK, stat.ContentType, obj) } -// ensureBucket creates the upload bucket with its expiry rule if it does not exist yet. +// ensureBucket creates the upload bucket if it does not exist yet and makes +// sure its expiry rule is set. func (s *server) ensureBucket(ctx context.Context) error { err := s.s3Client.MakeBucket(ctx, BUCKET_NAME, minio.MakeBucketOptions{}) if err != nil { // Check to see if we already own this bucket (which happens if you run this twice) exists, errBucketExists := s.s3Client.BucketExists(ctx, BUCKET_NAME) - if errBucketExists == nil && exists { - log.Printf("We already own %s\n", BUCKET_NAME) - return nil + if errBucketExists != nil { + return fmt.Errorf("could not create bucket: %w; could not check if bucket exists: %w", err, errBucketExists) } - return fmt.Errorf("could not check if bucket exists: %w", err) + if !exists { + return fmt.Errorf("could not create bucket: %w", err) + } + log.Printf("We already own %s\n", BUCKET_NAME) + } else { + log.Printf("Successfully created bucket %s\n", BUCKET_NAME) + s.lifecycleSet.Store(false) + } + if s.lifecycleSet.Load() { + return nil } - log.Printf("Successfully created bucket %s\n", BUCKET_NAME) config := lifecycle.NewConfiguration() config.Rules = []lifecycle.Rule{ { @@ -233,6 +246,7 @@ func (s *server) ensureBucket(ctx context.Context) error { if err := s.s3Client.SetBucketLifecycle(ctx, BUCKET_NAME, config); err != nil { return fmt.Errorf("could not set bucket lifecycle rule: %w", err) } + s.lifecycleSet.Store(true) return nil } @@ -242,7 +256,7 @@ func (s *server) handleHealth(c *echo.Context) error { ctx, cancel := context.WithTimeout(c.Request().Context(), 3*time.Second) defer cancel() exists, err := s.s3Client.BucketExists(ctx, BUCKET_NAME) - if err == nil && !exists { + if err == nil && (!exists || !s.lifecycleSet.Load()) { err = s.ensureBucket(ctx) } if err != nil { diff --git a/k8/file-deployment.yaml.tpl b/k8/file-deployment.yaml.tpl index 70e9800..aa7a1c4 100644 --- a/k8/file-deployment.yaml.tpl +++ b/k8/file-deployment.yaml.tpl @@ -38,7 +38,7 @@ spec: - name: http containerPort: 8080 # /api/v1/health checks (and recreates) the RustFS bucket, so the pod - # is not ready while object storage is unavailable. + # stays up but not ready while object storage is unavailable. readinessProbe: httpGet: path: /api/v1/health diff --git a/upgrade_k3s_containers.sh b/upgrade_k3s_containers.sh index 9c0fadd..433fdc8 100755 --- a/upgrade_k3s_containers.sh +++ b/upgrade_k3s_containers.sh @@ -201,18 +201,19 @@ for image in "${IMAGES[@]}"; do "${CRICTL[@]}" pull "$image" done -# Delete workers before bouncing control (including a rollout triggered by the -# apply below) so the new control process creates a fresh pool. Do not delete -# workers after control is ready. -log "removing worker pods before control restart" -kubectl delete pod -l role=worker --wait=false || true - log "applying k8/ manifests" kubectl apply -f "$REPO/k8/" log "deleting retired resources: ${RETIRED_RESOURCES[*]}" kubectl delete "${RETIRED_RESOURCES[@]}" --ignore-not-found kubectl rollout status deployment/rustfs --timeout=180s +# Delete workers before bouncing control so the new control process creates a +# fresh pool. Do not delete workers after control is ready. This stays after +# the apply: if apply fails, the old control keeps its working workers. A +# control rollout started by apply is superseded by the restart below. +log "removing worker pods before control restart" +kubectl delete pod -l role=worker --wait=false || true + log "restarting deployments: ${APP_DEPLOYMENTS[*]}" restart_args=() for deploy in "${APP_DEPLOYMENTS[@]}"; do