Harden autoupdate deploy and recover the RustFS bucket

file-service/main.go

@@ -13,6 +13,7 @@ import (
 	"regexp"
 	"slices"
 	"strconv"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -35,6 +36,8 @@ type server struct {
 	echo       *echo.Echo
 	httpServer *http.Server
 	s3Client   *minio.Client
+	// lifecycleSet is false until the bucket expiry rule has been applied.
+	lifecycleSet atomic.Bool
 }
 
 const BUCKET_NAME = "file-uploads"
@@ -68,34 +71,14 @@ func newServer() (*server, error) {
 	if err != nil {
 		return nil, fmt.Errorf("could not init S3 client: %w", err)
 	}
-	err = s3Client.MakeBucket(context.Background(), BUCKET_NAME, minio.MakeBucketOptions{})
-	if err != nil {
-		// Check to see if we already own this bucket (which happens if you run this twice)
-		exists, errBucketExists := s3Client.BucketExists(context.Background(), BUCKET_NAME)
-		if errBucketExists == nil && exists {
-			log.Printf("We already own %s\n", BUCKET_NAME)
-		} else {
-			return nil, fmt.Errorf("could not check if bucket exists: %w", err)
-		}
-	} else {
-		log.Printf("Successfully created bucket %s\n", BUCKET_NAME)
-		config := lifecycle.NewConfiguration()
-		config.Rules = []lifecycle.Rule{
-			{
-				ID:     "expire-bucket",
-				Status: "Enabled",
-				Expiration: lifecycle.Expiration{
-					Days: 1,
-				},
-			},
-		}
-		if err := s3Client.SetBucketLifecycle(context.Background(), BUCKET_NAME, config); err != nil {
-			return nil, fmt.Errorf("could not set bucket lifecycle rule: %w", err)
-		}
-	}
 	s := &server{
 		s3Client: s3Client,
 	}
+	// Storage may not be up yet (e.g. RustFS starting in parallel). Keep running;
+	// the health check reports not ready and retries until the bucket exists.
+	if err := s.ensureBucket(context.Background()); err != nil {
+		log.Printf("storage not ready yet: %v", err)
+	}
 
 	s.echo = echo.New()
 	s.echo.HTTPErrorHandler = echoutils.HTTPErrorHandler(s.echo)
@@ -229,7 +212,57 @@ func (s *server) handleDownloadFile(c *echo.Context) error {
 	return c.Stream(http.StatusOK, stat.ContentType, obj)
 }
 
+// ensureBucket creates the upload bucket if it does not exist yet and makes
+// sure its expiry rule is set.
+func (s *server) ensureBucket(ctx context.Context) error {
+	err := s.s3Client.MakeBucket(ctx, BUCKET_NAME, minio.MakeBucketOptions{})
+	if err != nil {
+		// Check to see if we already own this bucket (which happens if you run this twice)
+		exists, errBucketExists := s.s3Client.BucketExists(ctx, BUCKET_NAME)
+		if errBucketExists != nil {
+			return fmt.Errorf("could not create bucket: %w; could not check if bucket exists: %w", err, errBucketExists)
+		}
+		if !exists {
+			return fmt.Errorf("could not create bucket: %w", err)
+		}
+		log.Printf("We already own %s\n", BUCKET_NAME)
+	} else {
+		log.Printf("Successfully created bucket %s\n", BUCKET_NAME)
+		s.lifecycleSet.Store(false)
+	}
+	if s.lifecycleSet.Load() {
+		return nil
+	}
+	config := lifecycle.NewConfiguration()
+	config.Rules = []lifecycle.Rule{
+		{
+			ID:     "expire-bucket",
+			Status: "Enabled",
+			Expiration: lifecycle.Expiration{
+				Days: 1,
+			},
+		},
+	}
+	if err := s.s3Client.SetBucketLifecycle(ctx, BUCKET_NAME, config); err != nil {
+		return fmt.Errorf("could not set bucket lifecycle rule: %w", err)
+	}
+	s.lifecycleSet.Store(true)
+	return nil
+}
+
+// handleHealth reports unhealthy while object storage is unreachable. RustFS
+// keeps its data in an emptyDir, so the bucket is recreated after a restart.
 func (s *server) handleHealth(c *echo.Context) error {
+	ctx, cancel := context.WithTimeout(c.Request().Context(), 3*time.Second)
+	defer cancel()
+	exists, err := s.s3Client.BucketExists(ctx, BUCKET_NAME)
+	if err == nil && (!exists || !s.lifecycleSet.Load()) {
+		err = s.ensureBucket(ctx)
+	}
+	if err != nil {
+		log.Printf("health check failed: %v", err)
+		return c.String(http.StatusServiceUnavailable, "storage unavailable")
+	}
 	return c.String(http.StatusOK, "OK")
 }
 

k8/file-deployment.yaml.tpl

@@ -37,12 +37,13 @@ spec:
           ports:
             - name: http
               containerPort: 8080
-          # The HTTP server only starts after the storage bucket check passed, so
-          # a pod that cannot reach RustFS never becomes ready.
+          # /api/v1/health checks (and recreates) the RustFS bucket, so the pod
+          # stays up but not ready while object storage is unavailable.
           readinessProbe:
             httpGet:
               path: /api/v1/health
               port: http
             initialDelaySeconds: 2
             periodSeconds: 5
+            timeoutSeconds: 5
       restartPolicy: Always

upgrade_k3s_containers.sh

@@ -177,6 +177,20 @@ if [[ -z "$CONTINUE_SHA" ]]; then
   exec "$REPO/upgrade_k3s_containers.sh" --continue "$AFTER"
 fi
 
+# Without this, manifest changes (new services, env vars, ingress routes) never
+# reach the cluster and only image updates would be deployed. Generate before
+# pulling images so a missing or broken env file fails fast. The env file is
+# only sourced in the subshell so it cannot override this script's variables.
+[[ -f "$ENV_FILE" ]] || die "env file ${ENV_FILE} not found; see README.md for the required variables"
+log "generating k8/ manifests with ${ENV_FILE}"
+(
+  set -a
+  # shellcheck source=/dev/null
+  . "$ENV_FILE"
+  set +a
+  cd "$REPO" && bash k8/generate.sh
+) || die "k8/generate.sh failed"
+
 CRICTL=(k3s crictl)
 if ! command -v k3s >/dev/null; then
   CRICTL=(crictl)
@@ -187,24 +201,16 @@ for image in "${IMAGES[@]}"; do
   "${CRICTL[@]}" pull "$image"
 done
 
-# Without this, manifest changes (new services, env vars, ingress routes) never
-# reach the cluster and only image updates would be deployed.
-[[ -f "$ENV_FILE" ]] || die "env file ${ENV_FILE} not found; see README.md for the required variables"
-log "generating k8/ manifests with ${ENV_FILE}"
-set -a
-# shellcheck source=/dev/null
-. "$ENV_FILE"
-set +a
-(cd "$REPO" && bash k8/generate.sh) || die "k8/generate.sh failed"
-
 log "applying k8/ manifests"
 kubectl apply -f "$REPO/k8/"
 log "deleting retired resources: ${RETIRED_RESOURCES[*]}"
 kubectl delete "${RETIRED_RESOURCES[@]}" --ignore-not-found
 kubectl rollout status deployment/rustfs --timeout=180s
 
 # Delete workers before bouncing control so the new control process creates a
-# fresh pool. Do not delete workers after control is ready.
+# fresh pool. Do not delete workers after control is ready. This stays after
+# the apply: if apply fails, the old control keeps its working workers. A
+# control rollout started by apply is superseded by the restart below.
 log "removing worker pods before control restart"
 kubectl delete pod -l role=worker --wait=false || true
 
@@ -231,7 +237,7 @@ fi
 HEALTH_URL="${BASE_URL}/service/control/health"
 
 log "checking health at ${HEALTH_URL}"
-curl -fsS --retry 12 --retry-all-errors --retry-delay 5 "$HEALTH_URL" >/dev/null \
+curl -fsS --max-time 10 --retry 12 --retry-all-errors --retry-delay 5 "$HEALTH_URL" >/dev/null \
   || die "health check failed"
 
 # A missing object returns 404 only when file-service can reach object storage;
@@ -240,7 +246,7 @@ STORAGE_URL="${BASE_URL}/file-uploads/00000000-0000-0000-0000-000000000000.png"
 log "checking object storage via ${STORAGE_URL}"
 storage_status=""
 for _ in $(seq 1 12); do
-  storage_status="$(curl -s -o /dev/null -w '%{http_code}' -I "$STORAGE_URL" || true)"
+  storage_status="$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' -I "$STORAGE_URL" || true)"
   [[ "$storage_status" == "404" ]] && break
   sleep 5
 done