diff --git a/file-service/main.go b/file-service/main.go index 85d1dc2..97929e3 100644 --- a/file-service/main.go +++ b/file-service/main.go @@ -13,6 +13,7 @@ import ( "regexp" "slices" "strconv" + "sync/atomic" "syscall" "time" @@ -35,6 +36,8 @@ type server struct { echo *echo.Echo httpServer *http.Server s3Client *minio.Client + // lifecycleSet is false until the bucket expiry rule has been applied. + lifecycleSet atomic.Bool } const BUCKET_NAME = "file-uploads" @@ -68,34 +71,14 @@ func newServer() (*server, error) { if err != nil { return nil, fmt.Errorf("could not init S3 client: %w", err) } - err = s3Client.MakeBucket(context.Background(), BUCKET_NAME, minio.MakeBucketOptions{}) - if err != nil { - // Check to see if we already own this bucket (which happens if you run this twice) - exists, errBucketExists := s3Client.BucketExists(context.Background(), BUCKET_NAME) - if errBucketExists == nil && exists { - log.Printf("We already own %s\n", BUCKET_NAME) - } else { - return nil, fmt.Errorf("could not check if bucket exists: %w", err) - } - } else { - log.Printf("Successfully created bucket %s\n", BUCKET_NAME) - config := lifecycle.NewConfiguration() - config.Rules = []lifecycle.Rule{ - { - ID: "expire-bucket", - Status: "Enabled", - Expiration: lifecycle.Expiration{ - Days: 1, - }, - }, - } - if err := s3Client.SetBucketLifecycle(context.Background(), BUCKET_NAME, config); err != nil { - return nil, fmt.Errorf("could not set bucket lifecycle rule: %w", err) - } - } s := &server{ s3Client: s3Client, } + // Storage may not be up yet (e.g. RustFS starting in parallel). Keep running; + // the health check reports not ready and retries until the bucket exists. + if err := s.ensureBucket(context.Background()); err != nil { + log.Printf("storage not ready yet: %v", err) + } s.echo = echo.New() s.echo.HTTPErrorHandler = echoutils.HTTPErrorHandler(s.echo) @@ -229,7 +212,57 @@ func (s *server) handleDownloadFile(c *echo.Context) error { return c.Stream(http.StatusOK, stat.ContentType, obj) } +// ensureBucket creates the upload bucket if it does not exist yet and makes +// sure its expiry rule is set. +func (s *server) ensureBucket(ctx context.Context) error { + err := s.s3Client.MakeBucket(ctx, BUCKET_NAME, minio.MakeBucketOptions{}) + if err != nil { + // Check to see if we already own this bucket (which happens if you run this twice) + exists, errBucketExists := s.s3Client.BucketExists(ctx, BUCKET_NAME) + if errBucketExists != nil { + return fmt.Errorf("could not create bucket: %w; could not check if bucket exists: %w", err, errBucketExists) + } + if !exists { + return fmt.Errorf("could not create bucket: %w", err) + } + log.Printf("We already own %s\n", BUCKET_NAME) + } else { + log.Printf("Successfully created bucket %s\n", BUCKET_NAME) + s.lifecycleSet.Store(false) + } + if s.lifecycleSet.Load() { + return nil + } + config := lifecycle.NewConfiguration() + config.Rules = []lifecycle.Rule{ + { + ID: "expire-bucket", + Status: "Enabled", + Expiration: lifecycle.Expiration{ + Days: 1, + }, + }, + } + if err := s.s3Client.SetBucketLifecycle(ctx, BUCKET_NAME, config); err != nil { + return fmt.Errorf("could not set bucket lifecycle rule: %w", err) + } + s.lifecycleSet.Store(true) + return nil +} + +// handleHealth reports unhealthy while object storage is unreachable. RustFS +// keeps its data in an emptyDir, so the bucket is recreated after a restart. func (s *server) handleHealth(c *echo.Context) error { + ctx, cancel := context.WithTimeout(c.Request().Context(), 3*time.Second) + defer cancel() + exists, err := s.s3Client.BucketExists(ctx, BUCKET_NAME) + if err == nil && (!exists || !s.lifecycleSet.Load()) { + err = s.ensureBucket(ctx) + } + if err != nil { + log.Printf("health check failed: %v", err) + return c.String(http.StatusServiceUnavailable, "storage unavailable") + } return c.String(http.StatusOK, "OK") } diff --git a/k8/file-deployment.yaml.tpl b/k8/file-deployment.yaml.tpl index 76e9d43..aa7a1c4 100644 --- a/k8/file-deployment.yaml.tpl +++ b/k8/file-deployment.yaml.tpl @@ -37,12 +37,13 @@ spec: ports: - name: http containerPort: 8080 - # The HTTP server only starts after the storage bucket check passed, so - # a pod that cannot reach RustFS never becomes ready. + # /api/v1/health checks (and recreates) the RustFS bucket, so the pod + # stays up but not ready while object storage is unavailable. readinessProbe: httpGet: path: /api/v1/health port: http initialDelaySeconds: 2 periodSeconds: 5 + timeoutSeconds: 5 restartPolicy: Always diff --git a/upgrade_k3s_containers.sh b/upgrade_k3s_containers.sh index e9f4106..433fdc8 100755 --- a/upgrade_k3s_containers.sh +++ b/upgrade_k3s_containers.sh @@ -177,6 +177,20 @@ if [[ -z "$CONTINUE_SHA" ]]; then exec "$REPO/upgrade_k3s_containers.sh" --continue "$AFTER" fi +# Without this, manifest changes (new services, env vars, ingress routes) never +# reach the cluster and only image updates would be deployed. Generate before +# pulling images so a missing or broken env file fails fast. The env file is +# only sourced in the subshell so it cannot override this script's variables. +[[ -f "$ENV_FILE" ]] || die "env file ${ENV_FILE} not found; see README.md for the required variables" +log "generating k8/ manifests with ${ENV_FILE}" +( + set -a + # shellcheck source=/dev/null + . "$ENV_FILE" + set +a + cd "$REPO" && bash k8/generate.sh +) || die "k8/generate.sh failed" + CRICTL=(k3s crictl) if ! command -v k3s >/dev/null; then CRICTL=(crictl) @@ -187,16 +201,6 @@ for image in "${IMAGES[@]}"; do "${CRICTL[@]}" pull "$image" done -# Without this, manifest changes (new services, env vars, ingress routes) never -# reach the cluster and only image updates would be deployed. -[[ -f "$ENV_FILE" ]] || die "env file ${ENV_FILE} not found; see README.md for the required variables" -log "generating k8/ manifests with ${ENV_FILE}" -set -a -# shellcheck source=/dev/null -. "$ENV_FILE" -set +a -(cd "$REPO" && bash k8/generate.sh) || die "k8/generate.sh failed" - log "applying k8/ manifests" kubectl apply -f "$REPO/k8/" log "deleting retired resources: ${RETIRED_RESOURCES[*]}" @@ -204,7 +208,9 @@ kubectl delete "${RETIRED_RESOURCES[@]}" --ignore-not-found kubectl rollout status deployment/rustfs --timeout=180s # Delete workers before bouncing control so the new control process creates a -# fresh pool. Do not delete workers after control is ready. +# fresh pool. Do not delete workers after control is ready. This stays after +# the apply: if apply fails, the old control keeps its working workers. A +# control rollout started by apply is superseded by the restart below. log "removing worker pods before control restart" kubectl delete pod -l role=worker --wait=false || true @@ -231,7 +237,7 @@ fi HEALTH_URL="${BASE_URL}/service/control/health" log "checking health at ${HEALTH_URL}" -curl -fsS --retry 12 --retry-all-errors --retry-delay 5 "$HEALTH_URL" >/dev/null \ +curl -fsS --max-time 10 --retry 12 --retry-all-errors --retry-delay 5 "$HEALTH_URL" >/dev/null \ || die "health check failed" # A missing object returns 404 only when file-service can reach object storage; @@ -240,7 +246,7 @@ STORAGE_URL="${BASE_URL}/file-uploads/00000000-0000-0000-0000-000000000000.png" log "checking object storage via ${STORAGE_URL}" storage_status="" for _ in $(seq 1 12); do - storage_status="$(curl -s -o /dev/null -w '%{http_code}' -I "$STORAGE_URL" || true)" + storage_status="$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' -I "$STORAGE_URL" || true)" [[ "$storage_status" == "404" ]] && break sleep 5 done