feat: gate backups on repository health from the last check

api/v1/backup_types.go

@@ -49,6 +49,13 @@ type BackupSpec struct {
 	// When defined, only PVCs and PreBackupPods matching them are backed up.
 	// +optional
 	LabelSelectors []metav1.LabelSelector `json:"labelSelectors,omitempty"`
+
+	// SkipOnFailedCheck skips the backup when the last finished Check of its repository failed,
+	// instead of writing more data into a repository that restic reported as damaged.
+	// The Backup then completes with reason RepositoryUnhealthy and no backup Job is started.
+	// The operator flag --global-skip-backup-on-failed-check enables this for all Backups.
+	// +optional
+	SkipOnFailedCheck bool `json:"skipOnFailedCheck,omitempty"`
 }
 
 type BackupTemplate struct {

api/v1/check_types.go

@@ -5,6 +5,7 @@ import (
 	"reflect"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -128,6 +129,39 @@ func (c *CheckList) GetJobObjects() JobObjectList {
 	return items
 }
 
+// RepositoryOrDefault returns the restic repository this Check runs against.
+// A Check without a backend runs against the operator's global repository, given as defaultRepository.
+func (c *Check) RepositoryOrDefault(defaultRepository string) string {
+	if c.Spec.Backend == nil {
+		return defaultRepository
+	}
+	return c.Spec.Backend.String()
+}
+
+// FinishedAt returns when the Check completed, successfully or not, or nil if it has not finished yet.
+func (c *Check) FinishedAt() *metav1.Time {
+	if !c.Status.HasFinished() {
+		return nil
+	}
+	completedCond := meta.FindStatusCondition(c.Status.Conditions, ConditionCompleted.String())
+	return &completedCond.LastTransitionTime
+}
+
+// LatestFinished returns the Check that finished most recently, or nil if none of them has finished.
+func (c *CheckList) LatestFinished() *Check {
+	var latest *Check
+	for i := range c.Items {
+		finishedAt := c.Items[i].FinishedAt()
+		if finishedAt == nil {
+			continue
+		}
+		if latest == nil || finishedAt.After(latest.FinishedAt().Time) {
+			latest = &c.Items[i]
+		}
+	}
+	return latest
+}
+
 // GetDeepCopy returns a deep copy
 func (in *CheckSchedule) GetDeepCopy() ScheduleSpecInterface {
 	return in.DeepCopy()

api/v1/check_types_test.go

@@ -0,0 +1,86 @@
+package v1
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+)
+
+func newFinishedCheck(name string, reason ConditionReason, finishedAt time.Time) Check {
+	return Check{
+		ObjectMeta: metav1.ObjectMeta{Name: name},
+		Status: Status{Conditions: []metav1.Condition{{
+			Type:               ConditionCompleted.String(),
+			Status:             metav1.ConditionTrue,
+			Reason:             reason.String(),
+			LastTransitionTime: metav1.NewTime(finishedAt),
+		}}},
+	}
+}
+
+func TestCheck_RepositoryOrDefault(t *testing.T) {
+	check := Check{}
+	assert.Equal(t, "s3:https://global/bucket", check.RepositoryOrDefault("s3:https://global/bucket"))
+
+	check.Spec.Backend = &Backend{S3: &S3Spec{Endpoint: "https://endpoint", Bucket: "bucket"}}
+	assert.Equal(t, "s3:https://endpoint/bucket", check.RepositoryOrDefault("s3:https://global/bucket"))
+}
+
+func TestCheck_FinishedAt(t *testing.T) {
+	finishedAt := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
+
+	running := Check{}
+	running.Status.SetStarted("running")
+	assert.Nil(t, running.FinishedAt())
+
+	failed := newFinishedCheck("failed", ReasonFailed, finishedAt)
+	assert.Equal(t, finishedAt, failed.FinishedAt().Time)
+}
+
+func TestCheckList_LatestFinished(t *testing.T) {
+	base := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
+	running := Check{ObjectMeta: metav1.ObjectMeta{Name: "running"}}
+	running.Status.SetStarted("running")
+
+	tests := map[string]struct {
+		givenChecks  []Check
+		expectedName *string
+	}{
+		"GivenNoChecks_ThenExpectNil": {},
+		"GivenOnlyRunningCheck_ThenExpectNil": {
+			givenChecks: []Check{running},
+		},
+		"GivenFinishedChecks_ThenExpectMostRecent": {
+			givenChecks: []Check{
+				newFinishedCheck("old", ReasonSucceeded, base),
+				newFinishedCheck("new", ReasonFailed, base.Add(24*time.Hour)),
+				running,
+			},
+			expectedName: ptr.To("new"),
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			latest := (&CheckList{Items: tt.givenChecks}).LatestFinished()
+			if tt.expectedName == nil {
+				assert.Nil(t, latest)
+				return
+			}
+			assert.Equal(t, *tt.expectedName, latest.Name)
+		})
+	}
+}
+
+func TestSnapshotList_Latest(t *testing.T) {
+	base := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
+	list := &SnapshotList{Items: []Snapshot{
+		{ObjectMeta: metav1.ObjectMeta{Name: "old"}, Spec: SnapshotSpec{Date: &metav1.Time{Time: base}}},
+		{ObjectMeta: metav1.ObjectMeta{Name: "undated"}},
+		{ObjectMeta: metav1.ObjectMeta{Name: "new"}, Spec: SnapshotSpec{Date: &metav1.Time{Time: base.Add(time.Hour)}}},
+	}}
+	assert.Equal(t, "new", list.Latest().Name)
+	assert.Nil(t, (&SnapshotList{}).Latest())
+}

api/v1/common_types.go

@@ -47,6 +47,8 @@ const (
 	ConditionProgressing ConditionType = "Progressing"
 	// ConditionPreBackupPodReady is True if Deployments for all Container definitions were created and are ready
 	ConditionPreBackupPodReady ConditionType = "PreBackupPodReady"
+	// ConditionRepositoryHealthy reflects the result of the last finished Check against the repository a Backup writes to.
+	ConditionRepositoryHealthy ConditionType = "RepositoryHealthy"
 
 	// ReasonReady indicates the condition is ready for work
 	ReasonReady ConditionReason = "Ready"
@@ -72,6 +74,15 @@ const (
 	// ReasonWaiting is given when PreBackupPods are waiting to be started
 	ReasonWaiting ConditionReason = "Waiting"
 
+	// ReasonLastCheckSucceeded is given when the last finished Check of the repository succeeded
+	ReasonLastCheckSucceeded ConditionReason = "LastCheckSucceeded"
+	// ReasonLastCheckFailed is given when the last finished Check of the repository failed
+	ReasonLastCheckFailed ConditionReason = "LastCheckFailed"
+	// ReasonNoCheckFound is given when no Check of the repository has finished yet
+	ReasonNoCheckFound ConditionReason = "NoCheckFound"
+	// ReasonRepositoryUnhealthy is given when a Backup was skipped because the last Check of its repository failed
+	ReasonRepositoryUnhealthy ConditionReason = "RepositoryUnhealthy"
+
 	// LabelK8upType is the label key that identifies the job type
 	LabelK8upType = "k8up.io/type"
 	// LabelK8upOwnedBy is a label used to indicated which resource owns this resource to make it easy to fetch owned resources.

api/v1/snapshot_types.go

@@ -47,3 +47,18 @@ type SnapshotList struct {
 func init() {
 	SchemeBuilder.Register(&Snapshot{}, &SnapshotList{})
 }
+
+// Latest returns the Snapshot that was taken most recently, or nil if the list has no Snapshot with a date.
+func (s *SnapshotList) Latest() *Snapshot {
+	var latest *Snapshot
+	for i := range s.Items {
+		date := s.Items[i].Spec.Date
+		if date == nil {
+			continue
+		}
+		if latest == nil || date.After(latest.Spec.Date.Time) {
+			latest = &s.Items[i]
+		}
+	}
+	return latest
+}

charts/k8up/crds/k8up.io_backups.yaml

@@ -885,6 +885,13 @@ spec:
                       More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
                     type: object
                 type: object
+              skipOnFailedCheck:
+                description: |-
+                  SkipOnFailedCheck skips the backup when the last finished Check of its repository failed,
+                  instead of writing more data into a repository that restic reported as damaged.
+                  The Backup then completes with reason RepositoryUnhealthy and no backup Job is started.
+                  The operator flag --global-skip-backup-on-failed-check enables this for all Backups.
+                type: boolean
               statsURL:
                 description: |-
                   StatsURL sets an arbitrary URL where the restic container posts metrics and

charts/k8up/crds/k8up.io_schedules.yaml

@@ -2443,6 +2443,13 @@ spec:
                     description: ScheduleDefinition is the actual cron-type expression
                       that defines the interval of the actions.
                     type: string
+                  skipOnFailedCheck:
+                    description: |-
+                      SkipOnFailedCheck skips the backup when the last finished Check of its repository failed,
+                      instead of writing more data into a repository that restic reported as damaged.
+                      The Backup then completes with reason RepositoryUnhealthy and no backup Job is started.
+                      The operator flag --global-skip-backup-on-failed-check enables this for all Backups.
+                    type: boolean
                   statsURL:
                     description: |-
                       StatsURL sets an arbitrary URL where the restic container posts metrics and

cmd/operator/main.go

@@ -87,6 +87,7 @@ var (
 			&cli.BoolFlag{Destination: &cfg.Config.EnableRelaxedScheduling, Name: "enable-relaxed-scheduling", EnvVars: []string{"BACKUP_ENABLE_RELAXED_SCHEDULING"}, Value: false, DefaultText: "disabled", Usage: "enable relaxed scheduling of backup jobs relying on the Kubernetes scheduler"},
 			&cli.BoolFlag{Destination: &cfg.Config.SkipWithoutAnnotation, Name: "skip-pvcs-without-annotation", EnvVars: []string{"BACKUP_SKIP_WITHOUT_ANNOTATION"}, Value: false, DefaultText: "disabled", Usage: "skip selecting PVCs that don't have the BACKUP_ANNOTATION"},
 			&cli.BoolFlag{Destination: &cfg.Config.SkipSnapshotSync, Name: "global-skip-snapshot-sync", EnvVars: []string{"BACKUP_GLOBAL_SKIP_SNAPSHOT_SYNC"}, Value: false, DefaultText: "disabled", Usage: "if set, skip synchronizing Snapshot custom resources to the cluster after backup or prune operations. This sets the BACKUP_SKIP_SNAPSHOT_SYNC env var on backup and prune job pods."},
+			&cli.BoolFlag{Destination: &cfg.Config.SkipBackupOnFailedCheck, Name: "global-skip-backup-on-failed-check", EnvVars: []string{"BACKUP_GLOBAL_SKIP_BACKUP_ON_FAILED_CHECK"}, Value: false, DefaultText: "disabled", Usage: "if set, skip backups whose repository failed its last check, as if every Backup set skipOnFailedCheck"},
 			&cli.StringFlag{Destination: &cfg.Config.BackupCheckSchedule, Name: "checkschedule", EnvVars: []string{"BACKUP_CHECKSCHEDULE"}, Value: "0 0 * * 0", Usage: "the default check schedule"},
 			&cli.StringFlag{Destination: &cfg.Config.OperatorNamespace, Name: "operator-namespace", EnvVars: []string{"BACKUP_OPERATOR_NAMESPACE"}, Required: true, Usage: "set the namespace in which the K8up operator itself runs"},
 

config/crd/apiextensions.k8s.io/v1/k8up.io_backups.yaml

@@ -885,6 +885,13 @@ spec:
                       More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
                     type: object
                 type: object
+              skipOnFailedCheck:
+                description: |-
+                  SkipOnFailedCheck skips the backup when the last finished Check of its repository failed,
+                  instead of writing more data into a repository that restic reported as damaged.
+                  The Backup then completes with reason RepositoryUnhealthy and no backup Job is started.
+                  The operator flag --global-skip-backup-on-failed-check enables this for all Backups.
+                type: boolean
               statsURL:
                 description: |-
                   StatsURL sets an arbitrary URL where the restic container posts metrics and

config/crd/apiextensions.k8s.io/v1/k8up.io_schedules.yaml

@@ -2443,6 +2443,13 @@ spec:
                     description: ScheduleDefinition is the actual cron-type expression
                       that defines the interval of the actions.
                     type: string
+                  skipOnFailedCheck:
+                    description: |-
+                      SkipOnFailedCheck skips the backup when the last finished Check of its repository failed,
+                      instead of writing more data into a repository that restic reported as damaged.
+                      The Backup then completes with reason RepositoryUnhealthy and no backup Job is started.
+                      The operator flag --global-skip-backup-on-failed-check enables this for all Backups.
+                    type: boolean
                   statsURL:
                     description: |-
                       StatsURL sets an arbitrary URL where the restic container posts metrics and

docs/modules/ROOT/examples/usage/operator.txt

@@ -50,6 +50,7 @@ OPTIONS:
    --enable-relaxed-scheduling                         enable relaxed scheduling of backup jobs relying on the Kubernetes scheduler (default: disabled) [$BACKUP_ENABLE_RELAXED_SCHEDULING]
    --skip-pvcs-without-annotation                      skip selecting PVCs that don't have the BACKUP_ANNOTATION (default: disabled) [$BACKUP_SKIP_WITHOUT_ANNOTATION]
    --global-skip-snapshot-sync                         if set, skip synchronizing Snapshot custom resources to the cluster after backup or prune operations. This sets the BACKUP_SKIP_SNAPSHOT_SYNC env var on backup and prune job pods. (default: disabled) [$BACKUP_GLOBAL_SKIP_SNAPSHOT_SYNC]
+   --global-skip-backup-on-failed-check                if set, skip backups whose repository failed its last check, as if every Backup set skipOnFailedCheck (default: disabled) [$BACKUP_GLOBAL_SKIP_BACKUP_ON_FAILED_CHECK]
    --checkschedule value                               the default check schedule (default: "0 0 * * 0") [$BACKUP_CHECKSCHEDULE]
    --operator-namespace value                          set the namespace in which the K8up operator itself runs [$BACKUP_OPERATOR_NAMESPACE]
    --insecure-allow-podexec-spdy-fallback              enable fallback to SPDY connections for data streaming used by application aware backups. Might need to be enabled if the cluster has Kubernetes version 1.30 or lower. K8up uses WebSockets by default. CAUTION: Has been observed to cause silent data corruption in some network setups, use at own risk! (default: false) [$INSECURE_ALLOW_PODEXEC_SPDY_FALLBACK]

docs/modules/ROOT/pages/references/metrics.adoc

@@ -40,6 +40,16 @@ metrics:
 | Gauge
 | `namespace`
 | Number of active schedules managed by the operator
+
+| `k8up_backup_repository_healthy`
+| Gauge
+| `namespace`
+| 1 if the last finished check of the repository a backup writes to succeeded, 0 if it failed
+
+| `k8up_backups_skipped_unhealthy_repository_total`
+| Counter
+| `namespace`
+| Total number of backups skipped because the last check of their repository failed
 |===
 
 The `jobType` label can be one of: `backup`, `check`, `archive`, `restore`, `prune`.

docs/modules/ROOT/pages/references/status.adoc

@@ -90,4 +90,18 @@ Each condition also contains a human-readable message that describes the status
 | Finished
 | `PreBackupPod` deployments cleaned up after backup finished.
 
+.3+| `RepositoryHealthy`
+| LastCheckSucceeded
+| The last finished `Check` of the repository succeeded.
+
+| LastCheckFailed
+| The last finished `Check` of the repository failed.
+
+| NoCheckFound
+| No `Check` of the repository has finished yet.
+
+| `Completed`
+| RepositoryUnhealthy
+| The backup was skipped because the last `Check` of its repository failed and `skipOnFailedCheck` is set.
+
 |===

operator/backupcontroller/controller.go

@@ -71,6 +71,15 @@ func (r *BackupReconciler) Provision(ctx context.Context, obj *k8upv1.Backup) (r
 		return controllerruntime.Result{RequeueAfter: time.Second * 30}, nil
 	}
 
+	skip, err := r.gateOnRepositoryHealth(ctx, obj, repository)
+	if err != nil {
+		return controllerruntime.Result{}, err
+	}
+	if skip {
+		log.Info("Skipping backup because the last check of its repository failed")
+		return controllerruntime.Result{}, nil
+	}
+
 	lock := locker.GetForRepository(r.Kube, repository)
 	didRun, err := lock.TryRun(ctx, config, executor.GetConcurrencyLimit(), executor.Execute)
 	if !didRun && err == nil {

operator/backupcontroller/health_gate.go

@@ -0,0 +1,53 @@
+package backupcontroller
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
+	"github.com/k8up-io/k8up/v2/operator/cfg"
+	"github.com/k8up-io/k8up/v2/operator/monitoring"
+	"github.com/k8up-io/k8up/v2/restic/kubernetes"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// gateOnRepositoryHealth records the health of the backup's repository in the ConditionRepositoryHealthy
+// condition and reports whether the backup must be skipped.
+// A backup is skipped when the last finished Check of its repository failed and either the Backup sets
+// skipOnFailedCheck or the operator skips such backups globally. A skipped Backup is marked as finished.
+func (r *BackupReconciler) gateOnRepositoryHealth(ctx context.Context, obj *k8upv1.Backup, repository string) (bool, error) {
+	health, err := kubernetes.ReadRepositoryHealth(ctx, r.Kube, obj.Namespace, repository, cfg.Config.GetGlobalRepository())
+	if err != nil {
+		return false, fmt.Errorf("read repository health: %w", err)
+	}
+
+	status := obj.Status
+	switch {
+	case health.LastCheck == nil:
+		status.SetCondition(k8upv1.ConditionRepositoryHealthy, k8upv1.ReasonNoCheckFound, metav1.ConditionUnknown, health.Message())
+	case health.LastCheckFailed():
+		status.SetCondition(k8upv1.ConditionRepositoryHealthy, k8upv1.ReasonLastCheckFailed, metav1.ConditionFalse, health.Message())
+		monitoring.SetBackupRepositoryHealthy(obj.Namespace, false)
+	default:
+		status.SetCondition(k8upv1.ConditionRepositoryHealthy, k8upv1.ReasonLastCheckSucceeded, metav1.ConditionTrue, health.Message())
+		monitoring.SetBackupRepositoryHealthy(obj.Namespace, true)
+	}
+
+	skip := health.LastCheckFailed() && (obj.Spec.SkipOnFailedCheck || cfg.Config.SkipBackupOnFailedCheck)
+	if skip {
+		status.SetCondition(k8upv1.ConditionCompleted, k8upv1.ReasonRepositoryUnhealthy, metav1.ConditionTrue,
+			"the backup was skipped because the last check of its repository failed")
+		status.SetFinished("the backup was skipped")
+		monitoring.IncBackupsSkippedCounter(obj.Namespace)
+	}
+
+	if reflect.DeepEqual(status, obj.Status) {
+		return skip, nil
+	}
+	obj.SetStatus(status)
+	if err := r.Kube.Status().Update(ctx, obj); err != nil {
+		return false, fmt.Errorf("backup status update failed: %w", err)
+	}
+	return skip, nil
+}