diff --git a/api/v1/backup_types.go b/api/v1/backup_types.go index 62b4a1ae9..a36cef577 100644 --- a/api/v1/backup_types.go +++ b/api/v1/backup_types.go @@ -49,6 +49,13 @@ type BackupSpec struct { // When defined, only PVCs and PreBackupPods matching them are backed up. // +optional LabelSelectors []metav1.LabelSelector `json:"labelSelectors,omitempty"` + + // SkipOnFailedCheck skips the backup when the last finished Check of its repository failed, + // instead of writing more data into a repository that restic reported as damaged. + // The Backup then completes with reason RepositoryUnhealthy and no backup Job is started. + // The operator flag --global-skip-backup-on-failed-check enables this for all Backups. + // +optional + SkipOnFailedCheck bool `json:"skipOnFailedCheck,omitempty"` } type BackupTemplate struct { diff --git a/api/v1/check_types.go b/api/v1/check_types.go index e35cace20..4206d773a 100644 --- a/api/v1/check_types.go +++ b/api/v1/check_types.go @@ -5,6 +5,7 @@ import ( "reflect" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -128,6 +129,39 @@ func (c *CheckList) GetJobObjects() JobObjectList { return items } +// RepositoryOrDefault returns the restic repository this Check runs against. +// A Check without a backend runs against the operator's global repository, given as defaultRepository. +func (c *Check) RepositoryOrDefault(defaultRepository string) string { + if c.Spec.Backend == nil { + return defaultRepository + } + return c.Spec.Backend.String() +} + +// FinishedAt returns when the Check completed, successfully or not, or nil if it has not finished yet. +func (c *Check) FinishedAt() *metav1.Time { + if !c.Status.HasFinished() { + return nil + } + completedCond := meta.FindStatusCondition(c.Status.Conditions, ConditionCompleted.String()) + return &completedCond.LastTransitionTime +} + +// LatestFinished returns the Check that finished most recently, or nil if none of them has finished. +func (c *CheckList) LatestFinished() *Check { + var latest *Check + for i := range c.Items { + finishedAt := c.Items[i].FinishedAt() + if finishedAt == nil { + continue + } + if latest == nil || finishedAt.After(latest.FinishedAt().Time) { + latest = &c.Items[i] + } + } + return latest +} + // GetDeepCopy returns a deep copy func (in *CheckSchedule) GetDeepCopy() ScheduleSpecInterface { return in.DeepCopy() diff --git a/api/v1/check_types_test.go b/api/v1/check_types_test.go new file mode 100644 index 000000000..b3f415d47 --- /dev/null +++ b/api/v1/check_types_test.go @@ -0,0 +1,86 @@ +package v1 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func newFinishedCheck(name string, reason ConditionReason, finishedAt time.Time) Check { + return Check{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: Status{Conditions: []metav1.Condition{{ + Type: ConditionCompleted.String(), + Status: metav1.ConditionTrue, + Reason: reason.String(), + LastTransitionTime: metav1.NewTime(finishedAt), + }}}, + } +} + +func TestCheck_RepositoryOrDefault(t *testing.T) { + check := Check{} + assert.Equal(t, "s3:https://global/bucket", check.RepositoryOrDefault("s3:https://global/bucket")) + + check.Spec.Backend = &Backend{S3: &S3Spec{Endpoint: "https://endpoint", Bucket: "bucket"}} + assert.Equal(t, "s3:https://endpoint/bucket", check.RepositoryOrDefault("s3:https://global/bucket")) +} + +func TestCheck_FinishedAt(t *testing.T) { + finishedAt := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) + + running := Check{} + running.Status.SetStarted("running") + assert.Nil(t, running.FinishedAt()) + + failed := newFinishedCheck("failed", ReasonFailed, finishedAt) + assert.Equal(t, finishedAt, failed.FinishedAt().Time) +} + +func TestCheckList_LatestFinished(t *testing.T) { + base := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) + running := Check{ObjectMeta: metav1.ObjectMeta{Name: "running"}} + running.Status.SetStarted("running") + + tests := map[string]struct { + givenChecks []Check + expectedName *string + }{ + "GivenNoChecks_ThenExpectNil": {}, + "GivenOnlyRunningCheck_ThenExpectNil": { + givenChecks: []Check{running}, + }, + "GivenFinishedChecks_ThenExpectMostRecent": { + givenChecks: []Check{ + newFinishedCheck("old", ReasonSucceeded, base), + newFinishedCheck("new", ReasonFailed, base.Add(24*time.Hour)), + running, + }, + expectedName: ptr.To("new"), + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + latest := (&CheckList{Items: tt.givenChecks}).LatestFinished() + if tt.expectedName == nil { + assert.Nil(t, latest) + return + } + assert.Equal(t, *tt.expectedName, latest.Name) + }) + } +} + +func TestSnapshotList_Latest(t *testing.T) { + base := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) + list := &SnapshotList{Items: []Snapshot{ + {ObjectMeta: metav1.ObjectMeta{Name: "old"}, Spec: SnapshotSpec{Date: &metav1.Time{Time: base}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "undated"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "new"}, Spec: SnapshotSpec{Date: &metav1.Time{Time: base.Add(time.Hour)}}}, + }} + assert.Equal(t, "new", list.Latest().Name) + assert.Nil(t, (&SnapshotList{}).Latest()) +} diff --git a/api/v1/common_types.go b/api/v1/common_types.go index 17fc23cb5..c8d803aa0 100644 --- a/api/v1/common_types.go +++ b/api/v1/common_types.go @@ -47,6 +47,8 @@ const ( ConditionProgressing ConditionType = "Progressing" // ConditionPreBackupPodReady is True if Deployments for all Container definitions were created and are ready ConditionPreBackupPodReady ConditionType = "PreBackupPodReady" + // ConditionRepositoryHealthy reflects the result of the last finished Check against the repository a Backup writes to. + ConditionRepositoryHealthy ConditionType = "RepositoryHealthy" // ReasonReady indicates the condition is ready for work ReasonReady ConditionReason = "Ready" @@ -72,6 +74,15 @@ const ( // ReasonWaiting is given when PreBackupPods are waiting to be started ReasonWaiting ConditionReason = "Waiting" + // ReasonLastCheckSucceeded is given when the last finished Check of the repository succeeded + ReasonLastCheckSucceeded ConditionReason = "LastCheckSucceeded" + // ReasonLastCheckFailed is given when the last finished Check of the repository failed + ReasonLastCheckFailed ConditionReason = "LastCheckFailed" + // ReasonNoCheckFound is given when no Check of the repository has finished yet + ReasonNoCheckFound ConditionReason = "NoCheckFound" + // ReasonRepositoryUnhealthy is given when a Backup was skipped because the last Check of its repository failed + ReasonRepositoryUnhealthy ConditionReason = "RepositoryUnhealthy" + // LabelK8upType is the label key that identifies the job type LabelK8upType = "k8up.io/type" // LabelK8upOwnedBy is a label used to indicated which resource owns this resource to make it easy to fetch owned resources. diff --git a/api/v1/snapshot_types.go b/api/v1/snapshot_types.go index e5f019daf..f4b439a5d 100644 --- a/api/v1/snapshot_types.go +++ b/api/v1/snapshot_types.go @@ -47,3 +47,18 @@ type SnapshotList struct { func init() { SchemeBuilder.Register(&Snapshot{}, &SnapshotList{}) } + +// Latest returns the Snapshot that was taken most recently, or nil if the list has no Snapshot with a date. +func (s *SnapshotList) Latest() *Snapshot { + var latest *Snapshot + for i := range s.Items { + date := s.Items[i].Spec.Date + if date == nil { + continue + } + if latest == nil || date.After(latest.Spec.Date.Time) { + latest = &s.Items[i] + } + } + return latest +} diff --git a/charts/k8up/crds/k8up.io_backups.yaml b/charts/k8up/crds/k8up.io_backups.yaml index 9a03def0e..26dd4e2f7 100644 --- a/charts/k8up/crds/k8up.io_backups.yaml +++ b/charts/k8up/crds/k8up.io_backups.yaml @@ -885,6 +885,13 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + skipOnFailedCheck: + description: |- + SkipOnFailedCheck skips the backup when the last finished Check of its repository failed, + instead of writing more data into a repository that restic reported as damaged. + The Backup then completes with reason RepositoryUnhealthy and no backup Job is started. + The operator flag --global-skip-backup-on-failed-check enables this for all Backups. + type: boolean statsURL: description: |- StatsURL sets an arbitrary URL where the restic container posts metrics and diff --git a/charts/k8up/crds/k8up.io_schedules.yaml b/charts/k8up/crds/k8up.io_schedules.yaml index 476f9e9f7..cabb1cc35 100644 --- a/charts/k8up/crds/k8up.io_schedules.yaml +++ b/charts/k8up/crds/k8up.io_schedules.yaml @@ -2443,6 +2443,13 @@ spec: description: ScheduleDefinition is the actual cron-type expression that defines the interval of the actions. type: string + skipOnFailedCheck: + description: |- + SkipOnFailedCheck skips the backup when the last finished Check of its repository failed, + instead of writing more data into a repository that restic reported as damaged. + The Backup then completes with reason RepositoryUnhealthy and no backup Job is started. + The operator flag --global-skip-backup-on-failed-check enables this for all Backups. + type: boolean statsURL: description: |- StatsURL sets an arbitrary URL where the restic container posts metrics and diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 17b8b096f..376766e27 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -87,6 +87,7 @@ var ( &cli.BoolFlag{Destination: &cfg.Config.EnableRelaxedScheduling, Name: "enable-relaxed-scheduling", EnvVars: []string{"BACKUP_ENABLE_RELAXED_SCHEDULING"}, Value: false, DefaultText: "disabled", Usage: "enable relaxed scheduling of backup jobs relying on the Kubernetes scheduler"}, &cli.BoolFlag{Destination: &cfg.Config.SkipWithoutAnnotation, Name: "skip-pvcs-without-annotation", EnvVars: []string{"BACKUP_SKIP_WITHOUT_ANNOTATION"}, Value: false, DefaultText: "disabled", Usage: "skip selecting PVCs that don't have the BACKUP_ANNOTATION"}, &cli.BoolFlag{Destination: &cfg.Config.SkipSnapshotSync, Name: "global-skip-snapshot-sync", EnvVars: []string{"BACKUP_GLOBAL_SKIP_SNAPSHOT_SYNC"}, Value: false, DefaultText: "disabled", Usage: "if set, skip synchronizing Snapshot custom resources to the cluster after backup or prune operations. This sets the BACKUP_SKIP_SNAPSHOT_SYNC env var on backup and prune job pods."}, + &cli.BoolFlag{Destination: &cfg.Config.SkipBackupOnFailedCheck, Name: "global-skip-backup-on-failed-check", EnvVars: []string{"BACKUP_GLOBAL_SKIP_BACKUP_ON_FAILED_CHECK"}, Value: false, DefaultText: "disabled", Usage: "if set, skip backups whose repository failed its last check, as if every Backup set skipOnFailedCheck"}, &cli.StringFlag{Destination: &cfg.Config.BackupCheckSchedule, Name: "checkschedule", EnvVars: []string{"BACKUP_CHECKSCHEDULE"}, Value: "0 0 * * 0", Usage: "the default check schedule"}, &cli.StringFlag{Destination: &cfg.Config.OperatorNamespace, Name: "operator-namespace", EnvVars: []string{"BACKUP_OPERATOR_NAMESPACE"}, Required: true, Usage: "set the namespace in which the K8up operator itself runs"}, diff --git a/config/crd/apiextensions.k8s.io/v1/k8up.io_backups.yaml b/config/crd/apiextensions.k8s.io/v1/k8up.io_backups.yaml index 9a03def0e..26dd4e2f7 100644 --- a/config/crd/apiextensions.k8s.io/v1/k8up.io_backups.yaml +++ b/config/crd/apiextensions.k8s.io/v1/k8up.io_backups.yaml @@ -885,6 +885,13 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + skipOnFailedCheck: + description: |- + SkipOnFailedCheck skips the backup when the last finished Check of its repository failed, + instead of writing more data into a repository that restic reported as damaged. + The Backup then completes with reason RepositoryUnhealthy and no backup Job is started. + The operator flag --global-skip-backup-on-failed-check enables this for all Backups. + type: boolean statsURL: description: |- StatsURL sets an arbitrary URL where the restic container posts metrics and diff --git a/config/crd/apiextensions.k8s.io/v1/k8up.io_schedules.yaml b/config/crd/apiextensions.k8s.io/v1/k8up.io_schedules.yaml index 476f9e9f7..cabb1cc35 100644 --- a/config/crd/apiextensions.k8s.io/v1/k8up.io_schedules.yaml +++ b/config/crd/apiextensions.k8s.io/v1/k8up.io_schedules.yaml @@ -2443,6 +2443,13 @@ spec: description: ScheduleDefinition is the actual cron-type expression that defines the interval of the actions. type: string + skipOnFailedCheck: + description: |- + SkipOnFailedCheck skips the backup when the last finished Check of its repository failed, + instead of writing more data into a repository that restic reported as damaged. + The Backup then completes with reason RepositoryUnhealthy and no backup Job is started. + The operator flag --global-skip-backup-on-failed-check enables this for all Backups. + type: boolean statsURL: description: |- StatsURL sets an arbitrary URL where the restic container posts metrics and diff --git a/docs/modules/ROOT/examples/usage/operator.txt b/docs/modules/ROOT/examples/usage/operator.txt index 80215a7de..c158aedf1 100644 --- a/docs/modules/ROOT/examples/usage/operator.txt +++ b/docs/modules/ROOT/examples/usage/operator.txt @@ -50,6 +50,7 @@ OPTIONS: --enable-relaxed-scheduling enable relaxed scheduling of backup jobs relying on the Kubernetes scheduler (default: disabled) [$BACKUP_ENABLE_RELAXED_SCHEDULING] --skip-pvcs-without-annotation skip selecting PVCs that don't have the BACKUP_ANNOTATION (default: disabled) [$BACKUP_SKIP_WITHOUT_ANNOTATION] --global-skip-snapshot-sync if set, skip synchronizing Snapshot custom resources to the cluster after backup or prune operations. This sets the BACKUP_SKIP_SNAPSHOT_SYNC env var on backup and prune job pods. (default: disabled) [$BACKUP_GLOBAL_SKIP_SNAPSHOT_SYNC] + --global-skip-backup-on-failed-check if set, skip backups whose repository failed its last check, as if every Backup set skipOnFailedCheck (default: disabled) [$BACKUP_GLOBAL_SKIP_BACKUP_ON_FAILED_CHECK] --checkschedule value the default check schedule (default: "0 0 * * 0") [$BACKUP_CHECKSCHEDULE] --operator-namespace value set the namespace in which the K8up operator itself runs [$BACKUP_OPERATOR_NAMESPACE] --insecure-allow-podexec-spdy-fallback enable fallback to SPDY connections for data streaming used by application aware backups. Might need to be enabled if the cluster has Kubernetes version 1.30 or lower. K8up uses WebSockets by default. CAUTION: Has been observed to cause silent data corruption in some network setups, use at own risk! (default: false) [$INSECURE_ALLOW_PODEXEC_SPDY_FALLBACK] diff --git a/docs/modules/ROOT/pages/references/metrics.adoc b/docs/modules/ROOT/pages/references/metrics.adoc index 72b7770f4..16993f00c 100644 --- a/docs/modules/ROOT/pages/references/metrics.adoc +++ b/docs/modules/ROOT/pages/references/metrics.adoc @@ -40,6 +40,16 @@ metrics: | Gauge | `namespace` | Number of active schedules managed by the operator + +| `k8up_backup_repository_healthy` +| Gauge +| `namespace` +| 1 if the last finished check of the repository a backup writes to succeeded, 0 if it failed + +| `k8up_backups_skipped_unhealthy_repository_total` +| Counter +| `namespace` +| Total number of backups skipped because the last check of their repository failed |=== The `jobType` label can be one of: `backup`, `check`, `archive`, `restore`, `prune`. diff --git a/docs/modules/ROOT/pages/references/status.adoc b/docs/modules/ROOT/pages/references/status.adoc index cc9c3aad3..13d15abab 100644 --- a/docs/modules/ROOT/pages/references/status.adoc +++ b/docs/modules/ROOT/pages/references/status.adoc @@ -90,4 +90,18 @@ Each condition also contains a human-readable message that describes the status | Finished | `PreBackupPod` deployments cleaned up after backup finished. +.3+| `RepositoryHealthy` +| LastCheckSucceeded +| The last finished `Check` of the repository succeeded. + +| LastCheckFailed +| The last finished `Check` of the repository failed. + +| NoCheckFound +| No `Check` of the repository has finished yet. + +| `Completed` +| RepositoryUnhealthy +| The backup was skipped because the last `Check` of its repository failed and `skipOnFailedCheck` is set. + |=== diff --git a/operator/backupcontroller/controller.go b/operator/backupcontroller/controller.go index 346163b3e..4054603a3 100644 --- a/operator/backupcontroller/controller.go +++ b/operator/backupcontroller/controller.go @@ -71,6 +71,15 @@ func (r *BackupReconciler) Provision(ctx context.Context, obj *k8upv1.Backup) (r return controllerruntime.Result{RequeueAfter: time.Second * 30}, nil } + skip, err := r.gateOnRepositoryHealth(ctx, obj, repository) + if err != nil { + return controllerruntime.Result{}, err + } + if skip { + log.Info("Skipping backup because the last check of its repository failed") + return controllerruntime.Result{}, nil + } + lock := locker.GetForRepository(r.Kube, repository) didRun, err := lock.TryRun(ctx, config, executor.GetConcurrencyLimit(), executor.Execute) if !didRun && err == nil { diff --git a/operator/backupcontroller/health_gate.go b/operator/backupcontroller/health_gate.go new file mode 100644 index 000000000..5c5818a1b --- /dev/null +++ b/operator/backupcontroller/health_gate.go @@ -0,0 +1,53 @@ +package backupcontroller + +import ( + "context" + "fmt" + "reflect" + + k8upv1 "github.com/k8up-io/k8up/v2/api/v1" + "github.com/k8up-io/k8up/v2/operator/cfg" + "github.com/k8up-io/k8up/v2/operator/monitoring" + "github.com/k8up-io/k8up/v2/restic/kubernetes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// gateOnRepositoryHealth records the health of the backup's repository in the ConditionRepositoryHealthy +// condition and reports whether the backup must be skipped. +// A backup is skipped when the last finished Check of its repository failed and either the Backup sets +// skipOnFailedCheck or the operator skips such backups globally. A skipped Backup is marked as finished. +func (r *BackupReconciler) gateOnRepositoryHealth(ctx context.Context, obj *k8upv1.Backup, repository string) (bool, error) { + health, err := kubernetes.ReadRepositoryHealth(ctx, r.Kube, obj.Namespace, repository, cfg.Config.GetGlobalRepository()) + if err != nil { + return false, fmt.Errorf("read repository health: %w", err) + } + + status := obj.Status + switch { + case health.LastCheck == nil: + status.SetCondition(k8upv1.ConditionRepositoryHealthy, k8upv1.ReasonNoCheckFound, metav1.ConditionUnknown, health.Message()) + case health.LastCheckFailed(): + status.SetCondition(k8upv1.ConditionRepositoryHealthy, k8upv1.ReasonLastCheckFailed, metav1.ConditionFalse, health.Message()) + monitoring.SetBackupRepositoryHealthy(obj.Namespace, false) + default: + status.SetCondition(k8upv1.ConditionRepositoryHealthy, k8upv1.ReasonLastCheckSucceeded, metav1.ConditionTrue, health.Message()) + monitoring.SetBackupRepositoryHealthy(obj.Namespace, true) + } + + skip := health.LastCheckFailed() && (obj.Spec.SkipOnFailedCheck || cfg.Config.SkipBackupOnFailedCheck) + if skip { + status.SetCondition(k8upv1.ConditionCompleted, k8upv1.ReasonRepositoryUnhealthy, metav1.ConditionTrue, + "the backup was skipped because the last check of its repository failed") + status.SetFinished("the backup was skipped") + monitoring.IncBackupsSkippedCounter(obj.Namespace) + } + + if reflect.DeepEqual(status, obj.Status) { + return skip, nil + } + obj.SetStatus(status) + if err := r.Kube.Status().Update(ctx, obj); err != nil { + return false, fmt.Errorf("backup status update failed: %w", err) + } + return skip, nil +} diff --git a/operator/backupcontroller/health_gate_test.go b/operator/backupcontroller/health_gate_test.go new file mode 100644 index 000000000..a18374920 --- /dev/null +++ b/operator/backupcontroller/health_gate_test.go @@ -0,0 +1,114 @@ +package backupcontroller + +import ( + "context" + "testing" + "time" + + k8upv1 "github.com/k8up-io/k8up/v2/api/v1" + "github.com/k8up-io/k8up/v2/operator/cfg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestBackupReconciler_gateOnRepositoryHealth(t *testing.T) { + backend := &k8upv1.Backend{S3: &k8upv1.S3Spec{Endpoint: "https://endpoint", Bucket: "bucket"}} + finishedCheck := func(reason k8upv1.ConditionReason) *k8upv1.Check { + return &k8upv1.Check{ + ObjectMeta: metav1.ObjectMeta{Name: "weekly", Namespace: "app"}, + Spec: k8upv1.CheckSpec{RunnableSpec: k8upv1.RunnableSpec{Backend: backend}}, + Status: k8upv1.Status{Conditions: []metav1.Condition{{ + Type: k8upv1.ConditionCompleted.String(), + Status: metav1.ConditionTrue, + Reason: reason.String(), + LastTransitionTime: metav1.NewTime(time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)), + }}}, + } + } + + tests := map[string]struct { + givenCheck *k8upv1.Check + givenSkipOnFailed bool + givenGlobalSkip bool + expectedSkip bool + expectedHealthy metav1.ConditionStatus + expectedReason k8upv1.ConditionReason + expectedFinished bool + }{ + "GivenNoCheck_ThenExpectUnknownAndNoSkip": { + expectedHealthy: metav1.ConditionUnknown, + expectedReason: k8upv1.ReasonNoCheckFound, + }, + "GivenSucceededCheck_WhenSkipEnabled_ThenExpectHealthyAndNoSkip": { + givenCheck: finishedCheck(k8upv1.ReasonSucceeded), + givenSkipOnFailed: true, + expectedHealthy: metav1.ConditionTrue, + expectedReason: k8upv1.ReasonLastCheckSucceeded, + }, + "GivenFailedCheck_WhenSkipDisabled_ThenExpectUnhealthyAndNoSkip": { + givenCheck: finishedCheck(k8upv1.ReasonFailed), + expectedHealthy: metav1.ConditionFalse, + expectedReason: k8upv1.ReasonLastCheckFailed, + }, + "GivenFailedCheck_WhenSkipOnFailedCheck_ThenExpectSkipped": { + givenCheck: finishedCheck(k8upv1.ReasonFailed), + givenSkipOnFailed: true, + expectedSkip: true, + expectedHealthy: metav1.ConditionFalse, + expectedReason: k8upv1.ReasonLastCheckFailed, + expectedFinished: true, + }, + "GivenFailedCheck_WhenGlobalSkip_ThenExpectSkipped": { + givenCheck: finishedCheck(k8upv1.ReasonFailed), + givenGlobalSkip: true, + expectedSkip: true, + expectedHealthy: metav1.ConditionFalse, + expectedReason: k8upv1.ReasonLastCheckFailed, + expectedFinished: true, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + previousConfig := cfg.Config + t.Cleanup(func() { cfg.Config = previousConfig }) + cfg.Config = &cfg.Configuration{SkipBackupOnFailedCheck: tt.givenGlobalSkip} + + backup := &k8upv1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "nightly", Namespace: "app"}, + Spec: k8upv1.BackupSpec{ + RunnableSpec: k8upv1.RunnableSpec{Backend: backend}, + SkipOnFailedCheck: tt.givenSkipOnFailed, + }, + } + objects := []client.Object{backup} + if tt.givenCheck != nil { + objects = append(objects, tt.givenCheck) + } + scheme := runtime.NewScheme() + require.NoError(t, k8upv1.AddToScheme(scheme)) + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).WithStatusSubresource(backup).Build() + r := &BackupReconciler{Kube: kube} + + skip, err := r.gateOnRepositoryHealth(context.Background(), backup, backend.String()) + require.NoError(t, err) + assert.Equal(t, tt.expectedSkip, skip) + + stored := &k8upv1.Backup{} + require.NoError(t, kube.Get(context.Background(), client.ObjectKeyFromObject(backup), stored)) + healthy := meta.FindStatusCondition(stored.Status.Conditions, k8upv1.ConditionRepositoryHealthy.String()) + require.NotNil(t, healthy) + assert.Equal(t, tt.expectedHealthy, healthy.Status) + assert.Equal(t, tt.expectedReason.String(), healthy.Reason) + assert.Equal(t, tt.expectedFinished, stored.Status.HasFinished()) + if tt.expectedFinished { + completed := meta.FindStatusCondition(stored.Status.Conditions, k8upv1.ConditionCompleted.String()) + assert.Equal(t, k8upv1.ReasonRepositoryUnhealthy.String(), completed.Reason) + } + }) + } +} diff --git a/operator/cfg/config.go b/operator/cfg/config.go index 3741f9e05..d59cb4eaa 100644 --- a/operator/cfg/config.go +++ b/operator/cfg/config.go @@ -86,6 +86,7 @@ type Configuration struct { RepositorySecretName string SkipWithoutAnnotation bool SkipSnapshotSync bool + SkipBackupOnFailedCheck bool EnableRelaxedScheduling bool InsecureAllowPodExecSPDYFallback bool diff --git a/operator/monitoring/prometheus.go b/operator/monitoring/prometheus.go index 12f84015b..c78ce5c9e 100644 --- a/operator/monitoring/prometheus.go +++ b/operator/monitoring/prometheus.go @@ -38,6 +38,16 @@ var ( Name: "k8up_schedule_last_job_succeeded", Help: "1 if the most recent job triggered by this schedule succeeded, 0 if it failed.", }, []string{"namespace", "schedule", "jobType"}) + + backupRepositoryHealthy = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "k8up_backup_repository_healthy", + Help: "1 if the last finished check of the repository a backup writes to succeeded, 0 if it failed.", + }, []string{"namespace"}) + + backupsSkippedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "k8up_backups_skipped_unhealthy_repository_total", + Help: "The total number of backups skipped because the last check of their repository failed", + }, []string{"namespace"}) ) func CreateLabels(namespace string, jobType v1.JobType) { @@ -74,6 +84,20 @@ func SetScheduleLastJobStatus(namespace, scheduleName string, jobType v1.JobType scheduleLastJobSucceeded.WithLabelValues(namespace, scheduleName, jobType.String()).Set(val) } +// SetBackupRepositoryHealthy records whether the last finished check of a backup's repository succeeded (1) or failed (0). +func SetBackupRepositoryHealthy(namespace string, healthy bool) { + val := 0.0 + if healthy { + val = 1.0 + } + backupRepositoryHealthy.WithLabelValues(namespace).Set(val) +} + +// IncBackupsSkippedCounter counts a backup that was skipped because its repository is unhealthy. +func IncBackupsSkippedCounter(namespace string) { + backupsSkippedCounter.WithLabelValues(namespace).Inc() +} + func init() { // Register custom metrics with the global prometheus registry metrics.Registry.MustRegister( @@ -82,5 +106,7 @@ func init() { metricsTotalCounter, scheduleGauge, scheduleLastJobSucceeded, + backupRepositoryHealthy, + backupsSkippedCounter, ) } diff --git a/restic/kubernetes/repository_health.go b/restic/kubernetes/repository_health.go new file mode 100644 index 000000000..878cb20b0 --- /dev/null +++ b/restic/kubernetes/repository_health.go @@ -0,0 +1,81 @@ +package kubernetes + +import ( + "context" + "fmt" + + k8upv1 "github.com/k8up-io/k8up/v2/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// RepositoryHealth is what the cluster knows about a restic repository, read from the +// Check and Snapshot resources in one namespace. +type RepositoryHealth struct { + // LastCheck is the Check against the repository that finished most recently, nil if none has finished. + LastCheck *k8upv1.Check + // SnapshotCount is the number of Snapshot resources synced for the repository. + SnapshotCount int + // LatestSnapshot is when the newest Snapshot of the repository was taken, nil if there is none. + LatestSnapshot *metav1.Time +} + +// LastCheckFailed returns true if the last finished Check of the repository failed. +func (h RepositoryHealth) LastCheckFailed() bool { + return h.LastCheck != nil && h.LastCheck.Status.HasFailed() +} + +// Message describes the repository health for a status condition. +func (h RepositoryHealth) Message() string { + snapshots := fmt.Sprintf("%d snapshots", h.SnapshotCount) + if h.LatestSnapshot != nil { + snapshots += fmt.Sprintf(", latest taken %s", h.LatestSnapshot.UTC().Format("2006-01-02T15:04:05Z")) + } + if h.LastCheck == nil { + return fmt.Sprintf("no check of the repository has finished yet (%s)", snapshots) + } + result := "succeeded" + if h.LastCheckFailed() { + result = "failed" + } + return fmt.Sprintf("last check %q %s at %s (%s)", h.LastCheck.Name, result, + h.LastCheck.FinishedAt().UTC().Format("2006-01-02T15:04:05Z"), snapshots) +} + +// ReadRepositoryHealth reads the Checks and Snapshots of the given repository in the namespace. +// Checks without a backend run against the operator's global repository, given as defaultRepository. +func ReadRepositoryHealth(ctx context.Context, kube client.Reader, namespace, repository, defaultRepository string) (RepositoryHealth, error) { + checks := &k8upv1.CheckList{} + if err := kube.List(ctx, checks, client.InNamespace(namespace)); err != nil { + return RepositoryHealth{}, fmt.Errorf("list checks: %w", err) + } + checks = filterChecksByRepo(checks, repository, defaultRepository) + + snapshots := &k8upv1.SnapshotList{} + if err := kube.List(ctx, snapshots, client.InNamespace(namespace)); err != nil { + return RepositoryHealth{}, fmt.Errorf("list snapshots: %w", err) + } + snapshots = filterByRepo(snapshots, repository) + + health := RepositoryHealth{ + LastCheck: checks.LatestFinished(), + SnapshotCount: len(snapshots.Items), + } + if latest := snapshots.Latest(); latest != nil { + health.LatestSnapshot = latest.Spec.Date + } + return health, nil +} + +// filterChecksByRepo will filter the list according to the given repository. +func filterChecksByRepo(list *k8upv1.CheckList, repo, defaultRepo string) *k8upv1.CheckList { + filteredList := &k8upv1.CheckList{Items: []k8upv1.Check{}} + + for _, check := range list.Items { + if check.RepositoryOrDefault(defaultRepo) == repo { + filteredList.Items = append(filteredList.Items, check) + } + } + + return filteredList +} diff --git a/restic/kubernetes/repository_health_test.go b/restic/kubernetes/repository_health_test.go new file mode 100644 index 000000000..fcfc40024 --- /dev/null +++ b/restic/kubernetes/repository_health_test.go @@ -0,0 +1,131 @@ +package kubernetes + +import ( + "context" + "testing" + "time" + + k8upv1 "github.com/k8up-io/k8up/v2/api/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const ( + healthNamespace = "app" + healthRepository = "s3:https://endpoint/bucket" + globalRepository = "s3:https://global/bucket" +) + +func newCheck(name string, backend *k8upv1.Backend, reason k8upv1.ConditionReason, finishedAt time.Time) *k8upv1.Check { + return &k8upv1.Check{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: healthNamespace}, + Spec: k8upv1.CheckSpec{RunnableSpec: k8upv1.RunnableSpec{Backend: backend}}, + Status: k8upv1.Status{Conditions: []metav1.Condition{{ + Type: k8upv1.ConditionCompleted.String(), + Status: metav1.ConditionTrue, + Reason: reason.String(), + LastTransitionTime: metav1.NewTime(finishedAt), + }}}, + } +} + +func newSnapshot(name, repository string, date time.Time) *k8upv1.Snapshot { + return &k8upv1.Snapshot{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: healthNamespace}, + Spec: k8upv1.SnapshotSpec{ + ID: ptr.To(name), + Date: &metav1.Time{Time: date}, + Repository: ptr.To(repository), + }, + } +} + +func TestReadRepositoryHealth(t *testing.T) { + base := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) + backend := &k8upv1.Backend{S3: &k8upv1.S3Spec{Endpoint: "https://endpoint", Bucket: "bucket"}} + otherBackend := &k8upv1.Backend{S3: &k8upv1.S3Spec{Endpoint: "https://endpoint", Bucket: "other"}} + + tests := map[string]struct { + givenRepository string + givenObjects []client.Object + expectedLastCheck *string + expectedFailed bool + expectedSnapshotCount int + expectedLatest *time.Time + }{ + "GivenNoResources_ThenExpectNoCheckAndNoSnapshots": { + givenRepository: healthRepository, + }, + "GivenFailedCheckAfterSucceededCheck_ThenExpectFailed": { + givenRepository: healthRepository, + givenObjects: []client.Object{ + newCheck("weekly-1", backend, k8upv1.ReasonSucceeded, base), + newCheck("weekly-2", backend, k8upv1.ReasonFailed, base.Add(7*24*time.Hour)), + newSnapshot("aaaaaaaa", healthRepository, base), + newSnapshot("bbbbbbbb", healthRepository, base.Add(time.Hour)), + }, + expectedLastCheck: ptr.To("weekly-2"), + expectedFailed: true, + expectedSnapshotCount: 2, + expectedLatest: ptr.To(base.Add(time.Hour)), + }, + "GivenChecksOfOtherRepository_ThenExpectThemIgnored": { + givenRepository: healthRepository, + givenObjects: []client.Object{ + newCheck("ours", backend, k8upv1.ReasonSucceeded, base), + newCheck("theirs", otherBackend, k8upv1.ReasonFailed, base.Add(time.Hour)), + newSnapshot("cccccccc", "s3:https://endpoint/other", base), + }, + expectedLastCheck: ptr.To("ours"), + }, + "GivenCheckWithoutBackend_WhenGlobalRepository_ThenExpectItMatched": { + givenRepository: globalRepository, + givenObjects: []client.Object{ + newCheck("global", nil, k8upv1.ReasonFailed, base), + }, + expectedLastCheck: ptr.To("global"), + expectedFailed: true, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, k8upv1.AddToScheme(scheme)) + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.givenObjects...).Build() + + health, err := ReadRepositoryHealth(context.Background(), kube, healthNamespace, tt.givenRepository, globalRepository) + require.NoError(t, err) + + if tt.expectedLastCheck == nil { + assert.Nil(t, health.LastCheck) + } else { + require.NotNil(t, health.LastCheck) + assert.Equal(t, *tt.expectedLastCheck, health.LastCheck.Name) + } + assert.Equal(t, tt.expectedFailed, health.LastCheckFailed()) + assert.Equal(t, tt.expectedSnapshotCount, health.SnapshotCount) + if tt.expectedLatest == nil { + assert.Nil(t, health.LatestSnapshot) + } else { + assert.True(t, tt.expectedLatest.Equal(health.LatestSnapshot.Time)) + } + }) + } +} + +func TestRepositoryHealth_Message(t *testing.T) { + base := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) + assert.Equal(t, "no check of the repository has finished yet (0 snapshots)", RepositoryHealth{}.Message()) + + health := RepositoryHealth{ + LastCheck: newCheck("weekly", nil, k8upv1.ReasonFailed, base), + SnapshotCount: 3, + LatestSnapshot: &metav1.Time{Time: base.Add(-time.Hour)}, + } + assert.Equal(t, `last check "weekly" failed at 2026-10-20T03:00:00Z (3 snapshots, latest taken 2026-10-20T02:00:00Z)`, health.Message()) +}