From 77bb568da227bc30997ef3d75cb1f71ccb3ce5cb Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Mon, 24 Aug 2026 15:47:44 +0300 Subject: [PATCH] operator: set status for child objects if no parent objects exist --- docs/CHANGELOG.md | 1 + internal/controller/operator/controllers.go | 45 ++++++ .../operator/factory/reconcile/status.go | 43 +++++- .../operator/factory/reconcile/status_test.go | 134 +++++++++++++++++- .../operator/factory/vmalert/rules.go | 7 +- .../controller/operator/vmagent_controller.go | 16 ++- .../controller/operator/vmalert_controller.go | 21 ++- .../operator/vmalert_controller_test.go | 49 +++++++ .../operator/vmalertmanager_controller.go | 21 ++- .../vmalertmanagerconfig_controller.go | 6 +- .../operator/vmanomaly_controller.go | 21 ++- .../operator/vmanomalyconfig_controller.go | 29 ++-- .../controller/operator/vmauth_controller.go | 17 ++- .../operator/vmnodescrape_controller.go | 9 +- .../operator/vmpodscrape_controller.go | 7 +- .../controller/operator/vmprobe_controller.go | 7 +- .../controller/operator/vmrule_controller.go | 4 + .../operator/vmscrapeconfig_controller.go | 7 +- .../operator/vmservicescrape_controller.go | 7 +- .../operator/vmsingle_controller.go | 6 + .../operator/vmstaticscrape_controller.go | 7 +- .../controller/operator/vmuser_controller.go | 4 + 22 files changed, 387 insertions(+), 81 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b1323c8676..375d7b46c1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -39,6 +39,7 @@ aliases: * BUGFIX: [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/): allow `spec.unauthorizedUserAccessSpec` with only `access_log` set, without requiring `url_map`, `url_prefix`, or `targetRefs`. See [#2551](https://github.com/VictoriaMetrics/operator/issues/2551). * BUGFIX: [vmoperator](https://docs.victoriametrics.com/operator/): a `VolumeClaimTemplate` size decrease, which Kubernetes cannot apply in-place to a bound `PersistentVolumeClaim`, was only logged and otherwise reported as a fully successful, `operational` reconcile. It now surfaces as a failed reconcile with the decline reason in `status.reason`, so the divergence between spec and actual PVC size is queryable and can be alerted on. See [#2512](https://github.com/VictoriaMetrics/operator/issues/2512). * BUGFIX: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): pass previously skipped spec.extraEnvsFrom to anomaly pods. See [#2567](https://github.com/VictoriaMetrics/operator/issues/2567). +* BUGFIX: [vmnodescrape](https://docs.victoriametrics.com/operator/resources/vmnodescrape/), [vmservicescrape](https://docs.victoriametrics.com/operator/resources/vmservicescrape/), [vmpodscrape](https://docs.victoriametrics.com/operator/resources/vmpodscrape/), [vmprobe](https://docs.victoriametrics.com/operator/resources/vmprobe/), [vmscrapeconfig](https://docs.victoriametrics.com/operator/resources/vmscrapeconfig/), [vmstaticscrape](https://docs.victoriametrics.com/operator/resources/vmstaticscrape/), [vmrule](https://docs.victoriametrics.com/operator/resources/vmrule/), [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/), [vmalertmanagerconfig](https://docs.victoriametrics.com/operator/resources/vmalertmanagerconfig/), [vmanomalyconfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/): populate `status.updateStatus` (`Operational`/`Ignored`/`Failed`) after every reconcile instead of leaving it permanently unset, which was causing ArgoCD's health check to report these resources as stuck `Progressing` forever. ArgoCD's health check also needs to recognize the new `Ignored` value, see [argo-cd#29351](https://github.com/argoproj/argo-cd/pull/29351). See [#1181](https://github.com/VictoriaMetrics/operator/issues/1181). ## [v0.74.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.74.1) **Release date:** 04 Aug 2026 diff --git a/internal/controller/operator/controllers.go b/internal/controller/operator/controllers.go index 8c2b1e246c..d8fe5072ad 100644 --- a/internal/controller/operator/controllers.go +++ b/internal/controller/operator/controllers.go @@ -11,6 +11,7 @@ import ( "time" "uuid" + "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -25,6 +26,8 @@ import ( k8sreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" @@ -404,3 +407,45 @@ func reconcileAndTrackStatus[T client.Object, ST reconcile.StatusWithMetadata[ST } return result, nil } + +// releaseScrapeChildStatuses releases parentObject's Applied condition from every +// VMServiceScrape/VMPodScrape/VMNodeScrape/VMProbe/VMStaticScrape/VMScrapeConfig still +// carrying it, for use on VMAgent/VMSingle deletion, since no further reconcile of the +// deleted parent will ever release these otherwise. +func releaseScrapeChildStatuses(ctx context.Context, rclient client.Client, parentObject string) error { + var errs []error + if !build.IsControllerDisabled("VMServiceScrape") { + errs = append(errs, reconcile.StatusForChildObjects(ctx, rclient, parentObject, []*vmv1beta1.VMServiceScrape(nil))) + } + if !build.IsControllerDisabled("VMPodScrape") { + errs = append(errs, reconcile.StatusForChildObjects(ctx, rclient, parentObject, []*vmv1beta1.VMPodScrape(nil))) + } + if !build.IsControllerDisabled("VMNodeScrape") { + errs = append(errs, reconcile.StatusForChildObjects(ctx, rclient, parentObject, []*vmv1beta1.VMNodeScrape(nil))) + } + if !build.IsControllerDisabled("VMProbe") { + errs = append(errs, reconcile.StatusForChildObjects(ctx, rclient, parentObject, []*vmv1beta1.VMProbe(nil))) + } + if !build.IsControllerDisabled("VMStaticScrape") { + errs = append(errs, reconcile.StatusForChildObjects(ctx, rclient, parentObject, []*vmv1beta1.VMStaticScrape(nil))) + } + if !build.IsControllerDisabled("VMScrapeConfig") { + errs = append(errs, reconcile.StatusForChildObjects(ctx, rclient, parentObject, []*vmv1beta1.VMScrapeConfig(nil))) + } + return errors.Join(errs...) +} + +// collectAndSyncScrapeChildStatus collects instance's VMAgent/VMSingle selection and syncs its aggregated Applied condition. +func collectAndSyncScrapeChildStatus[T any, PT interface { + *T + client.Object + GetStatusMetadata() *vmv1beta1.StatusMetadata +}](l logr.Logger, ctx context.Context, rclient client.Client, cfg *config.BaseOperatorConf, instance PT) error { + agentErr := collectVMAgentScrapes(l, ctx, rclient, cfg, instance) + singleErr := collectVMSingleScrapes(l, ctx, rclient, cfg, instance) + errs := []error{agentErr, singleErr} + if agentErr == nil && singleErr == nil { + errs = append(errs, reconcile.SyncAggregatedChildStatus(ctx, rclient, instance)) + } + return errors.Join(errs...) +} diff --git a/internal/controller/operator/factory/reconcile/status.go b/internal/controller/operator/factory/reconcile/status.go index a90e0c00fa..549d7aed64 100644 --- a/internal/controller/operator/factory/reconcile/status.go +++ b/internal/controller/operator/factory/reconcile/status.go @@ -35,7 +35,8 @@ type objectWithStatus interface { } func childConditionType(parentObjectName string) string { - if len(strings.Split(parentObjectName, ".")) != 3 { + // < 3, not != 3: name or namespace may itself contain dots. + if len(strings.Split(parentObjectName, ".")) < 3 { panic(fmt.Sprintf("BUG: unexpected format for parentObjectName=%q, want name.namespace.resource", parentObjectName)) } return parentObjectName + vmv1beta1.ConditionDomainTypeAppliedSuffix @@ -240,7 +241,7 @@ func releaseChildStatusCondition[T any, PT interface { st.Conditions = removeConditionByType(st.Conditions, typeName) st.ObservedGeneration = dst.GetGeneration() - writeAggregatedStatus(st, vmv1beta1.ConditionDomainTypeAppliedSuffix) + writeAggregatedStatus(st) if !reflect.DeepEqual(prevSt, st) { if err := rclient.Status().Update(ctx, dst); err != nil { if k8serrors.IsNotFound(err) { @@ -285,7 +286,7 @@ func updateChildStatusConditions[T any, PT interface { st.Conditions = setConditionTo(st.Conditions, currCond) st.Conditions = removeStaleConditionsBySuffix(st.Conditions, vmv1beta1.ConditionDomainTypeAppliedSuffix) st.ObservedGeneration = dst.GetGeneration() - writeAggregatedStatus(st, vmv1beta1.ConditionDomainTypeAppliedSuffix) + writeAggregatedStatus(st) if !reflect.DeepEqual(prevSt, st) { if err := rclient.Status().Update(ctx, dst); err != nil { if k8serrors.IsNotFound(err) { @@ -344,11 +345,11 @@ func removeStaleConditionsBySuffix(src []vmv1beta1.Condition, domainTypeSuffix s // writeAggregatedStatus derives status from per-parent conditions; a child selected by // multiple parents is only Failed if it fails on all of them, not just one. -func writeAggregatedStatus(stm *vmv1beta1.StatusMetadata, domainTypeSuffix string) { +func writeAggregatedStatus(stm *vmv1beta1.StatusMetadata) { var appliedCount, failedCount int var errorMessages []string for _, c := range stm.Conditions { - if !strings.HasSuffix(c.Type, domainTypeSuffix) { + if !strings.HasSuffix(c.Type, vmv1beta1.ConditionDomainTypeAppliedSuffix) { continue } if c.Status == "False" { @@ -375,6 +376,38 @@ func writeAggregatedStatus(stm *vmv1beta1.StatusMetadata, domainTypeSuffix strin } } +// SyncAggregatedChildStatus recomputes status.updateStatus/reason for a config-selector child +// object (VMServiceScrape and friends) from its already-recorded per-parent conditions. +func SyncAggregatedChildStatus[T any, PT interface { + *T + objectWithStatus +}](ctx context.Context, rclient client.Client, instance PT) error { + nsn := types.NamespacedName{Namespace: instance.GetNamespace(), Name: instance.GetName()} + return retryOnConflict(func() error { + dst := PT(new(T)) + if err := rclient.Get(ctx, nsn, dst); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + st := dst.GetStatusMetadata() + prevSt := st.DeepCopy() + st.ObservedGeneration = dst.GetGeneration() + writeAggregatedStatus(st) + if reflect.DeepEqual(prevSt, st) { + return nil + } + if err := rclient.Status().Update(ctx, dst); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + return nil + }) +} + // adds 50% jitter to the given duration func jitterForDuration(d time.Duration) time.Duration { dv := d / 2 diff --git a/internal/controller/operator/factory/reconcile/status_test.go b/internal/controller/operator/factory/reconcile/status_test.go index 3867462df6..99c049908d 100644 --- a/internal/controller/operator/factory/reconcile/status_test.go +++ b/internal/controller/operator/factory/reconcile/status_test.go @@ -14,15 +14,24 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) +func TestChildConditionType_NameWithDots(t *testing.T) { + var got string + assert.NotPanics(t, func() { + got = childConditionType("my.alert.ns.vmalert") + }) + assert.Equal(t, "my.alert.ns.vmalert"+vmv1beta1.ConditionDomainTypeAppliedSuffix, got) +} + func TestWriteAggregatedStatus(t *testing.T) { f := func(conditions []vmv1beta1.Condition, expectedStatus vmv1beta1.UpdateStatus, expectedReasonContains string) { t.Helper() stm := &vmv1beta1.StatusMetadata{Conditions: conditions} - writeAggregatedStatus(stm, vmv1beta1.ConditionDomainTypeAppliedSuffix) + writeAggregatedStatus(stm) assert.Equal(t, expectedStatus, stm.UpdateStatus) if expectedReasonContains == "" { assert.Empty(t, stm.Reason) @@ -212,3 +221,126 @@ func TestStatusForChildObjects_FallsBackWithoutIndexedClient(t *testing.T) { assert.Equal(t, vmv1beta1.UpdateStatusIgnored, got.Status.UpdateStatus) assert.Empty(t, got.Status.Conditions) } + +func TestSyncAggregatedChildStatus(t *testing.T) { + ctx := context.Background() + scrape := &vmv1beta1.VMServiceScrape{ObjectMeta: metav1.ObjectMeta{Name: "scrape", Namespace: "ns"}} + rclient := k8stools.GetTestClientWithObjects([]runtime.Object{scrape}) + + // no parent ever wrote a condition for it: must become Ignored + require.NoError(t, SyncAggregatedChildStatus(ctx, rclient, scrape)) + var got vmv1beta1.VMServiceScrape + require.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: "ns", Name: "scrape"}, &got)) + assert.Equal(t, vmv1beta1.UpdateStatusIgnored, got.Status.UpdateStatus) + + // a parent selects it and applies it successfully + require.NoError(t, StatusForChildObjects(ctx, rclient, "vmagent1.ns.vmagent", []*vmv1beta1.VMServiceScrape{scrape})) + require.NoError(t, SyncAggregatedChildStatus(ctx, rclient, scrape)) + require.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: "ns", Name: "scrape"}, &got)) + assert.Equal(t, vmv1beta1.UpdateStatusOperational, got.Status.UpdateStatus) + + // dropped by that same parent again: back to Ignored + require.NoError(t, StatusForChildObjects(ctx, rclient, "vmagent1.ns.vmagent", []*vmv1beta1.VMServiceScrape{})) + require.NoError(t, SyncAggregatedChildStatus(ctx, rclient, scrape)) + require.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: "ns", Name: "scrape"}, &got)) + assert.Equal(t, vmv1beta1.UpdateStatusIgnored, got.Status.UpdateStatus) + + // object gone: not found is not an error + require.NoError(t, SyncAggregatedChildStatus(ctx, rclient, &vmv1beta1.VMServiceScrape{ + ObjectMeta: metav1.ObjectMeta{Name: "missing", Namespace: "ns"}, + })) +} + +// assertSyncAggregatedChildStatusRoundTrip exercises the same Ignored -> Operational -> Ignored +// cycle as TestSyncAggregatedChildStatus against a concrete child kind, to guard each controller +// that actually calls SyncAggregatedChildStatus (VMRule, VMUser, VMAlertmanagerConfig, +// VMAnomalyConfig, and the scrape kinds tested below). +func assertSyncAggregatedChildStatusRoundTrip[T any, PT interface { + *T + objectWithStatus +}](t *testing.T, obj PT, parent string) { + t.Helper() + ctx := context.Background() + rclient := k8stools.GetTestClientWithObjects([]runtime.Object{obj}) + nsn := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()} + + require.NoError(t, SyncAggregatedChildStatus(ctx, rclient, obj)) + got := PT(new(T)) + require.NoError(t, rclient.Get(ctx, nsn, got)) + assert.Equal(t, vmv1beta1.UpdateStatusIgnored, got.GetStatusMetadata().UpdateStatus) + + require.NoError(t, StatusForChildObjects(ctx, rclient, parent, []PT{obj})) + require.NoError(t, SyncAggregatedChildStatus(ctx, rclient, obj)) + got = PT(new(T)) + require.NoError(t, rclient.Get(ctx, nsn, got)) + assert.Equal(t, vmv1beta1.UpdateStatusOperational, got.GetStatusMetadata().UpdateStatus) + + corrupted := PT(new(T)) + require.NoError(t, rclient.Get(ctx, nsn, corrupted)) + corrupted.GetStatusMetadata().UpdateStatus = vmv1beta1.UpdateStatusFailed + require.NoError(t, rclient.Status().Update(ctx, corrupted)) + require.NoError(t, SyncAggregatedChildStatus(ctx, rclient, obj)) + got = PT(new(T)) + require.NoError(t, rclient.Get(ctx, nsn, got)) + assert.Equal(t, vmv1beta1.UpdateStatusOperational, got.GetStatusMetadata().UpdateStatus) + + require.NoError(t, StatusForChildObjects(ctx, rclient, parent, []PT{})) + require.NoError(t, SyncAggregatedChildStatus(ctx, rclient, obj)) + got = PT(new(T)) + require.NoError(t, rclient.Get(ctx, nsn, got)) + assert.Equal(t, vmv1beta1.UpdateStatusIgnored, got.GetStatusMetadata().UpdateStatus) +} + +func TestSyncAggregatedChildStatus_VMRule(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1beta1.VMRule](t, + &vmv1beta1.VMRule{ObjectMeta: metav1.ObjectMeta{Name: "rule", Namespace: "ns"}}, + "vmalert1.ns.vmalert") +} + +func TestSyncAggregatedChildStatus_VMUser(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1beta1.VMUser](t, + &vmv1beta1.VMUser{ObjectMeta: metav1.ObjectMeta{Name: "user", Namespace: "ns"}}, + "vmauth1.ns.vmauth") +} + +func TestSyncAggregatedChildStatus_VMAlertmanagerConfig(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1beta1.VMAlertmanagerConfig](t, + &vmv1beta1.VMAlertmanagerConfig{ObjectMeta: metav1.ObjectMeta{Name: "cfg", Namespace: "ns"}}, + "vmalertmanager1.ns.vmalertmanager") +} + +func TestSyncAggregatedChildStatus_VMAnomalyConfig(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1.VMAnomalyConfig](t, + &vmv1.VMAnomalyConfig{ObjectMeta: metav1.ObjectMeta{Name: "cfg", Namespace: "ns"}}, + "vmanomaly1.ns.vmanomaly") +} + +func TestSyncAggregatedChildStatus_VMNodeScrape(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1beta1.VMNodeScrape](t, + &vmv1beta1.VMNodeScrape{ObjectMeta: metav1.ObjectMeta{Name: "scrape", Namespace: "ns"}}, + "vmagent1.ns.vmagent") +} + +func TestSyncAggregatedChildStatus_VMPodScrape(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1beta1.VMPodScrape](t, + &vmv1beta1.VMPodScrape{ObjectMeta: metav1.ObjectMeta{Name: "scrape", Namespace: "ns"}}, + "vmagent1.ns.vmagent") +} + +func TestSyncAggregatedChildStatus_VMProbe(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1beta1.VMProbe](t, + &vmv1beta1.VMProbe{ObjectMeta: metav1.ObjectMeta{Name: "probe", Namespace: "ns"}}, + "vmagent1.ns.vmagent") +} + +func TestSyncAggregatedChildStatus_VMScrapeConfig(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1beta1.VMScrapeConfig](t, + &vmv1beta1.VMScrapeConfig{ObjectMeta: metav1.ObjectMeta{Name: "scrapeconfig", Namespace: "ns"}}, + "vmagent1.ns.vmagent") +} + +func TestSyncAggregatedChildStatus_VMStaticScrape(t *testing.T) { + assertSyncAggregatedChildStatusRoundTrip[vmv1beta1.VMStaticScrape](t, + &vmv1beta1.VMStaticScrape{ObjectMeta: metav1.ObjectMeta{Name: "scrape", Namespace: "ns"}}, + "vmagent1.ns.vmagent") +} diff --git a/internal/controller/operator/factory/vmalert/rules.go b/internal/controller/operator/factory/vmalert/rules.go index 6a177e110b..591d43bb66 100644 --- a/internal/controller/operator/factory/vmalert/rules.go +++ b/internal/controller/operator/factory/vmalert/rules.go @@ -25,9 +25,12 @@ import ( // Alerting rules are dropped when hasNotifiers is false, since vmalert would have nowhere to // send them; recording rules are unaffected. func CreateOrUpdateRuleConfigMaps(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert, childCR *vmv1beta1.VMRule, hasNotifiers bool) ([]string, error) { - // fast path if cr.IsUnmanaged() { - return nil, nil + if build.IsControllerDisabled("VMRule") { + return nil, nil + } + parentObject := fmt.Sprintf("%s.%s.vmalert", cr.Name, cr.Namespace) + return nil, reconcile.StatusForChildObjects(ctx, rclient, parentObject, []*vmv1beta1.VMRule(nil)) } return reconcileVMAlertConfig(ctx, rclient, cr, childCR, hasNotifiers) } diff --git a/internal/controller/operator/vmagent_controller.go b/internal/controller/operator/vmagent_controller.go index 7b1b2f3931..afd4f4783b 100644 --- a/internal/controller/operator/vmagent_controller.go +++ b/internal/controller/operator/vmagent_controller.go @@ -98,17 +98,23 @@ func (r *VMAgentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re return } - if !instance.IsUnmanaged(nil) { - agentSync.RLock() - defer agentSync.RUnlock() - } - RegisterObjectStat(&instance, r.name) if !instance.DeletionTimestamp.IsZero() { + agentSync.Lock() + defer agentSync.Unlock() + parentObject := fmt.Sprintf("%s.%s.vmagent", instance.Name, instance.Namespace) + if err = releaseScrapeChildStatuses(ctx, r.Client, parentObject); err != nil { + return + } err = finalize.OnVMAgentDelete(ctx, r.Client, &instance) return } + if !instance.IsUnmanaged(nil) { + agentSync.RLock() + defer agentSync.RUnlock() + } + if instance.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(instance.Status.ParsingSpecError) { err = newParsingError(instance.Status.ParsingSpecError) return diff --git a/internal/controller/operator/vmalert_controller.go b/internal/controller/operator/vmalert_controller.go index 6824bab669..07969998b2 100644 --- a/internal/controller/operator/vmalert_controller.go +++ b/internal/controller/operator/vmalert_controller.go @@ -18,6 +18,7 @@ package operator import ( "context" + "fmt" "strings" "sync" @@ -31,9 +32,11 @@ import ( vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/config" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/finalize" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/limiter" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/vmalert" ) @@ -87,17 +90,25 @@ func (r *VMAlertReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re return } - if !instance.IsUnmanaged() { - alertSync.RLock() - defer alertSync.RUnlock() - } - RegisterObjectStat(&instance, r.name) if !instance.DeletionTimestamp.IsZero() { + alertSync.Lock() + defer alertSync.Unlock() + if !build.IsControllerDisabled("VMRule") { + parentObject := fmt.Sprintf("%s.%s.vmalert", instance.Name, instance.Namespace) + if err = reconcile.StatusForChildObjects(ctx, r.Client, parentObject, []*vmv1beta1.VMRule(nil)); err != nil { + return + } + } err = finalize.OnVMAlertDelete(ctx, r.Client, &instance) return } + if !instance.IsUnmanaged() { + alertSync.RLock() + defer alertSync.RUnlock() + } + if instance.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(instance.Status.ParsingSpecError) { err = newParsingError(instance.Status.ParsingSpecError) return diff --git a/internal/controller/operator/vmalert_controller_test.go b/internal/controller/operator/vmalert_controller_test.go index 7f7c0326a5..4656564362 100644 --- a/internal/controller/operator/vmalert_controller_test.go +++ b/internal/controller/operator/vmalert_controller_test.go @@ -87,6 +87,55 @@ var _ = Describe("VMAlert Controller", func() { }) }) +func TestVMAlert_Reconcile_DeleteReleasesChildStatus(t *testing.T) { + g := NewWithT(t) + parent := &vmv1beta1.VMAlert{ + ObjectMeta: metav1.ObjectMeta{ + Name: "parent1", + Namespace: "default", + Finalizers: []string{vmv1beta1.FinalizerName}, + }, + Spec: vmv1beta1.VMAlertSpec{ + SelectAllByDefault: true, + }, + } + conditionType := "parent1.default.vmalert" + vmv1beta1.ConditionDomainTypeAppliedSuffix + child := &vmv1beta1.VMRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "child1", + Namespace: "default", + }, + Status: vmv1beta1.VMRuleStatus{ + StatusMetadata: vmv1beta1.StatusMetadata{ + UpdateStatus: vmv1beta1.UpdateStatusOperational, + Conditions: []vmv1beta1.Condition{ + {Type: conditionType, Status: "True", Reason: vmv1beta1.ConditionParsingReason}, + }, + }, + }, + } + + fclient := k8stools.GetTestClientWithObjects([]runtime.Object{parent, child}) + g.Expect(fclient.Delete(context.TODO(), parent)).To(Succeed()) + + r := &VMAlertReconciler{ + Client: fclient, + BaseConf: &config.BaseOperatorConf{}, + Log: ctrl.Log.WithName("test"), + OriginScheme: fclient.Scheme(), + } + nsn := types.NamespacedName{Name: parent.Name, Namespace: parent.Namespace} + _, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsn}) + g.Expect(err).NotTo(HaveOccurred()) + + var got vmv1beta1.VMRule + g.Expect(fclient.Get(context.TODO(), types.NamespacedName{Name: child.Name, Namespace: child.Namespace}, &got)).To(Succeed()) + for _, c := range got.Status.Conditions { + g.Expect(c.Type).NotTo(Equal(conditionType)) + } + g.Expect(got.Status.UpdateStatus).To(Equal(vmv1beta1.UpdateStatusIgnored)) +} + func TestVMAlert_Reconcile_AgentSync_Managed(t *testing.T) { g := NewWithT(t) managed := &vmv1beta1.VMAlert{ diff --git a/internal/controller/operator/vmalertmanager_controller.go b/internal/controller/operator/vmalertmanager_controller.go index 87bd28bf4c..dffe3b84c5 100644 --- a/internal/controller/operator/vmalertmanager_controller.go +++ b/internal/controller/operator/vmalertmanager_controller.go @@ -18,6 +18,7 @@ package operator import ( "context" + "fmt" "strings" "sync" @@ -31,9 +32,11 @@ import ( vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/config" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/finalize" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/limiter" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/vmalertmanager" ) @@ -90,16 +93,24 @@ func (r *VMAlertmanagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque return } - if !instance.IsUnmanaged() { - alertmanagerSync.RLock() - defer alertmanagerSync.RUnlock() - } - RegisterObjectStat(&instance, r.name) if !instance.DeletionTimestamp.IsZero() { + alertmanagerSync.Lock() + defer alertmanagerSync.Unlock() + if !build.IsControllerDisabled("VMAlertmanagerConfig") { + parentObject := fmt.Sprintf("%s.%s.vmalertmanager", instance.Name, instance.Namespace) + if err = reconcile.StatusForChildObjects(ctx, r.Client, parentObject, []*vmv1beta1.VMAlertmanagerConfig(nil)); err != nil { + return + } + } err = finalize.OnVMAlertManagerDelete(ctx, r.Client, &instance) return } + + if !instance.IsUnmanaged() { + alertmanagerSync.RLock() + defer alertmanagerSync.RUnlock() + } if instance.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(instance.Status.ParsingSpecError) { err = newParsingError(instance.Status.ParsingSpecError) return diff --git a/internal/controller/operator/vmalertmanagerconfig_controller.go b/internal/controller/operator/vmalertmanagerconfig_controller.go index 78921ea05e..91775ac83d 100644 --- a/internal/controller/operator/vmalertmanagerconfig_controller.go +++ b/internal/controller/operator/vmalertmanagerconfig_controller.go @@ -18,6 +18,7 @@ package operator import ( "context" + "errors" "fmt" "strings" @@ -33,6 +34,7 @@ import ( "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/vmalertmanager" ) @@ -83,6 +85,7 @@ func (r *VMAlertmanagerConfigReconciler) Reconcile(ctx context.Context, req ctrl return } if alertmanagerReconcileLimit.Throttle() { + err = reconcile.SyncAggregatedChildStatus(ctx, r.Client, &instance) return } @@ -100,7 +103,7 @@ func (r *VMAlertmanagerConfigReconciler) Reconcile(ctx context.Context, req ctrl g.SetLimit(childReconcileConcurrencyLimit) for i := range objects.Items { item := &objects.Items[i] - if !item.DeletionTimestamp.IsZero() || (item.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(item.Status.ParsingSpecError)) || item.IsUnmanaged() { + if !item.DeletionTimestamp.IsZero() || (item.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(item.Status.ParsingSpecError)) { continue } @@ -137,6 +140,7 @@ func (r *VMAlertmanagerConfigReconciler) Reconcile(ctx context.Context, req ctrl }) } err = g.Wait() + err = errors.Join(err, reconcile.SyncAggregatedChildStatus(ctx, r.Client, &instance)) return } diff --git a/internal/controller/operator/vmanomaly_controller.go b/internal/controller/operator/vmanomaly_controller.go index f2e386c78c..542e92555f 100644 --- a/internal/controller/operator/vmanomaly_controller.go +++ b/internal/controller/operator/vmanomaly_controller.go @@ -18,6 +18,7 @@ package operator import ( "context" + "fmt" "strings" "sync" @@ -32,9 +33,11 @@ import ( vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/config" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/finalize" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/limiter" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/vmanomaly" ) @@ -88,17 +91,25 @@ func (r *VMAnomalyReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return } - if !instance.IsUnmanaged() { - anomalySync.Lock() - defer anomalySync.Unlock() - } - RegisterObjectStat(&instance, r.name) if !instance.DeletionTimestamp.IsZero() { + anomalySync.Lock() + defer anomalySync.Unlock() + if !build.IsControllerDisabled("VMAnomalyConfig") { + parentObject := fmt.Sprintf("%s.%s.vmanomaly", instance.Name, instance.Namespace) + if err = reconcile.StatusForChildObjects(ctx, r.Client, parentObject, []*vmv1.VMAnomalyConfig(nil)); err != nil { + return + } + } err = finalize.OnVMAnomalyDelete(ctx, r.Client, &instance) return } + if !instance.IsUnmanaged() { + anomalySync.Lock() + defer anomalySync.Unlock() + } + if instance.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(instance.Status.ParsingSpecError) { err = newParsingError(instance.Status.ParsingSpecError) return diff --git a/internal/controller/operator/vmanomalyconfig_controller.go b/internal/controller/operator/vmanomalyconfig_controller.go index f51b4e1765..c01a059cb3 100644 --- a/internal/controller/operator/vmanomalyconfig_controller.go +++ b/internal/controller/operator/vmanomalyconfig_controller.go @@ -18,10 +18,12 @@ package operator import ( "context" + "errors" "fmt" "strings" "github.com/go-logr/logr" + "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" @@ -33,6 +35,7 @@ import ( "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/vmanomaly" ) @@ -81,7 +84,7 @@ func (r *VMAnomalyConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ RegisterObjectStat(&instance, r.name) if anomalyReconcileLimit.Throttle() { - // fast path, rate limited + err = reconcile.SyncAggregatedChildStatus(ctx, r.Client, &instance) return } @@ -95,13 +98,15 @@ func (r *VMAnomalyConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ return } + var g errgroup.Group + g.SetLimit(childReconcileConcurrencyLimit) for i := range objects.Items { item := &objects.Items[i] - if !item.DeletionTimestamp.IsZero() || (item.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(item.Status.ParsingSpecError)) || item.IsUnmanaged() { + if !item.DeletionTimestamp.IsZero() || (item.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(item.Status.ParsingSpecError)) { continue } - l := l.WithValues("vmanomaly", item.Name, "parent_namespace", item.Namespace) - ctx := logger.AddToContext(ctx, l) + itemLog := l.WithValues("vmanomaly", item.Name, "parent_namespace", item.Namespace) + itemCtx := logger.AddToContext(ctx, itemLog) // only check selector when deleting object, // since labels can be changed when updating and we can't tell if it was selected before, and we can't tell if it's creating or updating. if !instance.DeletionTimestamp.IsZero() { @@ -111,9 +116,9 @@ func (r *VMAnomalyConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ ObjectSelector: item.Spec.ConfigSelector, NamespaceSelector: item.Spec.ConfigNamespaceSelector, } - match, err := isSelectorsMatchesTargetCRD(ctx, r.Client, &instance, item, opts, r.BaseConf.WatchNamespaces) + match, err := isSelectorsMatchesTargetCRD(itemCtx, r.Client, &instance, item, opts, r.BaseConf.WatchNamespaces) if err != nil { - l.Error(err, "cannot match vmanomaly and vmanomalyconfig") + itemLog.Error(err, "cannot match vmanomaly and vmanomalyconfig") continue } if !match { @@ -121,10 +126,16 @@ func (r *VMAnomalyConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } - if err := vmanomaly.CreateOrUpdateConfig(ctx, r, item, &instance); err != nil { - l.Error(err, "failed to update vmanomaly config") - } + g.Go(func() error { + if configErr := vmanomaly.CreateOrUpdateConfig(itemCtx, r, item, &instance); configErr != nil { + itemLog.Error(configErr, "failed to update vmanomaly config") + return configErr + } + return nil + }) } + err = g.Wait() + err = errors.Join(err, reconcile.SyncAggregatedChildStatus(ctx, r.Client, &instance)) return } diff --git a/internal/controller/operator/vmauth_controller.go b/internal/controller/operator/vmauth_controller.go index 8a4b5eef9c..965eb096fe 100644 --- a/internal/controller/operator/vmauth_controller.go +++ b/internal/controller/operator/vmauth_controller.go @@ -34,6 +34,7 @@ import ( "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/finalize" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/limiter" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/vmauth" ) @@ -89,18 +90,24 @@ func (r *VMAuthReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res return result, newGetError(err) } - if !instance.IsUnmanaged() { - authSync.RLock() - defer authSync.RUnlock() - } - RegisterObjectStat(&instance, r.name) if !instance.DeletionTimestamp.IsZero() { + authSync.Lock() + defer authSync.Unlock() + parentObject := fmt.Sprintf("%s.%s.vmauth", instance.Name, instance.Namespace) + if err = reconcile.StatusForChildObjects(ctx, r.Client, parentObject, []*vmv1beta1.VMUser(nil)); err != nil { + return + } if err = finalize.OnVMAuthDelete(ctx, r, &instance); err != nil { err = fmt.Errorf("cannot remove finalizer from vmauth: %w", err) } return } + + if !instance.IsUnmanaged() { + authSync.RLock() + defer authSync.RUnlock() + } if instance.Status.ParsingSpecError != "" && !vmv1beta1.HasUnknownFields(instance.Status.ParsingSpecError) { err = newParsingError(instance.Status.ParsingSpecError) return diff --git a/internal/controller/operator/vmnodescrape_controller.go b/internal/controller/operator/vmnodescrape_controller.go index 45cbdbdd6b..af16573c94 100644 --- a/internal/controller/operator/vmnodescrape_controller.go +++ b/internal/controller/operator/vmnodescrape_controller.go @@ -81,14 +81,7 @@ func (r *VMNodeScrapeReconciler) Reconcile(ctx context.Context, req ctrl.Request return } - if err = collectVMAgentScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } - - if err = collectVMSingleScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } - + err = collectAndSyncScrapeChildStatus(l, ctx, r.Client, r.BaseConf, &instance) return } diff --git a/internal/controller/operator/vmpodscrape_controller.go b/internal/controller/operator/vmpodscrape_controller.go index 17e53a81a8..3449b3f747 100644 --- a/internal/controller/operator/vmpodscrape_controller.go +++ b/internal/controller/operator/vmpodscrape_controller.go @@ -80,12 +80,7 @@ func (r *VMPodScrapeReconciler) Reconcile(ctx context.Context, req ctrl.Request) err = newParsingError(instance.Status.ParsingSpecError) return } - if err = collectVMAgentScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } - if err = collectVMSingleScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } + err = collectAndSyncScrapeChildStatus(l, ctx, r.Client, r.BaseConf, &instance) return } diff --git a/internal/controller/operator/vmprobe_controller.go b/internal/controller/operator/vmprobe_controller.go index 8ee30f0fef..26a9bcc633 100644 --- a/internal/controller/operator/vmprobe_controller.go +++ b/internal/controller/operator/vmprobe_controller.go @@ -80,12 +80,7 @@ func (r *VMProbeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re err = newParsingError(instance.Status.ParsingSpecError) return } - if err = collectVMAgentScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } - if err = collectVMSingleScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } + err = collectAndSyncScrapeChildStatus(l, ctx, r.Client, r.BaseConf, &instance) return } diff --git a/internal/controller/operator/vmrule_controller.go b/internal/controller/operator/vmrule_controller.go index fc8af3a048..68ed1fe665 100644 --- a/internal/controller/operator/vmrule_controller.go +++ b/internal/controller/operator/vmrule_controller.go @@ -18,6 +18,7 @@ package operator import ( "context" + "errors" "fmt" "strings" @@ -33,6 +34,7 @@ import ( "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/vmalert" ) @@ -86,6 +88,7 @@ func (r *VMRuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res return } if alertReconcileLimit.Throttle() { + err = reconcile.SyncAggregatedChildStatus(ctx, r.Client, &instance) return } @@ -137,6 +140,7 @@ func (r *VMRuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res }) } err = g.Wait() + err = errors.Join(err, reconcile.SyncAggregatedChildStatus(ctx, r.Client, &instance)) return } diff --git a/internal/controller/operator/vmscrapeconfig_controller.go b/internal/controller/operator/vmscrapeconfig_controller.go index ca77ff155f..7405c57ce8 100644 --- a/internal/controller/operator/vmscrapeconfig_controller.go +++ b/internal/controller/operator/vmscrapeconfig_controller.go @@ -80,12 +80,7 @@ func (r *VMScrapeConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reque err = newParsingError(instance.Status.ParsingSpecError) return } - if err = collectVMAgentScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } - if err = collectVMSingleScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } + err = collectAndSyncScrapeChildStatus(l, ctx, r.Client, r.BaseConf, &instance) return } diff --git a/internal/controller/operator/vmservicescrape_controller.go b/internal/controller/operator/vmservicescrape_controller.go index 1a77f8ec3a..f76bd5fb4b 100644 --- a/internal/controller/operator/vmservicescrape_controller.go +++ b/internal/controller/operator/vmservicescrape_controller.go @@ -80,12 +80,7 @@ func (r *VMServiceScrapeReconciler) Reconcile(ctx context.Context, req ctrl.Requ err = newParsingError(instance.Status.ParsingSpecError) return } - if err = collectVMAgentScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } - if err = collectVMSingleScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } + err = collectAndSyncScrapeChildStatus(l, ctx, r.Client, r.BaseConf, &instance) return } diff --git a/internal/controller/operator/vmsingle_controller.go b/internal/controller/operator/vmsingle_controller.go index 96ea686aa3..19853f0c40 100644 --- a/internal/controller/operator/vmsingle_controller.go +++ b/internal/controller/operator/vmsingle_controller.go @@ -104,6 +104,12 @@ func (r *VMSingleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r RegisterObjectStat(&instance, r.name) if !instance.DeletionTimestamp.IsZero() { + vmsingleSync.Lock() + defer vmsingleSync.Unlock() + parentObject := fmt.Sprintf("%s.%s.vmsingle", instance.Name, instance.Namespace) + if err = releaseScrapeChildStatuses(ctx, r.Client, parentObject); err != nil { + return + } err = finalize.OnVMSingleDelete(ctx, r.Client, &instance) return } diff --git a/internal/controller/operator/vmstaticscrape_controller.go b/internal/controller/operator/vmstaticscrape_controller.go index 15e109b94f..c92e306b34 100644 --- a/internal/controller/operator/vmstaticscrape_controller.go +++ b/internal/controller/operator/vmstaticscrape_controller.go @@ -59,12 +59,7 @@ func (r *VMStaticScrapeReconciler) Reconcile(ctx context.Context, req ctrl.Reque err = newParsingError(instance.Status.ParsingSpecError) return } - if err = collectVMAgentScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } - if err = collectVMSingleScrapes(l, ctx, r.Client, r.BaseConf, &instance); err != nil { - return - } + err = collectAndSyncScrapeChildStatus(l, ctx, r.Client, r.BaseConf, &instance) return } diff --git a/internal/controller/operator/vmuser_controller.go b/internal/controller/operator/vmuser_controller.go index f88204a6c4..729b84734d 100644 --- a/internal/controller/operator/vmuser_controller.go +++ b/internal/controller/operator/vmuser_controller.go @@ -18,6 +18,7 @@ package operator import ( "context" + "errors" "fmt" "strings" @@ -36,6 +37,7 @@ import ( "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/finalize" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/vmauth" ) @@ -97,6 +99,7 @@ func (r *VMUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res } if authReconcileLimit.Throttle() { + err = reconcile.SyncAggregatedChildStatus(ctx, r.Client, &instance) return } @@ -149,6 +152,7 @@ func (r *VMUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res }) } err = g.Wait() + err = errors.Join(err, reconcile.SyncAggregatedChildStatus(ctx, r.Client, &instance)) return }