From 0a493a78bc20d86eef9f5bf8929a262770108552 Mon Sep 17 00:00:00 2001 From: Mangirdas Judeikis Date: Tue, 22 Sep 2026 13:37:29 +0300 Subject: [PATCH 1/2] bump kcp to 0.33.1 --- internal/resources/resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/resources/resources.go b/internal/resources/resources.go index 7269e07d..b6e8269c 100644 --- a/internal/resources/resources.go +++ b/internal/resources/resources.go @@ -44,7 +44,7 @@ const ( // // When changing this to a new minor version, you must also update // the .prow.yaml accordingly and shift the jobs. - ImageTag = "v0.33.0" + ImageTag = "v0.33.1" appNameLabel = "app.kubernetes.io/name" appInstanceLabel = "app.kubernetes.io/instance" From 684b2cd543a1cee88a0d357e6a2381bac26f6589 Mon Sep 17 00:00:00 2001 From: Mangirdas Judeikis Date: Wed, 23 Sep 2026 09:04:36 +0300 Subject: [PATCH 2/2] Add upgrade handler --- pkg/client/clients.go | 29 ++++++++++++- pkg/controller/shard/controller.go | 69 +++++++++++++++++++++++------- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/pkg/client/clients.go b/pkg/client/clients.go index 2d5da397..86de6a14 100644 --- a/pkg/client/clients.go +++ b/pkg/client/clients.go @@ -52,6 +52,23 @@ func NewShardClient(ctx context.Context, c ctrlruntimeclient.Client, addr Addres return newClient(ctx, c, addr.Shard(shard), cluster, scheme, rootShard) } +// NewShardIdentityClient returns a new client for talking to the given endpoint as the shard +// itself, using the shard's own (system:masters) client certificate. This is required for +// managing the shard's own Shard object, which kcp protects against writes by anyone else. +func NewShardIdentityClient(ctx context.Context, c ctrlruntimeclient.Client, endpoint Endpoint, shard *operatorv1alpha1.Shard, cluster logicalcluster.Path, scheme *runtime.Scheme) (ctrlruntimeclient.Client, error) { + key := types.NamespacedName{ + Namespace: shard.Namespace, + Name: resources.GetShardCertificateName(shard, operatorv1alpha1.ClientCertificate), + } + + tlsConfig, err := getTLSConfigFromSecret(ctx, c, key) + if err != nil { + return nil, fmt.Errorf("failed to determine TLS settings: %w", err) + } + + return newClientWithTLS(endpoint, cluster, scheme, tlsConfig) +} + func newClient( ctx context.Context, c ctrlruntimeclient.Client, @@ -64,6 +81,11 @@ func newClient( if err != nil { return nil, fmt.Errorf("failed to determine TLS settings: %w", err) } + + return newClientWithTLS(endpoint, cluster, scheme, tlsConfig) +} + +func newClientWithTLS(endpoint Endpoint, cluster logicalcluster.Path, scheme *runtime.Scheme, tlsConfig rest.TLSClientConfig) (ctrlruntimeclient.Client, error) { tlsConfig.ServerName = endpoint.ServerName host := endpoint.URL @@ -90,9 +112,14 @@ func getTLSConfig(ctx context.Context, c ctrlruntimeclient.Client, rootShard *op Name: resources.GetRootShardCertificateName(rootShard, operatorv1alpha1.OperatorCertificate), } + return getTLSConfigFromSecret(ctx, c, key) +} + +// getTLSConfigFromSecret returns the CA and client certificate stored in a cert-manager Secret. +func getTLSConfigFromSecret(ctx context.Context, c ctrlruntimeclient.Client, key types.NamespacedName) (rest.TLSClientConfig, error) { certSecret := &corev1.Secret{} if err := c.Get(ctx, key, certSecret); err != nil { - return rest.TLSClientConfig{}, fmt.Errorf("failed to get root shard proxy Secret: %w", err) + return rest.TLSClientConfig{}, fmt.Errorf("failed to get client certificate Secret %s: %w", key.Name, err) } return rest.TLSClientConfig{ diff --git a/pkg/controller/shard/controller.go b/pkg/controller/shard/controller.go index e48791a2..b6c22f50 100644 --- a/pkg/controller/shard/controller.go +++ b/pkg/controller/shard/controller.go @@ -57,7 +57,17 @@ import ( operatorv1alpha1 "github.com/kcp-dev/kcp-operator/sdk/apis/operator/v1alpha1" ) -const cleanupFinalizer = "operator.kcp.io/cleanup-shard" +const ( + cleanupFinalizer = "operator.kcp.io/cleanup-shard" + + // shardRepresentationAnnotation marks a Shard in the root workspace as a read-only + // representation mirrored by kcp from the shard-owned object. kcp removes it once the + // shard-owned object is gone. + shardRepresentationAnnotation = "core.kcp.io/shard-representation" +) + +// systemShardCluster is the logical cluster on every shard holding its own Shard object. +var systemShardCluster = logicalcluster.NewPath("system:shard") // ShardReconciler reconciles a Shard object type ShardReconciler struct { @@ -318,22 +328,19 @@ func (r *ShardReconciler) handleDeletion(ctx context.Context, client ctrlruntime return []metav1.Condition{cond}, nil } - // Create client to root shard - kcpClient, err := operatorclient.NewRootShardClient(ctx, client, r.Address, rootShard, logicalcluster.NewPath("root"), scheme) - if err != nil { - return nil, fmt.Errorf("failed to create root shard client: %w", err) + // Since kcp 0.34, each shard owns its Shard object in its own system:shard logical cluster, + // from where it is replicated to the cache server; the root workspace only holds a read-only + // representation maintained by kcp. Older kcp versions register shards directly in root. + // Both objects are protected against writes by anyone but system:masters, so the shard is + // deregistered using the shard's own client certificate. + // + // TODO: This will go away once shards owns their owns full lifecycle. + if err := r.deleteKCPShard(ctx, client, scheme, r.Address.Shard(s), s, systemShardCluster); err != nil { + return nil, err } - // Delete the kcp Shard object - kcpShard := &kcpcorev1alpha1.Shard{} - kcpShard.Name = s.Name - - logger.Info("Deleting kcp Shard object from root workspace", "name", s.Name) - if err := kcpClient.Delete(ctx, kcpShard); err != nil { - if !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to delete kcp Shard: %w", err) - } - logger.V(2).Info("kcp Shard object already deleted") + if err := r.deleteKCPShard(ctx, client, scheme, r.Address.RootShard(rootShard), s, logicalcluster.NewPath("root")); err != nil { + return nil, err } // Remove finalizer @@ -344,6 +351,38 @@ func (r *ShardReconciler) handleDeletion(ctx context.Context, client ctrlruntime return nil, nil } +// deleteKCPShard deletes the kcp Shard object for s from the given logical cluster. Kcp-managed +// representations are left alone, since kcp removes them itself. +func (r *ShardReconciler) deleteKCPShard(ctx context.Context, client ctrlruntimeclient.Client, scheme *runtime.Scheme, endpoint operatorclient.Endpoint, s *operatorv1alpha1.Shard, cluster logicalcluster.Path) error { + logger := log.FromContext(ctx).WithValues("name", s.Name, "cluster", cluster.String()) + + kcpClient, err := operatorclient.NewShardIdentityClient(ctx, client, endpoint, s, cluster, scheme) + if err != nil { + return fmt.Errorf("failed to create client for %s: %w", cluster, err) + } + + kcpShard := &kcpcorev1alpha1.Shard{} + if err := kcpClient.Get(ctx, types.NamespacedName{Name: s.Name}, kcpShard); err != nil { + if apierrors.IsNotFound(err) { + logger.V(2).Info("kcp Shard object does not exist") + return nil + } + return fmt.Errorf("failed to get kcp Shard in %s: %w", cluster, err) + } + + if kcpShard.Annotations[shardRepresentationAnnotation] == "true" { + logger.V(2).Info("Leaving kcp Shard representation to kcp") + return nil + } + + logger.Info("Deleting kcp Shard object") + if err := kcpClient.Delete(ctx, kcpShard); ctrlruntimeclient.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete kcp Shard in %s: %w", cluster, err) + } + + return nil +} + func (r *ShardReconciler) ensureFinalizer(ctx context.Context, client ctrlruntimeclient.Client, s *operatorv1alpha1.Shard) (bool, error) { finalizers := sets.New(s.GetFinalizers()...) if finalizers.Has(cleanupFinalizer) {