[wip][3- De-Rooting] Move serviceaccount of root shard

cmd/sharded-test-server/frontproxy.go

@@ -137,7 +137,6 @@ func startFrontProxy(
 		"--bind-address="+hostIP,
 		fmt.Sprintf("--mapping-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy", "mapping.yaml")),
 		fmt.Sprintf("--root-directory=%s", filepath.Join(workDirPath, ".kcp-front-proxy")),
-		fmt.Sprintf("--root-kubeconfig=%s", filepath.Join(workDirPath, ".kcp", "root.kubeconfig")),
 		fmt.Sprintf("--shards-kubeconfig=%s", filepath.Join(workDirPath, ".kcp-front-proxy", "shards.kubeconfig")),
 		fmt.Sprintf("--shard-peer-kubeconfig=%s", filepath.Join(workDirPath, ".kcp-front-proxy", "peers.kubeconfig")),
 		fmt.Sprintf("--client-ca-file=%s", filepath.Join(workDirPath, ".kcp", "client-ca.crt")),

config/crds/admin.kcp.io_apiidentities.yaml

@@ -0,0 +1,192 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.17.3
+  name: apiidentities.admin.kcp.io
+spec:
+  group: admin.kcp.io
+  names:
+    categories:
+    - kcp
+    kind: APIIdentity
+    listKind: APIIdentityList
+    plural: apiidentities
+    singular: apiidentity
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: APIExport owning the identity
+      jsonPath: .status.owner.name
+      name: Owner
+      type: string
+    - description: Logical cluster of the owner
+      jsonPath: .status.owner.cluster
+      name: Cluster
+      type: string
+    - description: Identity of a core APIExport in root
+      jsonPath: .status.core
+      name: Core
+      type: boolean
+    - description: True while other APIExports hold the identity
+      jsonPath: .status.conditions[?(@.type=="Shared")].status
+      name: Shared
+      priority: 1
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          APIIdentity is the registry entry of one API identity: the record of which
+          APIExport owns the identity and which other APIExports hold it. There is
+          exactly one entry per identity in an installation, named after the
+          identity value, living on the cache server where every shard can read it
+          and served read-only through the Admin workspace. Entries are written by
+          kcp only.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          status:
+            description: |-
+              APIIdentityStatus is the derived state of an identity. The object has no
+              spec: nothing about it is user intent.
+            properties:
+              conditions:
+                description: conditions is a list of conditions that apply to the
+                  identity.
+                items:
+                  description: Condition defines an observation of a object operational
+                    state.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        Last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed. If that is not known, then using the time when
+                        the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        A human readable message indicating details about the transition.
+                        This field may be empty.
+                      type: string
+                    reason:
+                      description: |-
+                        The reason for the condition's last transition in CamelCase.
+                        The specific API may choose whether or not this field is considered a guaranteed API.
+                        This field may not be empty.
+                      type: string
+                    severity:
+                      description: |-
+                        Severity provides an explicit classification of Reason code, so the users or machines can immediately
+                        understand the current situation and act accordingly.
+                        The Severity field MUST be set only when Status=False.
+                      type: string
+                    status:
+                      description: Status of the condition, one of True, False, Unknown.
+                      type: string
+                    type:
+                      description: |-
+                        Type of condition in CamelCase or in foo.example.com/CamelCase.
+                        Many .condition.type values are consistent across resources like Available, but because arbitrary conditions
+                        can be useful (see .node.status.conditions), the ability to deconflict is important.
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - status
+                  - type
+                  type: object
+                type: array
+              core:
+                description: |-
+                  core is true for the identities of the core APIExports in the root
+                  workspace, which every shard needs before it can serve.
+                type: boolean
+              holders:
+                description: |-
+                  holders lists the APIExports, other than the owner, that serve the
+                  identity: those whose identity secret carries the same key. A holder
+                  records itself when it gets the identity and is dropped when it is
+                  deleted.
+                items:
+                  description: Holder is an APIExport, other than the owner, that
+                    serves the identity.
+                  properties:
+                    cluster:
+                      description: cluster is the logical cluster name hosting the
+                        APIExport.
+                      minLength: 1
+                      type: string
+                    name:
+                      description: name is the name of the APIExport.
+                      minLength: 1
+                      type: string
+                    since:
+                      description: since is when the holder was recorded.
+                      format: date-time
+                      type: string
+                    uid:
+                      description: |-
+                        uid is the UID of the APIExport, so that an export deleted and
+                        recreated under the same name can be told apart from the original.
+                      type: string
+                  required:
+                  - cluster
+                  - name
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - cluster
+                - name
+                x-kubernetes-list-type: map
+              owner:
+                description: |-
+                  owner is the APIExport that holds the identity authoritatively: the
+                  first one to use it. It is written by kcp when the identity is claimed
+                  and changed by kcp only when the owner is deleted and a holder
+                  succeeds it. An entry whose owner is gone and that has no holder left
+                  is removed.
+                properties:
+                  cluster:
+                    description: cluster is the logical cluster name hosting the APIExport.
+                    minLength: 1
+                    type: string
+                  name:
+                    description: name is the name of the APIExport.
+                    minLength: 1
+                    type: string
+                  uid:
+                    description: |-
+                      uid is the UID of the APIExport, so that an export deleted and
+                      recreated under the same name can be told apart from the original.
+                    type: string
+                required:
+                - cluster
+                - name
+                type: object
+            required:
+            - owner
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources: {}

docs/content/concepts/apis/exporting-apis.md

@@ -163,15 +163,34 @@ above), kcp automatically generates one for you. The identity is always stored i
 the `APIExport.` By default, it is created in the `kcp-system` namespace in a secret whose name is the name of
 the `APIExport`.
 
-An `APIExport`'s identity is similar to a private key; you should never share it with anyone.
+An `APIExport`'s identity key is similar to a private key; you should never share it with anyone.
 
-The identity's **hash** is similar to a public key; it is not private and there are times when other consumers of your
-exported APIs need to know and reference it.
+The identity's **hash** (`status.identityHash`) is similar to a public key; it is not private and there are times
+when other consumers of your exported APIs need to know and reference it.
 
 Given 2 Workspaces, each with its own `APIExport` that exports `widgets.example.kcp.io`, kcp uses the identity hash (in
 a mostly transparent manner) to ensure the correct instances associated with the appropriate `APIResourceSchema` are
 served to clients. See [Build Your Controller](#build-your-controller) for more information.
 
+#### Sharing an identity with another APIExport
+
+Two `APIExports` whose secrets carry the same `key` have the same identity: they serve the same resources out of the
+same storage, which is how an `APIExport` is split into two or moved to another workspace without touching the
+consumers' objects. Today that works by copying the secret, and it keeps working.
+
+kcp records every identity in a registry (`APIIdentity` in `admin.kcp.io`, on the cache server, listed read-only
+in the [Admin workspace](../sharding/shards.md#the-admin-workspace-admin) with `kubectl ws use :admin`): the first
+`APIExport` to use an identity is its **owner**, every other `APIExport` whose secret carries the same `key` is a
+**holder**, and the entry names both. An identity with holders carries the `Shared` condition. The registry does
+not change who may serve an identity, which is still whoever has the secret; it makes sharing visible and gives the
+identity a lifecycle of its own.
+
+When the owner's `APIExport` is deleted, the identity passes to the oldest holder. The decision is made from the
+`APIExports` replicated to the cache server: an owner missing from the cache is marked `OwnerMissing`, and after a
+grace period (an hour by default, `--api-identity-missing-grace-period`) it is taken as gone. An identity with no owner and no holder left is removed from the registry; the
+next `APIExport` whose secret carries the original `key` registers it anew.
+`status.identityHash` is set once the identity is recorded in the registry and is immutable afterwards.
+
 ### Permission Claims
 
 When a consumer creates an `APIBinding` that binds to an `APIExport`, the API provider who owns the `APIExport`

docs/content/concepts/sharding/shards.md

@@ -108,14 +108,31 @@ root      us-east-2   https://127.0.0.1:6444   ...
 shard-1   us-east-1   https://127.0.0.1:6445   ...
 ```
 
+The Admin workspace also serves the API identity registry read-only:
+`apiidentities.admin.kcp.io` lists every APIExport identity in the
+installation, named by the identity hash, with the owning `APIExport`, the
+other exports holding it and whether it is a core identity. See
+[Exporting APIs](../apis/exporting-apis.md#sharing-an-identity-with-another-apiexport).
+
+```sh
+$ kubectl get apiidentities
+NAME                                                               OWNER             CLUSTER            CORE    AGE
+0e2fc4327ee7af2b2b9a0f2b4b1a8c4c7a0d6c2f0f1b6e5a9d8c7b6a5f4e3d2c   tenancy.kcp.io    root               true    3d
+5fdf7c7aaf407fd1594566869803f565bb84d22156cef5c445d2ee13ac2cfca6   widgets           2cynbfy2m0wtjqcs   false   2h
+$ kubectl get apiidentities -l apiidentities.admin.kcp.io/core=true           # the core identities
+$ kubectl get apiidentities -l apiidentities.admin.kcp.io/owner-cluster=root  # owned by exports in root
+$ kubectl get apiidentities -l apiidentities.admin.kcp.io/owner=widgets       # owned by an export named widgets
+```
+
 Access requires membership in the `system:kcp:admin` group; the serving
 path does not depend on the root shard. To reach the view through a
 front-proxy, add a `/services/` path mapping; any shard can be the backend.
 
-The front-proxy itself discovers shards through the Admin workspace: the
-kubeconfigs given via `--shard-peer-kubeconfig` (falling back to the
-`--root-kubeconfig` server) are seeds, used to bootstrap discovery and as a
-last-resort fallback only. Every shard observed through the view joins the
+The front-proxy itself discovers shards through the Admin workspace, and
+reads the identities of the core APIs from the `apiidentities` registry
+there, so it starts and runs without the root shard: the kubeconfigs given
+via `--shard-peer-kubeconfig` are seeds, used to bootstrap discovery and as
+a last-resort fallback only. Every shard observed through the view joins the
 failover peer set at runtime and decommissioned shards leave it; requests are
 distributed across the discovered shards, not the seeds. With short-lived
 shards the discovery channel therefore keeps working as long as any currently

docs/content/setup/production/metrics.md

@@ -134,3 +134,10 @@ scrape_configs:
       - targets:
           - cache.kcp.svc:6443
 ```
+
+## Scraping the front-proxy
+
+The front-proxy's own `/metrics` is authorized by group: members of
+`system:kcp:metrics-reader`, `system:kcp:admin` or `system:masters` may read
+it. Give the scraper identity the `system:kcp:metrics-reader` group, for
+example as a client certificate organization or an OIDC group claim.

hack/update-codegen-clients.sh

@@ -51,6 +51,7 @@ mkdir -p ${SDK_PKG}/client/{clientset,applyconfiguration,listers,informers}
   github.com/kcp-dev/sdk/apis/apis/v1alpha1 \
   github.com/kcp-dev/sdk/apis/apis/v1alpha2 \
   github.com/kcp-dev/sdk/apis/cache/v1alpha1 \
+  github.com/kcp-dev/sdk/apis/admin/v1alpha1 \
   github.com/kcp-dev/sdk/apis/migration/v1alpha1 \
   github.com/kcp-dev/sdk/apis/topology/v1alpha1 \
   github.com/kcp-dev/sdk/apis/third_party/conditions/apis/conditions/v1alpha1 \
@@ -68,6 +69,7 @@ mkdir -p ${SDK_PKG}/client/{clientset,applyconfiguration,listers,informers}
   --input github.com/kcp-dev/sdk/apis/apis/v1alpha2 \
   --input github.com/kcp-dev/sdk/apis/topology/v1alpha1 \
   --input github.com/kcp-dev/sdk/apis/cache/v1alpha1 \
+  --input github.com/kcp-dev/sdk/apis/admin/v1alpha1 \
   --input github.com/kcp-dev/sdk/apis/migration/v1alpha1 \
   --input-base="" \
   --apply-configuration-package=github.com/kcp-dev/sdk/client/applyconfiguration \
@@ -151,6 +153,7 @@ go install "${OPENAPI_PKG}"/cmd/openapi-gen
   github.com/kcp-dev/sdk/apis/apis/v1alpha2 \
   github.com/kcp-dev/sdk/apis/topology/v1alpha1 \
   github.com/kcp-dev/sdk/apis/cache/v1alpha1 \
+  github.com/kcp-dev/sdk/apis/admin/v1alpha1 \
   github.com/kcp-dev/sdk/apis/migration/v1alpha1 \
   github.com/kcp-dev/sdk/apis/third_party/conditions/apis/conditions/v1alpha1
 
@@ -165,6 +168,7 @@ go install "${OPENAPI_PKG}"/cmd/openapi-gen
   github.com/kcp-dev/sdk/apis/apis/v1alpha2 \
   github.com/kcp-dev/sdk/apis/topology/v1alpha1 \
   github.com/kcp-dev/sdk/apis/cache/v1alpha1 \
+  github.com/kcp-dev/sdk/apis/admin/v1alpha1 \
   github.com/kcp-dev/sdk/apis/migration/v1alpha1 \
   github.com/kcp-dev/sdk/apis/third_party/conditions/apis/conditions/v1alpha1 \
   k8s.io/apimachinery/pkg/apis/meta/v1 \

hack/update-codegen-crds.sh

@@ -47,7 +47,8 @@ done
 
 (
   ${KCP_APIGEN_GEN} --input-dir "${REPO_ROOT}"/config/crds --output-dir "${REPO_ROOT}"/config/root-phase0 \
-  --ignore-export-schemas cachedobjects.cache.kcp.io
+  --ignore-export-schemas cachedobjects.cache.kcp.io \
+  --ignore-schemas apiidentities.admin.kcp.io
 )
 
 

pkg/admission/apiexport/admission.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/authentication/user"
 
 	"github.com/kcp-dev/sdk/apis/apis"
 	apisv1alpha1 "github.com/kcp-dev/sdk/apis/apis/v1alpha1"
@@ -75,6 +76,13 @@ func (e *APIExportAdmission) Validate(ctx context.Context, a admission.Attribute
 		return fmt.Errorf("unexpected type %T", a.GetObject())
 	}
 
+	if a.GetSubresource() == "status" {
+		if a.GetOperation() == admission.Update {
+			return validateStatusUpdate(a, u)
+		}
+		return nil
+	}
+
 	switch a.GetKind().GroupVersion().Version {
 	case apisv1alpha1.SchemeGroupVersion.Version:
 		// v1alpha1 is deprecated, but we still need to support it for a while
@@ -124,6 +132,46 @@ func (e *APIExportAdmission) Validate(ctx context.Context, a admission.Attribute
 	return nil
 }
 
+// validateStatusUpdate keeps status.identityHash immutable for everyone but
+// kcp itself. The identity keys the storage of every consumer's bound
+// instances and is what permission claims and wildcard clients address the
+// export by; a status write that changed it would make the writer's export
+// serve another export's consumers' data. kcp changes it only to withdraw an
+// authorization. The field sits at the same path in every served version.
+func validateStatusUpdate(a admission.Attributes, u *unstructured.Unstructured) error {
+	old, ok := a.GetOldObject().(*unstructured.Unstructured)
+	if !ok {
+		return fmt.Errorf("unexpected type %T", a.GetOldObject())
+	}
+	oldHash, _, err := unstructured.NestedString(old.Object, "status", "identityHash")
+	if err != nil {
+		return fmt.Errorf("failed to read status.identityHash: %w", err)
+	}
+	newHash, _, err := unstructured.NestedString(u.Object, "status", "identityHash")
+	if err != nil {
+		return fmt.Errorf("failed to read status.identityHash: %w", err)
+	}
+	if oldHash == "" || newHash == oldHash || isSystemPrivileged(a) {
+		return nil
+	}
+	return admission.NewForbidden(a,
+		field.Forbidden(
+			field.NewPath("status").Child("identityHash"),
+			"identityHash is immutable once set"))
+}
+
+func isSystemPrivileged(a admission.Attributes) bool {
+	if a.GetUserInfo() == nil {
+		return false
+	}
+	for _, g := range a.GetUserInfo().GetGroups() {
+		if g == user.SystemPrivilegedGroup {
+			return true
+		}
+	}
+	return false
+}
+
 func (e *APIExportAdmission) validatev1alpha2(_ context.Context, a admission.Attributes, ae *apisv1alpha2.APIExport) (err error) {
 	for i, pc := range ae.Spec.PermissionClaims {
 		if pc.IdentityHash == "" && !e.isBuiltIn(pc.GroupResource) && pc.Group != apis.GroupName {