Merge pull request #18146 from rifelpet/gce-cilium-etcd

Fix cilium-etcd on GCE

Author: k8s-ci-robot

cmd/kops/integration_test.go

@@ -409,6 +409,20 @@ func TestMinimalGCEInternalLoadBalancer(t *testing.T) {
 		runTestTerraformGCE(t)
 }
 
+// TestMinimalGCEInternalLoadBalancerCiliumEtcd runs tests on a minimal GCE configuration with an internal load balancer and cilium-etcd.
+func TestMinimalGCEInternalLoadBalancerCiliumEtcd(t *testing.T) {
+	newIntegrationTest("minimal-gce-ilb-cilium-etcd.example.com", "minimal_gce_ilb_cilium_etcd").
+		withCiliumEtcd().
+		withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium-master-us-test1-a").
+		withAddons(
+			ciliumAddon,
+			dnsControllerAddon,
+			gcpCCMAddon,
+			gcpPDCSIAddon,
+		).
+		runTestTerraformGCE(t)
+}
+
 // TestMinimalGCEPublicLoadBalancer runs tests on a minimal GCE configuration with a public load balancer.
 func TestMinimalGCEPublicLoadBalancer(t *testing.T) {
 	newIntegrationTest("minimal-gce-plb.example.com", "minimal_gce_plb").

pkg/model/gcemodel/api_loadbalancer.go

@@ -22,6 +22,7 @@ import (
 
 	"golang.org/x/exp/slices"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/pkg/wellknownservices"
 	"k8s.io/kops/upup/pkg/fi"
@@ -127,6 +128,16 @@ func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderConte
 				Allowed:      []string{"tcp:" + strconv.Itoa(wellknownports.KopsControllerPort)},
 			})
 		}
+
+		if model.UseCiliumEtcd(b.Cluster) {
+			b.AddFirewallRulesTasks(c, "cilium-etcd", &gcetasks.FirewallRule{
+				Lifecycle:    b.Lifecycle,
+				Network:      network,
+				SourceRanges: b.Cluster.Spec.API.Access,
+				TargetTags:   []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane)},
+				Allowed:      []string{"tcp:" + strconv.Itoa(wellknownports.EtcdCiliumClientPort)},
+			})
+		}
 	}
 	return nil
 
@@ -234,6 +245,24 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
 
 			c.AddTask(fr)
 		}
+
+		if model.UseCiliumEtcd(b.Cluster) {
+			c.AddTask(&gcetasks.ForwardingRule{
+				Name:                s(b.NameForForwardingRule("cilium-etcd-" + sn.Name)),
+				Lifecycle:           b.Lifecycle,
+				BackendService:      bs,
+				Ports:               []string{strconv.Itoa(wellknownports.EtcdCiliumClientPort)},
+				IPAddress:           ipAddress,
+				IPProtocol:          "TCP",
+				LoadBalancingScheme: s("INTERNAL"),
+				Network:             network,
+				Subnetwork:          subnet,
+				Labels: map[string]string{
+					clusterLabel.Key: clusterLabel.Value,
+					"name":           "cilium-etcd-" + sn.Name,
+				},
+			})
+		}
 	}
 	return nil
 }

tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_cluster-completed.spec_content

@@ -0,0 +1,255 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: minimal-gce-ilb-cilium-etcd.example.com
+spec:
+  api:
+    loadBalancer:
+      subnets:
+      - name: us-test-1
+      type: Internal
+      useForInternalApi: true
+  authorization:
+    rbac: {}
+  channel: stable
+  cloudConfig:
+    gceServiceAccount: default
+    gcpPDCSIDriver:
+      defaultStorageClassName: balanced-csi
+      enabled: true
+      version: v1.22.1
+    manageStorageClasses: true
+    multizone: true
+    nodeTags: minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node
+  cloudControllerManager:
+    allocateNodeCIDRs: true
+    cidrAllocatorType: CloudAllocator
+    clusterCIDR: 100.96.0.0/11
+    clusterName: minimal-gce-ilb-cilium-etcd-example-com
+    controllers:
+    - '*'
+    image: registry.k8s.io/cloud-provider-gcp/cloud-controller-manager:v35.0.0
+    leaderElection:
+      leaderElect: true
+  cloudProvider: gce
+  clusterDNSDomain: cluster.local
+  configBase: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com
+  containerd:
+    logLevel: info
+    runc:
+      version: 1.3.4
+    sandboxImage: registry.k8s.io/pause:3.10.1
+    version: 2.1.6
+  dnsZone: "1"
+  etcdClusters:
+  - backups:
+      backupStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/main
+    cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    manager:
+      backupRetentionDays: 90
+    memoryRequest: 100Mi
+    name: main
+    version: 3.5.25
+  - backups:
+      backupStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/events
+    cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    manager:
+      backupRetentionDays: 90
+    memoryRequest: 100Mi
+    name: events
+    version: 3.5.25
+  - backups:
+      backupStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/cilium
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    manager:
+      backupRetentionDays: 90
+    name: cilium
+    version: 3.5.25
+  externalDns:
+    provider: dns-controller
+  iam:
+    legacy: false
+  keyStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/pki
+  kubeAPIServer:
+    allowPrivileged: true
+    anonymousAuth: false
+    apiAudiences:
+    - kubernetes.svc.default
+    apiServerCount: 1
+    authorizationMode: Node,RBAC
+    bindAddress: 0.0.0.0
+    cloudProvider: external
+    enableAdmissionPlugins:
+    - DefaultStorageClass
+    - DefaultTolerationSeconds
+    - LimitRanger
+    - MutatingAdmissionWebhook
+    - NamespaceLifecycle
+    - NodeRestriction
+    - ResourceQuota
+    - RuntimeClass
+    - ServiceAccount
+    - ValidatingAdmissionPolicy
+    - ValidatingAdmissionWebhook
+    etcdServers:
+    - https://127.0.0.1:4001
+    etcdServersOverrides:
+    - /events#https://127.0.0.1:4002
+    image: registry.k8s.io/kube-apiserver:v1.32.0
+    kubeletPreferredAddressTypes:
+    - InternalIP
+    - Hostname
+    - ExternalIP
+    logLevel: 2
+    requestheaderAllowedNames:
+    - aggregator
+    requestheaderExtraHeaderPrefixes:
+    - X-Remote-Extra-
+    requestheaderGroupHeaders:
+    - X-Remote-Group
+    requestheaderUsernameHeaders:
+    - X-Remote-User
+    securePort: 443
+    serviceAccountIssuer: https://api.internal.minimal-gce-ilb-cilium-etcd.example.com
+    serviceAccountJWKSURI: https://api.internal.minimal-gce-ilb-cilium-etcd.example.com/openid/v1/jwks
+    serviceClusterIPRange: 100.64.0.0/13
+    storageBackend: etcd3
+  kubeControllerManager:
+    allocateNodeCIDRs: true
+    attachDetachReconcileSyncPeriod: 1m0s
+    cloudProvider: external
+    clusterCIDR: 100.96.0.0/11
+    clusterName: minimal-gce-ilb-cilium-etcd.example.com
+    configureCloudRoutes: false
+    image: registry.k8s.io/kube-controller-manager:v1.32.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+    useServiceAccountCredentials: true
+  kubeDNS:
+    cacheMaxConcurrent: 150
+    cacheMaxSize: 1000
+    cpuRequest: 100m
+    domain: cluster.local
+    memoryLimit: 170Mi
+    memoryRequest: 70Mi
+    nodeLocalDNS:
+      cpuRequest: 25m
+      enabled: false
+      image: registry.k8s.io/dns/k8s-dns-node-cache:1.26.0
+      memoryRequest: 5Mi
+    provider: CoreDNS
+    serverIP: 100.64.0.10
+  kubeProxy:
+    clusterCIDR: 100.96.0.0/11
+    cpuRequest: 100m
+    enabled: false
+    image: registry.k8s.io/kube-proxy:v1.32.0
+    logLevel: 2
+  kubeScheduler:
+    image: registry.k8s.io/kube-scheduler:v1.32.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+  kubelet:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: external
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hairpinMode: promiscuous-bridge
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
+    registerSchedulable: true
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  - ::/0
+  kubernetesVersion: 1.32.0
+  masterKubelet:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: external
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hairpinMode: promiscuous-bridge
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
+    registerSchedulable: true
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
+  masterPublicName: api.minimal-gce-ilb-cilium-etcd.example.com
+  networking:
+    cilium:
+      agentPrometheusPort: 9090
+      bpfCTGlobalAnyMax: 262144
+      bpfCTGlobalTCPMax: 524288
+      bpfLBAlgorithm: random
+      bpfLBMaglevTableSize: "16381"
+      bpfLBMapMax: 65536
+      bpfNATGlobalMax: 524288
+      bpfNeighGlobalMax: 524288
+      bpfPolicyMapMax: 16384
+      clusterName: default
+      cniExclusive: true
+      cpuRequest: 25m
+      disableCNPStatusUpdates: true
+      disableMasquerade: false
+      enableBPFMasquerade: false
+      enableEndpointHealthChecking: true
+      enableL7Proxy: true
+      enableLocalRedirectPolicy: false
+      enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
+      etcdManaged: true
+      gatewayAPI:
+        enabled: false
+      hubble:
+        enabled: false
+      identityAllocationMode: crd
+      identityChangeGracePeriod: 5s
+      ingress:
+        enabled: false
+      ipam: kubernetes
+      memoryRequest: 128Mi
+      monitorAggregation: medium
+      sidecarIstioProxyImage: cilium/istio_proxy
+      toFqdnsDnsRejectResponseCode: refused
+      tunnel: vxlan
+      version: v1.18.6
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podCIDR: 100.96.0.0/11
+  project: testproject
+  secretStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/secrets
+  serviceClusterIPRange: 100.64.0.0/13
+  sshAccess:
+  - 0.0.0.0/0
+  - ::/0
+  subnets:
+  - cidr: 10.0.16.0/20
+    name: us-test1
+    region: us-test1
+    type: Private
+  topology:
+    dns:
+      type: Public

tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-cilium_content

@@ -0,0 +1,4 @@
+{
+  "memberCount": 1,
+  "etcdVersion": "3.5.25"
+}

tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-events_content

@@ -0,0 +1,4 @@
+{
+  "memberCount": 1,
+  "etcdVersion": "3.5.25"
+}

tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-main_content

@@ -0,0 +1,4 @@
+{
+  "memberCount": 1,
+  "etcdVersion": "3.5.25"
+}

tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_kops-version.txt_content

@@ -0,0 +1 @@
+1.34.0-beta.1