Skip to content

Commit bcab14d

Browse files
rifelpetrifelpet
committed
Add integration test for role=apiserver on GCE
1 parent 1b5ef8e commit bcab14d

26 files changed

Lines changed: 4834 additions & 0 deletions

File tree

‎cmd/kops/integration_test.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,13 @@ type integrationTest struct {
7777
startupScript bool
7878
// verify "kops get assets" functionality
7979
testGetAssets bool
80+
// gceAPIServerIGs is a list of APIServer instance group names and their zones for GCE
81+
gceAPIServerIGs []gceAPIServerIG
82+
}
83+
84+
type gceAPIServerIG struct {
85+
name string
86+
zone string
8087
}
8188

8289
func newIntegrationTest(clusterName, srcDir string) *integrationTest {
@@ -151,6 +158,11 @@ func (i *integrationTest) withCiliumEtcd() *integrationTest {
151158
return i
152159
}
153160

161+
func (i *integrationTest) withGCEDedicatedAPIServer(name, zone string) *integrationTest {
162+
i.gceAPIServerIGs = append(i.gceAPIServerIGs, gceAPIServerIG{name: name, zone: zone})
163+
return i
164+
}
165+
154166
func (i *integrationTest) withDedicatedAPIServer() *integrationTest {
155167
i.expectTerraformFilenames = append(i.expectTerraformFilenames,
156168
"aws_iam_role_apiservers."+i.clusterName+"_policy",
@@ -408,6 +420,21 @@ func TestMinimalGCEPublicLoadBalancer(t *testing.T) {
408420
runTestTerraformGCE(t)
409421
}
410422

423+
// TestMinimalGCEPublicLoadBalancerAPIServer runs tests on a minimal GCE configuration with a public load balancer and an APIServer instance group.
424+
func TestMinimalGCEPublicLoadBalancerAPIServer(t *testing.T) {
425+
featureflag.ParseFlags("+APIServerNodes")
426+
defer featureflag.ParseFlags("-APIServerNodes")
427+
428+
newIntegrationTest("minimal-gce-plb-apiserver.example.com", "minimal_gce_plb_apiserver").
429+
withAddons(
430+
dnsControllerAddon,
431+
gcpCCMAddon,
432+
gcpPDCSIAddon,
433+
).
434+
withGCEDedicatedAPIServer("apiserver-us-test1-a", "us-test1-a").
435+
runTestTerraformGCE(t)
436+
}
437+
411438
// TestMinimalGCELongClusterName runs tests on a minimal GCE configuration with a very long cluster name
412439
func TestMinimalGCELongClusterName(t *testing.T) {
413440
newIntegrationTest("minimal-gce-with-a-very-very-very-very-very-long-name.example.com", "minimal_gce_longclustername").
@@ -1681,6 +1708,17 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
16811708
}
16821709
}
16831710

1711+
for _, ig := range i.gceAPIServerIGs {
1712+
expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-"+ig.name+"_content")
1713+
1714+
prefix := "google_compute_instance_template_" + ig.name + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_"
1715+
if !i.startupScript {
1716+
expectedFilenames = append(expectedFilenames, prefix+"user-data")
1717+
} else {
1718+
expectedFilenames = append(expectedFilenames, prefix+"startup-script")
1719+
}
1720+
}
1721+
16841722
i.runTest(t, ctx, h, expectedFilenames, "", "", nil)
16851723
}
16861724

‎tests/integration/update_cluster/minimal_gce_plb_apiserver/data/aws_s3_object_cluster-completed.spec_content‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
apiVersion: kops.k8s.io/v1alpha2
2+
kind: Cluster
3+
metadata:
4+
creationTimestamp: "2017-01-01T00:00:00Z"
5+
name: minimal-gce-plb-apiserver.example.com
6+
spec:
7+
api:
8+
loadBalancer:
9+
subnets:
10+
- name: us-test-1
11+
type: Public
12+
useForInternalApi: true
13+
authorization:
14+
rbac: {}
15+
channel: stable
16+
cloudConfig:
17+
gceServiceAccount: default
18+
gcpPDCSIDriver:
19+
defaultStorageClassName: balanced-storage
20+
enabled: true
21+
version: v1.22.1
22+
manageStorageClasses: true
23+
multizone: true
24+
nodeTags: minimal-gce-plb-apiserver-example-com-k8s-io-role-node
25+
cloudControllerManager:
26+
allocateNodeCIDRs: true
27+
cidrAllocatorType: CloudAllocator
28+
clusterCIDR: 100.96.0.0/11
29+
clusterName: minimal-gce-plb-apiserver-example-com
30+
controllers:
31+
- '*'
32+
image: registry.k8s.io/cloud-provider-gcp/cloud-controller-manager:v35.0.0
33+
leaderElection:
34+
leaderElect: true
35+
cloudProvider: gce
36+
clusterDNSDomain: cluster.local
37+
configBase: memfs://tests/minimal-gce-plb-apiserver.example.com
38+
containerd:
39+
logLevel: info
40+
runc:
41+
version: 1.3.4
42+
sandboxImage: registry.k8s.io/pause:3.10.1
43+
version: 2.1.6
44+
dnsZone: "1"
45+
etcdClusters:
46+
- backups:
47+
backupStore: memfs://tests/minimal-gce-plb-apiserver.example.com/backups/etcd/main
48+
cpuRequest: 200m
49+
etcdMembers:
50+
- instanceGroup: master-us-test1-a
51+
name: a
52+
manager:
53+
backupRetentionDays: 90
54+
memoryRequest: 100Mi
55+
name: main
56+
version: 3.5.25
57+
- backups:
58+
backupStore: memfs://tests/minimal-gce-plb-apiserver.example.com/backups/etcd/events
59+
cpuRequest: 100m
60+
etcdMembers:
61+
- instanceGroup: master-us-test1-a
62+
name: a
63+
manager:
64+
backupRetentionDays: 90
65+
memoryRequest: 100Mi
66+
name: events
67+
version: 3.5.25
68+
externalDns:
69+
provider: dns-controller
70+
iam:
71+
legacy: false
72+
keyStore: memfs://tests/minimal-gce-plb-apiserver.example.com/pki
73+
kubeAPIServer:
74+
allowPrivileged: true
75+
anonymousAuth: false
76+
apiAudiences:
77+
- kubernetes.svc.default
78+
apiServerCount: 1
79+
authorizationMode: Node,RBAC
80+
bindAddress: 0.0.0.0
81+
cloudProvider: external
82+
enableAdmissionPlugins:
83+
- DefaultStorageClass
84+
- DefaultTolerationSeconds
85+
- LimitRanger
86+
- MutatingAdmissionWebhook
87+
- NamespaceLifecycle
88+
- NodeRestriction
89+
- ResourceQuota
90+
- RuntimeClass
91+
- ServiceAccount
92+
- ValidatingAdmissionPolicy
93+
- ValidatingAdmissionWebhook
94+
etcdServers:
95+
- https://127.0.0.1:4001
96+
etcdServersOverrides:
97+
- /events#https://127.0.0.1:4002
98+
image: registry.k8s.io/kube-apiserver:v1.32.0
99+
kubeletPreferredAddressTypes:
100+
- InternalIP
101+
- Hostname
102+
- ExternalIP
103+
logLevel: 2
104+
requestheaderAllowedNames:
105+
- aggregator
106+
requestheaderExtraHeaderPrefixes:
107+
- X-Remote-Extra-
108+
requestheaderGroupHeaders:
109+
- X-Remote-Group
110+
requestheaderUsernameHeaders:
111+
- X-Remote-User
112+
securePort: 443
113+
serviceAccountIssuer: https://api.internal.minimal-gce-plb-apiserver.example.com
114+
serviceAccountJWKSURI: https://api.internal.minimal-gce-plb-apiserver.example.com/openid/v1/jwks
115+
serviceClusterIPRange: 100.64.0.0/13
116+
storageBackend: etcd3
117+
kubeControllerManager:
118+
allocateNodeCIDRs: true
119+
attachDetachReconcileSyncPeriod: 1m0s
120+
cloudProvider: external
121+
clusterCIDR: 100.96.0.0/11
122+
clusterName: minimal-gce-plb-apiserver.example.com
123+
configureCloudRoutes: false
124+
image: registry.k8s.io/kube-controller-manager:v1.32.0
125+
leaderElection:
126+
leaderElect: true
127+
logLevel: 2
128+
useServiceAccountCredentials: true
129+
kubeDNS:
130+
cacheMaxConcurrent: 150
131+
cacheMaxSize: 1000
132+
cpuRequest: 100m
133+
domain: cluster.local
134+
memoryLimit: 170Mi
135+
memoryRequest: 70Mi
136+
nodeLocalDNS:
137+
cpuRequest: 25m
138+
enabled: false
139+
image: registry.k8s.io/dns/k8s-dns-node-cache:1.26.0
140+
memoryRequest: 5Mi
141+
provider: CoreDNS
142+
serverIP: 100.64.0.10
143+
kubeProxy:
144+
clusterCIDR: 100.96.0.0/11
145+
cpuRequest: 100m
146+
image: registry.k8s.io/kube-proxy:v1.32.0
147+
logLevel: 2
148+
kubeScheduler:
149+
image: registry.k8s.io/kube-scheduler:v1.32.0
150+
leaderElection:
151+
leaderElect: true
152+
logLevel: 2
153+
kubelet:
154+
anonymousAuth: false
155+
cgroupDriver: systemd
156+
cgroupRoot: /
157+
cloudProvider: external
158+
clusterDNS: 100.64.0.10
159+
clusterDomain: cluster.local
160+
enableDebuggingHandlers: true
161+
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
162+
hairpinMode: promiscuous-bridge
163+
kubeconfigPath: /var/lib/kubelet/kubeconfig
164+
logLevel: 2
165+
podManifestPath: /etc/kubernetes/manifests
166+
protectKernelDefaults: true
167+
registerSchedulable: true
168+
shutdownGracePeriod: 30s
169+
shutdownGracePeriodCriticalPods: 10s
170+
kubernetesApiAccess:
171+
- 0.0.0.0/0
172+
- ::/0
173+
kubernetesVersion: 1.32.0
174+
masterKubelet:
175+
anonymousAuth: false
176+
cgroupDriver: systemd
177+
cgroupRoot: /
178+
cloudProvider: external
179+
clusterDNS: 100.64.0.10
180+
clusterDomain: cluster.local
181+
enableDebuggingHandlers: true
182+
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
183+
hairpinMode: promiscuous-bridge
184+
kubeconfigPath: /var/lib/kubelet/kubeconfig
185+
logLevel: 2
186+
podManifestPath: /etc/kubernetes/manifests
187+
protectKernelDefaults: true
188+
registerSchedulable: true
189+
shutdownGracePeriod: 30s
190+
shutdownGracePeriodCriticalPods: 10s
191+
masterPublicName: api.minimal-gce-plb-apiserver.example.com
192+
networking:
193+
cni: {}
194+
nonMasqueradeCIDR: 100.64.0.0/10
195+
podCIDR: 100.96.0.0/11
196+
project: testproject
197+
secretStore: memfs://tests/minimal-gce-plb-apiserver.example.com/secrets
198+
serviceClusterIPRange: 100.64.0.0/13
199+
sshAccess:
200+
- 0.0.0.0/0
201+
- ::/0
202+
subnets:
203+
- cidr: 10.0.16.0/20
204+
name: us-test1
205+
region: us-test1
206+
type: Private
207+
topology:
208+
dns:
209+
type: Public

‎tests/integration/update_cluster/minimal_gce_plb_apiserver/data/aws_s3_object_etcd-cluster-spec-events_content‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"memberCount": 1,
3+
"etcdVersion": "3.5.25"
4+
}

‎tests/integration/update_cluster/minimal_gce_plb_apiserver/data/aws_s3_object_etcd-cluster-spec-main_content‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"memberCount": 1,
3+
"etcdVersion": "3.5.25"
4+
}

‎tests/integration/update_cluster/minimal_gce_plb_apiserver/data/aws_s3_object_kops-version.txt_content‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1.34.0-beta.1

0 commit comments

Comments
 (0)