Skip to content

Commit df62871

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #18147 from rifelpet/gce-apiserver
gce: support for role=apiserver
2 parents 3f293c5 + 7823daf commit df62871

31 files changed

Lines changed: 5031 additions & 5 deletions

File tree

‎cmd/kops/integration_test.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,13 @@ type integrationTest struct {
7777
startupScript bool
7878
// verify "kops get assets" functionality
7979
testGetAssets bool
80+
// gceAPIServerIGs is a list of APIServer instance group names and their zones for GCE
81+
gceAPIServerIGs []gceAPIServerIG
82+
}
83+
84+
type gceAPIServerIG struct {
85+
name string
86+
zone string
8087
}
8188

8289
func newIntegrationTest(clusterName, srcDir string) *integrationTest {
@@ -151,6 +158,11 @@ func (i *integrationTest) withCiliumEtcd() *integrationTest {
151158
return i
152159
}
153160

161+
func (i *integrationTest) withGCEDedicatedAPIServer(name, zone string) *integrationTest {
162+
i.gceAPIServerIGs = append(i.gceAPIServerIGs, gceAPIServerIG{name: name, zone: zone})
163+
return i
164+
}
165+
154166
func (i *integrationTest) withDedicatedAPIServer() *integrationTest {
155167
i.expectTerraformFilenames = append(i.expectTerraformFilenames,
156168
"aws_iam_role_apiservers."+i.clusterName+"_policy",
@@ -408,6 +420,21 @@ func TestMinimalGCEPublicLoadBalancer(t *testing.T) {
408420
runTestTerraformGCE(t)
409421
}
410422

423+
// TestMinimalGCEPublicLoadBalancerAPIServer runs tests on a minimal GCE configuration with a public load balancer and an APIServer instance group.
424+
func TestMinimalGCEPublicLoadBalancerAPIServer(t *testing.T) {
425+
featureflag.ParseFlags("+APIServerNodes")
426+
defer featureflag.ParseFlags("-APIServerNodes")
427+
428+
newIntegrationTest("minimal-gce-plb-apiserver.example.com", "minimal_gce_plb_apiserver").
429+
withAddons(
430+
dnsControllerAddon,
431+
gcpCCMAddon,
432+
gcpPDCSIAddon,
433+
).
434+
withGCEDedicatedAPIServer("apiserver-us-test1-a", "us-test1-a").
435+
runTestTerraformGCE(t)
436+
}
437+
411438
// TestMinimalGCELongClusterName runs tests on a minimal GCE configuration with a very long cluster name
412439
func TestMinimalGCELongClusterName(t *testing.T) {
413440
newIntegrationTest("minimal-gce-with-a-very-very-very-very-very-long-name.example.com", "minimal_gce_longclustername").
@@ -1681,6 +1708,17 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
16811708
}
16821709
}
16831710

1711+
for _, ig := range i.gceAPIServerIGs {
1712+
expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-"+ig.name+"_content")
1713+
1714+
prefix := "google_compute_instance_template_" + ig.name + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_"
1715+
if !i.startupScript {
1716+
expectedFilenames = append(expectedFilenames, prefix+"user-data")
1717+
} else {
1718+
expectedFilenames = append(expectedFilenames, prefix+"startup-script")
1719+
}
1720+
}
1721+
16841722
i.runTest(t, ctx, h, expectedFilenames, "", "", nil)
16851723
}
16861724

‎pkg/apis/kops/validation/instancegroup.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,8 +239,8 @@ func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, cl
239239
}
240240

241241
if g.Spec.Role == kops.InstanceGroupRoleAPIServer {
242-
if cluster.GetCloudProvider() != kops.CloudProviderAWS {
243-
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "APIServer role only supported on AWS"))
242+
if cluster.GetCloudProvider() != kops.CloudProviderAWS && cluster.GetCloudProvider() != kops.CloudProviderGCE {
243+
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "APIServer role only supported on AWS and GCE"))
244244
}
245245
if cluster.UsesNoneDNS() {
246246
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "APIServer cannot be used with topology.dns.type=None"))

‎pkg/apis/kops/validation/instancegroup_test.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -510,3 +510,69 @@ func createMinimalInstanceGroup() *kops.InstanceGroup {
510510
}
511511
return ig
512512
}
513+
514+
func TestCrossValidateAPIServerRole(t *testing.T) {
515+
grid := []struct {
516+
Description string
517+
Cluster *kops.Cluster
518+
ExpectedErrors int
519+
}{
520+
{
521+
Description: "APIServer role allowed on AWS",
522+
Cluster: &kops.Cluster{
523+
Spec: kops.ClusterSpec{
524+
CloudProvider: kops.CloudProviderSpec{
525+
AWS: &kops.AWSSpec{},
526+
},
527+
},
528+
},
529+
ExpectedErrors: 0,
530+
},
531+
{
532+
Description: "APIServer role allowed on GCE",
533+
Cluster: &kops.Cluster{
534+
Spec: kops.ClusterSpec{
535+
CloudProvider: kops.CloudProviderSpec{
536+
GCE: &kops.GCESpec{},
537+
},
538+
},
539+
},
540+
ExpectedErrors: 0,
541+
},
542+
{
543+
Description: "APIServer role forbidden on DO",
544+
Cluster: &kops.Cluster{
545+
Spec: kops.ClusterSpec{
546+
CloudProvider: kops.CloudProviderSpec{
547+
DO: &kops.DOSpec{},
548+
},
549+
},
550+
},
551+
ExpectedErrors: 1,
552+
},
553+
}
554+
555+
for _, g := range grid {
556+
t.Run(g.Description, func(t *testing.T) {
557+
ig := &kops.InstanceGroup{
558+
ObjectMeta: v1.ObjectMeta{
559+
Name: "apiserver",
560+
},
561+
Spec: kops.InstanceGroupSpec{
562+
Role: kops.InstanceGroupRoleAPIServer,
563+
Subnets: []string{"eu-central-1a"},
564+
MaxSize: fi.PtrTo(int32(1)),
565+
MinSize: fi.PtrTo(int32(1)),
566+
Image: "my-image",
567+
},
568+
}
569+
g.Cluster.Spec.Networking.Subnets = []kops.ClusterSubnetSpec{
570+
{Name: "eu-central-1a", Region: "eu-central-1"},
571+
}
572+
errs := CrossValidateInstanceGroup(ig, g.Cluster, nil, true)
573+
if len(errs) != g.ExpectedErrors {
574+
t.Errorf("expected %d errors, got %d: %v", g.ExpectedErrors, len(errs), errs)
575+
}
576+
})
577+
}
578+
}

‎pkg/model/gcemodel/api_loadbalancer.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
146146
c.AddTask(hc)
147147
var igms []*gcetasks.InstanceGroupManager
148148
for _, ig := range b.InstanceGroups {
149-
if ig.Spec.Role != kops.InstanceGroupRoleControlPlane {
149+
if !ig.HasAPIServer() {
150150
continue
151151
}
152152
if len(ig.Spec.Zones) > 1 {

‎pkg/model/gcemodel/autoscalinggroup.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,10 @@ func (b *AutoscalingGroupModelBuilder) buildInstanceTemplate(c *fi.CloudupModelB
214214
t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleControlPlane))
215215
t.Tags = append(t.Tags, b.GCETagForRole("master"))
216216

217+
case kops.InstanceGroupRoleAPIServer:
218+
t.Scopes = append(t.Scopes, "https://www.googleapis.com/auth/ndev.clouddns.readwrite")
219+
t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleControlPlane))
220+
217221
case kops.InstanceGroupRoleNode:
218222
t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleNode))
219223

@@ -349,8 +353,8 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) e
349353
ListManagedInstancesResults: "PAGINATED",
350354
}
351355

352-
// Attach masters to load balancer if we're using one
353-
if ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
356+
// Attach API server instances to load balancer if we're using one
357+
if ig.HasAPIServer() {
354358
if b.UseLoadBalancerForAPI() {
355359
lbSpec := b.Cluster.Spec.API.LoadBalancer
356360
if lbSpec != nil {

‎tests/e2e/templates/apiserver-gce.yaml.tmpl‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
{{$zone := index .zones 0}}
2+
apiVersion: kops.k8s.io/v1alpha2
3+
kind: Cluster
4+
metadata:
5+
name: {{.clusterName}}
6+
spec:
7+
kubernetesApiAccess:
8+
- {{.publicIP}}
9+
api:
10+
loadBalancer:
11+
type: Public
12+
authorization:
13+
rbac: {}
14+
channel: stable
15+
cloudConfig:
16+
gceServiceAccount: default
17+
cloudProvider: {{.cloudProvider}}
18+
configBase: "{{.stateStore}}/{{.clusterName}}"
19+
containerd:
20+
configAdditions:
21+
plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.runtime_type: io.containerd.runc.v2
22+
etcdClusters:
23+
- etcdMembers:
24+
- instanceGroup: control-plane-{{$zone}}
25+
name: {{$zone}}
26+
name: main
27+
- etcdMembers:
28+
- instanceGroup: control-plane-{{$zone}}
29+
name: {{$zone}}
30+
name: events
31+
iam:
32+
allowContainerRegistry: true
33+
legacy: false
34+
kubelet:
35+
anonymousAuth: false
36+
kubernetesVersion: {{.kubernetesVersion}}
37+
networking:
38+
kubenet: {}
39+
nodePortAccess:
40+
- 0.0.0.0/0
41+
nonMasqueradeCIDR: 100.64.0.0/10
42+
sshAccess:
43+
- {{.publicIP}}
44+
subnets:
45+
- cidr: 10.0.16.0/20
46+
name: {{$zone}}
47+
region: {{$zone}}
48+
type: Public
49+
topology:
50+
dns:
51+
type: Public
52+
53+
---
54+
55+
apiVersion: kops.k8s.io/v1alpha2
56+
kind: SSHCredential
57+
metadata:
58+
name: admin
59+
labels:
60+
kops.k8s.io/cluster: {{.clusterName}}
61+
spec:
62+
publicKey: {{.sshPublicKey}}
63+
64+
---
65+
66+
apiVersion: kops.k8s.io/v1alpha2
67+
kind: InstanceGroup
68+
metadata:
69+
name: nodes-{{$zone}}
70+
labels:
71+
kops.k8s.io/cluster: {{.clusterName}}
72+
spec:
73+
image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606
74+
machineType: e2-standard-2
75+
maxSize: 4
76+
minSize: 4
77+
role: Node
78+
rootVolumeSize: 100
79+
subnets:
80+
- {{$zone}}
81+
zones:
82+
- {{$zone}}
83+
84+
---
85+
86+
apiVersion: kops.k8s.io/v1alpha2
87+
kind: InstanceGroup
88+
metadata:
89+
name: control-plane-{{$zone}}
90+
labels:
91+
kops.k8s.io/cluster: {{.clusterName}}
92+
spec:
93+
image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606
94+
machineType: e2-standard-2
95+
maxSize: 1
96+
minSize: 1
97+
role: Master
98+
rootVolumeSize: 48
99+
subnets:
100+
- {{$zone}}
101+
zones:
102+
- {{$zone}}
103+
104+
---
105+
106+
apiVersion: kops.k8s.io/v1alpha2
107+
kind: InstanceGroup
108+
metadata:
109+
name: apiserver-{{$zone}}
110+
labels:
111+
kops.k8s.io/cluster: {{.clusterName}}
112+
spec:
113+
image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606
114+
machineType: e2-standard-2
115+
maxSize: 1
116+
minSize: 1
117+
role: APIServer
118+
rootVolumeSize: 48
119+
subnets:
120+
- {{$zone}}
121+
zones:
122+
- {{$zone}}

0 commit comments

Comments
 (0)