Skip to content

Commit deb9064

Browse files
mtuliokmala
mtulio
authored and
kmala
committed
e2e/loadbalancer: implement hairpin connection cases
Implementing the hairpin connection test cases, and exposing an issue on NLB with internal scheme which fails when the client is trying to access a service loadbalancer which is hosted in the same node. The hairpin connection is caused by the client IP preservation attribute is set to true (default), and the service does not provide an interface to prevent the issue. The e2e is expecting to pass to prevent permanent failures in CI, but it is tracked by an issue #1160.
1 parent 8bf34a3 commit deb9064

1 file changed

Lines changed: 117 additions & 21 deletions

File tree

‎tests/e2e/loadbalancer.go‎

Lines changed: 117 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,10 @@ import (
3838
)
3939

4040
const (
41-
annotationLBType = "service.beta.kubernetes.io/aws-load-balancer-type"
42-
annotationLBInternal = "service.beta.kubernetes.io/aws-load-balancer-internal"
43-
annotationLBTargetNodeLabels = "service.beta.kubernetes.io/aws-load-balancer-target-node-labels"
41+
annotationLBType = "service.beta.kubernetes.io/aws-load-balancer-type"
42+
annotationLBInternal = "service.beta.kubernetes.io/aws-load-balancer-internal"
43+
annotationLBTargetNodeLabels = "service.beta.kubernetes.io/aws-load-balancer-target-node-labels"
44+
annotationLBTargetGroupAttributes = "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes"
4445
)
4546

4647
var (
@@ -142,29 +143,106 @@ var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() {
142143
requireAffinity: true,
143144
},
144145
// Hairpining traffic test for NLB.
145-
// Hairpin connection work with target type as instance only when preserve client IP is disabled.
146-
// Currently CCM does not provide an interface to create a service with that setup, making an internal
147-
// Service to fail.
148-
// FIXME: https://github.com/kubernetes/cloud-provider-aws/issues/1160
149-
// Once issue 1160 is fixed, the skipTestFailure must be unset/false.
146+
// The target type instance (default) sets the preserve client IP attribute to true,
147+
// the NLB target group attributes are set to preserve_client_ip.enabled=false to allow hairpining traffic.
148+
// The test also validates the target group attributes are set correctly to AWS resource.
150149
{
151150
name: "NLB internal should be reachable with hairpinning traffic",
152151
resourceSuffix: "hp-nlb-int",
153152
extraAnnotations: map[string]string{
154-
annotationLBType: "nlb",
155-
annotationLBInternal: "true",
153+
annotationLBType: "nlb",
154+
annotationLBInternal: "true",
155+
annotationLBTargetGroupAttributes: "preserve_client_ip.enabled=false",
156156
},
157-
listenerCount: 1,
157+
listenerCount: 1,
158+
overrideTestRunInClusterReachableHTTP: true,
159+
requireAffinity: true,
158160
hookPostServiceConfig: func(cfg *e2eTestConfig) {
159161
framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample)
160162
if cfg.svc.Annotations == nil {
161163
cfg.svc.Annotations = map[string]string{}
162164
}
163165
cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample)
164166
},
165-
overrideTestRunInClusterReachableHTTP: true,
166-
requireAffinity: true,
167-
skipTestFailure: true,
167+
hookPreTest: func(e2e *e2eTestConfig) {
168+
framework.Logf("running hook pre-test: verify target group attributes are set correctly to AWS resource")
169+
170+
if e2e.svc.Status.LoadBalancer.Ingress[0].Hostname == "" && e2e.svc.Status.LoadBalancer.Ingress[0].IP == "" {
171+
framework.Failf("LoadBalancer ingress is empty (no hostname or IP) for service %s/%s", e2e.svc.Namespace, e2e.svc.Name)
172+
}
173+
174+
hostAddr := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0])
175+
framework.Logf("Load balancer's ingress address: %s", hostAddr)
176+
177+
if hostAddr == "" {
178+
framework.Failf("Unable to get LoadBalancer ingress address for service %s/%s", e2e.svc.Namespace, e2e.svc.Name)
179+
}
180+
181+
elbClient, err := getAWSClientLoadBalancer(e2e.ctx)
182+
framework.ExpectNoError(err, "failed to create AWS ELB client")
183+
184+
// DescribeLoadBalancers API doesn't support filtering by DNS name directly
185+
// Use AWS SDK paginator to search through all load balancers
186+
foundLB, err := getAWSLoadBalancerFromDNSName(e2e.ctx, elbClient, hostAddr)
187+
framework.ExpectNoError(err, "failed to find load balancer with DNS name %s", hostAddr)
188+
if foundLB == nil {
189+
framework.Failf("Found load balancer is nil for DNS name %s", hostAddr)
190+
}
191+
192+
lbARN := aws.ToString(foundLB.LoadBalancerArn)
193+
if lbARN == "" {
194+
framework.Failf("Load balancer ARN is empty for DNS name %s", hostAddr)
195+
}
196+
framework.Logf("Found load balancer: %s with ARN: %s", aws.ToString(foundLB.LoadBalancerName), lbARN)
197+
198+
// lookup target group ARN from load balancer ARN
199+
targetGroups, err := elbClient.DescribeTargetGroups(e2e.ctx, &elbv2.DescribeTargetGroupsInput{
200+
LoadBalancerArn: aws.String(lbARN),
201+
})
202+
framework.ExpectNoError(err, "failed to describe target groups")
203+
framework.ExpectEqual(len(targetGroups.TargetGroups), 1)
204+
205+
targetGroupAttributes, err := elbClient.DescribeTargetGroupAttributes(e2e.ctx, &elbv2.DescribeTargetGroupAttributesInput{
206+
TargetGroupArn: aws.String(aws.ToString(targetGroups.TargetGroups[0].TargetGroupArn)),
207+
})
208+
framework.ExpectNoError(err, "failed to describe target group attributes")
209+
210+
// verify if the target group attributes are set correctly
211+
212+
annotationToDict := map[string]string{}
213+
for _, v := range strings.Split(e2e.svc.Annotations[annotationLBTargetGroupAttributes], ",") {
214+
parts := strings.Split(v, "=")
215+
annotationToDict[parts[0]] = parts[1]
216+
}
217+
framework.Logf("TG attribute Annotation to dict: %v", annotationToDict)
218+
219+
framework.Logf("=== All Target Group Attributes from AWS ===")
220+
for _, attr := range targetGroupAttributes.Attributes {
221+
framework.Logf(" %s=%s", aws.ToString(attr.Key), aws.ToString(attr.Value))
222+
}
223+
224+
framework.Logf("=== Expected Target Group Attributes from Annotation ===")
225+
for key, value := range annotationToDict {
226+
framework.Logf(" %s=%s", key, value)
227+
}
228+
229+
// Check if our expected attributes are present and match
230+
framework.Logf("=== Verifying Target Group Attributes ===")
231+
for _, attr := range targetGroupAttributes.Attributes {
232+
if expectedValue, ok := annotationToDict[aws.ToString(attr.Key)]; ok {
233+
actualValue := aws.ToString(attr.Value)
234+
framework.Logf("Checking attribute: %s", aws.ToString(attr.Key))
235+
framework.Logf(" Expected: %s", expectedValue)
236+
framework.Logf(" Actual: %s", actualValue)
237+
238+
if actualValue != expectedValue {
239+
framework.Failf("Target group attribute mismatch for %s: expected %s, got %s", aws.ToString(attr.Key), expectedValue, actualValue)
240+
} else {
241+
framework.Logf("✓ Target group attribute %s matches expected value %s", aws.ToString(attr.Key), expectedValue)
242+
}
243+
}
244+
}
245+
},
168246
},
169247
}
170248

@@ -208,7 +286,23 @@ var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() {
208286
By("waiting for AWS load balancer provisioning")
209287
var err error
210288
e2e.svc, err = e2e.LBJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
211-
framework.ExpectNoError(err)
289+
// Collect comprehensive debugging information when LoadBalancer provisioning fails
290+
if err != nil {
291+
serviceName := e2e.LBJig.Name
292+
if e2e.svc != nil {
293+
serviceName = e2e.svc.Name
294+
}
295+
framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err)
296+
framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout)
297+
298+
// Ensure we have detailed debugging information before failing
299+
framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===")
300+
gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name)
301+
framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===")
302+
303+
// Fail the test immediately to prevent further execution
304+
framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above")
305+
}
212306
framework.Logf("[AWS] Load balancer provisioned successfully")
213307

214308
By("creating backend server pods")
@@ -244,7 +338,6 @@ var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() {
244338
framework.Logf("=== End of Service Validation Error Debug Information ===")
245339
framework.Failf("Service is nil after LoadBalancer provisioning for service %s", e2e.LBJig.Name)
246340
}
247-
248341
if len(e2e.svc.Spec.Ports) == 0 {
249342
framework.Logf("=== Service Ports Error Debug Information ===")
250343
framework.Logf("Service spec: %+v", e2e.svc.Spec)
@@ -259,6 +352,7 @@ var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() {
259352
framework.Logf("=== End of LoadBalancer Ingress Error Debug Information ===")
260353
framework.Failf("No ingress found in LoadBalancer status for service %s/%s", e2e.svc.Namespace, e2e.svc.Name)
261354
}
355+
262356
svcPort := int(e2e.svc.Spec.Ports[0].Port)
263357
ingressAddress := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0])
264358
framework.Logf("[LB-INFO] Ingress address: %s, port: %d", ingressAddress, svcPort)
@@ -278,15 +372,15 @@ var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() {
278372

279373
// overrideTestRunInClusterReachableHTTP changes the default test function to run the client in the cluster.
280374
if tc.overrideTestRunInClusterReachableHTTP {
281-
By("testing HTTP connectivity from internal network")
375+
By("testing HTTP connectivity for internal load balancer")
282376
framework.Logf("[TEST] Running internal connectivity test from node: %s", e2e.nodeSingleSample)
283377
err := inClusterTestReachableHTTP(cs, ns.Name, e2e.nodeSingleSample, ingressAddress, svcPort)
284378
if err != nil && tc.skipTestFailure {
285379
Skip(err.Error())
286380
}
287381
framework.ExpectNoError(err)
288382
} else {
289-
By("testing HTTP connectivity from external client")
383+
By("testing HTTP connectivity for external/internet-facing load balancer")
290384
framework.Logf("[TEST] Running external connectivity test to %s:%d", ingressAddress, svcPort)
291385
e2eservice.TestReachableHTTP(ingressAddress, svcPort, e2eservice.LoadBalancerLagTimeoutAWS)
292386
}
@@ -570,9 +664,11 @@ func getAWSLoadBalancerFromDNSName(ctx context.Context, elbClient *elbv2.Client,
570664
paginator := elbv2.NewDescribeLoadBalancersPaginator(elbClient, &elbv2.DescribeLoadBalancersInput{})
571665
for paginator.HasMorePages() {
572666
page, err := paginator.NextPage(ctx)
573-
framework.ExpectNoError(err)
667+
if err != nil {
668+
return nil, fmt.Errorf("failed to describe load balancers: %v", err)
669+
}
574670

575-
framework.Logf("found %d load balancers", len(page.LoadBalancers))
671+
framework.Logf("found %d load balancers in page", len(page.LoadBalancers))
576672
// Search for the load balancer with matching DNS name in this page
577673
for i := range page.LoadBalancers {
578674
if aws.ToString(page.LoadBalancers[i].DNSName) == lbDNSName {
@@ -587,7 +683,7 @@ func getAWSLoadBalancerFromDNSName(ctx context.Context, elbClient *elbv2.Client,
587683
}
588684

589685
if foundLB == nil {
590-
framework.Failf("No load balancer found with DNS name: %s", lbDNSName)
686+
return nil, fmt.Errorf("no load balancer found with DNS name: %s", lbDNSName)
591687
}
592688

593689
return foundLB, nil

0 commit comments

Comments
 (0)