diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index fd408beee0ed..932f8872e2d3 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -72,6 +72,8 @@ func AllTests() []upgrades.Test { &prometheus.MetricsAvailableAfterUpgradeTest{}, &dns.UpgradeTest{}, &router.GatewayAPIUpgradeTest{}, + &router.HAProxyVersionUpgradeTest{Pinned: false}, + &router.HAProxyVersionUpgradeTest{Pinned: true}, } } diff --git a/test/extended/router/haproxyversion_upgrade.go b/test/extended/router/haproxyversion_upgrade.go new file mode 100644 index 000000000000..9562d9e67324 --- /dev/null +++ b/test/extended/router/haproxyversion_upgrade.go @@ -0,0 +1,191 @@ +package router + +import ( + "context" + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/upgrades" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + operatorv1 "github.com/openshift/api/operator/v1" + operatorv1client "github.com/openshift/client-go/operator/clientset/versioned" + exutil "github.com/openshift/origin/test/extended/util" +) + +// HAProxyVersionUpgradeTest verifies if HAProxy version selection behaves +// as expected during upgrades. +// Pinned is a test parameter that should define whether the test uses a +// pinned version during upgrades, or it should upgrade leaving version unset. +type HAProxyVersionUpgradeTest struct { + Pinned bool + // + oc *exutil.CLI + operatorClient operatorv1client.Interface + controllers *ingressControllers + precheckErr error + ic types.NamespacedName + pinnedVersion operatorv1.HAProxyVersion // only used if Pinned is true +} + +func (h *HAProxyVersionUpgradeTest) Name() string { + if h.Pinned { + return "haproxy-pinned-version-upgrade" + } + return "haproxy-unset-version-upgrade" +} + +func (h *HAProxyVersionUpgradeTest) DisplayName() string { + if h.Pinned { + return "[sig-network-edge][Feature:Router][apigroup:route.openshift.io] Verify HAProxy pinned version state during upgrade" + } + return "[sig-network-edge][Feature:Router][apigroup:route.openshift.io] Verify HAProxy unset version state during upgrade" +} + +// Skip defines if the test should be skipped. HAProxy version test is skipped +// if the HAProxy version field cannot be found in the API. +func (h *HAProxyVersionUpgradeTest) Skip(_ upgrades.UpgradeContext) bool { + oc := exutil.NewCLIForMonitorTest(h.Name() + "-skip").AsAdmin() + hasField, err := apiHasHAProxyVersionField(context.Background(), oc) + if err != nil { + h.precheckErr = fmt.Errorf("error checking for HAProxy version API: %w", err) + return false + } + + h.precheckErr = nil + return !hasField +} + +// Setup configures all the test attributes and creates an IngressController +// resource that should be verified after the upgrade. +func (h *HAProxyVersionUpgradeTest) Setup(ctx context.Context, f *framework.Framework) { + o.Expect(h.precheckErr).NotTo(o.HaveOccurred(), "Skip() precheck failed: could not determine if HAProxy version upgrade test should run") + + g.By("Setting up HAProxy version test") + + h.oc = exutil.NewCLIWithFramework(f).AsAdmin() + h.operatorClient = h.oc.AdminOperatorClient() + h.controllers = &ingressControllers{} + + var customIngress func(*operatorv1.IngressController) + versions, err := getHAProxyVersionParams(ctx, h.oc) + o.Expect(err).NotTo(o.HaveOccurred(), "error getting HAProxy versions") + if h.Pinned { + customIngress = func(ic *operatorv1.IngressController) { + ic.Spec.HAProxyVersion = versions.defaultVersion + } + h.pinnedVersion = versions.defaultVersion + } else { + h.pinnedVersion = "" + } + + g.By("Creating the IngressController resource") + + const createControllerTimeout = 2 * time.Minute + ic, err := h.controllers.createIngressController(ctx, h.oc, createControllerTimeout, customIngress) + o.Expect(err).NotTo(o.HaveOccurred(), "error creating IngressController resource") + h.ic = types.NamespacedName{ + Namespace: ic.Namespace, + Name: ic.Name, + } + + g.By("Checking HAProxy version for Ingress " + ic.Name) + + err = waitForHAProxyVersion(ctx, h.oc, ic.Name, versions.defaultVersion) + o.Expect(err).NotTo(o.HaveOccurred(), "error getting HAProxy version from runtime API") +} + +// Test verifies if the expected HAProxy version is found after the upgrade. +// Current version is read from the IngressController status and from the +// HAProxy's runtime API. +func (h *HAProxyVersionUpgradeTest) Test(ctx context.Context, f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) { + g.By("Waiting for upgrade to complete") + <-done + + g.By("Validating HAProxy version after upgrade") + + var expectedVersion operatorv1.HAProxyVersion + if h.Pinned { + expectedVersion = h.pinnedVersion + } else { + versions, err := getHAProxyVersionParams(ctx, h.oc) + o.Expect(err).NotTo(o.HaveOccurred(), "error getting HAProxy versions") + expectedVersion = versions.defaultVersion + } + + const rollingOutTimeout = 15 * time.Minute + err := wait.PollUntilContextTimeout(ctx, time.Second, rollingOutTimeout, true, func(ctx context.Context) (ready bool, err error) { + ic, err := h.operatorClient.OperatorV1().IngressControllers(h.ic.Namespace).Get(ctx, h.ic.Name, metav1.GetOptions{}) + if err != nil { + framework.Logf("error getting IngressController resource: %s", err.Error()) + return false, nil + } + if ic.Status.EffectiveHAProxyVersion != expectedVersion { + framework.Logf("HAProxy version from IngressResource status %q does not match expected value %q", ic.Status.EffectiveHAProxyVersion, expectedVersion) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "timed out waiting for EffectiveHAProxyVersion to match expected version") + + g.By("Validating HAProxy version from runtime API") + + err = waitForHAProxyVersion(ctx, h.oc, h.ic.Name, expectedVersion) + o.Expect(err).NotTo(o.HaveOccurred(), "error getting HAProxy version from runtime API") +} + +// Teardown removes the configured IngressController after the test runs. +func (h *HAProxyVersionUpgradeTest) Teardown(ctx context.Context, f *framework.Framework) { + if h.operatorClient == nil { + framework.Logf("Skipping cleanup because setup did not initialize test resources") + return + } + if err := h.controllers.deleteAll(ctx, h.operatorClient); err != nil { + framework.Logf("error deleting IngressController resource: %s", err.Error()) + } +} + +// haproxyVersionParams has HAProxy version parameters from the Ingress operator. +type haproxyVersionParams struct { + defaultVersion operatorv1.HAProxyVersion +} + +// getHAProxyVersionParams parses the current Ingress operator configuration +// and extracts HAProxy version parameters. +func getHAProxyVersionParams(ctx context.Context, oc *exutil.CLI) (haproxyVersionParams, error) { + operatorNamespace := "openshift-ingress-operator" + operatorName := "ingress-operator" + deploy, err := oc.AdminKubeClient().AppsV1().Deployments(operatorNamespace).Get(ctx, operatorName, metav1.GetOptions{}) + if err != nil { + return haproxyVersionParams{}, err + } + + containers := deploy.Spec.Template.Spec.Containers + if len(containers) < 1 { + return haproxyVersionParams{}, fmt.Errorf("ingress-operator deployment is missing the operator container") + } + + operator := containers[0] + if operator.Name != "ingress-operator" { + return haproxyVersionParams{}, fmt.Errorf("ingress-operator deployment has an unexpected container name: %s", operator.Name) + } + + defaultVersion := func() string { + for _, env := range operator.Env { + if env.Name == "DEFAULT_HAPROXY_VERSION" { + return env.Value + } + } + // DEFAULT_HAPROXY_VERSION envvar not found, so this is pre 4.23/5.0, assume "2.8" + return "2.8" + }() + + return haproxyVersionParams{ + defaultVersion: operatorv1.HAProxyVersion(defaultVersion), + }, nil +} diff --git a/test/extended/router/multi-haproxy.go b/test/extended/router/multi-haproxy.go index 7536293414a2..883f3c18b6ba 100644 --- a/test/extended/router/multi-haproxy.go +++ b/test/extended/router/multi-haproxy.go @@ -10,6 +10,7 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -19,7 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/openshift/api/operator/v1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + operatorv1client "github.com/openshift/client-go/operator/clientset/versioned" "github.com/openshift/origin/test/extended/router/shard" exutil "github.com/openshift/origin/test/extended/util" @@ -51,32 +52,14 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. exutil.DumpPodLogsStartingWithInNamespace(ic.controller.Name, ic.controller.Namespace, oc) } } - var errs []error - for _, ic := range controllers.items { - err := operatorClient.OperatorV1().IngressControllers(ic.controller.Namespace).Delete(ctx, ic.controller.Name, *metav1.NewDeleteOptions(1)) - errs = append(errs, client.IgnoreNotFound(err)) - } - o.Expect(errors.Join(errs...)).NotTo(o.HaveOccurred()) + err := controllers.deleteAll(ctx, operatorClient) + o.Expect(err).NotTo(o.HaveOccurred()) controllers.items = nil }) g.BeforeEach(func() { - - apiExtClient, err := apiextensionsclient.NewForConfig(oc.AdminConfig()) - o.Expect(err).NotTo(o.HaveOccurred()) - - crd, err := apiExtClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "ingresscontrollers.operator.openshift.io", metav1.GetOptions{}) + hasField, err := apiHasHAProxyVersionField(ctx, oc) o.Expect(err).NotTo(o.HaveOccurred()) - - // Check if haproxyVersion field exists in the CRD schema - hasField := false - for _, v := range crd.Spec.Versions { - if v.Name == "v1" && v.Schema != nil && v.Schema.OpenAPIV3Schema != nil { - if _, ok := v.Schema.OpenAPIV3Schema.Properties["spec"].Properties["haproxyVersion"]; ok { - hasField = true - } - } - } if !hasField { g.Skip("IngressController CRD does not have haproxyVersion field — operator not yet updated") } @@ -288,6 +271,15 @@ func (i *ingressControllers) createIngressController(ctx context.Context, oc *ex return ingress, nil } +func (i *ingressControllers) deleteAll(ctx context.Context, operatorClient operatorv1client.Interface) error { + var errs []error + for _, ic := range i.items { + err := operatorClient.OperatorV1().IngressControllers(ic.controller.Namespace).Delete(ctx, ic.controller.Name, *metav1.NewDeleteOptions(1)) + errs = append(errs, client.IgnoreNotFound(err)) + } + return errors.Join(errs...) +} + // poll the router pods HAProxy Container to check that the version is correctly asserted func waitForHAProxyVersion(ctx context.Context, oc *exutil.CLI, ingressName string, desiredVersion operatorv1.HAProxyVersion) error { if desiredVersion == "" { @@ -308,3 +300,25 @@ func waitForHAProxyVersion(ctx context.Context, oc *exutil.CLI, ingressName stri }) return err } + +func apiHasHAProxyVersionField(ctx context.Context, oc *exutil.CLI) (bool, error) { + apiExtClient, err := apiextensionsclient.NewForConfig(oc.AdminConfig()) + if err != nil { + return false, err + } + + crd, err := apiExtClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "ingresscontrollers.operator.openshift.io", metav1.GetOptions{}) + if err != nil { + return false, err + } + + // Check if haproxyVersion field exists in the CRD schema + for _, v := range crd.Spec.Versions { + if v.Name == "v1" && v.Schema != nil && v.Schema.OpenAPIV3Schema != nil { + if _, ok := v.Schema.OpenAPIV3Schema.Properties["spec"].Properties["haproxyVersion"]; ok { + return true, nil + } + } + } + return false, nil +}