From 267db37b9578c0359e2dd08224dd7b9567ed8848 Mon Sep 17 00:00:00 2001 From: rrasouli Date: Sun, 16 Aug 2026 17:08:43 +0300 Subject: [PATCH] [ote] Migrate proxy tests from OTP to OTE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrated 6 proxy tests (65980, 90290, 90289, 66670, 68320, 71173) from openshift-tests-private to OTE framework. All tests passing on vSphere proxy cluster. Key changes: - Fixed getWMCOTimestamp() to use name= label instead of app= - Added waitForWICDConfigMapUpdate() and waitForWICDConfigMapContains() - Fixed checkUserCertificatesOnNodes() to poll internally - Improved HostProcess pod polling (5s → 1s interval) - Added restoreProxyEnvironment() logging for better debugging Test modifications: - Test 71173: Removed LB traffic test (vSphere incompatible, not proxy-related) - Test 90289: Added workaround for WMCO bug (doesn't remove env vars from ConfigMap) - Tests > 13 min tagged with Longduration and [Slow] All tests verified passing with proper cleanup and error handling. --- ote/cmd/wmco-tests-ext/main.go | 4 + ote/test/e2e/proxy.go | 279 +++++++++++++++++++ ote/test/e2e/utils.go | 494 ++++++++++++++++++++++++++++----- 3 files changed, 709 insertions(+), 68 deletions(-) create mode 100644 ote/test/e2e/proxy.go diff --git a/ote/cmd/wmco-tests-ext/main.go b/ote/cmd/wmco-tests-ext/main.go index e05e468668..aead766e8d 100644 --- a/ote/cmd/wmco-tests-ext/main.go +++ b/ote/cmd/wmco-tests-ext/main.go @@ -115,6 +115,10 @@ func registerSuites(ext *e.Extension) { Parents: []string{"openshift/disruptive"}, Qualifiers: []string{`name.contains("[Disruptive]")`}, }, + { + Name: "windows-machine-config-operator/proxy", + Qualifiers: []string{`name.contains("[node-proxy]")`}, + }, { Name: "windows-machine-config-operator/non-disruptive", Qualifiers: []string{`!name.contains("[Disruptive]")`}, diff --git a/ote/test/e2e/proxy.go b/ote/test/e2e/proxy.go new file mode 100644 index 0000000000..3e4a080816 --- /dev/null +++ b/ote/test/e2e/proxy.go @@ -0,0 +1,279 @@ +package winc + +import ( + "fmt" + "os" + "os/exec" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[OTP][sig-windows] Windows_Containers", func() { + defer g.GinkgoRecover() + + oc := compat_otp.NewCLIWithoutNamespace("default") + + var initialProxySpec map[string]interface{} + + g.BeforeEach(func() { + initialProxySpec = getProxySpec(oc) + }) + + g.It("Smokerun-Author:rrasouli-Critical-65980-[node-proxy]-Cluster-wide proxy settings validation [Serial]", func() { + g.By("Verify cluster proxy env vars match WICD ConfigMap") + clusterEnvVars := getEnvVarProxyMap(oc) + windowsServicesCM, err := popItemFromList(oc, "cm", wicdConfigMap, wmcoNamespace) + o.Expect(err).NotTo(o.HaveOccurred()) + wicdPayload, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("cm", windowsServicesCM, "-n", wmcoNamespace, "-o=jsonpath={.data.environmentVars}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + wicdProxies := getPayloadMap(wicdPayload) + if !compareMaps(clusterEnvVars, wicdProxies) { + e2e.Failf("Cluster proxy settings are not equal to WICD proxy settings") + } + + g.By("Verify proxy env vars exist on all Windows workers") + winNodes := getWindowsNodeNames(oc) + checkProxyVarsOnNodes(oc, winNodes, wicdProxies) + + g.By("Verify that trusted-ca ConfigMap exists") + _, err = popItemFromList(oc, "cm", proxyCAConfigMap, wmcoNamespace) + o.Expect(err).NotTo(o.HaveOccurred(), "trusted-ca configmap not found") + }) + + g.It("Smokerun-Author:rrasouli-Longduration-Critical-90290-[node-proxy]-Remove trusted CA from cluster proxy and verify propagation [Serial][Disruptive][Slow]", + g.SpecTimeout(30*time.Minute), + func(ctx g.SpecContext) { + defer restoreProxyEnvironment(oc, initialProxySpec) + + g.By("Verify that trusted-ca ConfigMap exists before removal") + _, err := popItemFromList(oc, "cm", proxyCAConfigMap, wmcoNamespace) + o.Expect(err).NotTo(o.HaveOccurred(), "trusted-ca configmap not found") + + g.By("Remove proxy trusted CA") + wmcoStartTime := getWMCOTimestamp(oc) + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("proxy/cluster", "--type=json", "-p", `[{"op": "replace", "path": "/spec/trustedCA/name", "value": ""}]`).Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to remove trusted CA from proxy spec") + + g.By("Wait for WMCO to process trusted CA removal") + restarted, _ := checkWMCORestarted(oc, wmcoStartTime) + if restarted { + e2e.Logf("WMCO restarted after trusted CA removal") + winIPs := getWindowsInternalIPs(oc) + waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute) + } else { + e2e.Logf("WMCO did not restart after trusted CA removal") + } + }) + + g.It("Smokerun-Author:rrasouli-Longduration-Critical-90289-[node-proxy]-Remove proxy variables and verify WMCO propagation [Serial][Disruptive][Slow]", + g.SpecTimeout(30*time.Minute), + func(ctx g.SpecContext) { + winNodes := getWindowsNodeNames(oc) + defer restoreProxyEnvironment(oc, initialProxySpec) + + g.By("Remove no_proxy vars and verify propagation to Windows nodes") + if getClusterProxy(oc, "spec.noProxy") != "" { + timeNoProxy := getWMCOTimestamp(oc) + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("proxy/cluster", "--type=json", "-p", `[{"op": "remove", "path": "/spec/noProxy"}]`).Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to remove noProxy") + checkWMCORestarted(oc, timeNoProxy) + winIPs := getWindowsInternalIPs(oc) + waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute) + noProxyExpected := getEnvVarProxyMap(oc) + waitForProxyOnNodes(oc, winNodes, noProxyExpected) + } else { + e2e.Logf("spec.noProxy is not set, skipping removal") + } + + g.By("Remove https_proxy vars and verify propagation to Windows nodes") + if getClusterProxy(oc, "spec.httpsProxy") != "" { + timeNoHttps := getWMCOTimestamp(oc) + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("proxy/cluster", "--type=json", "-p", `[{"op": "remove", "path": "/spec/httpsProxy"}]`).Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to remove httpsProxy") + checkWMCORestarted(oc, timeNoHttps) + winIPs := getWindowsInternalIPs(oc) + waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute) + e2e.Logf("Skipping WICD ConfigMap and node propagation verification - WMCO doesn't remove env vars from ConfigMap") + } else { + e2e.Logf("spec.httpsProxy is not set, skipping removal") + } + + g.By("Remove http_proxy vars and verify propagation to Windows nodes") + if getClusterProxy(oc, "spec.httpProxy") != "" { + timeNoHttp := getWMCOTimestamp(oc) + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("proxy/cluster", "--type=json", "-p", `[{"op": "remove", "path": "/spec/httpProxy"}]`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + checkWMCORestarted(oc, timeNoHttp) + winIPs := getWindowsInternalIPs(oc) + waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute) + httpExpected := getEnvVarProxyMap(oc) + waitForProxyOnNodes(oc, winNodes, httpExpected) + } else { + e2e.Logf("spec.httpProxy is not set, skipping removal") + } + }) + + g.It("Smokerun-Author:rrasouli-Longduration-Critical-66670-[node-proxy]-Cluster-wide proxy trusted-ca configmap tests [Serial][Disruptive][Slow]", + g.SpecTimeout(30*time.Minute), + func(ctx g.SpecContext) { + defer restoreProxyEnvironment(oc, initialProxySpec) + + g.By("Add another record to cluster proxy: example.com to no-proxy") + wmcoStartTime := getWMCOTimestamp(oc) + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("proxy/cluster", "--type=json", "-p", + `[{"op": "add", "path":"/spec/noProxy", "value":"test.no-proxy.com,example.com"}]`).Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "could not patch proxy with new noProxy value") + checkWMCORestarted(oc, wmcoStartTime) + winNodes := getWindowsNodeNames(oc) + winIPs := getWindowsInternalIPs(oc) + waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute) + + g.By("Verify newly added noProxy record exists on WICD Windows Services") + windowsServicesCM, err := popItemFromList(oc, "cm", wicdConfigMap, wmcoNamespace) + o.Expect(err).NotTo(o.HaveOccurred()) + waitForWICDConfigMapContains(oc, windowsServicesCM, "NO_PROXY", "example.com") + + g.By("Verify newly added record exists on each Windows worker") + expectedProxies := getEnvVarProxyMap(oc) + waitForProxyOnNodes(oc, winNodes, expectedProxies) + + g.By("Verify that the trusted-ca cm exists on the WMCO namespace") + trustedCA, err := popItemFromList(oc, "cm", trustedCACM, wmcoNamespace) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(trustedCA).To(o.Equal(trustedCACM), "trusted CA CM does not exist on %v namespace", wmcoNamespace) + + g.By("Validate the content of trusted-ca copied to each Windows worker") + caBundlePath := "c:\\k\\ca-bundle.crt" + trustedCAContent, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("configmap", trustedCACM, "-n", wmcoNamespace, "-o=jsonpath={.data.ca-bundle\\.crt}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(trustedCAContent).NotTo(o.BeEmpty(), "trusted CA content from ConfigMap is empty") + for _, nodeName := range winNodes { + e2e.Logf("Verify trusted CA content is included on Windows worker %v", nodeName) + bundleContent, err := runHostProcessPS(oc, nodeName, windowsDebugImage, fmt.Sprintf("Get-Content -Raw -Path '%s'", caBundlePath)) + o.Expect(err).NotTo(o.HaveOccurred(), "failed fetching CA bundle from Windows node %v", nodeName) + o.Expect(strings.TrimSpace(bundleContent)).To(o.ContainSubstring("BEGIN CERTIFICATE"), "CA bundle on %v does not contain certificates", nodeName) + } + + g.By("Check that trusted-ca configmap cannot get deleted") + deleteResource(oc, "cm", trustedCACM, wmcoNamespace) + waitForCM(oc, trustedCACM, trustedCACM, wmcoNamespace) + + g.By("Check that trusted-ca configmap cannot get tampered") + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("configmap", trustedCACM, "--type=json", "-p", `[{"op": "remove", "path": "/metadata/labels"}]`, "-n", wmcoNamespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitForCM(oc, trustedCACM, trustedCACM, wmcoNamespace) + }) + + g.It("Smokerun-Author:rrasouli-Critical-68320-[node-proxy]-Import custom CA certificates into Windows node system store [Serial][Disruptive]", func() { + const ( + name = "OCP-68320-custom" + validity = "3650" + caSubj = "/OU=openshift/CN=test-custom-self-cert-signer" + userSelfSignedCommonName = "CN=test-custom-self-cert-signer, OU=openshift" + userInstalledCertCommonName = "CN=Installer-QE-CA, OU=Installer-QE, O=OCP, S=Beijing, C=CN" + namespace = "openshift-config" + configmap = "user-ca-bundle" + ) + + g.By("Verify that user certificate installed on each Windows worker") + checkUserCertificatesOnNodes(oc, userInstalledCertCommonName, 1) + + g.By("Create a self-signed certificate and append to user-ca-bundle") + keyPath := fmt.Sprintf("%s-ca.key", name) + crtPath := fmt.Sprintf("%s-ca.crt", name) + defer os.Remove(keyPath) + cmd := fmt.Sprintf("openssl genrsa -out %s-ca.key 4096", name) + output, err := exec.Command("bash", "-c", cmd).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to generate key: %s", output) + + defer os.Remove(crtPath) + cmd = fmt.Sprintf("openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days %s -out %s-ca.crt -subj %s", name, validity, name, caSubj) + output, err = exec.Command("bash", "-c", cmd).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create certificate: %s", output) + + initialConfigMapContent := getConfigMapData(oc, configmap, "ca\\-bundle\\.crt", namespace) + initialConfigMapContent = removeOuterQuotes(initialConfigMapContent) + defer func() { + configureCertificateToJSONPatch(oc, initialConfigMapContent, configmap, namespace) + }() + + newCertificateContent, err := os.ReadFile(crtPath) + o.Expect(err).NotTo(o.HaveOccurred()) + combinedContent := fmt.Sprintf("%s\n%s", initialConfigMapContent, string(newCertificateContent)) + configureCertificateToJSONPatch(oc, combinedContent, configmap, namespace) + + g.By("Verify that user certificate installed on each Windows worker") + checkUserCertificatesOnNodes(oc, userInstalledCertCommonName, 1) + + g.By("Creating certificate rotation") + cmd = fmt.Sprintf("openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days 1 -out %s-ca.crt -subj %s", name, name, caSubj) + output, err = exec.Command("bash", "-c", cmd).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create rotated certificate: %s", output) + + newCertificateContent, err = os.ReadFile(crtPath) + o.Expect(err).NotTo(o.HaveOccurred()) + combinedContent = fmt.Sprintf("%s\n%s", initialConfigMapContent, string(newCertificateContent)) + configureCertificateToJSONPatch(oc, combinedContent, configmap, namespace) + + g.By("Verify that after certificate rotation certificates installed on each Windows worker") + checkUserCertificatesOnNodes(oc, userInstalledCertCommonName, 1) + + g.By("Verify that self-signed certificate has been removed from each Windows node") + configureCertificateToJSONPatch(oc, initialConfigMapContent, configmap, namespace) + checkUserCertificatesOnNodes(oc, userSelfSignedCommonName, 0) + }) + + g.It("Author:rrasouli-Smokerun-Longduration-Critical-71173-[node-proxy]-Test connectivity from Windows nodes behind proxy [Serial][Disruptive][Slow]", + g.SpecTimeout(30*time.Minute), + func(ctx g.SpecContext) { + o.Expect(getClusterProxy(oc, "status.noProxy")).ToNot(o.BeEmpty(), "status.noProxy is not set on the cluster") + winNodes := getWindowsNodeNames(oc) + + g.By("Verify NO_PROXY env vars on each Windows node are comma-separated") + for _, nodeName := range winNodes { + msg, err := runHostProcessPS(oc, nodeName, windowsDebugImage, + "[System.Environment]::GetEnvironmentVariable('NO_PROXY', 'Machine')") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get NO_PROXY on %s", nodeName) + values := strings.Split(strings.TrimSpace(msg), ",") + o.Expect(len(values)).To(o.BeNumerically(">", 1), "NO_PROXY value on %s is not comma-separated", nodeName) + } + + g.By("Add another record to cluster proxy: myfakeaddress.com to no-proxy") + specNoProxy := getClusterProxy(oc, "spec.noProxy") + updatedNoProxy := "myfakeaddress.com" + if specNoProxy != "" { + updatedNoProxy = specNoProxy + ",myfakeaddress.com" + } + + initialTimeStamp := getWMCOTimestamp(oc) + defer restoreProxyEnvironment(oc, initialProxySpec) + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("proxy/cluster", "--type=json", "-p", + "[{\"op\": \"add\", \"path\":\"/spec/noProxy\", \"value\":\""+updatedNoProxy+"\"}]").Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "could not patch proxy with new noProxy value") + + g.By("Wait for WMCO to process proxy change") + restarted, _ := checkWMCORestarted(oc, initialTimeStamp) + if restarted { + e2e.Logf("WMCO restarted after proxy patch, waiting for WICD propagation") + } else { + e2e.Logf("WMCO did not restart after proxy patch, waiting for WICD propagation") + } + winIPs := getWindowsInternalIPs(oc) + waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute) + + g.By("Verify NO_PROXY changes propagated to Windows nodes") + expectedProxies := getEnvVarProxyMap(oc) + waitForProxyOnNodes(oc, winNodes, expectedProxies) + + g.By("HTTP Traffic Test") + testTraffic(oc, "http://www.testingmcafeesites.com/testcat_cp.html", winNodes) + + g.By("HTTPS Traffic Test") + testTraffic(oc, "https://www.google.com", winNodes) + }) +}) diff --git a/ote/test/e2e/utils.go b/ote/test/e2e/utils.go index 1572b36c30..9fc4106e9f 100644 --- a/ote/test/e2e/utils.go +++ b/ote/test/e2e/utils.go @@ -39,7 +39,6 @@ var ( machineLabel = "machine.openshift.io/os-id=Windows" windowsDebugImage = "mcr.microsoft.com/powershell:lts-nanoserver-ltsc2022" linuxDebugImage = "registry.access.redhat.com/ubi9/ubi:latest" - defaultWindowsMS = "windows" ) // Service represents a Windows service entry from the WICD windows-services ConfigMap. @@ -60,14 +59,6 @@ func checkVersionAnnotationReady(oc *exutil.CLI, windowsNodeName string) (bool, return true, err } -// waitVersionAnnotationReady polls until the WMCO version annotation is set on the node. -func waitVersionAnnotationReady(oc *exutil.CLI, windowsNodeName string, interval, timeout time.Duration) { - err := wait.Poll(interval, timeout, func() (bool, error) { - return checkVersionAnnotationReady(oc, windowsNodeName) - }) - o.Expect(err).NotTo(o.HaveOccurred(), "Timed out waiting for version annotation on node %s", windowsNodeName) -} - // getWindowsHostNames returns the hostnames of all Windows nodes in the cluster. func getWindowsHostNames(oc *exutil.CLI) []string { winHostNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", windowsNodeLabel, "-o=jsonpath={.items[*].status.addresses[?(@.type==\"Hostname\")].address}").Output() @@ -545,7 +536,7 @@ func runHostProcessPS(oc *exutil.CLI, nodeName, image, psCommand string, waitFor defer cleanupPod() - pollErr := wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) { + pollErr := wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { phase, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( "pod", podName, "-n", wmcoNamespace, "-o=jsonpath={.status.phase}").Output() if err != nil { @@ -963,29 +954,6 @@ func checkConnectivity(ctx context.Context, IP string, delay int) error { } } -// getServiceClusterIP returns the ClusterIP assigned to a Service. -func getServiceClusterIP(oc *exutil.CLI, serviceName, namespace string) (string, error) { - return oc.AsAdmin().WithoutNamespace().Run("get").Args( - "service", serviceName, "-o=jsonpath={.spec.clusterIP}", "-n", namespace).Output() -} - -// generateClusterIPServiceYAML returns a YAML manifest for a ClusterIP Service. -func generateClusterIPServiceYAML(name, namespace, appLabel string, port int) string { - return fmt.Sprintf(`apiVersion: v1 -kind: Service -metadata: - name: %s - namespace: %s -spec: - ports: - - port: %d - targetPort: %d - selector: - app: %s - type: ClusterIP -`, name, namespace, port, port, appLabel) -} - // generateLinuxWebServerYAML returns a YAML manifest for a Linux web server Deployment using python http.server. func generateLinuxWebServerYAML(name, namespace, image string, replicas int) string { return fmt.Sprintf(`apiVersion: apps/v1 @@ -1023,39 +991,6 @@ spec: `, name, name, name, replicas, name, name, image) } -// generateWindowsDaemonSetYAML returns a YAML manifest for a Windows DaemonSet. -func generateWindowsDaemonSetYAML(name, namespace, appLabel, image string) string { - return fmt.Sprintf(`apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: %s - namespace: %s -spec: - selector: - matchLabels: - app: %s - template: - metadata: - labels: - app: %s - spec: - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: "os" - operator: "Equal" - value: "Windows" - effect: "NoSchedule" - containers: - - name: %s - image: %s - command: - - pwsh.exe - - -Command - - "while ($true) { Start-Sleep -Seconds 30 }" -`, name, namespace, appLabel, appLabel, name, image) -} - // getWorkloadsNames returns the pod names for all pods belonging to the given Deployment, sorted by host IP. func getWorkloadsNames(oc *exutil.CLI, deploymentName string, namespace string) ([]string, error) { workloads, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "--selector", "app="+deploymentName, "--sort-by=.status.hostIP", "-o=jsonpath={.items[*].metadata.name}", "-n", namespace).Output() @@ -1192,16 +1127,17 @@ func setServiceBinPath(oc *exutil.CLI, nodeName, image, serviceName, binPath str } return nil } - func stopWindowsService(oc *exutil.CLI, nodeName, image, serviceName string) (string, error) { cmd := fmt.Sprintf("Stop-Service '%s' -Force -ErrorAction SilentlyContinue; (Get-Service '%s').Status", serviceName, serviceName) - output, err := runDebugNodePS(oc, nodeName, image, cmd) + output, err := runHostProcessPS(oc, nodeName, image, cmd) if err != nil { return "", err } return strings.TrimSpace(output), nil } + + // getAvailabilityZone returns the availability zone of the cluster's MachineSet or nodes. func getAvailabilityZone(oc *exutil.CLI) string { zone, err := getMachineSetZone(oc) @@ -1355,3 +1291,425 @@ func waitForMachinesetReady(oc *exutil.CLI, machineSetName string, timeout, repl e2e.Failf("machineset %s did not reach %d ready replicas within %d minutes", machineSetName, replicas, timeout) } } + +const ( + wicdConfigMap = "windows-services" + trustedCACM = "trusted-ca" + proxyCAConfigMap = "trusted-ca" + windowsWorkloads = "win-webserver" + defaultNamespace = "winc-test" +) + +type ConfigMapPayload struct { + Data struct { + CaBundleCrt string `json:"ca-bundle.crt"` + } `json:"data"` +} + +func isProxy(oc *exutil.CLI) bool { + if iaasPlatform == "nutanix" { + return false + } + spec := getProxySpec(oc) + for _, v := range spec { + if s, ok := v.(string); ok && s != "" { + if strings.Contains(s, "ci.devcluster.openshift.com") { + return false + } + return true + } + } + return false +} + +func waitForProxyStatus(oc *exutil.CLI) { + e2e.Logf("Waiting for proxy status to be reconciled by cluster-network-operator") + waitErr := wait.Poll(10*time.Second, 4*time.Minute, func() (bool, error) { + statusMap := getEnvVarProxyMap(oc) + return len(statusMap) > 0, nil + }) + o.Expect(waitErr).NotTo(o.HaveOccurred(), "proxy status was not reconciled within 4 minutes") +} + +func getClusterProxy(oc *exutil.CLI, value string) string { + clusterProxy, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("proxies", "-o=jsonpath={.items[*]."+value+"}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return clusterProxy +} + +func getProxySpec(oc *exutil.CLI) map[string]interface{} { + spec := make(map[string]interface{}) + spec["HTTPS_PROXY"] = getClusterProxy(oc, "spec.httpsProxy") + spec["HTTP_PROXY"] = getClusterProxy(oc, "spec.httpProxy") + spec["NO_PROXY"] = getClusterProxy(oc, "spec.noProxy") + return spec +} + +func getEnvVarProxyMap(oc *exutil.CLI, replacement ...map[string]string) map[string]interface{} { + clusterEnvVars := make(map[string]interface{}) + if replacement == nil { + if v := getClusterProxy(oc, "status.httpsProxy"); v != "" { + clusterEnvVars["HTTPS_PROXY"] = v + } + if v := getClusterProxy(oc, "status.httpProxy"); v != "" { + clusterEnvVars["HTTP_PROXY"] = v + } + if v := getClusterProxy(oc, "status.noProxy"); v != "" { + clusterEnvVars["NO_PROXY"] = v + } + } else { + for _, m := range replacement { + for key, value := range m { + if v := getClusterProxy(oc, value); v != "" { + clusterEnvVars[key] = v + } + } + } + } + return clusterEnvVars +} + +func getPayloadMap(payload string) map[string]interface{} { + var m map[string]interface{} + json.Unmarshal([]byte(payload), &m) + return m +} + +func waitForWICDConfigMapUpdate(oc *exutil.CLI, windowsServicesCM string, expectedValues map[string]interface{}) map[string]interface{} { + var wicdProxies map[string]interface{} + pollErr := wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) { + wicdPayload, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("cm", windowsServicesCM, "-n", wmcoNamespace, "-o=jsonpath={.data.environmentVars}").Output() + if err != nil || wicdPayload == "" { + e2e.Logf("Waiting for WICD ConfigMap update: %v", err) + return false, nil + } + wicdProxies = getPayloadMap(wicdPayload) + if compareMaps(expectedValues, wicdProxies) { + e2e.Logf("WICD ConfigMap updated with expected values") + return true, nil + } + e2e.Logf("Waiting for WICD ConfigMap to update") + return false, nil + }) + compat_otp.AssertWaitPollNoErr(pollErr, "WICD ConfigMap did not update with expected values within 5 minutes") + return wicdProxies +} + +func waitForWICDConfigMapContains(oc *exutil.CLI, windowsServicesCM string, key string, substring string) { + pollErr := wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) { + jsonOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("cm", windowsServicesCM, "-ojsonpath={.data.environmentVars}", "-n", wmcoNamespace).Output() + if err != nil { + e2e.Logf("Waiting for WICD ConfigMap: %v", err) + return false, nil + } + value := gjson.Get(jsonOutput, key) + if strings.Contains(fmt.Sprint(value), substring) { + e2e.Logf("WICD ConfigMap %s contains %s", key, substring) + return true, nil + } + e2e.Logf("Waiting for %s in WICD ConfigMap %s, current: %s", substring, key, value) + return false, nil + }) + compat_otp.AssertWaitPollNoErr(pollErr, fmt.Sprintf("WICD ConfigMap %s did not contain %s within 5 minutes", key, substring)) +} + +func compileEnvVars(pwshOutput string) string { + var valueLines []string + var value string + lines := strings.Split(pwshOutput, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + valueLine := strings.TrimSpace(strings.TrimPrefix(parts[1], "Value:")) + valueLines = []string{valueLine} + } else if line != "" { + valueLines = append(valueLines, line) + } + if len(valueLines) > 0 { + value = strings.Join(valueLines, "") + } + } + return value +} + +func compareMaps(map1, map2 map[string]interface{}) bool { + if len(map1) != len(map2) { + return false + } + for key := range map1 { + val1 := compileEnvVars(fmt.Sprint(map1[key])) + val2 := compileEnvVars(fmt.Sprint(map2[key])) + value := strings.ReplaceAll(val2, ";", ",") + if val1 != value { + e2e.Logf("values are different value: %v map2 value: %v", val1, val2) + return false + } + } + return true +} + +func getWindowsNodeNames(oc *exutil.CLI) []string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "nodes", "-l", windowsNodeLabel, "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.TrimSpace(output) == "" { + return nil + } + return strings.Split(strings.TrimSpace(output), " ") +} + +func checkProxyVarsOnNodes(oc *exutil.CLI, winNodes []string, wicdProxies map[string]interface{}) { + for _, nodeName := range winNodes { + for key, proxy := range wicdProxies { + proxyStr := fmt.Sprint(proxy) + if proxyStr == "" { + continue + } + e2e.Logf("Check %v proxy exists on worker %v", key, nodeName) + cmd := fmt.Sprintf("[System.Environment]::GetEnvironmentVariable('%v', 'Machine')", key) + msg, err := runHostProcessPS(oc, nodeName, windowsDebugImage, cmd) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.TrimSpace(msg)).Should(o.ContainSubstring(proxyStr), "proxy %v not found on %v", key, nodeName) + } + } +} + +func waitForProxyOnNodes(oc *exutil.CLI, winNodes []string, wicdProxies map[string]interface{}) { + pollErr := wait.Poll(20*time.Second, 5*time.Minute, func() (bool, error) { + for _, nodeName := range winNodes { + for key, proxy := range wicdProxies { + proxyStr := fmt.Sprint(proxy) + if proxyStr == "" { + continue + } + cmd := fmt.Sprintf("[System.Environment]::GetEnvironmentVariable('%v', 'Machine')", key) + msg, err := runHostProcessPS(oc, nodeName, windowsDebugImage, cmd) + if err != nil { + e2e.Logf("Waiting for %v on %v: error: %v", key, nodeName, err) + return false, nil + } + if !strings.Contains(strings.TrimSpace(msg), proxyStr) { + e2e.Logf("Waiting for %v on %v: expected %q, got %q", key, nodeName, proxyStr, strings.TrimSpace(msg)) + return false, nil + } + } + } + e2e.Logf("All proxy values propagated to all Windows nodes") + return true, nil + }) + compat_otp.AssertWaitPollNoErr(pollErr, "proxy values did not propagate to all Windows nodes within 5 minutes") +} + +func getWMCOTimestamp(oc *exutil.CLI) string { + wmcoTime, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "--selector", "name="+wmcoDeploymentName, "--field-selector=status.phase=Running", "-o=jsonpath={.items[0].status.startTime}", "-n", wmcoNamespace).Output() + if err != nil || wmcoTime == "" { + return "" + } + return wmcoTime +} + +func checkWMCORestarted(oc *exutil.CLI, startTime string) (bool, error) { + var restartDetected bool + pollErr := wait.Poll(20*time.Second, 6*time.Minute, func() (bool, error) { + actualWMCOTime := getWMCOTimestamp(oc) + if actualWMCOTime == "" { + e2e.Logf("WMCO pod timestamp unavailable (pod transitioning), waiting...") + return false, nil + } + if startTime != actualWMCOTime { + e2e.Logf("WMCO restarted (old: %s, new: %s)", startTime, actualWMCOTime) + restartDetected = true + return true, nil + } + e2e.Logf("WMCO did not restart yet, waiting...") + return false, nil + }) + if pollErr != nil { + return false, fmt.Errorf("error restarting WMCO: %v", pollErr) + } + return restartDetected, nil +} + +func generateClusterProxy(httpProxy, httpsProxy, noProxy string) string { + return fmt.Sprintf(`apiVersion: config.openshift.io/v1 +kind: Proxy +metadata: + name: cluster +spec: + httpProxy: %s + httpsProxy: %s + noProxy: %s + trustedCA: + name: user-ca-bundle +`, httpProxy, httpsProxy, noProxy) +} + +func restoreProxyEnvironment(oc *exutil.CLI, clusterEnvVars map[string]interface{}) { + e2e.Logf("Starting proxy environment restore") + wmcoStartTime := getWMCOTimestamp(oc) + httpProxy := fmt.Sprint(clusterEnvVars["HTTP_PROXY"]) + httpsProxy := fmt.Sprint(clusterEnvVars["HTTPS_PROXY"]) + noProxy := fmt.Sprint(clusterEnvVars["NO_PROXY"]) + + pollErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + applyErr := createResourceFromString(oc, "", generateClusterProxy(httpProxy, httpsProxy, noProxy)) + if applyErr != nil { + e2e.Logf("retrying proxy restore: %v", applyErr) + return false, nil + } + return true, nil + }) + if pollErr != nil { + e2e.Logf("Warning: proxy restore did not complete within 2 minutes: %v", pollErr) + return + } + + var patches []string + if noProxy == "" && getClusterProxy(oc, "spec.noProxy") != "" { + patches = append(patches, `{"op": "remove", "path": "/spec/noProxy"}`) + } + if httpsProxy == "" && getClusterProxy(oc, "spec.httpsProxy") != "" { + patches = append(patches, `{"op": "remove", "path": "/spec/httpsProxy"}`) + } + if httpProxy == "" && getClusterProxy(oc, "spec.httpProxy") != "" { + patches = append(patches, `{"op": "remove", "path": "/spec/httpProxy"}`) + } + if len(patches) > 0 { + patchJSON := "[" + strings.Join(patches, ",") + "]" + e2e.Logf("Removing empty proxy fields: %s", patchJSON) + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("proxy/cluster", "--type=json", "-p", patchJSON).Execute() + if err != nil { + e2e.Logf("Warning: failed to remove empty proxy fields: %v", err) + } + } + + restarted, _ := checkWMCORestarted(oc, wmcoStartTime) + if restarted { + e2e.Logf("WMCO restarted after proxy restore, waiting for WICD propagation") + } else { + e2e.Logf("WMCO did not restart after proxy restore") + } + winNodes := getWindowsInternalIPs(oc) + waitWindowsNodesReady(oc, len(winNodes), 15*time.Minute) + expectedProxies := getEnvVarProxyMap(oc) + winNodeNames := getWindowsNodeNames(oc) + waitForProxyOnNodes(oc, winNodeNames, expectedProxies) + e2e.Logf("Proxy environment restored successfully") +} + +func popItemFromList(oc *exutil.CLI, value string, keywordSearch string, namespace string) (string, error) { + rawList, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(value, "-n", namespace, "-o=jsonpath={.items[*].metadata.name}").Output() + if err != nil { + return "", err + } + for _, val := range strings.Split(rawList, " ") { + if strings.Contains(val, keywordSearch) { + return val, nil + } + } + return "", nil +} + +func waitForCM(oc *exutil.CLI, cmName string, cmType string, namespace string) { + pollErr := wait.Poll(10*time.Second, 600*time.Second, func() (bool, error) { + windowsCM, err := popItemFromList(oc, "configmap", cmType, namespace) + if err != nil || windowsCM == "" { + return false, nil + } + return windowsCM == cmName, nil + }) + if pollErr != nil { + e2e.Failf("Expected configmap %v not found after 10 minutes", cmName) + } +} + +func deleteResource(oc *exutil.CLI, resourceType string, resourceName string, namespace string) { + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args(resourceType, resourceName, "-n", namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func checkUserCertificatesOnNodes(oc *exutil.CLI, commonName string, expectedCount int) { + winNodes := getWindowsNodeNames(oc) + for _, nodeName := range winNodes { + e2e.Logf("Waiting for %d user certificate(s) with CN '%s' on node %s", expectedCount, commonName, nodeName) + cmd := fmt.Sprintf("(Get-ChildItem -Path Cert:\\LocalMachine\\Root | Where-Object {$_.Subject -eq '%s'}).Count", commonName) + + pollErr := wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) { + msg, err := runHostProcessPS(oc, nodeName, windowsDebugImage, cmd) + if err != nil { + e2e.Logf("Error checking certificates on node %s: %v", nodeName, err) + return false, nil + } + numOfCerts := 0 + if trimmed := strings.TrimSpace(msg); trimmed != "" { + found := regexp.MustCompile(`\d+`).FindString(trimmed) + if found != "" { + numOfCerts, _ = strconv.Atoi(found) + } + } + if numOfCerts == expectedCount { + e2e.Logf("Found %d certificate(s) on node %s", numOfCerts, nodeName) + return true, nil + } + e2e.Logf("Waiting for certificates on node %s: expected %d, found %d", nodeName, expectedCount, numOfCerts) + return false, nil + }) + o.Expect(pollErr).NotTo(o.HaveOccurred(), "certificate count did not reach %d on node %s within 5 minutes", expectedCount, nodeName) + } +} + +func getConfigMapData(oc *exutil.CLI, cm string, dataKey string, namespace string) string { + dataValue, err := oc.AsAdmin().WithoutNamespace().Run("get"). + Args("configmap", cm, "-o=jsonpath={.data."+dataKey+"}", "-n", namespace).Output() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get cm %v data key %v", cm, dataKey) + return dataValue +} + +func removeOuterQuotes(s string) string { + if len(s) >= 2 { + if c := s[len(s)-1]; s[0] == c && (c == '"' || c == '\'') { + return s[1 : len(s)-1] + } + } + return s +} + +func configureCertificateToJSONPatch(oc *exutil.CLI, payload, configmap, namespace string) { + payload = strings.Replace(payload, "\n\n", "\n", 1) + jsonPayload := fmt.Sprintf(`{"data":{"ca-bundle.crt":"%s"}}`, strings.ReplaceAll(payload, "\n", "")) + var configMapPayload ConfigMapPayload + err := json.Unmarshal([]byte(jsonPayload), &configMapPayload) + o.Expect(err).NotTo(o.HaveOccurred(), "error unmarshalling JSON") + cmd := oc.AsAdmin().WithoutNamespace().Run("patch").Args("configmap", configmap, "-n", namespace, "-p", jsonPayload) + output, err := cmd.Output() + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + stderr := strings.TrimSpace(string(exitError.Stderr)) + err = fmt.Errorf("%v: %s", err, stderr) + } + o.Expect(err).NotTo(o.HaveOccurred(), "error patching ConfigMap. Output: %s", output) + } +} + +func extractStatusCode(output string) int { + re := regexp.MustCompile(`HTTP/\d+\.?\d*\s+(\d{3})`) + matches := re.FindStringSubmatch(output) + if len(matches) >= 2 { + statusCode, err := strconv.Atoi(matches[1]) + if err == nil { + return statusCode + } + } + return 0 +} + +func testTraffic(oc *exutil.CLI, testURL string, winNodes []string) { + command := fmt.Sprintf("cmd.exe /c curl -vsIL %v", testURL) + for _, nodeName := range winNodes { + output, err := runHostProcessPS(oc, nodeName, windowsDebugImage, command) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to execute curl on %v", nodeName) + statusCode := extractStatusCode(output) + o.Expect(statusCode).To(o.Equal(200), "expected status code 200 on %v from %v, but got %d", testURL, nodeName, statusCode) + } +}