diff --git a/test/e2e/common/node/framework/podresize/resize.go b/test/e2e/common/node/framework/podresize/resize.go index 5065883f62bf0..8aae1d6c660c9 100644 --- a/test/e2e/common/node/framework/podresize/resize.go +++ b/test/e2e/common/node/framework/podresize/resize.go @@ -396,6 +396,11 @@ func WaitForPodResizeActuation(ctx context.Context, f *framework.Framework, podC if !podutils.IsPodReady(pod) { return func() string { return "pod is not ready" }, nil } + if errs := CheckPodResized(ctx, f, pod, expectedContainers); len(errs) != 0 { + return func() string { + return formatErrors(utilerrors.NewAggregate(errs)).Error() + }, nil + } return nil, nil })), ) @@ -405,7 +410,7 @@ func WaitForPodResizeActuation(ctx context.Context, f *framework.Framework, podC return resizedPod } -func ExpectPodResized(ctx context.Context, f *framework.Framework, resizedPod *v1.Pod, expectedContainers []ResizableContainerInfo) { +func CheckPodResized(ctx context.Context, f *framework.Framework, resizedPod *v1.Pod, expectedContainers []ResizableContainerInfo) []error { ginkgo.GinkgoHelper() // Verify Pod Containers Cgroup Values @@ -453,11 +458,7 @@ func ExpectPodResized(ctx context.Context, f *framework.Framework, resizedPod *v } } - if len(errs) > 0 { - resizedPod.ManagedFields = nil // Suppress managed fields in error output. - framework.ExpectNoError(formatErrors(utilerrors.NewAggregate(errs)), - "Verifying pod resources resize state. Pod: %s", framework.PrettyPrintJSON(resizedPod)) - } + return errs } func MakeResizePatch(originalContainers, desiredContainers []ResizableContainerInfo, originPodResources, desiredPodResources *v1.ResourceRequirements) []byte { diff --git a/test/e2e/common/node/pod_level_resources_resize.go b/test/e2e/common/node/pod_level_resources_resize.go index f855299456b25..ed428ebca3f21 100644 --- a/test/e2e/common/node/pod_level_resources_resize.go +++ b/test/e2e/common/node/pod_level_resources_resize.go @@ -559,8 +559,7 @@ func doPodLevelResourcesMemoryLimitDecreaseTest(f *framework.Framework) { podresize.VerifyPodResources(testPod, containers, viableLoweredLimitPLR) ginkgo.By("waiting for viable lowered limit to be actuated") - resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod, containers) - podresize.ExpectPodResized(ctx, f, resizedPod, containers) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod, containers) // There is some latency after container startup before memory usage is scraped. On CRI-O // this latency is much higher, so wait enough time for cAdvisor to scrape metrics twice. @@ -637,8 +636,7 @@ func doPodLevelResourcesMemoryLimitDecreaseTest(f *framework.Framework) { podresize.VerifyPodResources(testPod, containers, originalPLR) ginkgo.By("waiting for the original values to be actuated") - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod, containers) - podresize.ExpectPodResized(ctx, f, resizedPod, containers) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod, containers) ginkgo.By("deleting pod") podClient.DeleteSync(ctx, testPod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete) @@ -709,7 +707,6 @@ func patchAndVerifyPLR(ctx context.Context, f *framework.Framework, podClient *e podresize.VerifyPodResources(patchedPod, expected, expectedPodResources) resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, newPod, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) // Uncomment pod-level status verification after patch in 1.36 release. // convesion of cgroup values -> Pod.Status.Resources -> cgroup values is // resulting in values off by a small number. diff --git a/test/e2e/common/node/pod_resize.go b/test/e2e/common/node/pod_resize.go index 8e551ba2fb8db..e376b73b83599 100644 --- a/test/e2e/common/node/pod_resize.go +++ b/test/e2e/common/node/pod_resize.go @@ -610,8 +610,7 @@ func doPodResizeMemoryLimitDecreaseTest(f *framework.Framework) { podresize.VerifyPodResources(testPod, viableLoweredLimit, nil) ginkgo.By("waiting for viable lowered limit to be actuated") - resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod, viableLoweredLimit) - podresize.ExpectPodResized(ctx, f, resizedPod, viableLoweredLimit) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod, viableLoweredLimit) // There is some latency after container startup before memory usage is scraped. On CRI-O // this latency is much higher, so wait enough time for cAdvisor to scrape metrics twice. @@ -683,8 +682,7 @@ func doPodResizeMemoryLimitDecreaseTest(f *framework.Framework) { podresize.VerifyPodResources(testPod, original, nil) ginkgo.By("waiting for the original values to be actuated") - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod, original) - podresize.ExpectPodResized(ctx, f, resizedPod, original) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod, original) ginkgo.By("deleting pod") podClient.DeleteSync(ctx, testPod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete) @@ -816,8 +814,7 @@ func doPodResizeReadAndReplaceTests(f *framework.Framework) { ginkgo.By("verifying pod resources after patch") expected := podresize.UpdateExpectedContainerRestarts(ctx, updatedPod, desiredContainers) - resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, updatedPod, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + podresize.WaitForPodResizeActuation(ctx, f, podClient, updatedPod, expected) ginkgo.By("verifying pod fetched from resize subresource") framework.ExpectNoError(framework.Gomega(). @@ -922,7 +919,6 @@ func patchAndVerify(ctx context.Context, f *framework.Framework, podClient *e2ep podresize.VerifyPodResources(patchedPod, expected, expectedPodResources) resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, newPod, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) if expectedPodResources != nil { framework.ExpectNoError(podresize.VerifyPodCgroupValues(ctx, f, resizedPod)) } diff --git a/test/e2e/node/pod_resize.go b/test/e2e/node/pod_resize.go index 19e54e53cb0bb..33f512b673381 100644 --- a/test/e2e/node/pod_resize.go +++ b/test/e2e/node/pod_resize.go @@ -106,7 +106,6 @@ func doPodResizeResourceQuotaTests(f *framework.Framework) { ginkgo.By("waiting for resize to be actuated") resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, newPods[0], expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) ginkgo.By("verifying pod resources after resize") podresize.VerifyPodResources(resizedPod, expected, nil) @@ -296,7 +295,6 @@ func doPodResizeLimitRangerTests(f *framework.Framework) { ginkgo.By("waiting for resize to be actuated") resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, newPods[0], expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) ginkgo.By("verifying pod resources after resize") podresize.VerifyPodResources(resizedPod, expected, nil) @@ -672,8 +670,7 @@ func doPodResizeSchedulerTests(f *framework.Framework) { RestartCount: testPod1.Status.ContainerStatuses[0].RestartCount, }, } - resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod1, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod1, expected) ginkgo.By(fmt.Sprintf("TEST3: Resize pod '%s' to exceed the node capacity", testPod1.Name)) _, p1Err = f.ClientSet.CoreV1().Pods(testPod1.Namespace).Patch(ctx, @@ -696,158 +693,140 @@ func doPodResizeSchedulerTests(f *framework.Framework) { func doPodResizeRetryDeferredTests(f *framework.Framework) { ginkgo.It("pod-resize-retry-deferred-test-1", func(ctx context.Context) { - // Deferred resize E2E test case #1: - // 1. Create pod1 and pod2 and pod3 on node. - // 2. Resize pod3 to request more cpu than available, verify the resize is deferred. - // 3. Resize pod1 down to make space for pod3, verify pod3's resize has completed. + // Deferred resize E2E test case #1 (2-Pod Scenario): + // 1. Assume Node CPU is divided into 10 "parts". + // 2. Create Pod A (7 parts) and Pod B (2 parts). + // 3. Verify remaining free CPU is < 5 parts (cannot fit Pod B's requested increase). + // 4. Resize Pod B to 7 parts. Verify the resize is deferred. + // 5. Resize Pod A down to 2 parts (frees up exactly 5 parts). + // 6. Verify Pod B's deferred resize has now completed successfully. podClient := e2epod.NewPodClient(f) - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err, "failed to get running nodes") - gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty()) - framework.Logf("Found %d schedulable nodes", len(nodes.Items)) - ginkgo.By("Find node CPU resources available for allocation!") - node := nodes.Items[0] - nodeAllocatableCPU, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) - framework.ExpectNoError(err, "failed to get CPU resources available for allocation") + node, nodeAllocatableCPU, nodeAvailableCPU := getBestNodeForResizeTest(ctx, f) framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", node.Name, nodeAllocatableCPU.MilliValue(), nodeAvailableCPU.MilliValue()) - testPod1CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue()/2, resource.DecimalSI) - testPod2CPUQuantity := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/2, resource.DecimalSI) - framework.Logf("testPod1 initial CPU request is '%dm'", testPod1CPUQuantity.MilliValue()) - framework.Logf("testPod2 initial CPU request is '%dm'", testPod2CPUQuantity.MilliValue()) + // Conceptually divide the available CPU into 10 parts. + onePartCPU := nodeAvailableCPU.MilliValue() / 10 + framework.Logf("Node '%s' currently available MilliCPUs = %dm (1 part = %dm)", node.Name, nodeAvailableCPU.MilliValue(), onePartCPU) - c1 := []podresize.ResizableContainerInfo{ - { - Name: "c1", - Resources: &cgroups.ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()}, - }, - } - c2 := []podresize.ResizableContainerInfo{ - { - Name: "c2", - Resources: &cgroups.ContainerResources{CPUReq: testPod2CPUQuantity.String(), CPULim: testPod2CPUQuantity.String()}, - }, - } - tStamp := strconv.Itoa(time.Now().Nanosecond()) - testPod1 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, c1, nil) - testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1) - testPod2 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, c2, nil) - testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2) - e2epod.SetNodeAffinity(&testPod1.Spec, node.Name) - e2epod.SetNodeAffinity(&testPod2.Spec, node.Name) + // Pod A: 7 parts, Pod B: 2 parts + podACPU := resource.NewMilliQuantity(onePartCPU*7, resource.DecimalSI) + podBCPU := resource.NewMilliQuantity(onePartCPU*2, resource.DecimalSI) - ginkgo.By(fmt.Sprintf("Create pod '%s' that fits the node '%s'", testPod1.Name, node.Name)) - testPod1 = podClient.CreateSync(ctx, testPod1) - gomega.Expect(testPod1.Status.Phase).To(gomega.Equal(v1.PodRunning)) - gomega.Expect(testPod1.Generation).To(gomega.BeEquivalentTo(1)) + framework.Logf("Pod A initial CPU request is '%dm'", podACPU.MilliValue()) + framework.Logf("Pod B initial CPU request is '%dm'", podBCPU.MilliValue()) - ginkgo.By(fmt.Sprintf("Create pod '%s' that fits the node '%s'", testPod2.Name, node.Name)) - testPod2 = podClient.CreateSync(ctx, testPod2) - gomega.Expect(testPod2.Status.Phase).To(gomega.Equal(v1.PodRunning)) - gomega.Expect(testPod2.Generation).To(gomega.BeEquivalentTo(1)) - - nodeAllocatableCPU2, nodeAvailableCPU2, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) - framework.ExpectNoError(err, "failed to get CPU resources available for allocation") - framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", - node.Name, nodeAllocatableCPU2.MilliValue(), nodeAvailableCPU2.MilliValue()) + cA := []podresize.ResizableContainerInfo{ + {Name: "ca", Resources: &cgroups.ContainerResources{CPUReq: podACPU.String(), CPULim: podACPU.String()}}, + } + cB := []podresize.ResizableContainerInfo{ + {Name: "cb", Resources: &cgroups.ContainerResources{CPUReq: podBCPU.String(), CPULim: podBCPU.String()}}, + } - testPod3CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU2.MilliValue()/4, resource.DecimalSI) - testPod3CPUQuantityResized := resource.NewMilliQuantity(nodeAvailableCPU2.MilliValue()+testPod1CPUQuantity.MilliValue()/4, resource.DecimalSI) - framework.Logf("testPod3 MilliCPUs after resize '%dm'", testPod3CPUQuantityResized.MilliValue()) + tStamp := strconv.Itoa(time.Now().Nanosecond()) - testPod1CPUQuantityResizedCPU := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/3, resource.DecimalSI) - framework.Logf("testPod1 MilliCPUs after resize '%dm'", testPod1CPUQuantityResizedCPU.MilliValue()) + podA := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod-a", tStamp, cA, nil) + podA = e2epod.MustMixinRestrictedPodSecurity(podA) + e2epod.SetNodeAffinity(&podA.Spec, node.Name) - c3 := []podresize.ResizableContainerInfo{ - { - Name: "c3", - Resources: &cgroups.ContainerResources{CPUReq: testPod3CPUQuantity.String(), CPULim: testPod3CPUQuantity.String()}, - }, - } - patchTestpod3ToDeferred := fmt.Sprintf(`{ - "spec": { - "containers": [ - { - "name": "c3", - "resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}} - } - ] - } - }`, testPod3CPUQuantityResized.MilliValue(), testPod3CPUQuantityResized.MilliValue()) - patchTestpod1ToMakeSpaceForPod3 := fmt.Sprintf(`{ - "spec": { - "containers": [ - { - "name": "c1", - "resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}} - } - ] - } - }`, testPod1CPUQuantityResizedCPU.MilliValue(), testPod1CPUQuantityResizedCPU.MilliValue()) + podB := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod-b", tStamp, cB, nil) + podB = e2epod.MustMixinRestrictedPodSecurity(podB) + e2epod.SetNodeAffinity(&podB.Spec, node.Name) - tStamp = strconv.Itoa(time.Now().Nanosecond()) - testPod3 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod3", tStamp, c3, nil) - testPod3 = e2epod.MustMixinRestrictedPodSecurity(testPod3) - e2epod.SetNodeAffinity(&testPod3.Spec, node.Name) + ginkgo.By(fmt.Sprintf("Create Pod A '%s' and Pod B '%s'", podA.Name, podB.Name)) - ginkgo.By(fmt.Sprintf("Create pod '%s' that fits the node '%s'", testPod3.Name, node.Name)) - testPod3 = podClient.CreateSync(ctx, testPod3) - gomega.Expect(testPod3.Status.Phase).To(gomega.Equal(v1.PodRunning)) - gomega.Expect(testPod3.Generation).To(gomega.BeEquivalentTo(1)) + podA = podClient.CreateSync(ctx, podA) + gomega.Expect(podA.Status.Phase).To(gomega.Equal(v1.PodRunning)) + gomega.Expect(podA.Generation).To(gomega.BeEquivalentTo(1)) - ginkgo.By(fmt.Sprintf("Resize pod '%s' that cannot fit node due to insufficient CPU", testPod3.Name)) - testPod3, p3Err := f.ClientSet.CoreV1().Pods(testPod3.Namespace).Patch(ctx, - testPod3.Name, types.StrategicMergePatchType, []byte(patchTestpod3ToDeferred), metav1.PatchOptions{}, "resize") - framework.ExpectNoError(p3Err, "failed to patch pod for resize") - waitForPodDeferred(ctx, f, testPod3) + podB = podClient.CreateSync(ctx, podB) + gomega.Expect(podB.Status.Phase).To(gomega.Equal(v1.PodRunning)) + gomega.Expect(podB.Generation).To(gomega.BeEquivalentTo(1)) - ginkgo.By(fmt.Sprintf("Resize pod '%s' to make enough space for pod '%s'", testPod1.Name, testPod3.Name)) - testPod1, p1Err := f.ClientSet.CoreV1().Pods(testPod1.Namespace).Patch(ctx, - testPod1.Name, types.StrategicMergePatchType, []byte(patchTestpod1ToMakeSpaceForPod3), metav1.PatchOptions{}, "resize") - framework.ExpectNoError(p1Err, "failed to patch pod for resize") - gomega.Expect(testPod1.Generation).To(gomega.BeEquivalentTo(2)) + ginkgo.By("Verify remaining CPU is less than the 5 parts required for Pod B's resize") + _, nodeAvailableCPU2, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) + framework.ExpectNoError(err, "failed to get CPU resources available for allocation") - ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod3.Name, testPod1.Name)) + fivePartsCPU := onePartCPU * 5 + framework.Logf("Remaining CPU = %dm. Pod B needs %dm more to resize.", nodeAvailableCPU2.MilliValue(), fivePartsCPU) + gomega.Expect(nodeAvailableCPU2.MilliValue()).To(gomega.BeNumerically("<", fivePartsCPU), + "Available CPU should be strictly less than 5 parts to ensure Pod B's resize is deferred") + + // Calculations for Resizing + // Pod B: Resize to 7 parts (needs 5 more parts) + podBResizedCPU := resource.NewMilliQuantity(onePartCPU*7, resource.DecimalSI) + // Pod A: Resize to 2 parts (drops 5 parts, freeing them up) + podAResizedCPU := resource.NewMilliQuantity(onePartCPU*2, resource.DecimalSI) + + patchPodBToDeferred := fmt.Sprintf(`{ + "spec": { + "containers": [ + { + "name": "cb", + "resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}} + } + ] + } + }`, podBResizedCPU.MilliValue(), podBResizedCPU.MilliValue()) + + patchPodAToMakeSpace := fmt.Sprintf(`{ + "spec": { + "containers": [ + { + "name": "ca", + "resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}} + } + ] + } + }`, podAResizedCPU.MilliValue(), podAResizedCPU.MilliValue()) + + ginkgo.By(fmt.Sprintf("Resize Pod B '%s' to 7 parts (will be deferred due to insufficient CPU)", podB.Name)) + podB, err = f.ClientSet.CoreV1().Pods(podB.Namespace).Patch(ctx, + podB.Name, types.StrategicMergePatchType, []byte(patchPodBToDeferred), metav1.PatchOptions{}, "resize") + framework.ExpectNoError(err, "failed to patch Pod B for resize") + waitForPodDeferred(ctx, f, podB) + + ginkgo.By(fmt.Sprintf("Resize Pod A '%s' down to 2 parts, freeing up exactly 5 parts for Pod B", podA.Name)) + podA, err = f.ClientSet.CoreV1().Pods(podA.Namespace).Patch(ctx, + podA.Name, types.StrategicMergePatchType, []byte(patchPodAToMakeSpace), metav1.PatchOptions{}, "resize") + framework.ExpectNoError(err, "failed to patch Pod A to free up space") + gomega.Expect(podA.Generation).To(gomega.BeEquivalentTo(2)) + + ginkgo.By("Verify Pod B successfully actuates the deferred resize after space is freed") expected := []podresize.ResizableContainerInfo{ { - Name: "c3", - Resources: &cgroups.ContainerResources{CPUReq: testPod3CPUQuantityResized.String(), CPULim: testPod3CPUQuantityResized.String()}, + Name: "cb", + Resources: &cgroups.ContainerResources{CPUReq: podBResizedCPU.String(), CPULim: podBResizedCPU.String()}, }, } - resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + podresize.WaitForPodResizeActuation(ctx, f, podClient, podB, expected) - ginkgo.By("deleting pods") - e2epod.DeletePodsWithWait(ctx, f.ClientSet, []*v1.Pod{testPod1, testPod2, testPod3}) + ginkgo.By("Cleaning up test pods") + e2epod.DeletePodsWithWait(ctx, f.ClientSet, []*v1.Pod{podA, podB}) }) ginkgo.It("pod-resize-retry-deferred-test-2", func(ctx context.Context) { // Deferred resize E2E test case #2: - // 1. Create 5 pods on the node, where the first one has 2/3 of the node allocatable CPU, - // and the remaining ones each have 1/16 of the node allocatable CPU. - // 2. Resize all remaining pods to request 2/3 of the node allocatable CPU, verify deferred. + // 1. Create 5 pods on the node, where the first one has 15/20 of the available node CPU, + // and the remaining ones each have 1/20 of the available node CPU. + // 2. Resize all remaining pods to request 15/20 of the available node CPU, verify deferred. // 3. Delete the first pod, verify the pod with the highest priority has its resize accepted. // 4. Repeat step 3 until all but the last pod has been deleted. podClient := e2epod.NewPodClient(f) - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err, "failed to get running nodes") - gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty()) - framework.Logf("Found %d schedulable nodes", len(nodes.Items)) - ginkgo.By("Find node CPU and memory resources available for allocation!") - node := nodes.Items[0] - - nodeAllocatableCPU, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) - framework.ExpectNoError(err, "failed to get CPU resources available for allocation") + node, nodeAllocatableCPU, nodeAvailableCPU := getBestNodeForResizeTest(ctx, f) framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", node.Name, nodeAllocatableCPU.MilliValue(), nodeAvailableCPU.MilliValue()) - majorityCPUQuantity := resource.NewMilliQuantity(2*nodeAvailableCPU.MilliValue()/3, resource.DecimalSI) - littleCPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue()/16, resource.DecimalSI) + // Divide the available CPU into 20 parts: + // Pod 1 starts with 15 parts, Pods 2-5 start with 1 part each (19 parts total). + // When Pods 2-5 are resized to 15 parts, total requested becomes 33 parts (> 20 parts, deferred). + onePartCPU := nodeAvailableCPU.MilliValue() / 20 + littleCPUQuantity := resource.NewMilliQuantity(onePartCPU, resource.DecimalSI) + majorityCPUQuantity := resource.NewMilliQuantity(onePartCPU*15, resource.DecimalSI) containerWithMajorityCPU := []podresize.ResizableContainerInfo{ { Name: "c", @@ -877,11 +856,11 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1) e2epod.SetNodeAffinity(&testPod1.Spec, node.Name) - ginkgo.By(fmt.Sprintf("Create pod '%s' with 2/3 of the node cpu", testPod1.Name)) + ginkgo.By(fmt.Sprintf("Create pod '%s' with 15/20 of the available node cpu", testPod1.Name)) testPod1 = podClient.CreateSync(ctx, testPod1) gomega.Expect(testPod1.Status.Phase).To(gomega.Equal(v1.PodRunning)) - // Create pod2 with 1/16 of the node allocatable CPU, with high priority based on priority class. + // Create pod2 with 1/20 of the available node CPU, with high priority based on priority class. testPod2 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, containerWithLittleCPU, nil) testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2) pc, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ @@ -898,34 +877,34 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { testPod2.Spec.PriorityClassName = pc.Name e2epod.SetNodeAffinity(&testPod2.Spec, node.Name) - ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/16 of the node cpu and high priority class", testPod2.Name)) + ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/20 of the available node cpu and high priority class", testPod2.Name)) testPod2 = podClient.CreateSync(ctx, testPod2) gomega.Expect(testPod2.Status.Phase).To(gomega.Equal(v1.PodRunning)) - // Create pod3 with 1/16 of the node allocatable CPU, that is a "guaranteed" pod (all others should be "burstable"). + // Create pod3 with 1/20 of the available node CPU, that is a "guaranteed" pod (all others should be "burstable"). testPod3 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod3", tStamp, containerWithLittleCPUGuaranteedQoS, nil) testPod3 = e2epod.MustMixinRestrictedPodSecurity(testPod3) e2epod.SetNodeAffinity(&testPod3.Spec, node.Name) - ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/16 of the node cpu and guaranteed qos", testPod3.Name)) + ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/20 of the available node cpu and guaranteed qos", testPod3.Name)) testPod3 = podClient.CreateSync(ctx, testPod3) gomega.Expect(testPod3.Status.Phase).To(gomega.Equal(v1.PodRunning)) - // Create pod4 with 1/16 of the node allocatable CPU. + // Create pod4 with 1/20 of the available node CPU. testPod4 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod4", tStamp, containerWithLittleCPU, nil) testPod4 = e2epod.MustMixinRestrictedPodSecurity(testPod4) e2epod.SetNodeAffinity(&testPod4.Spec, node.Name) - ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/16 of the node cpu", testPod4.Name)) + ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/20 of the available node cpu", testPod4.Name)) testPod4 = podClient.CreateSync(ctx, testPod4) gomega.Expect(testPod4.Status.Phase).To(gomega.Equal(v1.PodRunning)) - // Create pod5 with 1/16 of the node allocatable CPU. + // Create pod5 with 1/20 of the available node CPU. testPod5 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod5", tStamp, containerWithLittleCPU, nil) testPod5 = e2epod.MustMixinRestrictedPodSecurity(testPod5) e2epod.SetNodeAffinity(&testPod5.Spec, node.Name) - ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/16 of the node cpu", testPod5.Name)) + ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/20 of the available node cpu", testPod5.Name)) testPod5 = podClient.CreateSync(ctx, testPod5) gomega.Expect(testPod5.Status.Phase).To(gomega.Equal(v1.PodRunning)) @@ -943,29 +922,31 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { // Resize requests are done in an arbitrary order, to verify that the priority based on priority class // or qos class takes precedent over the order of the requests. - // Attempt pod4 resize request to 2/3 of the node allocatable CPU, verify deferred. - ginkgo.By(fmt.Sprintf("Resize pod '%s'", testPod4.Name)) + // Attempt pod4 resize request to 15/20 of the available node CPU, verify deferred. + ginkgo.By(fmt.Sprintf("Resize pod '%s' to 15/20 of the available node cpu", testPod4.Name)) testPod4, err = f.ClientSet.CoreV1().Pods(testPod4.Namespace).Patch(ctx, testPod4.Name, types.StrategicMergePatchType, []byte(patchTestPod), metav1.PatchOptions{}, "resize") framework.ExpectNoError(err, "failed to patch pod for resize") waitForPodDeferred(ctx, f, testPod4) - // Attempt pod3 resize request to 2/3 of the node allocatable CPU, verify deferred. - ginkgo.By(fmt.Sprintf("Resize pod '%s'", testPod3.Name)) + // Attempt pod3 resize request to 15/20 of the available node CPU, verify deferred. + ginkgo.By(fmt.Sprintf("Resize pod '%s' to 15/20 of the available node cpu", testPod3.Name)) testPod3, err = f.ClientSet.CoreV1().Pods(testPod3.Namespace).Patch(ctx, testPod3.Name, types.StrategicMergePatchType, []byte(patchTestPod), metav1.PatchOptions{}, "resize") framework.ExpectNoError(err, "failed to patch pod for resize") waitForPodDeferred(ctx, f, testPod3) - // Attempt pod2 resize request to 2/3 of the node allocatable CPU, verify deferred. - ginkgo.By(fmt.Sprintf("Resize pod '%s'", testPod2.Name)) + // Attempt pod2 resize request to 15/20 of the available node CPU, verify deferred. + ginkgo.By(fmt.Sprintf("Resize pod '%s' to 15/20 of the available node cpu", testPod2.Name)) testPod2, err = f.ClientSet.CoreV1().Pods(testPod2.Namespace).Patch(ctx, testPod2.Name, types.StrategicMergePatchType, []byte(patchTestPod), metav1.PatchOptions{}, "resize") framework.ExpectNoError(err, "failed to patch pod for resize") waitForPodDeferred(ctx, f, testPod2) - // Attempt pod5 resize request to 2/3 of the node allocatable CPU, verify deferred. - ginkgo.By(fmt.Sprintf("Resize pod '%s'", testPod5.Name)) + time.Sleep(1 * time.Second) // sleep to ensure testPod5's LastTransitionTime is later than testPod4's. + + // Attempt pod5 resize request to 15/20 of the available node CPU, verify deferred. + ginkgo.By(fmt.Sprintf("Resize pod '%s' to 15/20 of the available node cpu", testPod5.Name)) testPod5, err = f.ClientSet.CoreV1().Pods(testPod5.Namespace).Patch(ctx, testPod5.Name, types.StrategicMergePatchType, []byte(patchTestPod), metav1.PatchOptions{}, "resize") framework.ExpectNoError(err, "failed to patch pod for resize") @@ -983,8 +964,7 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { Resources: &cgroups.ContainerResources{CPUReq: majorityCPUQuantity.String(), CPULim: majorityCPUQuantity.String()}, }, } - resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expected) waitForPodDeferred(ctx, f, testPod3) waitForPodDeferred(ctx, f, testPod4) waitForPodDeferred(ctx, f, testPod5) @@ -1006,8 +986,7 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { }, }, } - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expected) waitForPodDeferred(ctx, f, testPod4) waitForPodDeferred(ctx, f, testPod5) @@ -1023,8 +1002,7 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { Resources: &cgroups.ContainerResources{CPUReq: majorityCPUQuantity.String(), CPULim: majorityCPUQuantity.String()}, }, } - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expected) waitForPodDeferred(ctx, f, testPod5) // Delete pod4. Verify pod5's resize has completed. @@ -1039,8 +1017,7 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { Resources: &cgroups.ContainerResources{CPUReq: majorityCPUQuantity.String(), CPULim: majorityCPUQuantity.String()}, }, } - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod5, expected) - podresize.ExpectPodResized(ctx, f, resizedPod, expected) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod5, expected) ginkgo.By("deleting pod5") delErr5 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod5) @@ -1060,21 +1037,13 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { // 6. Delete pod1, verify the chain of deferred resizes is actuated. podClient := e2epod.NewPodClient(f) - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err, "failed to get running nodes") - gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty()) - framework.Logf("Found %d schedulable nodes", len(nodes.Items)) - ginkgo.By("Find node CPU and memory resources available for allocation!") - node := nodes.Items[0] - - nodeAllocatableCPU, initNodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) - framework.ExpectNoError(err, "failed to get CPU resources available for allocation") + node, nodeAllocatableCPU, initNodeAvailableCPU := getBestNodeForResizeTest(ctx, f) framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", node.Name, nodeAllocatableCPU.MilliValue(), initNodeAvailableCPU.MilliValue()) nodeAllocatableMem, initNodeAvailableMem, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceMemory) - framework.ExpectNoError(err, "failed to get CPU resources available for allocation") + framework.ExpectNoError(err, "failed to get Memory resources available for allocation") framework.Logf("Node '%s': NodeAllocatable Memory = %d. Memory currently available to allocate = %d.", node.Name, nodeAllocatableMem.Value(), initNodeAvailableMem.Value()) @@ -1172,16 +1141,13 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { framework.ExpectNoError(delErr1, "failed to delete pod %s", testPod1.Name) ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod deletion '%s'", testPod2.Name, testPod1.Name)) - resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expectedTestPod2Resized) - podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod2Resized) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expectedTestPod2Resized) ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod3.Name, testPod2.Name)) - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expectedTestPod3Resized) - podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod3Resized) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expectedTestPod3Resized) ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod4.Name, testPod3.Name)) - resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expectedTestPod4Resized) - podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod4Resized) + podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expectedTestPod4Resized) ginkgo.By("deleting pods") e2epod.DeletePodsWithWait(ctx, f.ClientSet, testPods) @@ -1248,6 +1214,34 @@ func waitForResourceQuota(ctx context.Context, c clientset.Interface, ns, quotaN })).WithTimeout(framework.PollShortTimeout).ShouldNot(gomega.BeEmpty()) } +// getBestNodeForResizeTest finds and returns the schedulable node with the most available CPU +// capacity, along with its allocatable and available CPU quantities. +func getBestNodeForResizeTest(ctx context.Context, f *framework.Framework) (v1.Node, resource.Quantity, resource.Quantity) { + nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) + framework.ExpectNoError(err, "failed to get running nodes") + gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty()) + + var bestNode v1.Node + var bestAllocatable, bestAvailable resource.Quantity + found := false + + for i := range nodes.Items { + n := nodes.Items[i] + allocatable, available, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &n, v1.ResourceCPU) + if err != nil { + continue + } + if !found || available.Cmp(bestAvailable) > 0 { + bestNode = n + bestAllocatable = allocatable + bestAvailable = available + found = true + } + } + gomega.Expect(found).To(gomega.BeTrue(), "failed to find any schedulable node with available resources") + return bestNode, bestAllocatable, bestAvailable +} + func waitForPodDeferred(ctx context.Context, f *framework.Framework, testPod *v1.Pod) { framework.ExpectNoError(e2epod.WaitForPodCondition(ctx, f.ClientSet, testPod.Namespace, testPod.Name, "display pod resize status as deferred", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) { return helpers.IsPodResizeDeferred(pod), nil