From 95083dceea3c5b5174b7649d2dc38444334324e3 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Thu, 29 Oct 2026 08:56:35 +0100 Subject: [PATCH] Report hugepage-backed allocatable memory as hypervisor capacity Memory capacity was derived from each NUMA cell's total physical memory reported by libvirt. That value fluctuates between otherwise-identical hosts by a few hundred MB because linux reserves/frees varying amounts during boot, making capacity reporting non-deterministic. All KVM nova flavors use hw:mem_page_size=large, so the memory that can actually be allocated to VMs is the hugepage-backed memory, which is explicitly configured and therefore stable. Report that as the memory capacity instead. The libvirt capabilities XML already exposes per-cell hugepage counts and the Go structs already parse them, so no CRD or schema change is needed. A new HugepageMemoryToResource helper sums all non-base page sizes. If a cell reports no hugepages, fall back to its total physical memory so we never report a spurious zero capacity. Status.Capabilities.HostMemory is left as the physical total for information; it will now intentionally exceed Status.Capacity[memory]. Signed-off-by: Philipp Matthes --- internal/libvirt/libvirt.go | 27 ++++++++-- internal/libvirt/libvirt_test.go | 80 +++++++++++++++++++++++++++++ internal/libvirt/utils.go | 54 ++++++++++++++++++++ internal/libvirt/utils_test.go | 88 +++++++++++++++++++++++++++++++- 4 files changed, 244 insertions(+), 5 deletions(-) diff --git a/internal/libvirt/libvirt.go b/internal/libvirt/libvirt.go index 52603e8..063f224 100644 --- a/internal/libvirt/libvirt.go +++ b/internal/libvirt/libvirt.go @@ -444,13 +444,32 @@ func (l *LibVirt) addAllocationCapacity(old v1.Hypervisor) (v1.Hypervisor, error totalCpuCapacity := resource.NewQuantity(0, resource.DecimalSI) cellsById := make(map[uint64]v1.Cell) for _, cell := range caps.Host.Topology.CellSpec.Cells { - memoryCapacity, err := MemoryToResource( - cell.Memory.Value, - cell.Memory.Unit, - ) + // Report the stable, hugepage-backed allocatable memory as the cell's + // memory capacity. All KVM nova flavors use hw:mem_page_size=large, so + // this reflects what can actually be allocated to VMs and, unlike the + // cell's total physical memory, does not fluctuate between hosts. + memoryCapacity, err := HugepageMemoryToResource(cell.Pages) if err != nil { return old, err } + // Fall back to the cell's total physical memory if libvirt reports no + // hugepages for the cell. This should not happen on a correctly + // configured KVM host, but guards against reporting a spurious zero + // capacity (e.g. before hugepages are allocated at boot). + if memoryCapacity.IsZero() { + logger.Log.Info( + "no hugepages reported for numa cell, "+ + "falling back to total physical memory", + "cell", cell.ID, + ) + memoryCapacity, err = MemoryToResource( + cell.Memory.Value, + cell.Memory.Unit, + ) + if err != nil { + return old, err + } + } totalMemoryCapacity.Add(memoryCapacity) cpuCapacity := *resource.NewQuantity( diff --git a/internal/libvirt/libvirt_test.go b/internal/libvirt/libvirt_test.go index 9e4da3d..4ff05ab 100644 --- a/internal/libvirt/libvirt_test.go +++ b/internal/libvirt/libvirt_test.go @@ -490,6 +490,11 @@ func TestAddAllocationCapacity_InstanceSpreadAcrossMultipleNumaCells(t *testing. Unit: "GiB", Value: 256, }, + // 131072 * 2 MiB hugepages = 256 GiB allocatable. + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 131072}, + }, CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{ Num: 64, }, @@ -500,6 +505,10 @@ func TestAddAllocationCapacity_InstanceSpreadAcrossMultipleNumaCells(t *testing. Unit: "GiB", Value: 256, }, + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 131072}, + }, CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{ Num: 64, }, @@ -510,6 +519,10 @@ func TestAddAllocationCapacity_InstanceSpreadAcrossMultipleNumaCells(t *testing. Unit: "GiB", Value: 256, }, + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 131072}, + }, CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{ Num: 64, }, @@ -520,6 +533,10 @@ func TestAddAllocationCapacity_InstanceSpreadAcrossMultipleNumaCells(t *testing. Unit: "GiB", Value: 256, }, + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 131072}, + }, CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{ Num: 64, }, @@ -706,6 +723,12 @@ func TestAddAllocationCapacity_Success(t *testing.T) { Unit: "GiB", Value: 64, }, + // 32768 * 2 MiB hugepages = 64 GiB allocatable. + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 32768}, + {Unit: "KiB", Size: 1048576, Value: 0}, + }, CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{ Num: 16, }, @@ -792,6 +815,63 @@ func TestAddAllocationCapacity_Success(t *testing.T) { } } +func TestAddAllocationCapacity_FallsBackToPhysicalMemoryWithoutHugepages(t *testing.T) { + // When libvirt reports no hugepages for a cell, the memory capacity should + // fall back to the cell's total physical memory instead of reporting zero. + caps := capabilities.Capabilities{ + Host: capabilities.CapabilitiesHost{ + Topology: capabilities.CapabilitiesHostTopology{ + CellSpec: capabilities.CapabilitiesHostTopologyCells{ + Num: 1, + Cells: []capabilities.CapabilitiesHostTopologyCell{ + { + ID: 0, + Memory: capabilities.CapabilitiesHostTopologyCellMemory{ + Unit: "GiB", + Value: 64, + }, + // Only the base page, no hugepages configured. + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 16777216}, + }, + CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{ + Num: 16, + }, + }, + }, + }, + }, + }, + } + + l := &LibVirt{ + capabilitiesClient: &mockCapabilitiesClient{caps: caps}, + domainInfoClient: &mockDomInfoClient{infos: []dominfo.DomainInfo{}}, + } + + result, err := l.addAllocationCapacity(v1.Hypervisor{}) + if err != nil { + t.Fatalf("addAllocationCapacity() returned unexpected error: %v", err) + } + + // Capacity should fall back to the 64 GiB physical memory, not 0. + expectedMemCapacity := resource.NewQuantity(64*1024*1024*1024, resource.BinarySI) + memCap := result.Status.Capacity["memory"] + if !memCap.Equal(*expectedMemCapacity) { + t.Errorf("Expected fallback memory capacity %s, got %s", + expectedMemCapacity.String(), memCap.String()) + } + + if len(result.Status.Cells) != 1 { + t.Fatalf("Expected 1 cell, got %d", len(result.Status.Cells)) + } + cellMemCap := result.Status.Cells[0].Capacity["memory"] + if !cellMemCap.Equal(*expectedMemCapacity) { + t.Errorf("Cell 0: Expected fallback memory capacity %s, got %s", + expectedMemCapacity.String(), cellMemCap.String()) + } +} + func TestProcess_Success(t *testing.T) { caps := capabilities.Capabilities{ Host: capabilities.CapabilitiesHost{ diff --git a/internal/libvirt/utils.go b/internal/libvirt/utils.go index 7e9ba88..baac145 100644 --- a/internal/libvirt/utils.go +++ b/internal/libvirt/utils.go @@ -20,8 +20,11 @@ package libvirt import ( "encoding/hex" "fmt" + "math" "k8s.io/apimachinery/pkg/api/resource" + + "github.com/cobaltcore-dev/kvm-node-agent/internal/libvirt/capabilities" ) const ( @@ -81,3 +84,54 @@ func MemoryToResource(value int64, unit string) (resource.Quantity, error) { // Set the value return *quantity, nil } + +// HugepageMemoryToResource returns the allocatable hugepage-backed memory of a +// NUMA cell as a resource.Quantity. +// +// Unlike the cell's total physical memory, which fluctuates between otherwise +// identical hosts because linux reserves/frees varying amounts during boot, the +// number of hugepages is explicitly configured and therefore stable. All KVM +// nova flavors use hw:mem_page_size=large, so only non-base (hugepage) page +// sizes are allocatable to VMs. The base page size is the smallest page size +// reported for the cell and is excluded. A cell with no pages, or only base +// pages, yields 0. +func HugepageMemoryToResource(pages []capabilities.CapabilitiesHostTopologyCellPages) (resource.Quantity, error) { + total := resource.NewQuantity(0, resource.BinarySI) + if len(pages) == 0 { + return *total, nil + } + + // The base page size is the smallest reported page size. Do not assume it + // is the first entry. + baseSize := pages[0].Size + for _, page := range pages[1:] { + if page.Size < baseSize { + baseSize = page.Size + } + } + + for _, page := range pages { + // Skip the base page size, only hugepages are allocatable to VMs. + if page.Size == baseSize { + continue + } + // Libvirt always reports page sizes in KiB. Be defensive in case this + // ever changes, so we don't silently report a wrong capacity. + if page.Unit != UnitKiB { + return resource.Quantity{}, fmt.Errorf( + "unexpected hugepage unit %s", page.Unit, + ) + } + // Guard the uint64 -> int64 conversion. Real hugepage counts are far + // below this bound, but guard anyway so an implausible value cannot + // wrap into a negative capacity. + if page.Value > math.MaxInt64 { + return resource.Quantity{}, fmt.Errorf( + "hugepage count %d exceeds int64", page.Value, + ) + } + bytes := int64(page.Value) * int64(page.Size) * 1024 + total.Add(*resource.NewQuantity(bytes, resource.BinarySI)) + } + return *total, nil +} diff --git a/internal/libvirt/utils_test.go b/internal/libvirt/utils_test.go index 20e90a3..2c15c18 100644 --- a/internal/libvirt/utils_test.go +++ b/internal/libvirt/utils_test.go @@ -21,6 +21,8 @@ import ( "testing" "k8s.io/apimachinery/pkg/api/resource" + + "github.com/cobaltcore-dev/kvm-node-agent/internal/libvirt/capabilities" ) func TestMemoryToResourceKiB(t *testing.T) { @@ -385,8 +387,92 @@ func TestMemoryToResourceRealWorldScenarios(t *testing.T) { } } +func TestHugepageMemoryToResource(t *testing.T) { + tests := []struct { + name string + pages []capabilities.CapabilitiesHostTopologyCellPages + expectedBytes int64 + expectError bool + }{ + { + name: "base page plus 2 MiB hugepages", + pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 10}, + {Unit: "KiB", Size: 1048576, Value: 0}, + }, + // 10 * 2 MiB + expectedBytes: 10 * 2048 * 1024, + }, + { + name: "1 GiB hugepages", + pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 1048576, Value: 10}, + }, + expectedBytes: 10 * 1024 * 1024 * 1024, + }, + { + name: "mixed 2 MiB and 1 GiB hugepages", + pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 100}, + {Unit: "KiB", Size: 1048576, Value: 2}, + }, + expectedBytes: 100*2048*1024 + 2*1024*1024*1024, + }, + { + name: "no pages", + pages: nil, + expectedBytes: 0, + }, + { + name: "only base page", + pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + }, + expectedBytes: 0, + }, + { + name: "base page not first element", + pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 2048, Value: 10}, + {Unit: "KiB", Size: 4, Value: 1000000}, + }, + expectedBytes: 10 * 2048 * 1024, + }, + { + name: "unexpected hugepage unit", + pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "MiB", Size: 2048, Value: 10}, + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := HugepageMemoryToResource(tt.pages) + if tt.expectError { + if err == nil { + t.Fatalf("Expected error, got result: %s", result.String()) + } + return + } + if err != nil { + t.Fatalf("HugepageMemoryToResource() returned unexpected error: %v", err) + } + + expectedQuantity := resource.NewQuantity(tt.expectedBytes, resource.BinarySI) + if !result.Equal(*expectedQuantity) { + t.Errorf("Expected quantity %s, got %s", expectedQuantity.String(), result.String()) + } + }) + } +} + func TestMemoryToResourceNegativeValues(t *testing.T) { - // Test behavior with negative values (edge case) tests := []struct { name string value int64