Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions internal/scheduling/compute/nova_adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ package compute
import (
"strings"

hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
"k8s.io/apimachinery/pkg/api/resource"

api "github.com/cobaltcore-dev/cortex/api/external/nova"
"github.com/cobaltcore-dev/cortex/api/v1alpha1"
)
Expand Down Expand Up @@ -52,11 +55,8 @@ func ToSchedulerRequest(
GlobalRequestID: derefString(wire.Context.GlobalRequestID),
}

// Flavor sizing.
req.Flavor = FlavorSpec{
VCPUs: spec.Flavor.Data.VCPUs,
MemoryMB: spec.Flavor.Data.MemoryMB,
}
// Resource sizing.
req.RequestedResources = NewResourceDemand(spec.Flavor.Data.VCPUs, spec.Flavor.Data.MemoryMB)

// Hypervisor family.
req.HypervisorFamily = toHypervisorFamily(wire)
Expand Down Expand Up @@ -201,6 +201,19 @@ func firstNonEmpty(a, b string) string {
// in-process reservation controllers (which hold a flavor's extra-spec map
// directly) call them, so the conventions are understood in exactly one place.

// NewResourceDemand builds a ResourceDemand from Nova's vCPU count and memory in
// mebibytes. Nova's memory_mb field is documented as "megabytes" but its values are
// mebibytes (1 MiB = 2^20 bytes); this converts to the bytes hv.Status and the
// capacity lib use, so the Nova unit convention is applied in exactly one place.
// Resources left at 0 are still written, so callers can rely on the keys being present.
func NewResourceDemand(vcpus, memoryMB uint64) ResourceDemand {
//nolint:gosec // vCPU and memory values are small, non-negative flavor counts.
return ResourceDemand{
hv1.ResourceCPU: *resource.NewQuantity(int64(vcpus), resource.DecimalSI),
hv1.ResourceMemory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI),
}
}

// unsupportedCapabilityOps are the Nova extra-spec comparison operators the
// capabilities filter does not implement. A capability value containing any of
// these means the request uses an advanced match the pipeline can't honor, so
Expand Down
62 changes: 48 additions & 14 deletions internal/scheduling/compute/request.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ package compute
import (
"strings"

hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
"k8s.io/apimachinery/pkg/api/resource"

"github.com/cobaltcore-dev/cortex/api/scheduling"
"github.com/cobaltcore-dev/cortex/api/v1alpha1"
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
Expand Down Expand Up @@ -74,16 +77,38 @@ type Capability struct {
Value string
}

// FlavorSpec is the typed, Nova-agnostic description of the requested flavor.
// It carries no extra-spec map: the adapter resolves every extra spec the
// pipeline needs into the typed fields on this struct and on PlacementRequest.
type FlavorSpec struct {
// MemoryMB is the requested memory per instance. 0 means a zero-memory
// instance to capacity/bin-pack steps (fits anywhere), so the adapter sets it.
MemoryMB uint64
// VCPUs is the requested vCPUs per instance. 0 means a zero-CPU instance to
// capacity/bin-pack steps (fits anywhere), so the adapter sets it.
VCPUs uint64
// ResourceDemand is the typed, Nova-agnostic description of the resources a
// placement requests, as a per-resource quantity map keyed by hv1.ResourceName and
// aligned with the units hypervisors report in their status: memory in bytes, CPU
// as a plain count. The adapter resolves the flavor sizing into this map; it carries
// no extra-spec map.
//
// A resource absent from the map is not requested — capacity and bin-pack steps
// treat a missing entry as a zero-sized requirement (fits anywhere).
type ResourceDemand map[hv1.ResourceName]resource.Quantity

// VCPUs returns the requested vCPU count (0 if unset).
func (d ResourceDemand) VCPUs() int64 {
q := d[hv1.ResourceCPU]
return q.Value()
}

// MemoryBytes returns the requested memory in bytes (0 if unset).
func (d ResourceDemand) MemoryBytes() int64 {
q := d[hv1.ResourceMemory]
return q.Value()
}

// Scaled returns a copy of the demand with every quantity multiplied by n. It is
// used to turn a per-instance demand into the total for a multi-instance request.
func (d ResourceDemand) Scaled(n int64) ResourceDemand {
scaled := make(ResourceDemand, len(d))
for rn, perInstance := range d {
total := perInstance.DeepCopy()
total.Mul(n)
scaled[rn] = total
}
return scaled
}

// RequestedDestination mirrors the subset of Nova's requested_destination that
Expand Down Expand Up @@ -203,10 +228,10 @@ type PlacementRequest struct {
// means no hardware-version constraint.
HardwareVersion string

// Flavor is the requested flavor sizing. A zero FlavorSpec means capacity and
// bin-pack steps see a zero-sized instance (fits anywhere) — the adapter
// always populates it for real placements.
Flavor FlavorSpec
// RequestedResources is the requested instance sizing as a per-resource quantity
// map. A nil or empty map means capacity and bin-pack steps see a zero-sized
// instance (fits anywhere) — the adapter always populates it for real placements.
RequestedResources ResourceDemand

// RequiredTraits are trait names the host must have (adapter stripped the
// "trait:" prefix and kept the "required" ones). Empty/nil means no required
Expand Down Expand Up @@ -259,6 +284,15 @@ var _ lib.FilterWeigherPipelineRequest = PlacementRequest{}
// GetIntent returns the resolved scheduling intent.
func (r PlacementRequest) GetIntent() v1alpha1.SchedulingIntent { return r.Intent }

// TotalResources returns the full per-request resource demand: the per-instance
// RequestedResources scaled by NumInstances. Capacity and bin-pack steps place all
// instances on one host, so they size against this total. NumInstances is used raw
// (not normalized to 1) — the adapter always sets it >= 1.
func (r PlacementRequest) TotalResources() ResourceDemand {
//nolint:gosec // instance count is a small, non-negative count.
return r.RequestedResources.Scaled(int64(r.NumInstances))
}

// GetHosts returns the compute host names of all candidates.
func (r PlacementRequest) GetHosts() []string {
hosts := make([]string, len(r.Hosts))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib/capacity"
hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
"k8s.io/apimachinery/pkg/api/resource"
)

type FilterHasEnoughCapacityOpts struct {
Expand Down Expand Up @@ -200,22 +199,17 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.Plac
usageByHost[hv.Name] = usage
}

// A host must fit the full request: flavor resources × NumInstances. We can't spread
// instances across hosts, so all requested instances must land on the same host.
if request.Flavor.VCPUs == 0 {
// A host must fit the full request: requested resources × NumInstances. We can't
// spread instances across hosts, so all requested instances must land on the same host.
if request.RequestedResources.VCPUs() == 0 {
return nil, errors.New("flavor has 0 vcpus")
}
if request.Flavor.MemoryMB == 0 {
if request.RequestedResources.MemoryBytes() == 0 {
return nil, errors.New("flavor has 0 memory")
}
// Nova's memory_mb field is named "megabytes" in the API docs, but the values are in
// mebibytes (1 MiB = 2^20 bytes). Multiply by 1<<20 to convert to the bytes used in
// hv.Status.
//nolint:gosec // flavor fields are small, non-negative counts.
required := map[hv1.ResourceName]resource.Quantity{
"cpu": *resource.NewQuantity(int64(request.Flavor.VCPUs*request.NumInstances), resource.DecimalSI),
"memory": *resource.NewQuantity(int64(request.Flavor.MemoryMB*request.NumInstances*(1<<20)), resource.BinarySI),
}
// RequestedResources holds per-instance quantities in the units hv.Status reports
// (memory in bytes); TotalResources scales by NumInstances for the full request.
required := request.TotalResources()

hostsEncountered := make(map[string]struct{})
for host, usage := range usageByHost {
Expand All @@ -227,8 +221,8 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.Plac
traceLog.V(1).Info(
"filtering host due to insufficient capacity",
"host", host,
"requested_cpus", request.Flavor.VCPUs,
"requested_memory_mb", request.Flavor.MemoryMB,
"requested_cpus", request.RequestedResources.VCPUs(),
"requested_memory_bytes", request.RequestedResources.MemoryBytes(),
"num_instances", request.NumInstances,
"available_cpus", freeCPU.String(),
"available_memory", freeMemory.String(),
Expand All @@ -238,9 +232,9 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.Plac
}
traceLog.V(1).Info(
"host has enough capacity", "host", host,
"requested_cpus", request.Flavor.VCPUs,
"requested_cpus", request.RequestedResources.VCPUs(),
"available_cpus", freeCPU.String(),
"requested_memory_mb", request.Flavor.MemoryMB,
"requested_memory_bytes", request.RequestedResources.MemoryBytes(),
"available_memory", freeMemory.String(),
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -260,18 +260,15 @@ func newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup string, vcpus
}

return compute.PlacementRequest{
ProjectID: projectID,
InstanceUUID: instanceUUID,
NumInstances: 1,
Intent: intent,
HardwareVersion: flavorGroup,
HypervisorFamily: compute.HypervisorFamilyKVM,
Flavor: compute.FlavorSpec{
VCPUs: uint64(vcpus), //nolint:gosec // test code
MemoryMB: memoryMB,
},
Hosts: hostList,
Weights: weights,
ProjectID: projectID,
InstanceUUID: instanceUUID,
NumInstances: 1,
Intent: intent,
HardwareVersion: flavorGroup,
HypervisorFamily: compute.HypervisorFamilyKVM,
RequestedResources: compute.NewResourceDemand(uint64(vcpus), memoryMB), //nolint:gosec // test code
Hosts: hostList,
Weights: weights,
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -179,12 +179,15 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request compute.Place
if numInstances == 0 {
numInstances = 1
}
memoryMB := request.Flavor.MemoryMB
vcpus := request.Flavor.VCPUs
// RequestedResources holds per-instance memory in bytes; convert to MiB for the
// liquid RAM math below.
memoryMB := request.RequestedResources.MemoryBytes() / (1 << 20)
vcpus := request.RequestedResources.VCPUs()
instances := int64(numInstances)
// MB to GiB: using 1024 as divisor is slightly conservative (overestimates GiB needed).
requestRAM := int64(math.Ceil(float64(memoryMB*numInstances) / 1024.0))
requestCores := int64(vcpus * numInstances) //nolint:gosec // VCPUs from flavor specs, realistically bounded
requestInstances := int64(numInstances)
requestRAM := int64(math.Ceil(float64(memoryMB*instances) / 1024.0))
requestCores := vcpus * instances
requestInstances := instances

// Step 4: Derive the liquid resource names.
resourceRAM := "hw_version_" + hwVersion + "_ram"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,12 @@ func intentFromHint(hints map[string]any) v1alpha1.SchedulingIntent {

func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) compute.PlacementRequest {
return compute.PlacementRequest{
ProjectID: projectID,
AvailabilityZone: az,
NumInstances: numInstances,
Intent: intentFromHint(hints),
HardwareVersion: hwVersion,
Flavor: compute.FlavorSpec{
VCPUs: 4,
MemoryMB: memoryMB,
},
ProjectID: projectID,
AvailabilityZone: az,
NumInstances: numInstances,
Intent: intentFromHint(hints),
HardwareVersion: hwVersion,
RequestedResources: compute.NewResourceDemand(4, memoryMB),
Hosts: []compute.Host{
{ComputeHost: "host1"},
{ComputeHost: "host2"},
Expand Down
17 changes: 1 addition & 16 deletions internal/scheduling/nova/plugins/weighers/kvm_binpack.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/api/resource"
)

type KVMBinpackStepOpts struct {
Expand Down Expand Up @@ -85,7 +84,7 @@ func (s *KVMBinpackStep) Run(traceLog logr.Logger, request compute.PlacementRequ
}
hvsByName[hv.Name] = hv
}
vmResources := s.calcVMResources(request)
vmResources := request.TotalResources()

for host := range result.Activations {
hv, ok := hvsByName[host]
Expand Down Expand Up @@ -140,20 +139,6 @@ func (s *KVMBinpackStep) Run(traceLog logr.Logger, request compute.PlacementRequ
return result, nil
}

// calcVMResources calculates the total resource requests for the VM to be scheduled.
func (s *KVMBinpackStep) calcVMResources(req compute.PlacementRequest) map[hv1.ResourceName]resource.Quantity {
resources := make(map[hv1.ResourceName]resource.Quantity)
resourcesMemBytes := int64(req.Flavor.MemoryMB) * (1024 * 1024) //nolint:gosec // memory values are bounded by Nova
resourcesMemBytes *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova
resources[hv1.ResourceMemory] = *resource.
NewQuantity(resourcesMemBytes, resource.BinarySI)
resourcesCPU := int64(req.Flavor.VCPUs) //nolint:gosec // vCPU values are bounded by Nova
resourcesCPU *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova
resources[hv1.ResourceCPU] = *resource.
NewQuantity(resourcesCPU, resource.DecimalSI)
return resources
}

func init() {
Index["kvm_binpack"] = func() NovaWeigher { return &KVMBinpackStep{} }
}
22 changes: 9 additions & 13 deletions internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,16 +46,13 @@ func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) com
}

return compute.PlacementRequest{
ProjectID: "project-A",
InstanceUUID: "instance-123",
NumInstances: numInstances,
HypervisorFamily: compute.HypervisorFamilyKVM,
Flavor: compute.FlavorSpec{
VCPUs: vcpus,
MemoryMB: memoryMB,
},
Hosts: hostList,
Weights: weights,
ProjectID: "project-A",
InstanceUUID: "instance-123",
NumInstances: numInstances,
HypervisorFamily: compute.HypervisorFamilyKVM,
RequestedResources: compute.NewResourceDemand(vcpus, memoryMB),
Hosts: hostList,
Weights: weights,
}
}

Expand Down Expand Up @@ -471,7 +468,7 @@ func TestKVMBinpackStep_Run(t *testing.T) {
}
}

func TestKVMBinpackStep_calcVMResources(t *testing.T) {
func TestKVMBinpackStep_TotalResources(t *testing.T) {
tests := []struct {
name string
request compute.PlacementRequest
Expand Down Expand Up @@ -512,8 +509,7 @@ func TestKVMBinpackStep_calcVMResources(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
step := &KVMBinpackStep{}
resources := step.calcVMResources(tt.request)
resources := tt.request.TotalResources()

memResource, ok := resources[hv1.ResourceMemory]
if !ok {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (s *KVMCommittedResourceReservationStep) Run(traceLog logr.Logger, request
}

// Flavor memory in bytes for free-capacity check.
flavorMemoryBytes := int64(request.Flavor.MemoryMB) * 1024 * 1024 //nolint:gosec
flavorMemoryBytes := request.RequestedResources.MemoryBytes()

var reservations v1alpha1.ReservationList
if err := s.Client.List(context.Background(), &reservations); err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,12 @@ func newCRRequest(hosts []string) compute.PlacementRequest {
weights[h] = 1.0
}
return compute.PlacementRequest{
ProjectID: "project-A",
AvailabilityZone: "qa-de-1a",
ResourceGroup: "group-2101",
Flavor: compute.FlavorSpec{
MemoryMB: 8 * 1024,
},
Hosts: hostList,
Weights: weights,
ProjectID: "project-A",
AvailabilityZone: "qa-de-1a",
ResourceGroup: "group-2101",
RequestedResources: compute.NewResourceDemand(0, 8*1024),
Hosts: hostList,
Weights: weights,
}
}

Expand Down Expand Up @@ -212,14 +210,12 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) {
newCRReservation("res-1", "host1", "project-A", "group-2101", "qa-de-1a", "16Gi"),
},
request: compute.PlacementRequest{
ProjectID: "project-A",
AvailabilityZone: "qa-de-1a",
Intent: api.ReserveForFailoverIntent,
ResourceGroup: "group-2101",
Flavor: compute.FlavorSpec{
MemoryMB: 4096,
},
Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}},
ProjectID: "project-A",
AvailabilityZone: "qa-de-1a",
Intent: api.ReserveForFailoverIntent,
ResourceGroup: "group-2101",
RequestedResources: compute.NewResourceDemand(0, 4096),
Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}},
},
opts: KVMCommittedResourceReservationOpts{ReservationHostWeight: floatPtr(1.0), DefaultHostWeight: floatPtr(0.1)},
expectedWeights: map[string]float64{"host1": 0.0, "host2": 0.0},
Expand Down
Loading
Loading