From 75b551f47a127e7efb5d649e4c40aa4741c46a4e Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 14:08:54 +0100 Subject: [PATCH 01/21] feat(compute): implement scheduling pipeline with request and result handling Signed-off-by: Markus Wieland --- internal/scheduling/compute/request.go | 382 ++++++++++++++++++ internal/scheduling/compute/request_test.go | 101 +++++ internal/scheduling/compute/result.go | 41 ++ internal/scheduling/compute/scheduler.go | 141 +++++++ internal/scheduling/compute/scheduler_test.go | 269 ++++++++++++ 5 files changed, 934 insertions(+) create mode 100644 internal/scheduling/compute/request.go create mode 100644 internal/scheduling/compute/request_test.go create mode 100644 internal/scheduling/compute/result.go create mode 100644 internal/scheduling/compute/scheduler.go create mode 100644 internal/scheduling/compute/scheduler_test.go diff --git a/internal/scheduling/compute/request.go b/internal/scheduling/compute/request.go new file mode 100644 index 000000000..e0ce6f8a4 --- /dev/null +++ b/internal/scheduling/compute/request.go @@ -0,0 +1,382 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +// Package compute is the in-process entrypoint for Nova placement scheduling. +// +// It defines a well-typed, Nova-agnostic SchedulerRequest that the +// filter-weigher pipeline consumes, and a Scheduler that owns the single shared +// scheduling lock and runs the pipeline plus caller-provided side-effect +// closures. Callers that run in the same process (the Nova HTTP adapter, the +// reservation/commitment/capacity/failover controllers) call the Scheduler +// directly instead of POSTing to the external HTTP endpoint. +// +// The spec deliberately contains NO loosely-typed maps or Nova string +// conventions (flavor extra-spec keys, scheduler-hint keys, image property +// keys). A Nova spec adapter is responsible for translating Nova's over-the-wire +// shape into these typed fields; the pipeline only ever sees well-specced +// values. +package compute + +import ( + "strings" + + "github.com/cobaltcore-dev/cortex/api/scheduling" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" +) + +// HypervisorFamily is the coarse hypervisor family a request targets. It is the +// distinction the pipeline and request routing care about; the more specific +// HypervisorType (QEMU vs CH within the KVM family) is only needed by Nova side +// effects. The adapter resolves the family from Nova's +// "capabilities:hypervisor_type" flavor extra spec. +type HypervisorFamily string + +const ( + // HypervisorFamilyKVM covers KVM-based hypervisors (QEMU and CH). + HypervisorFamilyKVM HypervisorFamily = "kvm" + // HypervisorFamilyVMware covers VMware hypervisors. + HypervisorFamilyVMware HypervisorFamily = "vmware" +) + +// HypervisorType is the specific hypervisor type of a request. The adapter +// resolves it from Nova's "capabilities:hypervisor_type" flavor extra spec. +type HypervisorType string + +const ( + // HypervisorTypeQEMU represents QEMU/KVM hypervisors. + HypervisorTypeQEMU HypervisorType = "qemu" + // HypervisorTypeCH represents Cloud-Hypervisor/KVM hypervisors. + HypervisorTypeCH HypervisorType = "ch" + // HypervisorTypeVMware represents VMware hypervisors. + HypervisorTypeVMware HypervisorType = "vmware" +) + +// FlavorType classifies the workload type. The adapter resolves it from Nova's +// "trait:CUSTOM_HANA_EXCLUSIVE_HOST" flavor extra spec. +type FlavorType string + +const ( + // FlavorTypeGeneralPurpose represents general purpose workloads. + FlavorTypeGeneralPurpose FlavorType = "general-purpose" + // FlavorTypeHANA represents HANA workloads. + FlavorTypeHANA FlavorType = "hana" +) + +// ImageHypervisorType is the hypervisor type declared by the image. The adapter +// resolves it from Nova's image properties ("img_hv_type"/"hypervisor_type"). +type ImageHypervisorType string + +const ( + // ImageHypervisorTypeVMware indicates a VMware image. + ImageHypervisorTypeVMware ImageHypervisorType = "vmware" + // ImageHypervisorTypeBaremetal indicates a baremetal image. + ImageHypervisorTypeBaremetal ImageHypervisorType = "baremetal" + // ImageHypervisorTypeKVM indicates a KVM image. + ImageHypervisorTypeKVM ImageHypervisorType = "kvm" + // ImageHypervisorTypeUnset indicates no image hypervisor type was declared. + ImageHypervisorTypeUnset ImageHypervisorType = "" +) + +// Host is a placement candidate. For KVM hosts the compute host name and the +// hypervisor hostname are identical; for others they may differ. +type Host struct { + // ComputeHost is the Nova compute host name, e.g. nova-compute-bb123. + ComputeHost string + // HypervisorHostname is the hypervisor hostname, e.g. domain-c123.. + HypervisorHostname string +} + +// Capability is a single required hypervisor capability as a typed key/value +// equality, e.g. {Name: "cpu_arch", Value: "x86_64"}. The adapter resolves these +// from Nova's "capabilities:" flavor extra specs and rejects any entry that +// uses an unsupported comparison operator before it reaches the pipeline. +type Capability struct { + // Name is the capability key without the "capabilities:" prefix. + Name string + // Value is the required exact value. + Value string +} + +// FlavorSpec is the typed, Nova-agnostic description of the requested flavor. +// It carries no extra-spec map: the adapter resolves every extra spec the +// pipeline needs into the typed fields on this struct and on SchedulerRequest. +type FlavorSpec struct { + // Name is the flavor name. Empty only affects CR/flavor-group lookups in + // side effects, not the pipeline math. + Name string + // FlavorID is the flavor ID. Empty only affects the VM resource written by + // side effects. + FlavorID string + // MemoryMB is the requested memory per instance. 0 means a zero-memory + // instance to capacity/bin-pack steps (fits anywhere), so the adapter sets it. + MemoryMB uint64 + // VCPUs is the requested vCPUs per instance. 0 means a zero-CPU instance to + // capacity/bin-pack steps (fits anywhere), so the adapter sets it. + VCPUs uint64 + // Type classifies the workload (general-purpose/hana), resolved by the + // adapter from the flavor's extra specs. Empty is treated as general-purpose + // by pipeline inference. + Type FlavorType +} + +// RequestedDestination mirrors the subset of Nova's requested_destination that +// the pipeline reads. A nil *RequestedDestination on the request means no +// destination constraint was requested. +type RequestedDestination struct { + // Host is the requested target host. Empty means no host was pinned (only an + // aggregate constraint applies). + Host string + // Aggregates are the host-aggregate names the destination must belong to. + // Empty/nil means no aggregate constraint. + Aggregates []string +} + +// InstanceGroupSpec is the typed subset of a Nova server group the pipeline +// reads. A nil *InstanceGroupSpec on the request means the instance is not part +// of a server group, so group affinity/anti-affinity filters are skipped. +type InstanceGroupSpec struct { + // UUID identifies the server group. Empty if unknown. + UUID string + // Policy is the group policy (e.g. "anti-affinity", "soft-affinity"). Empty + // means no policy-specific filter/weigher applies. + Policy string + // Hosts are the hosts already occupied by members of this group. Empty/nil + // means the group currently occupies no hosts. + Hosts []string + // Members are the instance UUIDs already in the group. Empty/nil means the + // group has no members yet, so anti-affinity has nothing to exclude. + Members []string + // MaxServerPerHost caps how many group members may share a host. 0 means + // unset, which the anti-affinity filter treats as the Nova default of 1. + MaxServerPerHost int +} + +// SchedulerRequest is the well-typed, Nova-agnostic input to the scheduling +// pipeline. Every field is a resolved scalar, enum, or typed collection — there +// are no maps of Nova-specific string keys and no values the pipeline must parse. +// +// Each field documents what leaving it at its zero value means, so the Nova spec +// adapter (and any other caller) can see exactly which fields are load-bearing. +// +// SchedulerRequest implements lib.FilterWeigherPipelineRequest so it can be the +// RequestType of the filter-weigher pipeline. +type SchedulerRequest struct { + // Pipeline is the name of the pipeline to execute. Required: an empty name + // resolves no pipeline and the run fails with "pipeline not found". + Pipeline string + // GatherAllCandidates, when true, tells the Scheduler to ignore the Hosts and + // Weights the caller provided and instead gather ALL available placement + // candidates (via the CandidateGatherer) before filtering. It is set for + // requests where cortex can enumerate the candidate hosts itself — today the + // KVM family, whose hypervisors cortex knows from the hypervisor operator; + // VMware placement leaves it false and relies on the caller's preselection. + // When true, Hosts/Weights below are overwritten before filtering. + GatherAllCandidates bool + // Hosts are the placement candidates. An empty list yields an empty result + // (no host found), unless GatherAllCandidates is set, in which case Hosts is + // overwritten before filtering. + Hosts []Host + // Weights are the per-host input weights, keyed by compute host name. A host + // absent from this map is weighed as 0. Empty means every candidate starts + // equally weighted. + Weights map[string]float64 + // Options configure the pipeline behavior for this run. The zero value is a + // normal (non-read-only) run with no candidate limit; see scheduling.Options + // for per-field defaults. + Options scheduling.Options + + // Intent is the resolved scheduling intent (replaces the "_nova_check_type" + // scheduler hint). The zero value is the empty intent; intent-gated + // filters/weighers treat it like an initial placement (create) and a forced + // destination is honored (see IsForcedDestination). + Intent v1alpha1.SchedulingIntent + + // InstanceUUID identifies the instance being placed. Empty is allowed for + // synthetic runs (e.g. capacity probes); it only appears in logs and side + // effects, so an empty value disables nothing in the pipeline itself. + InstanceUUID string + // ProjectID is the owning OpenStack project. Empty disables project-scoped + // filtering (e.g. allowed-projects, quota) — those steps treat it as "no + // project context". + ProjectID string + // UserID is the requesting user. Empty only affects trace logging and the VM + // ownership written by side effects. + UserID string + // AvailabilityZone is the target AZ. Empty means no AZ constraint, so the + // correct-AZ filter keeps all hosts. + AvailabilityZone string + // NumInstances is the number of instances this request represents. It is a + // MULTIPLIER in capacity and bin-pack math, so 0 makes a host look infinitely + // free and must not be left unset — the adapter always sets it (Nova sends + // >= 1). + NumInstances uint64 + + // IgnoreHosts are hosts to exclude from placement. Empty/nil means no hosts + // are excluded. + IgnoreHosts []string + // ForceHosts pins placement to these compute hosts. Empty/nil means no host + // is forced. When set (and the intent does not require filter validation), + // the pipeline is skipped and only forced hosts are returned. + ForceHosts []string + // ForceNodes pins placement to these hypervisor hostnames. Empty/nil means no + // node is forced. Semantics mirror ForceHosts. + ForceNodes []string + + // HypervisorFamily is the coarse family (kvm/vmware) used by the pipeline and + // routing. Empty means unresolved: KVM-only gates (e.g. VM creation) treat it + // as not-KVM and skip. + HypervisorFamily HypervisorFamily + // HypervisorType is the specific type (qemu/ch/vmware), needed only by Nova + // side effects. Empty means unresolved; the VM-upsert side effect errors + // rather than guessing. + HypervisorType HypervisorType + // ImageHypervisorType is the hypervisor type declared by the image. The empty + // value (ImageHypervisorTypeUnset) means the image declared none, which the + // image-properties filter treats as "no image constraint". + ImageHypervisorType ImageHypervisorType + // HardwareVersion is resolved from the "quota:hw_version" extra spec. Empty + // means no hardware-version constraint. + HardwareVersion string + + // Flavor is the requested flavor sizing. A zero FlavorSpec means capacity and + // bin-pack steps see a zero-sized instance (fits anywhere) — the adapter + // always populates it for real placements. + Flavor FlavorSpec + // ImageID is the Glance image UUID. Empty only affects the VM resource + // written by side effects, not pipeline decisions. + ImageID string + + // RequiredTraits are trait names the host must have (adapter stripped the + // "trait:" prefix and kept the "required" ones). Empty/nil means no required + // traits, so the traits filter imposes no inclusion constraint. + RequiredTraits []string + // ForbiddenTraits are trait names the host must NOT have. Empty/nil means no + // forbidden traits. If both trait lists are empty the traits filter is a no-op. + ForbiddenTraits []string + // RequestedCapabilities are required (key=value) hypervisor capabilities + // (adapter stripped the "capabilities:" prefix and rejected unsupported + // operators). Empty/nil means the capabilities filter is a no-op. + RequestedCapabilities []Capability + + // SourceHost is the migration source (was hint "source_host"). Empty means + // not a host-to-host move, so the live-migratable filter imposes no source + // constraint. + SourceHost string + // DomainName is the customer domain (was hint "domain_name"). Empty means no + // domain context, so the external-customer filter is skipped. + DomainName string + // ResourceGroup is the failover resource group (was hint + // "_cortex_resource_group"). Empty means no group, so group-consolidation + // weighing is skipped. + ResourceGroup string + + // RequestedDestination is the requested target host/aggregate. Nil means no + // destination constraint (see the type doc). + RequestedDestination *RequestedDestination + // InstanceGroup is the server group the instance belongs to. Nil means the + // instance is not grouped, so affinity/anti-affinity steps are skipped (see + // the type doc). + InstanceGroup *InstanceGroupSpec + + // RequestID identifies this scheduling request for tracing. Empty only + // affects log correlation. + RequestID string + // GlobalRequestID is the cross-service request ID for tracing. Empty only + // affects log correlation. + GlobalRequestID string +} + +// TODO: REMOVE ME Compile-time check that SchedulerRequest satisfies the pipeline request contract. +var _ lib.FilterWeigherPipelineRequest = SchedulerRequest{} + +// GetIntent returns the resolved scheduling intent. +func (r SchedulerRequest) GetIntent() v1alpha1.SchedulingIntent { return r.Intent } + +// GetHosts returns the compute host names of all candidates. +func (r SchedulerRequest) GetHosts() []string { + hosts := make([]string, len(r.Hosts)) + for i, h := range r.Hosts { + hosts[i] = h.ComputeHost + } + return hosts +} + +// GetWeights returns the per-host input weights. +func (r SchedulerRequest) GetWeights() map[string]float64 { return r.Weights } + +// GetOptions returns the call-time options for this run. +func (r SchedulerRequest) GetOptions() scheduling.Options { return r.Options } + +// GetTraceLogArgs returns alternating key/value pairs for the step trace log. +func (r SchedulerRequest) GetTraceLogArgs() []any { + return []any{ + "greq", r.GlobalRequestID, + "req", r.RequestID, + "user", r.UserID, + "project", r.ProjectID, + } +} + +// Filter returns a copy of the request narrowed to the given hosts, with the +// weights updated to the provided values. Hosts not present in includedHosts are +// dropped and won't be reconsidered by later steps. +func (r SchedulerRequest) Filter(includedHosts map[string]float64) lib.FilterWeigherPipelineRequest { + filteredHosts := make([]Host, 0, len(includedHosts)) + for _, h := range r.Hosts { + if _, ok := includedHosts[h.ComputeHost]; ok { + filteredHosts = append(filteredHosts, h) + } + } + r.Hosts = filteredHosts + filteredWeights := make(map[string]float64, len(includedHosts)) + for host, weight := range includedHosts { + filteredWeights[host] = weight + } + r.Weights = filteredWeights + return r +} + +// IsForcedDestination reports whether the request is forced onto specific +// hosts/nodes and the scheduler filters should be skipped entirely. A forced +// request that still carries a non-create intent (rebuild/evacuate/resize) must +// be validated by the filters, so it is not treated as forced here. Intent +// resolution happens in the adapter; this only inspects the typed Intent field. +func (r SchedulerRequest) IsForcedDestination() bool { + if len(r.ForceHosts) == 0 && len(r.ForceNodes) == 0 { + return false + } + return r.Intent == "" || r.Intent == v1alpha1.SchedulingIntentUnknown +} + +// ForcedHosts returns the subset of request hosts matching the forced +// destinations. force_hosts matches on compute host name (case-insensitive); +// force_nodes matches on hypervisor hostname (case-sensitive); when both are set +// a host must match both. Order is preserved and compute hosts are deduplicated. +func (r SchedulerRequest) ForcedHosts() []string { + forceHostSet := make(map[string]bool, len(r.ForceHosts)) + for _, h := range r.ForceHosts { + forceHostSet[strings.ToLower(h)] = true + } + forceNodeSet := make(map[string]bool, len(r.ForceNodes)) + for _, n := range r.ForceNodes { + forceNodeSet[n] = true + } + matched := make([]string, 0, len(r.Hosts)) + seen := make(map[string]bool, len(r.Hosts)) + for _, h := range r.Hosts { + if len(forceHostSet) > 0 && !forceHostSet[strings.ToLower(h.ComputeHost)] { + continue + } + if len(forceNodeSet) > 0 && !forceNodeSet[h.HypervisorHostname] { + continue + } + key := strings.ToLower(h.ComputeHost) + if seen[key] { + continue + } + seen[key] = true + matched = append(matched, h.ComputeHost) + } + return matched +} diff --git a/internal/scheduling/compute/request_test.go b/internal/scheduling/compute/request_test.go new file mode 100644 index 000000000..2d41642f5 --- /dev/null +++ b/internal/scheduling/compute/request_test.go @@ -0,0 +1,101 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "slices" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +func TestHypervisorFamily(t *testing.T) { + // The family is the distinction the pipeline uses; the specific type carries + // the QEMU/CH detail only Nova side effects need. + cases := []struct { + hvType HypervisorType + family HypervisorFamily + }{ + {HypervisorTypeQEMU, HypervisorFamilyKVM}, + {HypervisorTypeCH, HypervisorFamilyKVM}, + {HypervisorTypeVMware, HypervisorFamilyVMware}, + } + for _, c := range cases { + r := SchedulerRequest{HypervisorType: c.hvType, HypervisorFamily: c.family} + if r.HypervisorFamily != c.family { + t.Errorf("type %q: family = %q, want %q", c.hvType, r.HypervisorFamily, c.family) + } + } +} + +func TestIsForcedDestination(t *testing.T) { + cases := []struct { + name string + req SchedulerRequest + forced bool + }{ + {"no force", SchedulerRequest{}, false}, + {"force host, no intent", SchedulerRequest{ForceHosts: []string{"h1"}}, true}, + {"force node, no intent", SchedulerRequest{ForceNodes: []string{"n1"}}, true}, + {"force host, unknown intent", SchedulerRequest{ForceHosts: []string{"h1"}, Intent: v1alpha1.SchedulingIntentUnknown}, true}, + {"force host, concrete intent must still filter", SchedulerRequest{ForceHosts: []string{"h1"}, Intent: "evacuate"}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.req.IsForcedDestination(); got != c.forced { + t.Errorf("IsForcedDestination() = %v, want %v", got, c.forced) + } + }) + } +} + +func TestForcedHosts(t *testing.T) { + hosts := []Host{ + {ComputeHost: "Host-A", HypervisorHostname: "node-a"}, + {ComputeHost: "host-b", HypervisorHostname: "node-b"}, + {ComputeHost: "host-b", HypervisorHostname: "node-b2"}, // duplicate compute host + } + cases := []struct { + name string + force []string + nodes []string + want []string + }{ + {"force_hosts case-insensitive", []string{"host-a"}, nil, []string{"Host-A"}}, + {"force_nodes exact", nil, []string{"node-b"}, []string{"host-b"}}, + {"both intersect", []string{"host-b"}, []string{"node-b2"}, []string{"host-b"}}, + {"no match", []string{"nope"}, nil, []string{}}, + {"dedup compute host", []string{"host-b"}, nil, []string{"host-b"}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + r := SchedulerRequest{Hosts: hosts, ForceHosts: c.force, ForceNodes: c.nodes} + got := r.ForcedHosts() + if !slices.Equal(got, c.want) { + t.Errorf("ForcedHosts() = %v, want %v", got, c.want) + } + }) + } +} + +func TestFilterNarrowsHostsAndWeights(t *testing.T) { + r := SchedulerRequest{ + Hosts: []Host{ + {ComputeHost: "a"}, {ComputeHost: "b"}, {ComputeHost: "c"}, + }, + Weights: map[string]float64{"a": 1, "b": 2, "c": 3}, + } + filtered := r.Filter(map[string]float64{"a": 10, "c": 30}).(SchedulerRequest) + got := filtered.GetHosts() + slices.Sort(got) + if !slices.Equal(got, []string{"a", "c"}) { + t.Fatalf("Filter kept hosts %v, want [a c]", got) + } + if filtered.Weights["a"] != 10 || filtered.Weights["c"] != 30 { + t.Fatalf("Filter weights = %v, want a:10 c:30", filtered.Weights) + } + if _, ok := filtered.Weights["b"]; ok { + t.Fatalf("Filter should have dropped host b from weights: %v", filtered.Weights) + } +} diff --git a/internal/scheduling/compute/result.go b/internal/scheduling/compute/result.go new file mode 100644 index 000000000..c05b8d409 --- /dev/null +++ b/internal/scheduling/compute/result.go @@ -0,0 +1,41 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +// SchedulerResult is the well-defined output of a scheduling pipeline run: +// ordered hosts, the target host, and the input/output weights plus per-step +// activations used to rank them. +// +// It is currently an alias of v1alpha1.DecisionResult, the struct the generic +// pipeline returns today. Once the Decision CRD is removed the underlying type +// moves to lib.PipelineResult and this alias is re-pointed there; callers that +// read compute.SchedulerResult are unaffected by that move. +type SchedulerResult = v1alpha1.DecisionResult + +// MatchedReservation records a reservation that matched a candidate host during +// the run, for observability and downstream consumers. +type MatchedReservation struct { + // Host is the compute host the reservation applies to. + Host string + // ReservationName is the name of the matched Reservation resource. + ReservationName string + // Type is the reservation type (e.g. inflight, committed-resource, failover). + Type v1alpha1.ReservationType +} + +// RunOutcome bundles everything the pipeline produces for a scheduling run: the +// pipeline result and any matched reservations. The human-readable explanation +// is NOT part of the outcome — it is a concern of writing the decision onto the +// scheduled resource (e.g. the Nova VM's CortexSpec), so the relevant side +// effect generates it from the Result via lib.GenerateExplanation. +type RunOutcome struct { + // Result is the pipeline output (ordered hosts, target host, weights, steps). + Result SchedulerResult + // MatchedReservations lists reservations that matched candidate hosts. + MatchedReservations []MatchedReservation +} diff --git a/internal/scheduling/compute/scheduler.go b/internal/scheduling/compute/scheduler.go new file mode 100644 index 000000000..decfd7136 --- /dev/null +++ b/internal/scheduling/compute/scheduler.go @@ -0,0 +1,141 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "context" + "fmt" + "sync" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + ctrl "sigs.k8s.io/controller-runtime" +) + +// PipelineProvider supplies initialized pipelines by name. The Nova +// filter-weigher pipeline controller satisfies this interface via its embedded +// lib.BasePipelineController. +type PipelineProvider interface { + // GetPipeline returns the initialized pipeline for the given name, if ready. + GetPipeline(name string) (lib.FilterWeigherPipeline[SchedulerRequest], bool) +} + +// CandidateGatherer gathers all placement candidates for a request when the +// pipeline is configured to ignore the caller's preselection. +type CandidateGatherer interface { + // MutateWithAllCandidates overrides the request's hosts and weights with all + // available placement candidates. + MutateWithAllCandidates(ctx context.Context, request *SchedulerRequest) error +} + +// PreHook is a caller-provided hook executed under the scheduling lock before +// the pipeline runs. It may mutate the request (e.g. to create a pessimistic +// in-flight reservation and narrow the candidate set). Returning an error aborts +// the run before the pipeline executes. A nil PreHook is a no-op. +type PreHook func(ctx context.Context, req *SchedulerRequest) error + +// PostHook is a caller-provided side effect executed after the pipeline run, +// under the scheduling lock (e.g. upserting the VM resource, recording the +// decision). Returning an error aborts the run; non-fatal side effects should +// log and return nil so the placement decision still stands. A nil PostHook is a +// no-op. +type PostHook func(ctx context.Context, req SchedulerRequest, out RunOutcome) error + +// Scheduler is the single in-process owner of Nova placement decisions. It holds +// the one shared scheduling lock and runs the pipeline plus a caller-provided +// pre-hook and post-hook. It owns no side-effect policy itself — the caller +// decides, via the hooks it passes, what happens before and after the run. +type Scheduler struct { + // mu is the single shared scheduling lock, held across the pre-hook, the + // pipeline run, and the post-hook in Run. RunWithoutLock bypasses it. + mu sync.Mutex + + // pipelines resolves pipelines + their config by name. + pipelines PipelineProvider + // gatherer gathers all candidates when a pipeline ignores preselection. + gatherer CandidateGatherer +} + +// NewScheduler builds a Scheduler over the given pipeline provider and candidate +// gatherer. +func NewScheduler(pipelines PipelineProvider, gatherer CandidateGatherer) *Scheduler { + return &Scheduler{pipelines: pipelines, gatherer: gatherer} +} + +// Run executes the pipeline and hooks while holding the shared scheduling lock +// across the whole operation, so concurrent runs observe a consistent view of +// shared scheduling state. The pre-hook runs before the pipeline; the post-hook +// after. Either hook may be nil. Use this for any run that mutates shared state. +func (s *Scheduler) Run( + ctx context.Context, + req SchedulerRequest, + pre PreHook, + post PostHook, +) (RunOutcome, error) { + + s.mu.Lock() + defer s.mu.Unlock() + return s.run(ctx, req, pre, post) +} + +// RunWithoutLock executes the pipeline and hooks WITHOUT acquiring the shared +// lock. Use this only for read-only runs (e.g. capacity probes) that do not +// mutate shared scheduling state and can safely run concurrently with a locked +// run. Either hook may be nil. +func (s *Scheduler) RunWithoutLock( + ctx context.Context, + req SchedulerRequest, + pre PreHook, + post PostHook, +) (RunOutcome, error) { + + return s.run(ctx, req, pre, post) +} + +// run is the shared body of Run and RunWithoutLock. The caller is responsible +// for holding (or deliberately not holding) the lock. +func (s *Scheduler) run( + ctx context.Context, + req SchedulerRequest, + pre PreHook, + post PostHook, +) (RunOutcome, error) { + + log := ctrl.LoggerFrom(ctx) + + pipeline, ok := s.pipelines.GetPipeline(req.Pipeline) + if !ok { + return RunOutcome{}, fmt.Errorf("pipeline not found or not ready: %q", req.Pipeline) + } + + // The pre-hook runs before the pipeline and may mutate the request. + if pre != nil { + if err := pre(ctx, &req); err != nil { + return RunOutcome{}, err + } + } + + // When the request opts into gathering all candidates, override the hosts and + // weights with every available placement candidate before filtering. + if req.GatherAllCandidates { + log.V(1).Info("gathering all placement candidates before filtering") + if err := s.gatherer.MutateWithAllCandidates(ctx, &req); err != nil { + return RunOutcome{}, fmt.Errorf("failed to gather all placement candidates: %w", err) + } + log.V(1).Info("gathered all placement candidates", "numHosts", len(req.Hosts)) + } + + result, runErr := pipeline.Run(req) + if runErr != nil { + return RunOutcome{}, runErr + } + out := RunOutcome{Result: result} + + // The post-hook runs after a successful pipeline run, under the lock. + if post != nil { + if err := post(ctx, req, out); err != nil { + return out, err + } + } + return out, nil +} diff --git a/internal/scheduling/compute/scheduler_test.go b/internal/scheduling/compute/scheduler_test.go new file mode 100644 index 000000000..10193ab3f --- /dev/null +++ b/internal/scheduling/compute/scheduler_test.go @@ -0,0 +1,269 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" +) + +// fakePipeline is a stub lib.FilterWeigherPipeline[SchedulerRequest] that records +// the request it was run with and returns a configurable result/error. The run +// function, if set, is invoked inside Run (used to observe lock behavior). +type fakePipeline struct { + result SchedulerResult + err error + onRun func(SchedulerRequest) + lastReq SchedulerRequest +} + +func (p *fakePipeline) Run(req SchedulerRequest) (SchedulerResult, error) { + p.lastReq = req + if p.onRun != nil { + p.onRun(req) + } + return p.result, p.err +} + +// fakeProvider implements PipelineProvider. A pipeline is returned only for the +// configured name. +type fakeProvider struct { + name string + pipeline lib.FilterWeigherPipeline[SchedulerRequest] +} + +func (f *fakeProvider) GetPipeline(name string) (lib.FilterWeigherPipeline[SchedulerRequest], bool) { + if name != f.name || f.pipeline == nil { + return nil, false + } + return f.pipeline, true +} + +// fakeGatherer implements CandidateGatherer, recording whether it was called. +type fakeGatherer struct { + called bool + err error +} + +func (g *fakeGatherer) MutateWithAllCandidates(_ context.Context, req *SchedulerRequest) error { + g.called = true + if g.err != nil { + return g.err + } + req.Hosts = []Host{{ComputeHost: "gathered", HypervisorHostname: "gathered"}} + return nil +} + +// newTestScheduler builds a Scheduler whose only ready pipeline is named "p". +func newTestScheduler(p *fakePipeline, g CandidateGatherer) *Scheduler { + return NewScheduler(&fakeProvider{name: "p", pipeline: p}, g) +} + +func TestRunPipelineNotFound(t *testing.T) { + s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "missing"}, nil, nil) + if err == nil { + t.Fatal("expected error for missing pipeline, got nil") + } +} + +func TestRunPreThenPost(t *testing.T) { + s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + var order []string + pre := func(_ context.Context, _ *SchedulerRequest) error { + order = append(order, "pre") + return nil + } + post := func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { + order = append(order, "post") + return nil + } + if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, pre, post); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(order) != 2 || order[0] != "pre" || order[1] != "post" { + t.Fatalf("hooks ran out of order: %v", order) + } +} + +func TestRunNilHooks(t *testing.T) { + s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, nil); err != nil { + t.Fatalf("nil hooks should be a no-op, got error: %v", err) + } +} + +func TestRunPostHookErrorAborts(t *testing.T) { + s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + boom := errors.New("boom") + post := func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { return boom } + _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, post) + if !errors.Is(err, boom) { + t.Fatalf("expected boom error, got %v", err) + } +} + +func TestRunPipelineErrorSkipsPostHook(t *testing.T) { + boom := errors.New("pipeline failed") + s := newTestScheduler(&fakePipeline{err: boom}, &fakeGatherer{}) + var ran bool + post := func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { ran = true; return nil } + _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, post) + if !errors.Is(err, boom) { + t.Fatalf("expected pipeline error, got %v", err) + } + if ran { + t.Fatal("post-hook must not run when the pipeline errors") + } +} + +func TestRunPreHookRunsBeforePipelineAndMutates(t *testing.T) { + var sawHostInPipeline string + p := &fakePipeline{onRun: func(r SchedulerRequest) { + if len(r.Hosts) > 0 { + sawHostInPipeline = r.Hosts[0].ComputeHost + } + }} + s := newTestScheduler(p, &fakeGatherer{}) + pre := func(_ context.Context, r *SchedulerRequest) error { + r.Hosts = []Host{{ComputeHost: "from-prehook"}} + return nil + } + if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, pre, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if sawHostInPipeline != "from-prehook" { + t.Fatalf("pre-hook mutation not visible to pipeline: got %q", sawHostInPipeline) + } +} + +func TestRunPreHookErrorAbortsBeforePipeline(t *testing.T) { + var pipelineRan bool + p := &fakePipeline{onRun: func(SchedulerRequest) { pipelineRan = true }} + s := newTestScheduler(p, &fakeGatherer{}) + boom := errors.New("prehook boom") + pre := func(_ context.Context, _ *SchedulerRequest) error { return boom } + _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, pre, nil) + if !errors.Is(err, boom) { + t.Fatalf("expected prehook error, got %v", err) + } + if pipelineRan { + t.Fatal("pipeline must not run after a pre-hook error") + } +} + +func TestRunGathersCandidatesWhenRequested(t *testing.T) { + g := &fakeGatherer{} + p := &fakePipeline{} + s := newTestScheduler(p, g) + req := SchedulerRequest{Pipeline: "p", GatherAllCandidates: true} + if _, err := s.Run(context.Background(), req, nil, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !g.called { + t.Fatal("expected gatherer to be called when GatherAllCandidates is set") + } + if len(p.lastReq.Hosts) != 1 || p.lastReq.Hosts[0].ComputeHost != "gathered" { + t.Fatalf("pipeline did not see gathered candidates: %+v", p.lastReq.Hosts) + } +} + +func TestRunDoesNotGatherWithoutGatherAllCandidates(t *testing.T) { + g := &fakeGatherer{} + s := newTestScheduler(&fakePipeline{}, g) + if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if g.called { + t.Fatal("gatherer should not be called without GatherAllCandidates") + } +} + +// TestRunHoldsLockAcrossPostHook verifies that a second Run blocks until the +// first Run's post-hook completes — i.e. the lock spans the whole run, not just +// the pipeline. +func TestRunHoldsLockAcrossPostHook(t *testing.T) { + s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + + firstInPost := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondDone := make(chan struct{}) + + var slowPost PostHook = func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { + close(firstInPost) + <-releaseFirst + return nil + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, slowPost); err != nil { + t.Errorf("first Run: %v", err) + } + }() + + <-firstInPost // first Run is parked inside its post-hook, still holding the lock. + + go func() { + if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, nil); err != nil { + t.Errorf("second Run: %v", err) + } + close(secondDone) + }() + + // The second Run must be blocked on the lock held by the first. + select { + case <-secondDone: + t.Fatal("second Run completed while the first still held the lock") + case <-time.After(50 * time.Millisecond): + } + + close(releaseFirst) // let the first Run finish and release the lock. + wg.Wait() + <-secondDone // the second Run now proceeds. +} + +// TestRunWithoutLockDoesNotBlock verifies that RunWithoutLock proceeds even while +// a locked Run holds the lock inside its post-hook. +func TestRunWithoutLockDoesNotBlock(t *testing.T) { + s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + + firstInPost := make(chan struct{}) + releaseFirst := make(chan struct{}) + var slowPost PostHook = func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { + close(firstInPost) + <-releaseFirst + return nil + } + + go func() { + if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, slowPost); err != nil { + t.Errorf("locked Run: %v", err) + } + }() + <-firstInPost // locked Run holds the lock. + + done := make(chan struct{}) + go func() { + if _, err := s.RunWithoutLock(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, nil); err != nil { + t.Errorf("RunWithoutLock: %v", err) + } + close(done) + }() + + select { + case <-done: + // Expected: lock-free run completes without waiting for the lock. + case <-time.After(time.Second): + t.Fatal("RunWithoutLock blocked on the scheduling lock") + } + close(releaseFirst) +} From 4f0996794b4a6d7dbcc2fd93ade3cfdb06386ca0 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 14:27:33 +0100 Subject: [PATCH 02/21] feat(nova): rewrite filter/weigher plugins onto compute.SchedulerRequest Phase 2 of the pipeline cleanup. Flip the Nova filter-weigher pipeline's RequestType from the Nova wire type api.ExternalSchedulerRequest to the typed, Nova-agnostic compute.SchedulerRequest, and rewrite all 28 filter/weigher plugins (plus both zz_index.go registries) to read typed fields: - intent via request.Intent instead of GetIntent()+error - scheduler hints -> typed fields (SourceHost, DomainName, ResourceGroup) - trait:/capabilities: extra-spec iteration -> RequiredTraits/ForbiddenTraits/ RequestedCapabilities; accel:device_profile -> AcceleratorsRequested - flavor/hypervisor/image enums read from typed fields Spec refinements: FlavorSpec.MemoryMB (uint64) -> Memory (resource.Quantity, bytes), all consumers updated; add AcceleratorsRequested and fix the accelerators filter's request-side gate. WIP: the tree does not build at this commit. The plugins now require compute.SchedulerRequest but the Nova controller is still typed on api.ExternalSchedulerRequest; it builds once the wire->spec adapter lands (Phase 3). The compute and plugin packages build in isolation. Signed-off-by: Markus Wieland --- internal/scheduling/compute/request.go | 13 +++- .../filters/filter_aggregate_metadata.go | 13 ++-- .../filters/filter_allowed_projects.go | 13 ++-- .../plugins/filters/filter_capabilities.go | 50 +++---------- .../nova/plugins/filters/filter_correct_az.go | 12 +-- .../plugins/filters/filter_exclude_hosts.go | 6 +- .../filters/filter_external_customer.go | 17 +++-- .../filters/filter_has_accelerators.go | 9 +-- .../filters/filter_has_enough_capacity.go | 73 +++++++++---------- .../filters/filter_has_requested_traits.go | 36 ++------- .../filters/filter_host_instructions.go | 14 ++-- .../filters/filter_image_properties.go | 60 +++++---------- .../filters/filter_instance_group_affinity.go | 17 +++-- .../filter_instance_group_anti_affinity.go | 27 +++---- .../plugins/filters/filter_live_migratable.go | 19 ++--- .../filters/filter_quota_enforcement.go | 61 ++++++++-------- .../filters/filter_requested_destination.go | 17 +++-- .../filters/filter_status_conditions.go | 6 +- .../nova/plugins/filters/zz_index.go | 4 +- .../nova/plugins/weighers/kvm_binpack.go | 15 ++-- .../kvm_committed_resource_reservation.go | 19 ++--- .../plugins/weighers/kvm_cr_migration_slot.go | 14 ++-- .../weighers/kvm_failover_evacuation.go | 12 +-- .../kvm_failover_reservation_consolidation.go | 16 ++-- .../kvm_instance_group_soft_affinity.go | 14 ++-- .../weighers/kvm_prefer_smaller_hosts.go | 6 +- .../vmware_anti_affinity_noisy_projects.go | 10 +-- .../vmware_avoid_long_term_contended_hosts.go | 8 +- ...vmware_avoid_short_term_contended_hosts.go | 12 +-- .../nova/plugins/weighers/vmware_binpack.go | 15 ++-- .../nova/plugins/weighers/zz_index.go | 4 +- 31 files changed, 267 insertions(+), 345 deletions(-) diff --git a/internal/scheduling/compute/request.go b/internal/scheduling/compute/request.go index e0ce6f8a4..8e39b1e4e 100644 --- a/internal/scheduling/compute/request.go +++ b/internal/scheduling/compute/request.go @@ -23,6 +23,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "k8s.io/apimachinery/pkg/api/resource" ) // HypervisorFamily is the coarse hypervisor family a request targets. It is the @@ -108,9 +109,10 @@ type FlavorSpec struct { // FlavorID is the flavor ID. Empty only affects the VM resource written by // side effects. FlavorID string - // MemoryMB is the requested memory per instance. 0 means a zero-memory - // instance to capacity/bin-pack steps (fits anywhere), so the adapter sets it. - MemoryMB uint64 + // Memory is the requested memory per instance, as a resource.Quantity in + // bytes. The zero value is a zero-memory instance to capacity/bin-pack steps + // (fits anywhere), so the adapter sets it (from Nova's memory_mb * 2^20). + Memory resource.Quantity // VCPUs is the requested vCPUs per instance. 0 means a zero-CPU instance to // capacity/bin-pack steps (fits anywhere), so the adapter sets it. VCPUs uint64 @@ -258,6 +260,11 @@ type SchedulerRequest struct { // (adapter stripped the "capabilities:" prefix and rejected unsupported // operators). Empty/nil means the capabilities filter is a no-op. RequestedCapabilities []Capability + // AcceleratorsRequested reports whether the flavor requests a hardware + // accelerator (adapter resolves it from the presence of the + // "accel:device_profile" extra spec). When true, the accelerators filter keeps + // only hosts carrying the COMPUTE_ACCELERATORS trait; false is a no-op. + AcceleratorsRequested bool // SourceHost is the migration source (was hint "source_host"). Empty means // not a host-to-host move, so the live-migratable filter imposes no source diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go index d223dfe5e..a39bf83b5 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go @@ -10,23 +10,24 @@ import ( "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterAggregateMetadata struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Restrict hosts to specific projects if they are in an aggregate that has // the "filter_tenant_id" metadata key set. -func (s *FilterAggregateMetadata) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterAggregateMetadata) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Capacity probe calls are synthetic and not placed on behalf of a tenant // project, so they must ignore tenant pinning. Failover reservations DO respect // tenant pinning: a reservation on a host the VM's project cannot use is dead // weight and would be deleted at validation time. - if intent, err := request.GetIntent(); err == nil && intent == api.CapacityProbeIntent { + if request.Intent == api.CapacityProbeIntent { return result, nil } @@ -71,20 +72,20 @@ func (s *FilterAggregateMetadata) Run(traceLog logr.Logger, request api.External } for host, restrictedProjects := range restrictedProjectsByHost { - if !slices.Contains(restrictedProjects, request.Spec.Data.ProjectID) { + if !slices.Contains(restrictedProjects, request.ProjectID) { // Project is not allowed on this hypervisor, filter it out. delete(result.Activations, host) traceLog.V(1).Info( "filtering host not allowing project based on aggregate metadata", "host", host, - "project", request.Spec.Data.ProjectID, + "project", request.ProjectID, "restricted_projects", restrictedProjects, ) } else { traceLog.V(1).Info( "host allows project based on aggregate metadata, keeping", "host", host, - "project", request.Spec.Data.ProjectID, + "project", request.ProjectID, "restricted_projects", restrictedProjects, ) } diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go index 5e1e51b5a..101acf51a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go @@ -9,26 +9,27 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterAllowedProjectsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Lock certain hosts for certain projects, based on the hypervisor spec. // Note that hosts without specified projects are still accessible. -func (s *FilterAllowedProjectsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterAllowedProjectsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Capacity probe calls are synthetic and not placed on behalf of a tenant // project, so they must ignore tenant pinning. Failover reservations DO respect // tenant pinning: a reservation on a host the VM's project cannot use is dead // weight and would be deleted at validation time. - if intent, err := request.GetIntent(); err == nil && intent == api.CapacityProbeIntent { + if request.Intent == api.CapacityProbeIntent { return result, nil } - if request.Spec.Data.ProjectID == "" { + if request.ProjectID == "" { traceLog.V(1).Info("no project ID in request, skipping filter") return result, nil } @@ -48,13 +49,13 @@ func (s *FilterAllowedProjectsStep) Run(traceLog logr.Logger, request api.Extern traceLog.V(1).Info("host allows all projects, keeping", "host", hv.Name) continue } - if !slices.Contains(hv.Spec.AllowedProjects, request.Spec.Data.ProjectID) { + if !slices.Contains(hv.Spec.AllowedProjects, request.ProjectID) { // Project is not allowed on this hypervisor, filter it out. delete(result.Activations, hv.Name) traceLog.V(1).Info( "filtering host not allowing project", "host", hv.Name, - "project", request.Spec.Data.ProjectID, + "project", request.ProjectID, ) } } diff --git a/internal/scheduling/nova/plugins/filters/filter_capabilities.go b/internal/scheduling/nova/plugins/filters/filter_capabilities.go index 31277eeaa..2a204d963 100644 --- a/internal/scheduling/nova/plugins/filters/filter_capabilities.go +++ b/internal/scheduling/nova/plugins/filters/filter_capabilities.go @@ -7,15 +7,14 @@ import ( "context" "fmt" "github.com/go-logr/logr" - "strings" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterCapabilitiesStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Get the provided capabilities of a hypervisor resource in the format Nova expects. @@ -45,46 +44,14 @@ func hvToNovaCapabilities(hv hv1.Hypervisor) (map[string]string, error) { // Check the capabilities of each host and if they match the extra spec provided // in the request spec flavor. -func (s *FilterCapabilitiesStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterCapabilitiesStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - extraSpecs := request.Spec.Data.Flavor.Data.ExtraSpecs - if len(extraSpecs) == 0 { - traceLog.V(1).Info("no flavor extra spec capabilities in request, skipping filter") - return result, nil - } - - // Extract only capability-related extra specs into a separate map. - // We must not modify the original ExtraSpecs map as it's shared across filters. - requestedCapabilities := make(map[string]string) - for key, value := range extraSpecs { - if strings.HasPrefix(key, "capabilities:") { - requestedCapabilities[key] = value - } - } + requestedCapabilities := request.RequestedCapabilities if len(requestedCapabilities) == 0 { - traceLog.V(1).Info("no capabilities in flavor extra specs, skipping filter") + traceLog.V(1).Info("no capabilities requested, skipping filter") return result, nil } - // Note: currently none of the advanced operators for capabilities are - // supported because they are not used by any of our flavors in production. - // Ops: https://github.com/sapcc/nova/blob/3ebf80/nova/scheduler/filters/extra_specs_ops.py#L23 - unsupportedOps := []string{ - "=", "", "", "==", "!=", ">=", "<=", - "s==", "s!=", "s<", "s<=", "s>", "s>=", "", // or is special - } - for key, expr := range requestedCapabilities { - for _, op := range unsupportedOps { - if strings.Contains(expr, op) { - traceLog.V(1).Info( - "unsupported extra spec operator in capabilities filter, skipping filter", - "key", key, "expr", expr, "flavor", request.Spec.Data.Flavor, - ) - return result, nil - } - } - } - hvs := &hv1.HypervisorList{} if err := s.Client.List(context.Background(), hvs); err != nil { traceLog.Error(err, "failed to list hypervisors") @@ -118,11 +85,12 @@ func (s *FilterCapabilitiesStep) Run(traceLog logr.Logger, request api.ExternalS continue } // Check if the provided capabilities match the requested ones. - for keyRequested, valueRequested := range requestedCapabilities { - if providedValue, ok := provided[keyRequested]; !ok || providedValue != valueRequested { + for _, requested := range requestedCapabilities { + key := "capabilities:" + requested.Name + if providedValue, ok := provided[key]; !ok || providedValue != requested.Value { traceLog.V(1).Info( "filtering host with mismatched capabilities", "host", host, - "wantKey", keyRequested, "wantValue", valueRequested, + "wantKey", key, "wantValue", requested.Value, "haveKey?", ok, "haveValue", providedValue, ) delete(result.Activations, host) diff --git a/internal/scheduling/nova/plugins/filters/filter_correct_az.go b/internal/scheduling/nova/plugins/filters/filter_correct_az.go index 882cf261c..2b1a12d8d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_correct_az.go +++ b/internal/scheduling/nova/plugins/filters/filter_correct_az.go @@ -7,20 +7,20 @@ import ( "context" "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" ) type FilterCorrectAZStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Only get hosts in the requested az. -func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - if request.Spec.Data.AvailabilityZone == "" { + if request.AvailabilityZone == "" { traceLog.V(1).Info("no availability zone requested, skipping filter_correct_az step") return result, nil } @@ -42,7 +42,7 @@ func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request api.ExternalSche traceLog.V(1).Info("host missing zone label, keeping", "host", hv.Name) continue } - if az == request.Spec.Data.AvailabilityZone { + if az == request.AvailabilityZone { // We always assume the name of the resource corresponds // to the compute host name. computeHostsInAZ[hv.Name] = struct{}{} @@ -51,7 +51,7 @@ func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request api.ExternalSche traceLog.V(1).Info( "hosts inside requested az", - "availabilityZone", request.Spec.Data.AvailabilityZone, + "availabilityZone", request.AvailabilityZone, "hosts", computeHostsInAZ, ) for host := range result.Activations { diff --git a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go index c35a63cff..6bd31579a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go +++ b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go @@ -6,7 +6,7 @@ package filters import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -14,7 +14,7 @@ import ( // This step is useful to circumvent scheduling issues on specific hosts by // excluding them from all scheduling decisions. type FilterExcludeHostsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterExcludeHostsStepOpts] + lib.BaseFilter[compute.SchedulerRequest, FilterExcludeHostsStepOpts] } type FilterExcludeHostsStepOpts struct { @@ -29,7 +29,7 @@ func (opts FilterExcludeHostsStepOpts) Validate() error { return nil } func (s *FilterExcludeHostsStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.SchedulerRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer.go b/internal/scheduling/nova/plugins/filters/filter_external_customer.go index 19a412797..2697fe63c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer.go @@ -12,6 +12,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -29,29 +30,29 @@ func (opts FilterExternalCustomerStepOpts) Validate() error { } type FilterExternalCustomerStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterExternalCustomerStepOpts] + lib.BaseFilter[compute.SchedulerRequest, FilterExternalCustomerStepOpts] } // Prefix-match the domain name for external customer domains and filter out hosts // that are not intended for external customers. -func (s *FilterExternalCustomerStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterExternalCustomerStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Failover and capacity probe calls are not associated with a user domain; // domain-based host restrictions don't apply. CR scheduling does carry a real domain // and must be checked. - if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + if slices.Contains([]v1alpha1.SchedulingIntent{ api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent, api.CapacityProbeIntent, - }, intent) { - traceLog.V(1).Info("skipping external customer filter for non-placement intent", "intent", intent) + }, request.Intent) { + traceLog.V(1).Info("skipping external customer filter for non-placement intent", "intent", request.Intent) return result, nil } - domainName, err := request.Spec.Data.GetSchedulerHintStr("domain_name") - if err != nil { - traceLog.Error(err, "failed to get domain_name scheduler hint, skipping filter") + domainName := request.DomainName + if domainName == "" { + traceLog.V(1).Info("no domain name in request, skipping filter") return result, nil } if slices.Contains(s.Options.CustomerIgnoredDomainNames, domainName) { diff --git a/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go b/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go index ad3c20288..ed03766a1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go @@ -8,20 +8,19 @@ import ( "github.com/go-logr/logr" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterHasAcceleratorsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // If requested, only get hosts with accelerators. -func (s *FilterHasAcceleratorsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHasAcceleratorsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - extraSpecs := request.Spec.Data.Flavor.Data.ExtraSpecs - if _, ok := extraSpecs["accel:device_profile"]; !ok { + if !request.AcceleratorsRequested { traceLog.V(1).Info("no accelerators requested, skipping filter") return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index 6400d5362..b590ab495 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -12,6 +12,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" resv "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -36,7 +37,7 @@ type FilterHasEnoughCapacityOpts struct { func (FilterHasEnoughCapacityOpts) Validate() error { return nil } type FilterHasEnoughCapacity struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterHasEnoughCapacityOpts] + lib.BaseFilter[compute.SchedulerRequest, FilterHasEnoughCapacityOpts] } // Filter hosts that don't have enough capacity to run the requested flavor. @@ -63,7 +64,7 @@ type FilterHasEnoughCapacity struct { // known at this point. // // Please also note that disk space is currently not considered by this filter. -func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { opts := request.GetOptions() result := s.IncludeAllHostsFromRequest(request) @@ -157,22 +158,21 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // Check if this is a CR reservation scheduling request. // If so, we should NOT unlock any CR reservations to prevent overbooking. // CR capacity should only be unlocked for actual VM scheduling. - intent, err := request.GetIntent() switch { - case err == nil && intent == api.ReserveForCommittedResourceIntent: + case request.Intent == api.ReserveForCommittedResourceIntent: traceLog.V(1).Info("keeping CR reservation locked for CR reservation scheduling", "reservation", reservation.Name, - "intent", intent) + "intent", request.Intent) // Don't continue - fall through to block the resources case !s.Options.LockReserved && !opts.LockReservations && // For committed resource reservations: unlock resources only if: // 1. Project ID matches // 2. ResourceGroup matches the flavor's hw_version - reservation.Spec.CommittedResourceReservation.MatchesGroup(request.Spec.Data.ProjectID, request.Spec.Data.Flavor.Data.ExtraSpecs["quota:hw_version"]): + reservation.Spec.CommittedResourceReservation.MatchesGroup(request.ProjectID, request.HardwareVersion): traceLog.V(1).Info("unlocking resources reserved by matching committed resource reservation with allocation", "reservation", reservation.Name, - "instanceUUID", request.Spec.Data.InstanceUUID, - "projectID", request.Spec.Data.ProjectID, + "instanceUUID", request.InstanceUUID, + "projectID", request.ProjectID, "resourceGroup", reservation.Spec.CommittedResourceReservation.ResourceGroup) continue } @@ -184,25 +184,22 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // 1. Failover reservations are specifically for HA/evacuation scenarios. // 2. During live migrations or other operations, we don't want to use failover capacity. // Note: we cannot use failover reservations from other VMs, as that can invalidate our HA guarantees. - intent, err := request.GetIntent() - if err == nil { - switch intent { - case api.EvacuateIntent: - if reservation.Status.FailoverReservation != nil { - if _, contained := reservation.Status.FailoverReservation.Allocations[request.Spec.Data.InstanceUUID]; contained { - traceLog.V(1).Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", - "reservation", reservation.Name, - "instanceUUID", request.Spec.Data.InstanceUUID) - continue - } + switch request.Intent { + case api.EvacuateIntent: + if reservation.Status.FailoverReservation != nil { + if _, contained := reservation.Status.FailoverReservation.Allocations[request.InstanceUUID]; contained { + traceLog.V(1).Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", + "reservation", reservation.Name, + "instanceUUID", request.InstanceUUID) + continue } - case api.ReuseFailoverReservationIntent: - // Reuse check: the reservation already pre-blocks the right capacity for this VM. - // Don't subtract it from free capacity to avoid double-counting. - traceLog.V(1).Info("skipping failover reservation block for reuse compatibility check", - "reservation", reservation.Name) - continue } + case api.ReuseFailoverReservationIntent: + // Reuse check: the reservation already pre-blocks the right capacity for this VM. + // Don't subtract it from free capacity to avoid double-counting. + traceLog.V(1).Info("skipping failover reservation block for reuse compatibility check", + "reservation", reservation.Name) + continue } traceLog.V(1).Info("processing failover reservation", "reservation", reservation.Name) } @@ -283,7 +280,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External hostsEncountered[host] = struct{}{} // Check cpu capacity. - if request.Spec.Data.Flavor.Data.VCPUs == 0 { + if request.Flavor.VCPUs == 0 { return nil, errors.New("flavor has 0 vcpus") } freeCPU, ok := free["cpu"] @@ -298,11 +295,11 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // Calculate how many instances can fit on this host, based on cpu. //nolint:gosec // We're checking for underflows above (< 0). vcpuSlots := uint64(freeCPU.Value()) / - request.Spec.Data.Flavor.Data.VCPUs - if vcpuSlots < request.Spec.Data.NumInstances { + request.Flavor.VCPUs + if vcpuSlots < request.NumInstances { traceLog.V(1).Info( "filtering host due to insufficient CPU capacity", - "host", host, "requested", request.Spec.Data.Flavor.Data.VCPUs, + "host", host, "requested", request.Flavor.VCPUs, "available", freeCPU.String(), ) delete(result.Activations, host) @@ -310,7 +307,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External } // Check memory capacity. - if request.Spec.Data.Flavor.Data.MemoryMB == 0 { + if request.Flavor.Memory.IsZero() { return nil, errors.New("flavor has 0 memory") } freeMemory, ok := free["memory"] @@ -323,25 +320,23 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External continue } // Calculate how many instances can fit on this host, based on memory. - // Nova's memory_mb field is named "megabytes" in the API docs, but the values - // are in mebibytes (1 MiB = 2^20 bytes). Multiply by 1<<20 to convert to bytes. + // Flavor.Memory is already in bytes (resource.Quantity). //nolint:gosec // We're checking for underflows above (< 0). - memorySlots := uint64(freeMemory.Value()) / - (request.Spec.Data.Flavor.Data.MemoryMB * (1 << 20)) - if memorySlots < request.Spec.Data.NumInstances { + memorySlots := uint64(freeMemory.Value()) / uint64(request.Flavor.Memory.Value()) + if memorySlots < request.NumInstances { traceLog.V(1).Info( "filtering host due to insufficient RAM capacity", - "host", host, "requested_mb", request.Spec.Data.Flavor.Data.MemoryMB, - "available_mb", freeMemory.String(), + "host", host, "requested_memory", request.Flavor.Memory.String(), + "available_memory", freeMemory.String(), ) delete(result.Activations, host) continue } traceLog.V(1).Info( "host has enough capacity", "host", host, - "requested_cpus", request.Spec.Data.Flavor.Data.VCPUs, + "requested_cpus", request.Flavor.VCPUs, "available_cpus", freeCPU.String(), - "requested_memory_mb", request.Spec.Data.Flavor.Data.MemoryMB, + "requested_memory", request.Flavor.Memory.String(), "available_memory", freeMemory.String(), ) } diff --git a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go index d6d322707..55d087193 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go @@ -7,43 +7,23 @@ import ( "context" "github.com/go-logr/logr" "slices" - "strings" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterHasRequestedTraits struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } -// Filter hosts that do not have the requested traits given by the extra spec: -// - "trait:": "forbidden" means the host must not have the specified trait. -// - "trait:": "required" means the host must have the specified trait. -func (s *FilterHasRequestedTraits) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +// Filter hosts based on the traits requested in the flavor: +// - a forbidden trait means the host must not have the specified trait. +// - a required trait means the host must have the specified trait. +func (s *FilterHasRequestedTraits) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - var requiredTraits, forbiddenTraits []string - for key, value := range request.Spec.Data.Flavor.Data.ExtraSpecs { - if !strings.HasPrefix(key, "trait:") { - traceLog.V(1).Info("ignoring extra spec that does not start with trait:", "key", key) - continue - } - traitName := strings.TrimPrefix(key, "trait:") - switch value { - case "forbidden": - forbiddenTraits = append(forbiddenTraits, traitName) - traceLog.V(1).Info("trait marked forbidden, will filter out hosts with this trait", - "trait", traitName) - case "required": - requiredTraits = append(requiredTraits, traitName) - traceLog.V(1).Info("trait marked required, will filter out hosts without this trait", - "trait", traitName) - default: - traceLog.V(1).Info("ignoring trait with unsupported value", - "trait", traitName, "value", value) - } - } + requiredTraits := request.RequiredTraits + forbiddenTraits := request.ForbiddenTraits if len(requiredTraits) == 0 && len(forbiddenTraits) == 0 { traceLog.V(1).Info("no traits requested, skipping filter") return result, nil diff --git a/internal/scheduling/nova/plugins/filters/filter_host_instructions.go b/internal/scheduling/nova/plugins/filters/filter_host_instructions.go index c4121c22d..6b8c8a38e 100644 --- a/internal/scheduling/nova/plugins/filters/filter_host_instructions.go +++ b/internal/scheduling/nova/plugins/filters/filter_host_instructions.go @@ -7,30 +7,30 @@ import ( "github.com/go-logr/logr" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) type FilterHostInstructionsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Filter hosts based on instructions given in the request spec. Supported are: // - spec.ignore_hosts: Filter out all hosts in this list. // - spec.force_hosts: Include only hosts in this list. -func (s *FilterHostInstructionsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHostInstructionsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - if request.Spec.Data.IgnoreHosts != nil { - for _, host := range *request.Spec.Data.IgnoreHosts { + if len(request.IgnoreHosts) > 0 { + for _, host := range request.IgnoreHosts { delete(result.Activations, host) traceLog.V(1).Info("filtering host which is ignored", "host", host) } } else { traceLog.V(1).Info("no hosts to ignore specified") } - if request.Spec.Data.ForceHosts != nil { + if len(request.ForceHosts) > 0 { for host := range result.Activations { - if !slices.Contains(*request.Spec.Data.ForceHosts, host) { + if !slices.Contains(request.ForceHosts, host) { delete(result.Activations, host) traceLog.V(1).Info("filtering host which is not forced", "host", host) } diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties.go b/internal/scheduling/nova/plugins/filters/filter_image_properties.go index f4467c828..55dacf33f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties.go @@ -10,63 +10,41 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterImagePropertiesStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Filter hosts based on image properties given in the request spec. -func (s *FilterImagePropertiesStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterImagePropertiesStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - // Resolve the scheduling intent once and fall back to a readable default - // when it cannot be determined. - intent, intentErr := request.GetIntent() - intentLabel := "unknown" - if intentErr == nil { - intentLabel = string(intent) - } - // Apply this filter to all requests, unless we know from the request's // intent that image metadata is expected to not be set. - if intentErr == nil { - intentsExpectedToNotHaveImageMeta := []v1alpha1.SchedulingIntent{ - // Cortex-internal intents in which scheduling requests are sent - // mainly based on flavor-related metadata, independent of the - // image that'll land there. - api.ReserveForFailoverIntent, - api.ReuseFailoverReservationIntent, - api.ReserveForCommittedResourceIntent, - api.CapacityProbeIntent, - } - if slices.Contains(intentsExpectedToNotHaveImageMeta, intent) { - traceLog.V(1).Info("skipping filter: expected to have no image metadata", - "intent", intent) - return result, nil - } + intentsExpectedToNotHaveImageMeta := []v1alpha1.SchedulingIntent{ + // Cortex-internal intents in which scheduling requests are sent + // mainly based on flavor-related metadata, independent of the + // image that'll land there. + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, } - - // If the image properties indicate any other hypervisor type than kvm, - // we filter out all known kvm hypervisors. - hvType, err := request.Spec.Data.Image.Data.GetHypervisorType() - if err != nil { - // Hypervisor type could not be determined. In this case, just - // ignore the filter and return all hosts. - traceLog.Error(err, "could not determine hypervisor type from image properties") - // Expose this event through the step monitor to alert on high-frequency - // occurrences of this situation. Include the request intent as a dynamic - // label so the alert can pinpoint which scheduling intents are affected. - result.Events = append(result.Events, lib.FilterWeigherPipelineStepEvent{ - Name: "image_properties_hv_type_undetermined", - Labels: map[string]string{"intent": intentLabel}, - }) + if slices.Contains(intentsExpectedToNotHaveImageMeta, request.Intent) { + traceLog.V(1).Info("skipping filter: expected to have no image metadata", + "intent", request.Intent) return result, nil } - if hvType != api.NovaImageMetaHVTypeKVM { + // If the image properties indicate any other hypervisor type than kvm, + // we filter out all known kvm hypervisors. An unset image hypervisor type + // means the image declared none, so no image constraint applies. + hvType := request.ImageHypervisorType + if hvType != compute.ImageHypervisorTypeUnset && hvType != compute.ImageHypervisorTypeKVM { traceLog.V(1).Info("filtering out all known kvm hypervisors since image properties indicate a different hypervisor type", "image_hypervisor_type", hvType) hvs := new(hv1.HypervisorList) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go index a233bd37a..e2821f5b0 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go @@ -9,49 +9,50 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) type FilterInstanceGroupAffinityStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Select hosts in spec.instance_group. func (s *FilterInstanceGroupAffinityStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.SchedulerRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Instance group hints are only present in user-initiated VM placement requests; - if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + if slices.Contains([]v1alpha1.SchedulingIntent{ api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent, api.ReserveForCommittedResourceIntent, api.CapacityProbeIntent, - }, intent) { + }, request.Intent) { return result, nil } - ig := request.Spec.Data.InstanceGroup + ig := request.InstanceGroup if ig == nil { traceLog.V(1).Info("no instance group in request, skipping filter") return result, nil } - policy := ig.Data.Policy + policy := ig.Policy if policy != "affinity" { traceLog.V(1).Info("instance group policy is not 'affinity', skipping filter") return result, nil } - if len(ig.Data.Hosts) == 0 { + if len(ig.Hosts) == 0 { // Nothing to do. traceLog.V(1).Info("instance group has no hosts, skipping filter") return result, nil } for host := range result.Activations { - if slices.Contains(ig.Data.Hosts, host) { + if slices.Contains(ig.Hosts, host) { traceLog.V(1).Info("host is in instance group, keeping", "host", host) continue } diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go index ad17d8ab2..84036fb42 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go @@ -10,55 +10,52 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterInstanceGroupAntiAffinityStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Select hosts not in spec_obj.instance_group but only until // max_server_per_host is reached (by default = 1). func (s *FilterInstanceGroupAntiAffinityStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.SchedulerRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Instance group hints are only present in user-initiated VM placement requests; - if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + if slices.Contains([]v1alpha1.SchedulingIntent{ api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent, api.ReserveForCommittedResourceIntent, api.CapacityProbeIntent, - }, intent) { + }, request.Intent) { return result, nil } - ig := request.Spec.Data.InstanceGroup + ig := request.InstanceGroup if ig == nil { traceLog.V(1).Info("no instance group in request, skipping filter") return result, nil } - policy := ig.Data.Policy + policy := ig.Policy if policy != "anti-affinity" { traceLog.V(1).Info("instance group policy is not 'anti-affinity', skipping filter") return result, nil } - memberVMs := ig.Data.Members + memberVMs := ig.Members if len(memberVMs) == 0 { // Nothing to do. traceLog.V(1).Info("instance group has no members, skipping filter") return result, nil } - maxServersPerHost := 1 - if ig.Data.Rules != nil { - if maxServersPerHostAny, ok := ig.Data.Rules["max_server_per_host"]; ok { - if maxServersPerHostInt, ok := maxServersPerHostAny.(int); ok { - maxServersPerHost = maxServersPerHostInt - } - } + maxServersPerHost := ig.MaxServerPerHost + if maxServersPerHost == 0 { + maxServersPerHost = 1 } hvs := &hv1.HypervisorList{} @@ -84,7 +81,7 @@ func (s *FilterInstanceGroupAntiAffinityStep) Run( // Check if this host is already running the same vm (resizes). // In this case we should not filter it out. if slices.ContainsFunc(hv.Status.Instances, func(inst hv1.Instance) bool { - return inst.ID == request.Spec.Data.InstanceUUID + return inst.ID == request.InstanceUUID }) { traceLog.V(1).Info("host is running the same VM, not filtering out", "host", host) continue diff --git a/internal/scheduling/nova/plugins/filters/filter_live_migratable.go b/internal/scheduling/nova/plugins/filters/filter_live_migratable.go index 94c525f53..85abc3d97 100644 --- a/internal/scheduling/nova/plugins/filters/filter_live_migratable.go +++ b/internal/scheduling/nova/plugins/filters/filter_live_migratable.go @@ -10,12 +10,13 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterLiveMigratableStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Check if a vm can be live migrated from a source to a given target host. @@ -50,25 +51,19 @@ func (s *FilterLiveMigratableStep) checkHasSufficientFeatures( // Ensure the target host of a live migration can accept the migrating VM. func (s *FilterLiveMigratableStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.SchedulerRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - intent, err := request.GetIntent() - if err != nil { - traceLog.V(1).Info("failed to determine request intent, skipping filter", "error", err) - return result, nil - } - if intent != api.LiveMigrationIntent { + if request.Intent != api.LiveMigrationIntent { traceLog.V(1).Info("not a live migration request, skipping filter") return result, nil } - sourceHost, err := request.Spec.Data.GetSchedulerHintStr("source_host") - if err != nil || sourceHost == "" { - traceLog.V(1).Info("no source_host scheduler hint, skipping filter") - //nolint:nilerr // Not an error we want to fail the scheduling for. + sourceHost := request.SourceHost + if sourceHost == "" { + traceLog.V(1).Info("no source host set, skipping filter") return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 09bc67fa5..1cd315908 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -7,11 +7,11 @@ import ( "context" "fmt" "github.com/go-logr/logr" - "math" "time" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -71,7 +71,7 @@ func (FilterQuotaEnforcementOpts) Validate() error { return nil } // // Disabled by default; activated by adding "filter_quota_enforcement" to the pipeline config. type FilterQuotaEnforcement struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterQuotaEnforcementOpts] + lib.BaseFilter[compute.SchedulerRequest, FilterQuotaEnforcementOpts] } // recordDecision appends an accept/reject/skip outcome as a named pipeline @@ -92,7 +92,7 @@ func recordDecision(result *lib.FilterWeigherPipelineStepResult, mode, decision, }) } -func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) mode := "shadow" @@ -105,34 +105,31 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS // external non-consuming intents (evacuate / live-migrate — VM already exists, // being moved) apart from cortex-internal reservation intents (failover / // committed-resource reservations — new schedules driven by cortex itself). - intent, err := request.GetIntent() - if err == nil { - switch intent { - case api.EvacuateIntent, api.LiveMigrationIntent: - traceLog.V(1).Info("skipping quota enforcement for non-consuming intent", "intent", intent) - recordDecision(result, mode, "accept_skipped_intent_migration", "", "", "") - return result, nil - case api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent: - traceLog.V(1).Info("skipping quota enforcement for failover reservation intent") - recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") - return result, nil - case api.ReserveForCommittedResourceIntent: - traceLog.V(1).Info("skipping quota enforcement for committed resource reservation intent") - recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") - return result, nil - case api.CapacityProbeIntent: - traceLog.V(1).Info("skipping quota enforcement for capacity probe intent") - recordDecision(result, mode, "accept_skipped", "", "", "") - return result, nil - } + switch request.Intent { + case api.EvacuateIntent, api.LiveMigrationIntent: + traceLog.V(1).Info("skipping quota enforcement for non-consuming intent", "intent", request.Intent) + recordDecision(result, mode, "accept_skipped_intent_migration", "", "", "") + return result, nil + case api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent: + traceLog.V(1).Info("skipping quota enforcement for failover reservation intent") + recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") + return result, nil + case api.ReserveForCommittedResourceIntent: + traceLog.V(1).Info("skipping quota enforcement for committed resource reservation intent") + recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") + return result, nil + case api.CapacityProbeIntent: + traceLog.V(1).Info("skipping quota enforcement for capacity probe intent") + recordDecision(result, mode, "accept_skipped", "", "", "") + return result, nil } // Step 2: Extract project, AZ, and hw_version from the request. // Missing-data skips share a single decision bucket; the specific missing // field is in the trace log. - projectID := request.Spec.Data.ProjectID - az := request.Spec.Data.AvailabilityZone - hwVersion := request.Spec.Data.Flavor.Data.ExtraSpecs["quota:hw_version"] + projectID := request.ProjectID + az := request.AvailabilityZone + hwVersion := request.HardwareVersion if projectID == "" { traceLog.V(1).Info("no project ID in request, skipping quota enforcement") @@ -151,14 +148,16 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS } // Step 3: Compute resource demands from the request. - numInstances := request.Spec.Data.NumInstances + numInstances := request.NumInstances if numInstances == 0 { numInstances = 1 } - memoryMB := request.Spec.Data.Flavor.Data.MemoryMB - vcpus := request.Spec.Data.Flavor.Data.VCPUs - // MB to GiB: using 1024 as divisor is slightly conservative (overestimates GiB needed). - requestRAM := int64(math.Ceil(float64(memoryMB*numInstances) / 1024.0)) + vcpus := request.Flavor.VCPUs + // Total requested RAM across all instances, converted to GiB (ceiling, so we + // never under-count headroom demand). Flavor.Memory is bytes per instance. + //nolint:gosec // instance count and flavor memory are bounded by Nova. + totalMemBytes := request.Flavor.Memory.Value() * int64(numInstances) + requestRAM := (totalMemBytes + (1 << 30) - 1) / (1 << 30) requestCores := int64(vcpus * numInstances) //nolint:gosec // VCPUs from flavor specs, realistically bounded requestInstances := int64(numInstances) diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go index e4fa2af5d..540faf6b6 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go @@ -11,12 +11,13 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterRequestedDestinationStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // processRequestedAggregates filters hosts based on the requested aggregates. @@ -101,23 +102,23 @@ func (s *FilterRequestedDestinationStep) processRequestedHost( // The requested destination can include a specific host, aggregates, or both. // When both are specified, aggregate filtering is applied first, followed by // host filtering. -func (s *FilterRequestedDestinationStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterRequestedDestinationStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // The requested_destination hint is only set by Nova for user-directed placement; - if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + if slices.Contains([]v1alpha1.SchedulingIntent{ api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent, api.ReserveForCommittedResourceIntent, api.CapacityProbeIntent, - }, intent) { + }, request.Intent) { return result, nil } - rd := request.Spec.Data.RequestedDestination + rd := request.RequestedDestination if rd == nil { traceLog.V(1).Info("no requested_destination in request, skipping filter") return result, nil } - if len(rd.Data.Aggregates) == 0 && rd.Data.Host == "" { + if len(rd.Aggregates) == 0 && rd.Host == "" { traceLog.V(1).Info("requested_destination has no host or aggregates, skipping filter") return result, nil } @@ -133,8 +134,8 @@ func (s *FilterRequestedDestinationStep) Run(traceLog logr.Logger, request api.E } hvsByName[hv.Name] = hv } - s.processRequestedAggregates(traceLog, rd.Data.Aggregates, hvsByName, result.Activations) - s.processRequestedHost(traceLog, rd.Data.Host, result.Activations) + s.processRequestedAggregates(traceLog, rd.Aggregates, hvsByName, result.Activations) + s.processRequestedHost(traceLog, rd.Host, result.Activations) return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_status_conditions.go b/internal/scheduling/nova/plugins/filters/filter_status_conditions.go index 310a238c2..5097141b2 100644 --- a/internal/scheduling/nova/plugins/filters/filter_status_conditions.go +++ b/internal/scheduling/nova/plugins/filters/filter_status_conditions.go @@ -7,7 +7,7 @@ import ( "context" "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -15,12 +15,12 @@ import ( ) type FilterStatusConditionsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Check that all status conditions meet the expected values, for example, // that the hypervisor is ready and not disabled. -func (s *FilterStatusConditionsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterStatusConditionsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) hvs := &hv1.HypervisorList{} diff --git a/internal/scheduling/nova/plugins/filters/zz_index.go b/internal/scheduling/nova/plugins/filters/zz_index.go index 0f63f8842..afe9b1280 100644 --- a/internal/scheduling/nova/plugins/filters/zz_index.go +++ b/internal/scheduling/nova/plugins/filters/zz_index.go @@ -4,11 +4,11 @@ package filters import ( - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type NovaFilter = lib.Filter[api.ExternalSchedulerRequest] +type NovaFilter = lib.Filter[compute.SchedulerRequest] // Configuration of filters supported by the nova scheduler. var Index = map[string]func() NovaFilter{} diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go index a314dc5e8..85d1b5d12 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go @@ -9,7 +9,7 @@ import ( "fmt" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/go-logr/logr" @@ -65,11 +65,11 @@ func (o KVMBinpackStepOpts) Validate() error { // Explanation of the algorithm: https://volcano.sh/en/docs/plugins/#binpack type KVMBinpackStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMBinpackStepOpts] + lib.BaseWeigher[compute.SchedulerRequest, KVMBinpackStepOpts] } // Run this weigher in the pipeline after filters have been executed. -func (s *KVMBinpackStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMBinpackStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["binpack score"] = s.PrepareStats(request, "float") @@ -141,14 +141,13 @@ func (s *KVMBinpackStep) Run(traceLog logr.Logger, request api.ExternalScheduler } // calcVMResources calculates the total resource requests for the VM to be scheduled. -func (s *KVMBinpackStep) calcVMResources(req api.ExternalSchedulerRequest) map[hv1.ResourceName]resource.Quantity { +func (s *KVMBinpackStep) calcVMResources(req compute.SchedulerRequest) map[hv1.ResourceName]resource.Quantity { resources := make(map[hv1.ResourceName]resource.Quantity) - resourcesMemBytes := int64(req.Spec.Data.Flavor.Data.MemoryMB) * (1024 * 1024) //nolint:gosec // memory values are bounded by Nova - resourcesMemBytes *= int64(req.Spec.Data.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesMemBytes := req.Flavor.Memory.Value() * int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[hv1.ResourceMemory] = *resource. NewQuantity(resourcesMemBytes, resource.BinarySI) - resourcesCPU := int64(req.Spec.Data.Flavor.Data.VCPUs) //nolint:gosec // vCPU values are bounded by Nova - resourcesCPU *= int64(req.Spec.Data.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesCPU := int64(req.Flavor.VCPUs) //nolint:gosec // vCPU values are bounded by Nova + resourcesCPU *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[hv1.ResourceCPU] = *resource. NewQuantity(resourcesCPU, resource.DecimalSI) return resources diff --git a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go index fc0c8c6b9..4c3314840 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go @@ -12,6 +12,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -47,31 +48,31 @@ func (o KVMCommittedResourceReservationOpts) GetDefaultHostWeight() float64 { // Hosts that have a ready Reservation matching the request's project and flavor group, // with enough free memory capacity for the requested VM, receive a higher weight. type KVMCommittedResourceReservationStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMCommittedResourceReservationOpts] + lib.BaseWeigher[compute.SchedulerRequest, KVMCommittedResourceReservationOpts] } // Run the weigher step. // Hosts with a matching CommittedResourceReservation (project + flavor group + free capacity) get a higher weight. -func (s *KVMCommittedResourceReservationStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMCommittedResourceReservationStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Skip for failover reservation scheduling — CR reservations should not influence // where failover capacity is placed. - if intent, err := request.GetIntent(); err == nil && intent == api.ReserveForFailoverIntent { + if request.Intent == api.ReserveForFailoverIntent { traceLog.V(1).Info("skipping committed resource reservation weigher for failover reservation intent") return result, nil } - projectID := request.Spec.Data.ProjectID - az := request.Spec.Data.AvailabilityZone - resourceGroup, err := request.Spec.Data.GetSchedulerHintStr(api.HintKeyResourceGroup) - if err != nil || resourceGroup == "" { + projectID := request.ProjectID + az := request.AvailabilityZone + resourceGroup := request.ResourceGroup + if resourceGroup == "" { traceLog.V(1).Info("skipping committed resource reservation weigher: no resource group in request") - return result, nil //nolint:nilerr + return result, nil } // Flavor memory in bytes for free-capacity check. - flavorMemoryBytes := int64(request.Spec.Data.Flavor.Data.MemoryMB) * 1024 * 1024 //nolint:gosec + flavorMemoryBytes := request.Flavor.Memory.Value() var reservations v1alpha1.ReservationList if err := s.Client.List(context.Background(), &reservations); err != nil { diff --git a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go index 9c897a61c..e1b80b0c9 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go @@ -11,6 +11,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -58,24 +59,23 @@ func (o KVMCRMigrationSlotOpts) GetDefaultHostWeight() float64 { // // Only activates for LiveMigrationIntent. type KVMCRMigrationSlotStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMCRMigrationSlotOpts] + lib.BaseWeigher[compute.SchedulerRequest, KVMCRMigrationSlotOpts] } func (s *KVMCRMigrationSlotStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.SchedulerRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - intent, err := request.GetIntent() - if err != nil || intent != api.LiveMigrationIntent { + if request.Intent != api.LiveMigrationIntent { traceLog.V(1).Info("not a live migration, skipping CR migration slot weigher") - return result, nil //nolint:nilerr + return result, nil } - instanceUUID := request.Spec.Data.InstanceUUID - projectID := request.Spec.Data.ProjectID + instanceUUID := request.InstanceUUID + projectID := request.ProjectID var allReservations v1alpha1.ReservationList if err := s.Client.List(context.Background(), &allReservations, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go index 3088496b0..38b21fdd6 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go @@ -10,6 +10,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -45,22 +46,21 @@ func (o KVMFailoverEvacuationOpts) GetDefaultHostWeight() float64 { // Hosts that match a failover reservation for the VM being scheduled get a higher weight, // encouraging placement on pre-reserved failover capacity. type KVMFailoverEvacuationStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMFailoverEvacuationOpts] + lib.BaseWeigher[compute.SchedulerRequest, KVMFailoverEvacuationOpts] } // Run the weigher step. // For evacuation requests, hosts matching a failover reservation where the VM is in Allocations get a higher weight. // For non-evacuation requests (e.g., live migration, rebuild), this weigher has no effect. -func (s *KVMFailoverEvacuationStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMFailoverEvacuationStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - intent, err := request.GetIntent() - if err != nil || intent != api.EvacuateIntent { + if request.Intent != api.EvacuateIntent { traceLog.V(1).Info("skipping failover weigher for non-evacuation request") - return result, nil //nolint:nilerr // intentionally skip weigher on error + return result, nil } - instanceUUID := request.Spec.Data.InstanceUUID + instanceUUID := request.InstanceUUID // Fetch all reservations var reservations v1alpha1.ReservationList diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go index 270d3c34c..8aa42c766 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go @@ -11,6 +11,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -73,25 +74,24 @@ func (o KVMFailoverReservationConsolidationOpts) GetSameSpecPenalty() float64 { // // This produces bounded output (~0 to 1) that plays nicely with other weighers. type KVMFailoverReservationConsolidationStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMFailoverReservationConsolidationOpts] + lib.BaseWeigher[compute.SchedulerRequest, KVMFailoverReservationConsolidationOpts] } // Run the weigher step. // For reserve_for_failover requests, hosts are scored based on existing failover reservation density // and same-spec diversity. For all other request types, this weigher has no effect. -func (s *KVMFailoverReservationConsolidationStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMFailoverReservationConsolidationStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - intent, err := request.GetIntent() - if err != nil || intent != api.ReserveForFailoverIntent { + if request.Intent != api.ReserveForFailoverIntent { traceLog.V(1).Info("skipping failover reservation consolidation weigher for non-failover-reservation request") - return result, nil //nolint:nilerr // intentionally skip weigher on error + return result, nil } - // Extract the resource group from the scheduler hint. + // Extract the resource group from the request. // This identifies which "spec group" the incoming reservation belongs to. - // If the hint is missing, requestResourceGroup will be empty and the same-group penalty is skipped. - requestResourceGroup, _ := request.Spec.Data.GetSchedulerHintStr(api.HintKeyResourceGroup) //nolint:errcheck // missing hint is fine, same-group penalty is simply skipped + // If it is empty, the same-group penalty is skipped. + requestResourceGroup := request.ResourceGroup // Fetch all reservations. var reservations v1alpha1.ReservationList diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go index c95ff1b3c..60799ff59 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go @@ -7,7 +7,7 @@ import ( "context" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/go-logr/logr" @@ -23,19 +23,19 @@ import ( // respectively, for the scheduler to choose that host for new instances of // the same group. type KVMInstanceGroupSoftAffinityStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseWeigher[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] } -func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["affinity"] = s.PrepareStats(request, "float") - ig := request.Spec.Data.InstanceGroup + ig := request.InstanceGroup if ig == nil { traceLog.V(1).Info("no instance group in request, skipping weigher") return result, nil } - policy := ig.Data.Policy + policy := ig.Policy var factor float64 switch policy { case "soft-anti-affinity": @@ -46,7 +46,7 @@ func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request api traceLog.V(1).Info("instance group policy is not 'soft-affinity' or 'soft-anti-affinity', skipping weigher", "policy", policy) return result, nil } - if len(ig.Data.Members) == 0 { + if len(ig.Members) == 0 { traceLog.V(1).Info("instance group has no members, skipping weigher") return result, nil } @@ -72,7 +72,7 @@ func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request api } count := 0 for _, instance := range hv.Status.Instances { - if slices.Contains(ig.Data.Members, instance.ID) { + if slices.Contains(ig.Members, instance.ID) { count++ } } diff --git a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go index 52632df77..18c837741 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go @@ -9,7 +9,7 @@ import ( "fmt" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/go-logr/logr" @@ -53,11 +53,11 @@ func (o KVMPreferSmallerHostsStepOpts) Validate() error { // accommodate larger VMs when they need to be scheduled. type KVMPreferSmallerHostsStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMPreferSmallerHostsStepOpts] + lib.BaseWeigher[compute.SchedulerRequest, KVMPreferSmallerHostsStepOpts] } // Run this weigher in the pipeline after filters have been executed. -func (s *KVMPreferSmallerHostsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMPreferSmallerHostsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["small host score"] = s.PrepareStats(request, "float") diff --git a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go index 426526e2d..47bcfbb7a 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go @@ -9,9 +9,9 @@ import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,7 +38,7 @@ func (o VMwareAntiAffinityNoisyProjectsStepOpts) Validate() error { // Step to avoid noisy projects by downvoting the hosts they are running on. type VMwareAntiAffinityNoisyProjectsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[api.ExternalSchedulerRequest, VMwareAntiAffinityNoisyProjectsStepOpts] + lib.BaseWeigher[sched.SchedulerRequest, VMwareAntiAffinityNoisyProjectsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -53,7 +53,7 @@ func (s *VMwareAntiAffinityNoisyProjectsStep) Init(ctx context.Context, client c } // Downvote the hosts a project is currently running on if it's noisy. -func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request sched.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu usage of this project"] = s.PrepareStats(request, "%") @@ -72,7 +72,7 @@ func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request return nil, err } for _, p := range projectNoisinessOnHosts { - if p.Project != request.Spec.Data.ProjectID { + if p.Project != request.ProjectID { continue } // Only modify the weight if the host is in the scenario. @@ -92,7 +92,7 @@ func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request } func init() { - Index["vmware_anti_affinity_noisy_projects"] = func() lib.Weigher[api.ExternalSchedulerRequest] { + Index["vmware_anti_affinity_noisy_projects"] = func() lib.Weigher[sched.SchedulerRequest] { return &VMwareAntiAffinityNoisyProjectsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go index a12c35fe1..5d005a496 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go @@ -9,9 +9,9 @@ import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,7 +47,7 @@ func (o VMwareAvoidLongTermContendedHostsStepOpts) Validate() error { // Step to avoid long term contended hosts by downvoting them. type VMwareAvoidLongTermContendedHostsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[api.ExternalSchedulerRequest, VMwareAvoidLongTermContendedHostsStepOpts] + lib.BaseWeigher[sched.SchedulerRequest, VMwareAvoidLongTermContendedHostsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -62,7 +62,7 @@ func (s *VMwareAvoidLongTermContendedHostsStep) Init(ctx context.Context, client } // Downvote hosts that are highly contended. -func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, request sched.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu contention"] = s.PrepareStats(request, "%") @@ -110,7 +110,7 @@ func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, reques } func init() { - Index["vmware_avoid_long_term_contended_hosts"] = func() lib.Weigher[api.ExternalSchedulerRequest] { + Index["vmware_avoid_long_term_contended_hosts"] = func() lib.Weigher[sched.SchedulerRequest] { return &VMwareAvoidLongTermContendedHostsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go index 209eb97ba..779304247 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go @@ -9,9 +9,9 @@ import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + extractorcompute "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,7 +47,7 @@ func (o VMwareAvoidShortTermContendedHostsStepOpts) Validate() error { // Step to avoid recently contended hosts by downvoting them. type VMwareAvoidShortTermContendedHostsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[api.ExternalSchedulerRequest, VMwareAvoidShortTermContendedHostsStepOpts] + lib.BaseWeigher[compute.SchedulerRequest, VMwareAvoidShortTermContendedHostsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -62,7 +62,7 @@ func (s *VMwareAvoidShortTermContendedHostsStep) Init(ctx context.Context, clien } // Downvote hosts that are highly contended. -func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu contention"] = s.PrepareStats(request, "%") @@ -77,7 +77,7 @@ func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, reque return nil, err } highlyContendedHosts, err := v1alpha1. - UnboxFeatureList[compute.VROpsHostsystemContentionShortTerm](knowledge.Status.Raw) + UnboxFeatureList[extractorcompute.VROpsHostsystemContentionShortTerm](knowledge.Status.Raw) if err != nil { return nil, err } @@ -110,7 +110,7 @@ func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, reque } func init() { - Index["vmware_avoid_short_term_contended_hosts"] = func() lib.Weigher[api.ExternalSchedulerRequest] { + Index["vmware_avoid_short_term_contended_hosts"] = func() lib.Weigher[compute.SchedulerRequest] { return &VMwareAvoidShortTermContendedHostsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_binpack.go b/internal/scheduling/nova/plugins/weighers/vmware_binpack.go index 3f8b01fd8..d554588c7 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_binpack.go @@ -11,9 +11,9 @@ import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -69,7 +69,7 @@ func (o VMwareBinpackStepOpts) Validate() error { // Explanation of the algorithm: https://volcano.sh/en/docs/plugins/#binpack type VMwareBinpackStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[api.ExternalSchedulerRequest, VMwareBinpackStepOpts] + lib.BaseWeigher[sched.SchedulerRequest, VMwareBinpackStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -86,7 +86,7 @@ func (s *VMwareBinpackStep) Init(ctx context.Context, client client.Client, weig } // Run this weigher in the pipeline after filters have been executed. -func (s *VMwareBinpackStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareBinpackStep) Run(traceLog logr.Logger, request sched.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["binpack score"] = s.PrepareStats(request, "float") @@ -185,14 +185,13 @@ func (s *VMwareBinpackStep) calcHostAllocation(hostUtilization compute.HostUtili } // calcVMResources calculates the total resource requests for the VM to be scheduled. -func (s *VMwareBinpackStep) calcVMResources(req api.ExternalSchedulerRequest) map[corev1.ResourceName]resource.Quantity { +func (s *VMwareBinpackStep) calcVMResources(req sched.SchedulerRequest) map[corev1.ResourceName]resource.Quantity { resources := make(map[corev1.ResourceName]resource.Quantity) - resourcesMemBytes := int64(req.Spec.Data.Flavor.Data.MemoryMB) * (1 << 20) //nolint:gosec // memory values are bounded by Nova - resourcesMemBytes *= int64(req.Spec.Data.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesMemBytes := req.Flavor.Memory.Value() * int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[corev1.ResourceMemory] = *resource. NewQuantity(resourcesMemBytes, resource.BinarySI) - resourcesCPU := int64(req.Spec.Data.Flavor.Data.VCPUs) //nolint:gosec // vCPU values are bounded by Nova - resourcesCPU *= int64(req.Spec.Data.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesCPU := int64(req.Flavor.VCPUs) //nolint:gosec // vCPU values are bounded by Nova + resourcesCPU *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[corev1.ResourceCPU] = *resource. NewQuantity(resourcesCPU, resource.DecimalSI) return resources diff --git a/internal/scheduling/nova/plugins/weighers/zz_index.go b/internal/scheduling/nova/plugins/weighers/zz_index.go index d3136e724..6271a9b69 100644 --- a/internal/scheduling/nova/plugins/weighers/zz_index.go +++ b/internal/scheduling/nova/plugins/weighers/zz_index.go @@ -4,11 +4,11 @@ package weighers import ( - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type NovaWeigher = lib.Weigher[api.ExternalSchedulerRequest] +type NovaWeigher = lib.Weigher[compute.SchedulerRequest] // Configuration of weighers supported by the nova scheduler. var Index = map[string]func() NovaWeigher{} From 8fbb8f6101797dacad224d83c1babc19bb50e1dd Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 15:11:45 +0100 Subject: [PATCH 03/21] feat(nova): run the pipeline on typed compute.SchedulerRequest via a wire adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 of the pipeline cleanup — closes the Phase 2 seam so the tree builds and all tests pass. - Add internal/scheduling/nova/adapter.go: toSchedulerRequest translates the Nova wire ExternalSchedulerRequest into the typed compute.SchedulerRequest, resolving scheduler hints, flavor/image extra specs, traits, capabilities and accelerators into typed fields. Intent is taken from the already-resolved decision.Spec.Intent. - Flip the Nova FilterWeigherPipelineController and candidate gatherer to compute.SchedulerRequest; process() builds the typed request via the adapter. vm_handler and the CR recorder keep consuming the Nova wire request (they are Nova-owned side effects). - GatherAllCandidates is derived from the hypervisor family (KVM gathers, VMware uses the caller's preselection) in the adapter — the Pipeline CRD's IgnorePreselection field is removed entirely (Go type, CRD manifest, helm values). - FlavorSpec.Memory is a resource.Quantity (bytes); all consumers updated. - image-properties filter emits its undetermined-hv diagnostic via the new ImageHypervisorTypeUnparseable enum value. - Export the controller's Gatherer field and guard a nil gatherer with a clear error instead of panicking. - Migrate all Nova filter/weigher plus gatherer/controller/integration tests to typed fixtures. Fix test-handler drift in the failover/failoverhost integration tests (resolve intent onto the Decision, as the production handler does). go build ./..., go test ./..., go vet ./... and make lint are all clean. Signed-off-by: Markus Wieland --- api/v1alpha1/pipeline_types.go | 6 - .../cortex-nova/templates/pipelines_kvm.yaml | 4 - .../files/crds/cortex.cloud_pipelines.yaml | 7 - internal/scheduling/compute/request.go | 8 +- internal/scheduling/nova/adapter.go | 265 +++++++++++++ .../scheduling/nova/candidate_gatherer.go | 28 +- .../nova/candidate_gatherer_test.go | 184 ++------- internal/scheduling/nova/cr_metrics_test.go | 5 +- .../filter_weigher_pipeline_controller.go | 49 +-- ...filter_weigher_pipeline_controller_test.go | 137 +++---- internal/scheduling/nova/integration_test.go | 8 +- .../filters/filter_aggregate_metadata_test.go | 202 +++------- .../filters/filter_allowed_projects_test.go | 162 +++----- .../filters/filter_capabilities_test.go | 372 ++++-------------- .../plugins/filters/filter_correct_az_test.go | 64 +-- .../filters/filter_exclude_hosts_test.go | 68 ++-- .../filters/filter_external_customer_test.go | 220 +++-------- .../filters/filter_has_accelerators_test.go | 192 ++------- .../filter_has_enough_capacity_test.go | 69 ++-- .../filter_has_requested_traits_test.go | 266 +++---------- .../filters/filter_host_instructions_test.go | 205 +++------- .../filters/filter_image_properties.go | 19 +- .../filters/filter_image_properties_test.go | 55 +-- .../filter_instance_group_affinity_test.go | 223 +++-------- ...ilter_instance_group_anti_affinity_test.go | 337 ++++------------ .../filters/filter_live_migratable_test.go | 179 +++------ .../filters/filter_quota_enforcement.go | 4 +- .../filters/filter_quota_enforcement_test.go | 56 ++- .../filter_requested_destination_test.go | 267 ++++--------- .../filters/filter_status_conditions_test.go | 48 +-- .../nova/plugins/weighers/kvm_binpack_test.go | 43 +- ...kvm_committed_resource_reservation_test.go | 53 ++- .../weighers/kvm_cr_migration_slot_test.go | 39 +- .../weighers/kvm_failover_evacuation_test.go | 51 +-- ...failover_reservation_consolidation_test.go | 44 +-- .../kvm_instance_group_soft_affinity_test.go | 55 ++- .../weighers/kvm_prefer_smaller_hosts_test.go | 41 +- ...mware_anti_affinity_noisy_projects_test.go | 28 +- ...re_avoid_long_term_contended_hosts_test.go | 12 +- ...e_avoid_short_term_contended_hosts_test.go | 12 +- .../plugins/weighers/vmware_binpack_test.go | 75 ++-- .../reservations/failover/integration_test.go | 32 +- .../failoverhost/integration_test.go | 24 +- 43 files changed, 1505 insertions(+), 2713 deletions(-) create mode 100644 internal/scheduling/nova/adapter.go diff --git a/api/v1alpha1/pipeline_types.go b/api/v1alpha1/pipeline_types.go index 1e5cfdca3..252718a8d 100644 --- a/api/v1alpha1/pipeline_types.go +++ b/api/v1alpha1/pipeline_types.go @@ -78,12 +78,6 @@ type PipelineSpec struct { // +kubebuilder:validation:Optional Description string `json:"description,omitempty"` - // If this pipeline should ignore host preselection and gather all - // available placement candidates before applying filters, instead of - // relying on a pre-filtered set and weights. - // +kubebuilder:default=false - IgnorePreselection bool `json:"ignorePreselection,omitempty"` - // The type of the pipeline, used to differentiate between // filter-weigher and detector pipelines within the same // scheduling domain. diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index 817e75ab6..dab714b8c 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -15,7 +15,6 @@ spec: Specifically, this pipeline is used for general purpose workloads. type: filter-weigher # Fetch all placement candidates, ignoring nova's preselection. - ignorePreselection: true filters: - name: filter_status_conditions description: | @@ -189,7 +188,6 @@ spec: Specifically, this pipeline is used for hana virtual machines. type: filter-weigher # Fetch all placement candidates, ignoring nova's preselection. - ignorePreselection: true filters: - name: filter_status_conditions description: | @@ -355,7 +353,6 @@ spec: Deprecated: no longer used by the failover reservation controller. Kept for backwards compatibility with deployments that reference this pipeline by name in Helm values. type: filter-weigher - ignorePreselection: true filters: - name: filter_status_conditions description: | @@ -637,7 +634,6 @@ spec: considered. type: filter-weigher # Fetch all placement candidates, ignoring nova's preselection. - ignorePreselection: true filters: - name: filter_status_conditions description: | diff --git a/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml b/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml index 21a69b62f..f77625aaf 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml @@ -180,13 +180,6 @@ spec: - name type: object type: array - ignorePreselection: - default: false - description: |- - If this pipeline should ignore host preselection and gather all - available placement candidates before applying filters, instead of - relying on a pre-filtered set and weights. - type: boolean schedulingDomain: description: |- SchedulingDomain defines in which scheduling domain this pipeline diff --git a/internal/scheduling/compute/request.go b/internal/scheduling/compute/request.go index 8e39b1e4e..c8d9e7940 100644 --- a/internal/scheduling/compute/request.go +++ b/internal/scheduling/compute/request.go @@ -77,6 +77,8 @@ const ( ImageHypervisorTypeKVM ImageHypervisorType = "kvm" // ImageHypervisorTypeUnset indicates no image hypervisor type was declared. ImageHypervisorTypeUnset ImageHypervisorType = "" + // ImageHypervisorTypeUnparseable indicates an image hypervisor type was declared but could not be parsed. + ImageHypervisorTypeUnparseable ImageHypervisorType = "unparseable" ) // Host is a placement candidate. For KVM hosts the compute host name and the @@ -234,8 +236,10 @@ type SchedulerRequest struct { // rather than guessing. HypervisorType HypervisorType // ImageHypervisorType is the hypervisor type declared by the image. The empty - // value (ImageHypervisorTypeUnset) means the image declared none, which the - // image-properties filter treats as "no image constraint". + // value (ImageHypervisorTypeUnset) means the image declared none ("no image + // constraint"); ImageHypervisorTypeUnparseable means it declared one that + // could not be parsed, which the image-properties filter surfaces as a + // diagnostic. ImageHypervisorType ImageHypervisorType // HardwareVersion is resolved from the "quota:hw_version" extra spec. Empty // means no hardware-version constraint. diff --git a/internal/scheduling/nova/adapter.go b/internal/scheduling/nova/adapter.go new file mode 100644 index 000000000..a70e03327 --- /dev/null +++ b/internal/scheduling/nova/adapter.go @@ -0,0 +1,265 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package nova + +import ( + "strings" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "k8s.io/apimachinery/pkg/api/resource" +) + +// unsupportedCapabilityOps are the Nova extra-spec comparison operators the +// capabilities filter does not implement. A capability value containing any of +// these means the request uses an advanced match the pipeline can't honor, so +// we drop capabilities entirely (the filter then becomes a no-op), matching the +// old filter's "skip" behavior. +// See: https://github.com/sapcc/nova/blob/3ebf80/nova/scheduler/filters/extra_specs_ops.py#L23 +var unsupportedCapabilityOps = []string{ + "=", "", "", "==", "!=", ">=", "<=", + "s==", "s!=", "s<", "s<=", "s>", "s>=", "", +} + +// toSchedulerRequest translates the Nova over-the-wire request into the typed, +// Nova-agnostic compute.SchedulerRequest the pipeline consumes. This is the one +// place that reads Nova's loosely-typed scheduler hints and flavor/image extra +// specs; every downstream filter/weigher sees only resolved typed fields. +// +// intent and pipeline are resolved by the caller (the HTTP API layer) and passed +// in. +func toSchedulerRequest( + wire api.ExternalSchedulerRequest, + intent v1alpha1.SchedulingIntent, + pipeline string, +) compute.SchedulerRequest { + + spec := wire.Spec.Data + + req := compute.SchedulerRequest{ + Pipeline: pipeline, + Hosts: toComputeHosts(wire.Hosts), + Weights: wire.Weights, + Options: wire.Options, + Intent: intent, + + InstanceUUID: spec.InstanceUUID, + ProjectID: firstNonEmpty(spec.ProjectID, wire.Context.ProjectID), + UserID: spec.UserID, + AvailabilityZone: spec.AvailabilityZone, + NumInstances: spec.NumInstances, + + IgnoreHosts: derefStrings(spec.IgnoreHosts), + ForceHosts: derefStrings(spec.ForceHosts), + ForceNodes: derefStrings(spec.ForceNodes), + + HardwareVersion: spec.Flavor.Data.ExtraSpecs["quota:hw_version"], + + SourceHost: hintStr(spec, "source_host"), + DomainName: hintStr(spec, "domain_name"), + ResourceGroup: hintStr(spec, api.HintKeyResourceGroup), + + RequestID: wire.Context.RequestID, + GlobalRequestID: derefString(wire.Context.GlobalRequestID), + } + + // Flavor sizing + classification. + req.Flavor = compute.FlavorSpec{ + Name: spec.Flavor.Data.Name, + FlavorID: spec.Flavor.Data.FlavorID, + VCPUs: spec.Flavor.Data.VCPUs, + // Nova's memory_mb is in mebibytes (2^20 bytes). + Memory: *resource.NewQuantity(int64(spec.Flavor.Data.MemoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // flavor memory bounded by Nova + Type: toFlavorType(wire), + } + + // Hypervisor family + specific type. + req.HypervisorFamily, req.HypervisorType = toHypervisorFamilyAndType(wire) + + // Whether to gather all placement candidates (ignore the caller's host + // preselection) is determined by the hypervisor family: cortex can enumerate + // KVM hypervisors itself, whereas VMware placement relies on the host list + // Nova sent in the request. + req.GatherAllCandidates = req.HypervisorFamily == compute.HypervisorFamilyKVM + + // Image. + req.ImageID = spec.Image.Data.ID + req.ImageHypervisorType = toImageHypervisorType(spec.Image.Data) + + // Traits, capabilities, accelerators from flavor extra specs. + req.RequiredTraits, req.ForbiddenTraits = toTraits(spec.Flavor.Data.ExtraSpecs) + req.RequestedCapabilities = toCapabilities(spec.Flavor.Data.ExtraSpecs) + _, req.AcceleratorsRequested = spec.Flavor.Data.ExtraSpecs["accel:device_profile"] + + // Nested objects. + req.RequestedDestination = toRequestedDestination(spec.RequestedDestination) + req.InstanceGroup = toInstanceGroup(spec.InstanceGroup) + + return req +} + +func toComputeHosts(hosts []api.ExternalSchedulerHost) []compute.Host { + out := make([]compute.Host, len(hosts)) + for i, h := range hosts { + out[i] = compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname} + } + return out +} + +// toFlavorType maps the wire flavor type enum onto the compute enum. +func toFlavorType(wire api.ExternalSchedulerRequest) compute.FlavorType { + ft, err := wire.GetFlavorType() + if err != nil { + return compute.FlavorTypeGeneralPurpose + } + if ft == api.FlavorTypeHANA { + return compute.FlavorTypeHANA + } + return compute.FlavorTypeGeneralPurpose +} + +// toHypervisorFamilyAndType resolves the coarse family and the specific type +// from the wire hypervisor type. Both are empty when unresolved. +func toHypervisorFamilyAndType(wire api.ExternalSchedulerRequest) (compute.HypervisorFamily, compute.HypervisorType) { + hv, err := wire.GetHypervisorType() + if err != nil { + return "", "" + } + switch hv { + case api.HypervisorTypeQEMU: + return compute.HypervisorFamilyKVM, compute.HypervisorTypeQEMU + case api.HypervisorTypeCH: + return compute.HypervisorFamilyKVM, compute.HypervisorTypeCH + case api.HypervisorTypeVMware: + return compute.HypervisorFamilyVMware, compute.HypervisorTypeVMware + } + return "", "" +} + +// toImageHypervisorType maps the image-declared hypervisor type. It returns +// ImageHypervisorTypeUnparseable when the image declared a type that could not +// be parsed, distinct from ImageHypervisorTypeUnset when none was declared. +func toImageHypervisorType(image api.NovaImageMeta) compute.ImageHypervisorType { + hv, err := image.GetHypervisorType() + if err != nil { + // GetHypervisorType errors both when nothing is declared and when a + // declared value is unrecognized. Distinguish them: if a known key is + // present, the value was unparseable. + if image.Properties.Data != nil { + for _, key := range []string{"img_hv_type", "hypervisor_type"} { + if _, ok := image.Properties.Data[key]; ok { + return compute.ImageHypervisorTypeUnparseable + } + } + } + return compute.ImageHypervisorTypeUnset + } + switch hv { + case api.NovaImageMetaHVTypeVMware: + return compute.ImageHypervisorTypeVMware + case api.NovaImageMetaHVTypeBaremetal: + return compute.ImageHypervisorTypeBaremetal + case api.NovaImageMetaHVTypeKVM: + return compute.ImageHypervisorTypeKVM + } + return compute.ImageHypervisorTypeUnset +} + +// toTraits splits the flavor extra specs' "trait:" entries into required +// and forbidden trait names (classified by the "required"/"forbidden" value). +func toTraits(extraSpecs map[string]string) (required, forbidden []string) { + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "trait:") + if !ok { + continue + } + switch value { + case "required": + required = append(required, name) + case "forbidden": + forbidden = append(forbidden, name) + } + } + return required, forbidden +} + +// toCapabilities extracts the "capabilities:" extra specs as typed Capability +// equalities. If any value uses an unsupported comparison operator, it returns +// nil so the capabilities filter is skipped entirely (matching Nova behavior). +func toCapabilities(extraSpecs map[string]string) []compute.Capability { + var caps []compute.Capability + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "capabilities:") + if !ok { + continue + } + for _, op := range unsupportedCapabilityOps { + if strings.Contains(value, op) { + return nil // Unsupported operator: skip the whole filter. + } + } + caps = append(caps, compute.Capability{Name: name, Value: value}) + } + return caps +} + +func toRequestedDestination(rd *api.NovaObject[api.NovaRequestedDestination]) *compute.RequestedDestination { + if rd == nil { + return nil + } + return &compute.RequestedDestination{ + Host: rd.Data.Host, + Aggregates: rd.Data.Aggregates, + } +} + +func toInstanceGroup(ig *api.NovaObject[api.NovaInstanceGroup]) *compute.InstanceGroupSpec { + if ig == nil { + return nil + } + out := &compute.InstanceGroupSpec{ + UUID: ig.Data.UUID, + Policy: ig.Data.Policy, + Hosts: ig.Data.Hosts, + Members: ig.Data.Members, + } + if ig.Data.Rules != nil { + if v, ok := ig.Data.Rules["max_server_per_host"]; ok { + if n, ok := v.(int); ok { + out.MaxServerPerHost = n + } + } + } + return out +} + +func hintStr(spec api.NovaSpec, key string) string { + s, err := spec.GetSchedulerHintStr(key) + if err != nil { + return "" + } + return s +} + +func derefStrings(p *[]string) []string { + if p == nil { + return nil + } + return *p +} + +func derefString(p *string) string { + if p == nil { + return "" + } + return *p +} + +func firstNonEmpty(a, b string) string { + if a != "" { + return a + } + return b +} diff --git a/internal/scheduling/nova/candidate_gatherer.go b/internal/scheduling/nova/candidate_gatherer.go index a493aaf42..fee0abe51 100644 --- a/internal/scheduling/nova/candidate_gatherer.go +++ b/internal/scheduling/nova/candidate_gatherer.go @@ -6,9 +6,8 @@ package nova import ( "context" "fmt" - "strings" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -17,7 +16,7 @@ import ( // for a given Nova scheduling request. type CandidateGatherer interface { // Gather all placement candidates and mutate the request accordingly. - MutateWithAllCandidates(ctx context.Context, request *api.ExternalSchedulerRequest) error + MutateWithAllCandidates(ctx context.Context, request *compute.SchedulerRequest) error } // candidateGatherer is the default implementation of CandidateGatherer @@ -26,23 +25,12 @@ type candidateGatherer struct{ client.Client } // MutateWithAllCandidates gathers all placement candidates and mutates // the request accordingly. -func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request *api.ExternalSchedulerRequest) error { +func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.SchedulerRequest) error { // Currently we can only get candidates for kvm placements. - hvType, ok := request.Spec.Data.Flavor.Data.ExtraSpecs["capabilities:hypervisor_type"] - if !ok { + if request.HypervisorFamily != compute.HypervisorFamilyKVM { return fmt.Errorf( - "missing hypervisor_type in flavor extra specs: %v", - request.Spec.Data.Flavor.Data.ExtraSpecs, - ) - } - switch strings.ToLower(hvType) { - case "qemu", "ch": - // Supported hypervisor type. - default: - // Unsupported hypervisor type, do nothing. - return fmt.Errorf( - "cannot gather all placement candidates for hypervisor type %q", - request.Spec.Data.Flavor.Data.ExtraSpecs["capabilities:hypervisor_type"], + "cannot gather all placement candidates for hypervisor family %q", + request.HypervisorFamily, ) } @@ -51,10 +39,10 @@ func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request if err := g.List(ctx, hypervisorList); err != nil { return err } - hosts := make([]api.ExternalSchedulerHost, 0, len(hypervisorList.Items)) + hosts := make([]compute.Host, 0, len(hypervisorList.Items)) weights := make(map[string]float64, len(hypervisorList.Items)) for _, hv := range hypervisorList.Items { - host := api.ExternalSchedulerHost{ + host := compute.Host{ // For KVM hosts, compute host name and hypervisor hostname is identical. ComputeHost: hv.Name, HypervisorHostname: hv.Name, diff --git a/internal/scheduling/nova/candidate_gatherer_test.go b/internal/scheduling/nova/candidate_gatherer_test.go index 9f2b9a3b4..6f3694339 100644 --- a/internal/scheduling/nova/candidate_gatherer_test.go +++ b/internal/scheduling/nova/candidate_gatherer_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -26,69 +26,41 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { tests := []struct { name string hypervisors []client.Object - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectError bool errorContains string expectedHosts []string expectedWeight float64 }{ { - name: "missing hypervisor_type in flavor extra specs", + name: "unresolved hypervisor family", hypervisors: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, + request: compute.SchedulerRequest{ + HypervisorFamily: "", }, expectError: true, - errorContains: "missing hypervisor_type in flavor extra specs", + errorContains: "cannot gather all placement candidates for hypervisor family", }, { - name: "unsupported hypervisor type vmware", + name: "unsupported hypervisor family vmware", hypervisors: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "VMware vCenter Server", - }, - }, - }, - }, - }, + request: compute.SchedulerRequest{ + HypervisorFamily: compute.HypervisorFamilyVMware, }, expectError: true, - errorContains: "cannot gather all placement candidates for hypervisor type", + errorContains: "cannot gather all placement candidates for hypervisor family", }, { - name: "unsupported hypervisor type unknown", + name: "unsupported hypervisor family unknown", hypervisors: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "unknown-type", - }, - }, - }, - }, - }, + request: compute.SchedulerRequest{ + HypervisorFamily: "unknown-family", }, expectError: true, - errorContains: "cannot gather all placement candidates for hypervisor type", + errorContains: "cannot gather all placement candidates for hypervisor family", }, { - name: "successful gathering with qemu hypervisor type", + name: "successful gathering with kvm family (qemu)", hypervisors: []client.Object{ &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -106,25 +78,16 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "qemu", - }, - }, - }, - }, - }, + request: compute.SchedulerRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, }, expectError: false, expectedHosts: []string{"host1", "host2", "host3"}, expectedWeight: 0.0, }, { - name: "successful gathering with QEMU uppercase", + name: "successful gathering with kvm family single host", hypervisors: []client.Object{ &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -132,25 +95,16 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - }, - }, - }, - }, - }, + request: compute.SchedulerRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, }, expectError: false, expectedHosts: []string{"host1"}, expectedWeight: 0.0, }, { - name: "successful gathering with ch hypervisor type", + name: "successful gathering with kvm family (ch)", hypervisors: []client.Object{ &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -163,25 +117,16 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "ch", - }, - }, - }, - }, - }, + request: compute.SchedulerRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeCH, }, expectError: false, expectedHosts: []string{"ch-host1", "ch-host2"}, expectedWeight: 0.0, }, { - name: "successful gathering with CH uppercase", + name: "successful gathering with kvm family single ch host", hypervisors: []client.Object{ &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -189,18 +134,9 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, - }, + request: compute.SchedulerRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeCH, }, expectError: false, expectedHosts: []string{"ch-host1"}, @@ -209,49 +145,14 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { { name: "empty hypervisor list", hypervisors: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "qemu", - }, - }, - }, - }, - }, + request: compute.SchedulerRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, }, expectError: false, expectedHosts: []string{}, expectedWeight: 0.0, }, - { - name: "mixed case Qemu", - hypervisors: []client.Object{ - &hv1.Hypervisor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "host1", - }, - }, - }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "Qemu", - }, - }, - }, - }, - }, - }, - expectError: false, - expectedHosts: []string{"host1"}, - expectedWeight: 0.0, - }, { name: "request with existing hosts and weights are overwritten", hypervisors: []client.Object{ @@ -261,19 +162,10 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "qemu", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, + Hosts: []compute.Host{ {ComputeHost: "old-host", HypervisorHostname: "old-hv"}, }, Weights: map[string]float64{ @@ -321,7 +213,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { } // Verify each expected host is present - hostMap := make(map[string]api.ExternalSchedulerHost) + hostMap := make(map[string]compute.Host) for _, host := range tt.request.Hosts { hostMap[host.ComputeHost] = host } diff --git a/internal/scheduling/nova/cr_metrics_test.go b/internal/scheduling/nova/cr_metrics_test.go index 6b0ac83c3..f5553808b 100644 --- a/internal/scheduling/nova/cr_metrics_test.go +++ b/internal/scheduling/nova/cr_metrics_test.go @@ -21,6 +21,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -367,9 +368,9 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { reg.MustRegister(counter) controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[sched.SchedulerRequest]]{ Client: fakeClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, FeatureGates: FeatureGates{CommittedResourceTracking: enabled}, diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index a81074072..93862ed44 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" @@ -38,7 +39,7 @@ import ( // reconfigure the pipelines as needed. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. - lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]] + lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]] // Mutex to only allow one process at a time. // Read-only runs (opts.ReadOnly == true) acquire a read lock; write runs acquire the full lock. @@ -49,7 +50,7 @@ type FilterWeigherPipelineController struct { // Monitor to pass down to all pipelines. Monitor lib.FilterWeigherPipelineMonitor // Candidate gatherer to get all placement candidates if needed. - gatherer CandidateGatherer + Gatherer CandidateGatherer // vmHandler maps processed decisions into vm-operator VirtualMachine resources. vmHandler *VirtualMachineHandler @@ -153,24 +154,28 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision log.Error(nil, "skipping decision, no novaRaw spec defined") return nil, errors.New("no novaRaw spec defined") } - var request api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.NovaRaw.Raw, &request); err != nil { + var wire api.ExternalSchedulerRequest + if err := json.Unmarshal(decision.Spec.NovaRaw.Raw, &wire); err != nil { log.Error(err, "failed to unmarshal novaRaw spec") return nil, err } - // If necessary gather all placement candidates before filtering. - // This will override the hosts and weights in the nova request. - pipelineConf, ok := c.GetPipelineConfig(decision.Spec.PipelineRef.Name) - if !ok { - log.Error(nil, "pipeline config not found", "pipelineName", decision.Spec.PipelineRef.Name) - return &request, fmt.Errorf("pipeline config not found: %q", decision.Spec.PipelineRef.Name) - } - if pipelineConf.Spec.IgnorePreselection { + // Translate the Nova wire request into the typed pipeline request. Intent was + // resolved by the API layer and carried on the decision spec. Whether to + // gather all candidates is derived from the hypervisor family inside the + // adapter (KVM gathers all; VMware uses the caller's preselection). + request := toSchedulerRequest(wire, decision.Spec.Intent, decision.Spec.PipelineRef.Name) + + // If necessary gather all placement candidates before filtering. This + // overrides the hosts and weights in the request. + if request.GatherAllCandidates { + if c.Gatherer == nil { + return &wire, errors.New("candidate gathering requested but no gatherer configured") + } log.V(1).Info("gathering all placement candidates before filtering") - if err := c.gatherer.MutateWithAllCandidates(ctx, &request); err != nil { + if err := c.Gatherer.MutateWithAllCandidates(ctx, &request); err != nil { log.Error(err, "failed to gather all placement candidates") - return &request, err + return &wire, err } log.V(1).Info("gathered all placement candidates", "numHosts", len(request.Hosts)) } @@ -185,7 +190,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) // We create a virtual machine resource based on the scheduling decision, // logging the specific reason whenever we skip creation so it can be traced. - instanceUUID := request.Spec.Data.InstanceUUID + instanceUUID := request.InstanceUUID switch { case err != nil: log.V(1).Info("skipping virtual machine creation, pipeline run failed", "instanceUUID", instanceUUID) @@ -195,13 +200,13 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision log.V(1).Info("skipping virtual machine creation, feature gate disabled", "instanceUUID", instanceUUID) case !slices.Contains(api.NovaIntents, decision.Spec.Intent): log.V(1).Info("skipping virtual machine creation, intent not triggered by Nova", "instanceUUID", instanceUUID, "intent", decision.Spec.Intent) - case !request.IsKVMRequest(): + case request.HypervisorFamily != compute.HypervisorFamilyKVM: log.V(1).Info("skipping virtual machine creation, unsupported hypervisor type", "instanceUUID", instanceUUID) default: - vmUpsertErr := c.vmHandler.Upsert(ctx, decision, request) + vmUpsertErr := c.vmHandler.Upsert(ctx, decision, wire) if c.FeatureGates.VirtualMachineCreationMandatory && vmUpsertErr != nil { log.Error(vmUpsertErr, "failed to upsert virtual machine resource and it is mandatory", "instanceUUID", instanceUUID) - return &request, vmUpsertErr + return &wire, vmUpsertErr } if vmUpsertErr != nil { // Non-fatal: the placement decision itself succeeded. @@ -211,7 +216,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision if err != nil { log.Error(err, "failed to run pipeline") - return &request, err + return &wire, err } meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ Type: v1alpha1.DecisionConditionReady, @@ -220,7 +225,7 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision Message: "pipeline run succeeded", }) log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return &request, nil + return &wire, nil } // peekReadOnly determines whether a decision should use a read lock instead of @@ -239,7 +244,7 @@ func (c *FilterWeigherPipelineController) peekReadOnly(decision *v1alpha1.Decisi func (c *FilterWeigherPipelineController) InitPipeline( ctx context.Context, p v1alpha1.Pipeline, -) lib.PipelineInitResult[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]] { +) lib.PipelineInitResult[lib.FilterWeigherPipeline[compute.SchedulerRequest]] { return lib.InitNewFilterWeigherPipeline( ctx, c.Client, p.Name, @@ -252,7 +257,7 @@ func (c *FilterWeigherPipelineController) InitPipeline( func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainNova - c.gatherer = &candidateGatherer{Client: mcl} + c.Gatherer = &candidateGatherer{Client: mcl} c.vmHandler = &VirtualMachineHandler{Client: mcl} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { return err diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go index f7c4f5a41..817bfa994 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go @@ -21,6 +21,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -28,10 +29,10 @@ import ( type mockCandidateGatherer struct { called bool err error - gatheredHosts []api.ExternalSchedulerHost + gatheredHosts []compute.Host } -func (m *mockCandidateGatherer) MutateWithAllCandidates(ctx context.Context, request *api.ExternalSchedulerRequest) error { +func (m *mockCandidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.SchedulerRequest) error { m.called = true if m.err != nil { return m.err @@ -219,9 +220,9 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { Build() controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: lib.FilterWeigherPipelineMonitor{}, @@ -648,9 +649,9 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Build() controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: lib.FilterWeigherPipelineMonitor{}, @@ -710,90 +711,88 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) } } -func TestFilterWeigherPipelineController_IgnorePreselection(t *testing.T) { +func TestFilterWeigherPipelineController_GathersCandidatesForKVM(t *testing.T) { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { t.Fatalf("Failed to add v1alpha1 scheme: %v", err) } - // Create a request with initial hosts - novaRequest := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Name: "RequestSpec", - Namespace: "nova_object", - Version: "1.19", - Data: api.NovaSpec{ - ProjectID: "test-project", - UserID: "test-user", - InstanceUUID: "test-instance-uuid", - NumInstances: 1, + // Build a Nova request with the given flavor hypervisor type. Whether the + // controller gathers all candidates is derived from the hypervisor family in + // the adapter (KVM gathers; VMware / unset do not), NOT from pipeline config. + makeRaw := func(hypervisorType string) []byte { + extraSpecs := map[string]string{} + if hypervisorType != "" { + extraSpecs["capabilities:hypervisor_type"] = hypervisorType + } + novaRequest := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + ProjectID: "test-project", + UserID: "test-user", + InstanceUUID: "test-instance-uuid", + NumInstances: 1, + Flavor: api.NovaObject[api.NovaFlavor]{Data: api.NovaFlavor{ExtraSpecs: extraSpecs}}, + }, }, - }, - Context: api.NovaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: func() *string { s := "global-req-123"; return &s }(), - }, - Hosts: []api.ExternalSchedulerHost{ - {ComputeHost: "original-host-1", HypervisorHostname: "hv-1"}, - {ComputeHost: "original-host-2", HypervisorHostname: "hv-2"}, - }, - Weights: map[string]float64{"original-host-1": 1.0, "original-host-2": 0.5}, - Pipeline: "test-pipeline", - Options: scheduling.Options{}, - } - - novaRaw, err := json.Marshal(novaRequest) - if err != nil { - t.Fatalf("Failed to marshal nova request: %v", err) + Context: api.NovaRequestContext{RequestID: "req-123"}, + Hosts: []api.ExternalSchedulerHost{ + {ComputeHost: "original-host-1", HypervisorHostname: "hv-1"}, + {ComputeHost: "original-host-2", HypervisorHostname: "hv-2"}, + }, + Weights: map[string]float64{"original-host-1": 1.0, "original-host-2": 0.5}, + Pipeline: "test-pipeline", + Options: scheduling.Options{}, + } + raw, err := json.Marshal(novaRequest) + if err != nil { + t.Fatalf("Failed to marshal nova request: %v", err) + } + return raw } tests := []struct { name string - ignorePreselection bool + hypervisorType string gathererErr error - gatheredHosts []api.ExternalSchedulerHost + gatheredHosts []compute.Host expectGathererCall bool expectError bool errorContains string }{ { - name: "IgnorePreselection disabled - gatherer not called", - ignorePreselection: false, - gathererErr: nil, - gatheredHosts: nil, + name: "VMware request - gatherer not called", + hypervisorType: "vmware vcenter server", + expectGathererCall: false, + }, + { + name: "unset hypervisor type - gatherer not called", + hypervisorType: "", expectGathererCall: false, - expectError: false, }, { - name: "IgnorePreselection enabled - gatherer called and succeeds", - ignorePreselection: true, - gathererErr: nil, - gatheredHosts: []api.ExternalSchedulerHost{ + name: "KVM request - gatherer called and succeeds", + hypervisorType: "qemu", + gatheredHosts: []compute.Host{ {ComputeHost: "gathered-host-1", HypervisorHostname: "gathered-host-1"}, {ComputeHost: "gathered-host-2", HypervisorHostname: "gathered-host-2"}, {ComputeHost: "gathered-host-3", HypervisorHostname: "gathered-host-3"}, }, expectGathererCall: true, - expectError: false, }, { - name: "IgnorePreselection enabled - gatherer returns error", - ignorePreselection: true, + name: "KVM request - gatherer returns error", + hypervisorType: "qemu", gathererErr: errGathererFailed, - gatheredHosts: nil, expectGathererCall: true, expectError: true, errorContains: "gatherer failed", }, { - name: "IgnorePreselection enabled - gatherer returns empty hosts", - ignorePreselection: true, - gathererErr: nil, - gatheredHosts: []api.ExternalSchedulerHost{}, + name: "KVM request - gatherer returns empty hosts", + hypervisorType: "ch", + gatheredHosts: []compute.Host{}, expectGathererCall: true, - expectError: false, }, } @@ -810,38 +809,34 @@ func TestFilterWeigherPipelineController_IgnorePreselection(t *testing.T) { Build() controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ Client: fakeClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: lib.FilterWeigherPipelineMonitor{}, - gatherer: mockGatherer, + Gatherer: mockGatherer, } - // Setup pipeline config with IgnorePreselection flag pipelineConf := v1alpha1.Pipeline{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pipeline", }, Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - IgnorePreselection: tt.ignorePreselection, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, + Type: v1alpha1.PipelineTypeFilterWeigher, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + Filters: []v1alpha1.FilterSpec{}, + Weighers: []v1alpha1.WeigherSpec{}, }, } controller.PipelineConfigs["test-pipeline"] = pipelineConf - // Initialize the pipeline initResult := controller.InitPipeline(context.Background(), pipelineConf) if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { t.Fatalf("Failed to initialize pipeline: filter errors: %v, weigher errors: %v", initResult.FilterErrors, initResult.WeigherErrors) } controller.Pipelines["test-pipeline"] = initResult.Pipeline - // Create decision decision := &v1alpha1.Decision{ ObjectMeta: metav1.ObjectMeta{ Name: "test-decision-preselection", @@ -853,15 +848,13 @@ func TestFilterWeigherPipelineController_IgnorePreselection(t *testing.T) { Name: "test-pipeline", }, NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, + Raw: makeRaw(tt.hypervisorType), }, }, } - // Process the decision err := controller.ProcessNewDecisionFromAPI(context.Background(), decision) - // Verify gatherer was called (or not) as expected if tt.expectGathererCall && !mockGatherer.called { t.Error("Expected gatherer to be called but it was not") } @@ -869,7 +862,6 @@ func TestFilterWeigherPipelineController_IgnorePreselection(t *testing.T) { t.Error("Expected gatherer not to be called but it was") } - // Verify error expectations if tt.expectError && err == nil { t.Error("Expected error but got none") } @@ -880,7 +872,6 @@ func TestFilterWeigherPipelineController_IgnorePreselection(t *testing.T) { t.Errorf("Expected error to contain %q, got: %v", tt.errorContains, err) } - // Verify result is set when no error if !tt.expectError && decision.Status.Result == nil { t.Error("Expected result to be set but was nil") } diff --git a/internal/scheduling/nova/integration_test.go b/internal/scheduling/nova/integration_test.go index f22ff3523..4606850f1 100644 --- a/internal/scheduling/nova/integration_test.go +++ b/internal/scheduling/nova/integration_test.go @@ -15,6 +15,7 @@ import ( novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/weighers" @@ -248,12 +249,15 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec // Create the pipeline controller with shared monitor to avoid duplicate registration controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: getSharedMonitor(), + // KVM requests trigger candidate gathering; use a passthrough gatherer + // (nil gatheredHosts) so the test's preselected hosts are preserved. + Gatherer: &mockCandidateGatherer{}, } // Create and register the test pipeline diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go index 769d06729..fcc4e2774 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go @@ -9,6 +9,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -20,20 +22,16 @@ import ( func TestFilterAggregateMetadata_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string }{ { name: "No aggregates with filter_tenant_id - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -64,13 +62,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host with filter_tenant_id matching project - host passes", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -94,13 +88,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host with filter_tenant_id not matching project - host filtered", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -124,13 +114,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host in multiple aggregates with different filter_tenant_id - project matches one", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -150,13 +136,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host in multiple aggregates - project matches none", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-c", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-c", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -176,13 +158,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Mixed hosts - some restricted, some open", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -220,13 +198,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{}, }, hypervisors: []hv1.Hypervisor{ { @@ -241,13 +215,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "All hosts filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-nonexistent", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-nonexistent", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -271,13 +241,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host not in hypervisor list - passes (no restrictions found)", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, @@ -295,13 +261,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Aggregate with nil metadata - treated as no restriction", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -318,13 +280,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id - project matches one of many", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -341,13 +299,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id - project matches none", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-d", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-d", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -364,13 +318,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id with surrounding whitespace - project matches", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -387,13 +337,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys - project matches one of the numbered keys", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-c", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-c", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -414,13 +360,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys with comma-separated values - project matches", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-e", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-e", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -441,13 +383,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys - project matches none", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-z", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-z", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -510,13 +448,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { } func TestFilterAggregateMetadata_Run_ClientError(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request := compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, } @@ -575,14 +509,10 @@ func TestFilterAggregateMetadata_SkipsForCapacityProbeIntent(t *testing.T) { step := &FilterAggregateMetadata{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-y", - SchedulerHints: map[string]any{"_nova_check_type": "capacity_probe"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.SchedulerRequest{ + ProjectID: "project-y", + Intent: api.CapacityProbeIntent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { @@ -612,18 +542,14 @@ func TestFilterAggregateMetadata_AppliesForFailoverIntents(t *testing.T) { &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, } - for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation"} { - t.Run(intent, func(t *testing.T) { + for _, intent := range []v1alpha1.SchedulingIntent{api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent} { + t.Run(string(intent), func(t *testing.T) { step := &FilterAggregateMetadata{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-y", - SchedulerHints: map[string]any{"_nova_check_type": intent}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.SchedulerRequest{ + ProjectID: "project-y", + Intent: intent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go index 755ea9903..6bd68cf0d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go @@ -8,6 +8,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -67,19 +69,15 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "No project ID in request - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -90,13 +88,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches allowed projects on host2", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -107,13 +101,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches allowed projects on host3 only", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -124,13 +114,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project does not match any allowed projects", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-d", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-d", + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -141,13 +127,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Hosts without allowed projects are accessible", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-x", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-x", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -157,13 +139,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Mixed hosts - some with empty allowed projects, some with matching project", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -175,13 +153,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "All hosts filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-nonexistent", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-nonexistent", + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -192,26 +166,18 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Project matches on multiple hosts with allowed projects", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -224,13 +190,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches second item in allowed projects list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host2"}, }, }, @@ -239,13 +201,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "All hosts have empty allowed projects", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "any-project", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "any-project", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -255,13 +213,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Mixed allowed projects configuration", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-c", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "project-c", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -274,13 +228,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Case sensitive project matching", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "Project-A", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ProjectID: "Project-A", + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, }, @@ -340,14 +290,10 @@ func TestFilterAllowedProjectsStep_SkipsForCapacityProbeIntent(t *testing.T) { step := &FilterAllowedProjectsStep{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-y", - SchedulerHints: map[string]any{"_nova_check_type": "capacity_probe"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.SchedulerRequest{ + ProjectID: "project-y", + Intent: api.CapacityProbeIntent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { @@ -372,18 +318,14 @@ func TestFilterAllowedProjectsStep_AppliesForFailoverIntents(t *testing.T) { }, } - for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation"} { - t.Run(intent, func(t *testing.T) { + for _, intent := range []v1alpha1.SchedulingIntent{api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent} { + t.Run(string(intent), func(t *testing.T) { step := &FilterAllowedProjectsStep{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-y", - SchedulerHints: map[string]any{"_nova_check_type": intent}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.SchedulerRequest{ + ProjectID: "project-y", + Intent: intent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go b/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go index 622932bb8..36fe46525 100644 --- a/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -225,24 +225,15 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string expectedEvents []lib.FilterWeigherPipelineStepEvent }{ { name: "No extra specs in request - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -253,20 +244,8 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Non-capability extra specs - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "hw:mem_page_size": "large", - "hw:cpu_policy": "dedicated", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -276,19 +255,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match CH hypervisor type", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -299,19 +270,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match QEMU hypervisor type", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "QEMU"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -323,19 +286,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match x86_64 CPU architecture", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:cpu_arch": "x86_64", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "cpu_arch", Value: "x86_64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -347,19 +302,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match aarch64 CPU architecture", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:cpu_arch": "aarch64", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "cpu_arch", Value: "aarch64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -371,20 +318,12 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match both hypervisor type and CPU architecture", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - "capabilities:cpu_arch": "x86_64", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, + {Name: "cpu_arch", Value: "x86_64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -396,20 +335,12 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match QEMU with aarch64", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - "capabilities:cpu_arch": "aarch64", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "QEMU"}, + {Name: "cpu_arch", Value: "aarch64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -421,19 +352,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Ignore hypervisors not in request even with unknown type", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -443,19 +366,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Candidate hypervisor with empty hypervisor type emits event and is filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host-empty-type"}, }, }, @@ -467,19 +382,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Candidate hypervisor with unknown hypervisor type emits event and is filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host-unknown-type"}, }, }, @@ -491,19 +398,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "No matching hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "KVM", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "KVM"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -513,21 +412,12 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Mixed capability and non-capability extra specs", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - "hw:mem_page_size": "large", - "capabilities:cpu_arch": "x86_64", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, + {Name: "cpu_arch", Value: "x86_64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -538,38 +428,22 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Case sensitive capability matching", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "ch", // lowercase should not match - }, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "ch"}, // lowercase should not match }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -579,19 +453,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Unsupported operator in extra specs - skip filter", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:cpu_arch": "s>=x86_64", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + // An unsupported comparison operator is dropped by the adapter, + // so the filter sees no requested capabilities and is a no-op. + RequestedCapabilities: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -602,19 +468,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Unsupported operator - skip filter", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": " CH QEMU", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + // An unsupported comparison operator is dropped by the adapter, + // so the filter sees no requested capabilities and is a no-op. + RequestedCapabilities: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -624,17 +482,8 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "All hosts match when no capabilities requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -701,78 +550,3 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }) } } - -// TestFilterCapabilitiesStep_DoesNotMutateExtraSpecs verifies that the filter -// does not modify the original ExtraSpecs map from the request. This is important -// because subsequent filters (like filter_has_requested_traits) also need to access -// the full ExtraSpecs including non-capability keys like trait:*. -func TestFilterCapabilitiesStep_DoesNotMutateExtraSpecs(t *testing.T) { - scheme := runtime.NewScheme() - err := hv1.AddToScheme(scheme) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - hvs := []client.Object{ - &hv1.Hypervisor{ - ObjectMeta: v1.ObjectMeta{Name: "host1"}, - Status: hv1.HypervisorStatus{ - DomainCapabilities: hv1.DomainCapabilities{HypervisorType: "ch"}, - Capabilities: hv1.Capabilities{HostCpuArch: "x86_64"}, - }, - }, - } - - // Create a request with mixed extra specs (capabilities + traits + other) - originalExtraSpecs := map[string]string{ - "capabilities:hypervisor_type": "CH", - "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "forbidden", - "trait:CUSTOM_HW_SAPPHIRE_RAPIDS": "required", - "hw:mem_page_size": "large", - "hw:numa_nodes": "1", - } - - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: originalExtraSpecs, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}}, - } - - step := &FilterCapabilitiesStep{} - step.Client = fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(hvs...). - Build() - - _, err = step.Run(logr.Discard(), request) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - // Verify that the original ExtraSpecs map was NOT mutated - expectedKeys := []string{ - "capabilities:hypervisor_type", - "trait:CUSTOM_HANA_EXCLUSIVE_HOST", - "trait:CUSTOM_HW_SAPPHIRE_RAPIDS", - "hw:mem_page_size", - "hw:numa_nodes", - } - - if len(request.Spec.Data.Flavor.Data.ExtraSpecs) != len(expectedKeys) { - t.Errorf("ExtraSpecs was mutated: expected %d keys, got %d", - len(expectedKeys), len(request.Spec.Data.Flavor.Data.ExtraSpecs)) - } - - for _, key := range expectedKeys { - if _, ok := request.Spec.Data.Flavor.Data.ExtraSpecs[key]; !ok { - t.Errorf("ExtraSpecs was mutated: key %q was deleted", key) - } - } -} diff --git a/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go b/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go index 135766806..2fe0c0829 100644 --- a/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -57,19 +57,15 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "Filter hosts in az-1", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-1", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AvailabilityZone: "az-1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -82,13 +78,9 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Filter hosts in az-2", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-2", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AvailabilityZone: "az-2", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -100,13 +92,9 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Filter hosts in az-3", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-3", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AvailabilityZone: "az-3", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -118,13 +106,9 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "No hosts in requested AZ", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-nonexistent", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AvailabilityZone: "az-nonexistent", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -135,26 +119,18 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-1", - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + AvailabilityZone: "az-1", + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Host not in database", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-1", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AvailabilityZone: "az-1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go index 45676b731..a0c232b26 100644 --- a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -15,15 +15,15 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { tests := []struct { name string excludedHosts []string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "No excluded hosts - all hosts pass", excludedHosts: []string{}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -35,8 +35,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single excluded host", excludedHosts: []string{"host2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -48,8 +48,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Multiple excluded hosts", excludedHosts: []string{"host1", "host3"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -62,8 +62,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "All hosts excluded", excludedHosts: []string{"host1", "host2", "host3"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -75,8 +75,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Excluded host not in request - no effect", excludedHosts: []string{"host-nonexistent"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -87,8 +87,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Some excluded hosts in request, some not", excludedHosts: []string{"host1", "host-nonexistent", "host3"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -100,8 +100,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Empty host list in request", excludedHosts: []string{"host1", "host2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, @@ -109,8 +109,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Nil excluded hosts list", excludedHosts: nil, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -121,8 +121,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Duplicate excluded hosts", excludedHosts: []string{"host1", "host1", "host2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -134,8 +134,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single host request - excluded", excludedHosts: []string{"host1"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -145,8 +145,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single host request - not excluded", excludedHosts: []string{"host2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -156,8 +156,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Case sensitive matching", excludedHosts: []string{"Host1", "HOST2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "Host1"}, @@ -169,8 +169,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Hosts with special characters", excludedHosts: []string{"nova-compute-bb123.region.example.com"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "nova-compute-bb123.region.example.com"}, {ComputeHost: "nova-compute-bb124.region.example.com"}, }, @@ -181,8 +181,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Empty string in excluded hosts", excludedHosts: []string{"", "host1"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -193,8 +193,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Large number of excluded hosts", excludedHosts: []string{"host1", "host2", "host3", "host4", "host5", "host6", "host7", "host8", "host9", "host10"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, {ComputeHost: "host10"}, @@ -210,8 +210,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { step := &FilterExcludeHostsStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, FilterExcludeHostsStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, FilterExcludeHostsStepOpts]{}, + BaseFilter: lib.BaseFilter[compute.SchedulerRequest, FilterExcludeHostsStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, FilterExcludeHostsStepOpts]{}, }, } step.Options = FilterExcludeHostsStepOpts{ diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go index 4d2ebe784..c99e19a61 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go @@ -8,6 +8,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -77,7 +79,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { tests := []struct { name string opts FilterExternalCustomerStepOpts - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string expectError bool @@ -87,15 +89,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "ext-customer1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -110,15 +106,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "internal-customer", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "internal-customer", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -133,15 +123,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"external-", "ext-", "customer-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "customer-abc", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "customer-abc", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -156,15 +140,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { CustomerDomainNamePrefixes: []string{"ext-"}, CustomerIgnoredDomainNames: []string{"ext-special-domain"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-special-domain", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "ext-special-domain", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -178,15 +156,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer2", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "ext-customer2", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host5"}, @@ -200,15 +172,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer3", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "ext-customer3", + Hosts: []compute.Host{ {ComputeHost: "host3"}, {ComputeHost: "host4"}, }, @@ -221,15 +187,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + DomainName: "ext-customer", + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, @@ -239,15 +199,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": []any{"ext-customer", "other"}, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "ext-customer", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -260,13 +214,8 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -279,13 +228,8 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -298,15 +242,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "EXT-customer", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "EXT-customer", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -320,15 +258,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "ext", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -342,15 +274,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { CustomerDomainNamePrefixes: []string{"ext-"}, CustomerIgnoredDomainNames: []string{"ext-test", "ext-dev", "ext-staging"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-dev", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "ext-dev", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -363,16 +289,10 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForFailoverIntent), - "domain_name": "ext-customer1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.ReserveForFailoverIntent, + DomainName: "ext-customer1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -385,15 +305,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForCommittedResourceIntent), - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.ReserveForCommittedResourceIntent, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -406,16 +320,10 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForCommittedResourceIntent), - "domain_name": "ext-customer1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.ReserveForCommittedResourceIntent, + DomainName: "ext-customer1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -429,15 +337,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + DomainName: "ext-customer1", + Hosts: []compute.Host{ {ComputeHost: "host-custom-trait"}, {ComputeHost: "host4"}, }, @@ -565,16 +467,10 @@ func TestFilterExternalCustomerStep_SkipsForNonPlacementIntent(t *testing.T) { for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { t.Run(intent, func(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "iaas-customer", - "_nova_check_type": intent, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.SchedulerRequest{ + Intent: v1alpha1.SchedulingIntent(intent), + DomainName: "iaas-customer", + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go b/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go index 2c92cf81b..971845e87 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -67,23 +67,14 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "No accelerators requested - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -95,19 +86,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - filter to hosts with COMPUTE_ACCELERATORS trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -119,19 +100,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested with different device profile value", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "fpga-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, {ComputeHost: "host5"}, @@ -142,19 +113,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - all hosts have the trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host5"}, @@ -165,19 +126,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - no hosts have the trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host3"}, {ComputeHost: "host4"}, }, @@ -187,20 +138,8 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Other extra specs present but no accelerator request - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "hw:cpu_policy": "dedicated", - "quota:vif_inbound": "100000", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -211,19 +150,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested with empty device profile value", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -234,55 +163,26 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Empty host list with accelerators requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Empty host list without accelerators requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Mixed hosts with and without accelerators trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "custom-accelerator", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -295,21 +195,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators with additional extra specs", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - "hw:cpu_policy": "dedicated", - "hw:mem_page_size": "large", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -319,19 +207,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Host not in database with accelerators requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go index bdde34cca..0ed265cea 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go @@ -10,6 +10,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -248,58 +249,48 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { +func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) compute.SchedulerRequest { return newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup, vcpus, memory, "", evacuation, hosts) } // newNovaRequestWithIntent creates a nova request with a specific intent. // intentHint can be: "evacuate", "reserve_for_committed_resource", "reserve_for_failover", or "" for create. -func newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory, intentHint string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory, intentHint string, evacuation bool, hosts []string) compute.SchedulerRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + hostList[i] = compute.Host{ComputeHost: h} } - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", - "quota:hw_version": flavorGroup, - } - - var schedulerHints map[string]any - if evacuation { - schedulerHints = map[string]any{ - "_nova_check_type": []any{"evacuate"}, - } - } else if intentHint != "" { - schedulerHints = map[string]any{ - "_nova_check_type": intentHint, - } + var intent v1alpha1.SchedulingIntent + switch { + case evacuation: + intent = api.EvacuateIntent + case intentHint == "reserve_for_committed_resource": + intent = api.ReserveForCommittedResourceIntent + case intentHint == "reserve_for_failover": + intent = api.ReserveForFailoverIntent } memoryMB := parseMemoryToMB(memory) - spec := api.NovaSpec{ - ProjectID: projectID, - InstanceUUID: instanceUUID, - NumInstances: 1, - SchedulerHints: schedulerHints, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: flavorName, - VCPUs: uint64(vcpus), //nolint:gosec // test code - MemoryMB: memoryMB, - ExtraSpecs: extraSpecs, - }, - }, - } - weights := make(map[string]float64) for _, h := range hosts { weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.SchedulerRequest{ + ProjectID: projectID, + InstanceUUID: instanceUUID, + NumInstances: 1, + Intent: intent, + HardwareVersion: flavorGroup, + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, + Flavor: compute.FlavorSpec{ + Name: flavorName, + VCPUs: uint64(vcpus), //nolint:gosec // test code + Memory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // test code + }, Hosts: hostList, Weights: weights, } @@ -362,7 +353,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.SchedulerRequest opts FilterHasEnoughCapacityOpts expectedHosts []string filteredHosts []string @@ -774,7 +765,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name string hypervisors []*hv1.Hypervisor reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.SchedulerRequest ignoredReservationTypes []v1alpha1.ReservationType expectedHosts []string filteredHosts []string @@ -1087,7 +1078,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) name string hypervisors []*hv1.Hypervisor reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.SchedulerRequest opts FilterHasEnoughCapacityOpts expectedHosts []string filteredHosts []string @@ -1296,7 +1287,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ diff --git a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go index 719aa37fd..7ba8f7b4c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -75,23 +75,14 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "No traits requested - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -102,19 +93,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Single required trait - filter hosts with trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -126,20 +107,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Multiple required traits - filter hosts with all traits", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - "trait:CUSTOM_TRAIT_B": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -151,19 +121,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Single forbidden trait - filter hosts without trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForbiddenTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -176,20 +136,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Multiple forbidden traits - filter hosts without any of them", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "forbidden", - "trait:CUSTOM_TRAIT_B": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForbiddenTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -202,20 +151,10 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Mixed required and forbidden traits", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - "trait:CUSTOM_TRAIT_D": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + ForbiddenTraits: []string{"CUSTOM_TRAIT_D"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host4"}, @@ -227,19 +166,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Required trait that no host has", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_NONEXISTENT": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_NONEXISTENT"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -250,19 +179,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Forbidden trait that no host has - all pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_NONEXISTENT": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForbiddenTraits: []string{"CUSTOM_TRAIT_NONEXISTENT"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -273,19 +192,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Host with no traits - required trait filters it out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -295,19 +204,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Host with no traits - forbidden trait lets it pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForbiddenTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -317,20 +216,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Non-trait extra specs are ignored", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "hw:cpu_policy": "dedicated", - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -340,19 +228,8 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Invalid trait value (not required or forbidden) - ignored", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "invalid", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -363,38 +240,18 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Host not in database", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, @@ -404,21 +261,10 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Complex scenario with many traits", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - "trait:CUSTOM_TRAIT_B": "required", - "trait:CUSTOM_TRAIT_D": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, + ForbiddenTraits: []string{"CUSTOM_TRAIT_D"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -432,20 +278,10 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "All hosts match required and forbidden traits", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - "trait:CUSTOM_TRAIT_E": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + ForbiddenTraits: []string{"CUSTOM_TRAIT_E"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go b/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go index 349932c3c..da07468a5 100644 --- a/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go @@ -7,26 +7,20 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) func TestFilterHostInstructionsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "No host instructions - no filtering", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -38,14 +32,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore single host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host2"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -57,14 +46,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore multiple hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1", "host3", "host4"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host1", "host3", "host4"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -76,14 +60,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore all hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1", "host2", "host3", "host4"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host1", "host2", "host3", "host4"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -95,14 +74,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore non-existent host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host-nonexistent"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host-nonexistent"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -113,14 +87,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force single host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{"host2"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForceHosts: []string{"host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -132,14 +101,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force multiple hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{"host1", "host3"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForceHosts: []string{"host1", "host3"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -151,14 +115,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force non-existent host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{"host-nonexistent"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForceHosts: []string{"host-nonexistent"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -169,14 +128,10 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force and ignore same host - ignore takes precedence", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host2"}, - ForceHosts: &[]string{"host2"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host2"}, + ForceHosts: []string{"host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -187,14 +142,10 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force and ignore different hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1"}, - ForceHosts: &[]string{"host2", "host3"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host1"}, + ForceHosts: []string{"host2", "host3"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -206,14 +157,10 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Complex scenario - multiple ignore and force", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host2", "host5"}, - ForceHosts: &[]string{"host1", "host3", "host4"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host2", "host5"}, + ForceHosts: []string{"host1", "host3", "host4"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -227,28 +174,19 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1"}, - ForceHosts: &[]string{"host2"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host1"}, + ForceHosts: []string{"host2"}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Empty ignore list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -258,48 +196,36 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Empty force list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForceHosts: []string{}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, }, - expectedHosts: []string{}, - filteredHosts: []string{"host1", "host2"}, + // An empty force list is no constraint in the typed spec (it cannot be + // distinguished from "unset"), so all hosts are kept. + expectedHosts: []string{"host1", "host2"}, + filteredHosts: []string{}, }, { name: "Both lists empty", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{}, - ForceHosts: &[]string{}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{}, + ForceHosts: []string{}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, }, - expectedHosts: []string{}, - filteredHosts: []string{"host1", "host2"}, + expectedHosts: []string{"host1", "host2"}, + filteredHosts: []string{}, }, { name: "Duplicate hosts in ignore list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1", "host1", "host2"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + IgnoreHosts: []string{"host1", "host1", "host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -310,14 +236,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Duplicate hosts in force list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{"host1", "host1", "host2"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + ForceHosts: []string{"host1", "host1", "host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties.go b/internal/scheduling/nova/plugins/filters/filter_image_properties.go index 55dacf33f..944797d81 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties.go @@ -40,10 +40,27 @@ func (s *FilterImagePropertiesStep) Run(traceLog logr.Logger, request compute.Sc return result, nil } + hvType := request.ImageHypervisorType + + // The image declared a hypervisor type that could not be parsed. Ignore the + // filter and keep all hosts, but surface an event so high-frequency + // occurrences can be alerted on (labeled by intent). + if hvType == compute.ImageHypervisorTypeUnparseable { + traceLog.Error(nil, "could not determine hypervisor type from image properties") + intentLabel := string(request.Intent) + if intentLabel == "" { + intentLabel = "unknown" + } + result.Events = append(result.Events, lib.FilterWeigherPipelineStepEvent{ + Name: "image_properties_hv_type_undetermined", + Labels: map[string]string{"intent": intentLabel}, + }) + return result, nil + } + // If the image properties indicate any other hypervisor type than kvm, // we filter out all known kvm hypervisors. An unset image hypervisor type // means the image declared none, so no image constraint applies. - hvType := request.ImageHypervisorType if hvType != compute.ImageHypervisorTypeUnset && hvType != compute.ImageHypervisorTypeKVM { traceLog.V(1).Info("filtering out all known kvm hypervisors since image properties indicate a different hypervisor type", "image_hypervisor_type", hvType) diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go index 0c836a9e2..2e58e3b8d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go @@ -8,6 +8,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,24 +18,25 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -// requestWith builds an ExternalSchedulerRequest with the given hosts and image -// properties. -func requestWith(hosts []string, properties map[string]any) api.ExternalSchedulerRequest { - schedHosts := make([]api.ExternalSchedulerHost, 0, len(hosts)) +// requestWith builds a SchedulerRequest with the given hosts and image +// hypervisor type. +func requestWith(hosts []string, hvType compute.ImageHypervisorType) compute.SchedulerRequest { + schedHosts := make([]compute.Host, 0, len(hosts)) for _, h := range hosts { - schedHosts = append(schedHosts, api.ExternalSchedulerHost{ComputeHost: h}) + schedHosts = append(schedHosts, compute.Host{ComputeHost: h}) + } + return compute.SchedulerRequest{ + Hosts: schedHosts, + ImageHypervisorType: hvType, } - request := api.ExternalSchedulerRequest{Hosts: schedHosts} - request.Spec.Data.Image.Data.Properties = api.NovaObject[map[string]any]{Data: properties} - return request } -// requestWithIntent builds an ExternalSchedulerRequest like requestWith, but -// additionally sets the _nova_check_type scheduler hint so GetIntent resolves -// to the given intent. -func requestWithIntent(hosts []string, properties map[string]any, checkType string) api.ExternalSchedulerRequest { - request := requestWith(hosts, properties) - request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": checkType} +// requestWithIntent builds a SchedulerRequest with the given intent. The image +// hypervisor type is irrelevant to the intent-skip paths these callers exercise, +// so it is fixed to VMware. +func requestWithIntent(hosts []string, intent v1alpha1.SchedulingIntent) compute.SchedulerRequest { + request := requestWith(hosts, compute.ImageHypervisorTypeVMware) + request.Intent = intent return request } @@ -51,32 +54,32 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string expectedEvents []lib.FilterWeigherPipelineStepEvent }{ { name: "kvm image keeps all hosts", - request: requestWith([]string{"host1", "host2", "host3"}, map[string]any{"img_hv_type": "kvm"}), + request: requestWith([]string{"host1", "host2", "host3"}, compute.ImageHypervisorTypeKVM), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "vmware image filters out known kvm hypervisors", - request: requestWith([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}), + request: requestWith([]string{"host1", "host2", "host3"}, compute.ImageHypervisorTypeVMware), expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2"}, }, { name: "baremetal image filters out known kvm hypervisors", - request: requestWith([]string{"host1", "host3"}, map[string]any{"img_hv_type": "baremetal"}), + request: requestWith([]string{"host1", "host3"}, compute.ImageHypervisorTypeBaremetal), expectedHosts: []string{"host3"}, filteredHosts: []string{"host1"}, }, { name: "undeterminable hypervisor type keeps all hosts", - request: requestWith([]string{"host1", "host2", "host3"}, nil), + request: requestWith([]string{"host1", "host2", "host3"}, compute.ImageHypervisorTypeUnparseable), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, expectedEvents: []lib.FilterWeigherPipelineStepEvent{ @@ -85,7 +88,7 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { }, { name: "unsupported hypervisor type keeps all hosts", - request: requestWith([]string{"host1", "host2"}, map[string]any{"img_hv_type": "xen"}), + request: requestWith([]string{"host1", "host2"}, compute.ImageHypervisorTypeUnparseable), expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, expectedEvents: []lib.FilterWeigherPipelineStepEvent{ @@ -94,37 +97,37 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { }, { name: "empty host list", - request: requestWith([]string{}, map[string]any{"hypervisor_type": "vmware"}), + request: requestWith([]string{}, compute.ImageHypervisorTypeVMware), expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "reserve_for_failover intent skips filter and keeps all hosts", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "reserve_for_failover"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.ReserveForFailoverIntent), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "reuse_failover_reservation intent skips filter and keeps all hosts", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "reuse_failover_reservation"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.ReuseFailoverReservationIntent), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "reserve_for_committed_resource intent skips filter and keeps all hosts", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "reserve_for_committed_resource"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.ReserveForCommittedResourceIntent), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "capacity_probe intent skips filter and keeps all hosts", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "capacity_probe"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.CapacityProbeIntent), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "create intent still filters out known kvm hypervisors", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "create"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.CreateIntent), expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go index 5a955d515..029fe6d90 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go @@ -7,25 +7,22 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "No instance group - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -36,18 +33,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Instance group with anti-affinity policy - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Hosts: []string{"host1", "host2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Hosts: []string{"host1", "host2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -58,18 +46,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Instance group with soft-affinity policy - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "soft-affinity", - Hosts: []string{"host1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "soft-affinity", Hosts: []string{"host1"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -79,18 +58,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy with empty hosts list - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -101,18 +71,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - only hosts in instance group pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1", "host3"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host3"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -123,18 +84,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - single host in group", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -146,18 +98,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - all hosts filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host5", "host6"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host5", "host6"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -168,18 +111,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - all hosts in group pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1", "host2", "host3"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2", "host3"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -190,18 +124,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - partial overlap", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host2", "host3", "host5"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host2", "host3", "host5"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -213,36 +138,18 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - empty request hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1", "host2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2"}}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Affinity policy - case sensitive host matching", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"Host1", "HOST2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"Host1", "HOST2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -253,23 +160,11 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - with instance UUID and project ID", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-123", - ProjectID: "project-abc", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - UUID: "ig-uuid-456", - Policy: "affinity", - Hosts: []string{"host1", "host2"}, - Members: []string{"vm-uuid-789"}, - ProjectID: "project-abc", - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-123", + ProjectID: "project-abc", + InstanceGroup: &compute.InstanceGroupSpec{UUID: "ig-uuid-456", Policy: "affinity", Hosts: []string{"host1", "host2"}, Members: []string{"vm-uuid-789"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -281,18 +176,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - duplicate hosts in group list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1", "host2", "host1", "host2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2", "host1", "host2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -303,18 +189,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - single host scenario", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -358,10 +235,8 @@ func TestFilterInstanceGroupAffinityStep_SkipsForNonPlacementIntent(t *testing.T for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { t.Run(intent, func(t *testing.T) { request := newNovaRequest("vm", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) - request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{Policy: "affinity", Hosts: []string{"host1"}}, - } - request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": intent} + request.InstanceGroup = &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1"}} + request.Intent = v1alpha1.SchedulingIntent(intent) step := &FilterInstanceGroupAffinityStep{} result, err := step.Run(logr.Discard(), request) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go index bfbd1c475..27bce62c0 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go @@ -7,7 +7,8 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -79,20 +80,16 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "No instance group - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -103,19 +100,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Instance group with affinity policy - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -126,19 +114,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Instance group with soft-anti-affinity policy - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "soft-anti-affinity", - Members: []string{"vm-uuid-1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "soft-anti-affinity", Members: []string{"vm-uuid-1"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -148,19 +127,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy with empty members list - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -171,19 +141,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - default max_server_per_host=1", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -195,22 +156,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=2", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - Rules: map[string]any{ - "max_server_per_host": 2, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 2}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -222,22 +171,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=3", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-4", "vm-uuid-5", "vm-uuid-6"}, - Rules: map[string]any{ - "max_server_per_host": 3, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-4", "vm-uuid-5", "vm-uuid-6"}, MaxServerPerHost: 3}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -247,19 +184,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - host running same VM (resize scenario)", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-1", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-1", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host4"}, @@ -270,19 +198,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - all hosts filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}}, + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -293,19 +212,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - mixed hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3", "vm-uuid-4", "vm-uuid-5"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3", "vm-uuid-4", "vm-uuid-5"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -318,38 +228,20 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - empty request hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Anti-affinity policy - host with non-member instances", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-100", "vm-uuid-101"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-100", "vm-uuid-101"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -362,22 +254,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=2 with mixed instances", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2"}, - Rules: map[string]any{ - "max_server_per_host": 2, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}, MaxServerPerHost: 2}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -390,22 +270,11 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - with instance UUID and project ID", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - ProjectID: "project-abc", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - UUID: "ig-uuid-456", - Policy: "anti-affinity", - Members: []string{"vm-uuid-1"}, - ProjectID: "project-abc", - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + ProjectID: "project-abc", + InstanceGroup: &compute.InstanceGroupSpec{UUID: "ig-uuid-456", Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -416,22 +285,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - multiple members on same host with max=2", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - Rules: map[string]any{ - "max_server_per_host": 2, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 2}, + Hosts: []compute.Host{ {ComputeHost: "host3"}, {ComputeHost: "host4"}, }, @@ -441,19 +298,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - resize with VM on multiple hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-2", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-2", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -465,19 +313,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - single host scenario", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, + Hosts: []compute.Host{ {ComputeHost: "host2"}, }, }, @@ -486,22 +325,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - high max_server_per_host value", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - Rules: map[string]any{ - "max_server_per_host": 10, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 10}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -567,14 +394,12 @@ func TestFilterInstanceGroupAntiAffinityStep_SkipsForNonPlacementIntent(t *testi for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { t.Run(intent, func(t *testing.T) { request := newNovaRequest("vm-new", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) - request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-existing"}, - Rules: map[string]any{"max_server_per_host": 1}, - }, + request.InstanceGroup = &compute.InstanceGroupSpec{ + Policy: "anti-affinity", + Members: []string{"vm-existing"}, + MaxServerPerHost: 1, } - request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": intent} + request.Intent = v1alpha1.SchedulingIntent(intent) result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go b/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go index 8f866b309..e5b743a4f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go @@ -9,6 +9,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -200,7 +201,7 @@ func TestFilterLiveMigratableStep_checkHasSufficientFeatures(t *testing.T) { func TestFilterLiveMigratableStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string @@ -208,15 +209,9 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }{ { name: "Not a live migration - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "rebuild", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.EvacuateIntent, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -241,15 +236,9 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration without source_host - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -274,16 +263,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with compatible hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -330,16 +313,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with incompatible architecture", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -380,16 +357,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing CPU modes", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -436,16 +407,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing features", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -476,16 +441,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing devices", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -516,16 +475,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with non-existent target host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host-nonexistent"}, @@ -551,16 +504,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with all hosts filtered", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -592,16 +539,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{}, }, hypervisors: []hv1.Hypervisor{ { @@ -634,8 +575,8 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -677,16 +618,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { } func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "nonexistent-host", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request := compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "nonexistent-host", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -721,8 +656,8 @@ func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -738,16 +673,10 @@ func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { } func TestFilterLiveMigratableStep_Run_ClientError(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request := compute.SchedulerRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, } @@ -767,8 +696,8 @@ func TestFilterLiveMigratableStep_Run_ClientError(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 1cd315908..7fe3df57e 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -6,9 +6,10 @@ package filters import ( "context" "fmt" - "github.com/go-logr/logr" "time" + "github.com/go-logr/logr" + api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" @@ -155,7 +156,6 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request compute.Sched vcpus := request.Flavor.VCPUs // Total requested RAM across all instances, converted to GiB (ceiling, so we // never under-count headroom demand). Flavor.Memory is bytes per instance. - //nolint:gosec // instance count and flavor memory are bounded by Nova. totalMemBytes := request.Flavor.Memory.Value() * int64(numInstances) requestRAM := (totalMemBytes + (1 << 30) - 1) / (1 << 30) requestCores := int64(vcpus * numInstances) //nolint:gosec // VCPUs from flavor specs, realistically bounded diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 2543c5e20..039fa0c48 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -9,6 +9,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/resource" @@ -18,27 +19,44 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) api.ExternalSchedulerRequest { - flavor := api.NovaFlavor{ - MemoryMB: memoryMB, - VCPUs: 4, +// intentFromHint maps an old "_nova_check_type" scheduler hint value to the +// typed scheduling intent the plugin now reads. The hints map carries at most +// the "_nova_check_type" key in these tests. +func intentFromHint(hints map[string]any) v1alpha1.SchedulingIntent { + raw, ok := hints["_nova_check_type"] + if !ok { + return "" } - if hwVersion != "" { - flavor.ExtraSpecs = map[string]string{"quota:hw_version": hwVersion} - } else { - flavor.ExtraSpecs = map[string]string{} + switch raw { + case "evacuate": + return api.EvacuateIntent + case "live_migrate": + return api.LiveMigrationIntent + case "reserve_for_failover": + return api.ReserveForFailoverIntent + case "reserve_for_committed_resource": + return api.ReserveForCommittedResourceIntent + case "resize": + return api.ResizeIntent + case "capacity_probe": + return api.CapacityProbeIntent + default: + return "" } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: projectID, - AvailabilityZone: az, - NumInstances: numInstances, - SchedulerHints: hints, - Flavor: api.NovaObject[api.NovaFlavor]{Data: flavor}, - }, +} + +func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) compute.SchedulerRequest { + return compute.SchedulerRequest{ + ProjectID: projectID, + AvailabilityZone: az, + NumInstances: numInstances, + Intent: intentFromHint(hints), + HardwareVersion: hwVersion, + Flavor: compute.FlavorSpec{ + VCPUs: 4, + Memory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // test code }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -55,7 +73,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { type tc struct { name string objects []client.Object - request api.ExternalSchedulerRequest + request compute.SchedulerRequest enforce bool expectAccept bool diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go index a145190b9..fe1805e62 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go @@ -9,6 +9,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,7 +23,7 @@ import ( func TestFilterRequestedDestinationStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string @@ -29,13 +31,9 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }{ { name: "No requested_destination - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + RequestedDestination: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -61,18 +59,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Empty requested_destination - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{}, - Host: "", - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{}, + Host: "", }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -93,17 +85,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by specific host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Host: "host2", - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Host: "host2", }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -129,17 +115,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by single aggregate - hosts in spec aggregates", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -165,17 +145,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by single aggregate - hosts in status aggregates", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -201,18 +175,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by multiple aggregates with OR syntax (comma-separated)", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - // "aggregate1,aggregate3" means host must be in aggregate1 OR aggregate3 - Aggregates: []string{"aggregate1,aggregate3"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + // "aggregate1,aggregate3" means host must be in aggregate1 OR aggregate3 + Aggregates: []string{"aggregate1,aggregate3"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -243,18 +211,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by multiple aggregates with AND logic", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - // ["aggregate1", "aggregate2"] means host must be in aggregate1 AND aggregate2 - Aggregates: []string{"aggregate1", "aggregate2"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + // ["aggregate1", "aggregate2"] means host must be in aggregate1 AND aggregate2 + Aggregates: []string{"aggregate1", "aggregate2"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -280,18 +242,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by mixed AND/OR aggregates", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - // Host must be in (aggregate1 OR aggregate2) AND aggregate3 - Aggregates: []string{"aggregate1,aggregate2", "aggregate3"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + // Host must be in (aggregate1 OR aggregate2) AND aggregate3 + Aggregates: []string{"aggregate1,aggregate2", "aggregate3"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -322,18 +278,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by aggregates with OR syntax - hosts in both spec and status", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - // "aggregate1,aggregate2" means host must be in aggregate1 OR aggregate2 - Aggregates: []string{"aggregate1,aggregate2"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + // "aggregate1,aggregate2" means host must be in aggregate1 OR aggregate2 + Aggregates: []string{"aggregate1,aggregate2"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -359,18 +309,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by both host and aggregates - host takes precedence", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - Host: "host2", - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, + Host: "host2", }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -396,17 +340,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Host not found in hypervisors - filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-nonexistent"}, {ComputeHost: "host2"}, @@ -428,17 +366,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Host with no aggregates - filtered out when aggregates requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -464,17 +396,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "All hosts filtered by aggregate", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"nonexistent-aggregate"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"nonexistent-aggregate"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -495,17 +421,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Requested host not in candidates", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Host: "host-not-in-list", - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Host: "host-not-in-list", }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -530,17 +450,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{}, + Hosts: []compute.Host{}, }, hypervisors: []hv1.Hypervisor{ { @@ -571,8 +485,8 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { Build() step := &FilterRequestedDestinationStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -740,17 +654,11 @@ func TestFilterRequestedDestinationStep_processRequestedHost(t *testing.T) { } func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request := compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, } @@ -770,8 +678,8 @@ func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { Build() step := &FilterRequestedDestinationStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -796,18 +704,17 @@ func TestFilterRequestedDestinationStep_SkipsForNonPlacementIntent(t *testing.T) step := &FilterRequestedDestinationStep{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { - t.Run(intent, func(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{Host: "host1"}, - }, - SchedulerHints: map[string]any{"_nova_check_type": intent}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + for _, intent := range []v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + } { + t.Run(string(intent), func(t *testing.T) { + request := compute.SchedulerRequest{ + RequestedDestination: &compute.RequestedDestination{Host: "host1"}, + Intent: intent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go b/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go index 7862b6544..0b936ca77 100644 --- a/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -214,14 +214,14 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedHosts []string filteredHosts []string }{ { name: "Filter hosts with all conditions met", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -233,8 +233,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host not ready should be filtered", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host2"}, }, }, @@ -243,8 +243,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Terminating host should be filtered", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host3"}, }, }, @@ -253,8 +253,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Tainted host should not be filtered", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host4"}, }, }, @@ -263,8 +263,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Disabled hypervisor should be filtered", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host8"}, }, }, @@ -273,8 +273,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host with optional conditions in any state should pass", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host5"}, }, }, @@ -283,8 +283,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host missing Ready condition should be kept", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host6"}, }, }, @@ -293,8 +293,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host with no conditions should be kept", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host7"}, }, }, @@ -303,16 +303,16 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{}, + request: compute.SchedulerRequest{ + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Host not in database", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, @@ -322,8 +322,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Mixed condition states", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.SchedulerRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go index 421b53042..0435fbc60 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,28 +34,10 @@ func newHypervisor(name, capacityCPU, capacityMem, allocationCPU, allocationMem } } -func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) compute.SchedulerRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} - } - - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: "instance-123", - NumInstances: numInstances, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: vcpus, - MemoryMB: memoryMB, - ExtraSpecs: extraSpecs, - }, - }, + hostList[i] = compute.Host{ComputeHost: h} } weights := make(map[string]float64) @@ -63,8 +45,17 @@ func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) api weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.SchedulerRequest{ + ProjectID: "project-A", + InstanceUUID: "instance-123", + NumInstances: numInstances, + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, + Flavor: compute.FlavorSpec{ + Name: "m1.large", + VCPUs: vcpus, + Memory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // test code + }, Hosts: hostList, Weights: weights, } @@ -183,7 +174,7 @@ func TestKVMBinpackStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request api.ExternalSchedulerRequest + request compute.SchedulerRequest opts KVMBinpackStepOpts expectedWeights map[string]float64 wantErr bool @@ -485,7 +476,7 @@ func TestKVMBinpackStep_Run(t *testing.T) { func TestKVMBinpackStep_calcVMResources(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedMemBytes int64 expectedCPU int64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go index 421c6b75b..d80f8bb0f 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go @@ -9,6 +9,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -53,28 +54,22 @@ func newCRReservation(name, host, projectID, resourceGroup, az, totalMemory stri } } -func newCRRequest(hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newCRRequest(hosts []string) compute.SchedulerRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} - } - hints := map[string]any{ - api.HintKeyResourceGroup: []any{"group-2101"}, - } - spec := api.NovaSpec{ - ProjectID: "project-A", - AvailabilityZone: "qa-de-1a", - SchedulerHints: hints, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{MemoryMB: 8 * 1024}, - }, + hostList[i] = compute.Host{ComputeHost: h} } weights := make(map[string]float64, len(hosts)) for _, h := range hosts { weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.SchedulerRequest{ + ProjectID: "project-A", + AvailabilityZone: "qa-de-1a", + ResourceGroup: "group-2101", + Flavor: compute.FlavorSpec{ + Memory: *resource.NewQuantity(8*1024*(1<<20), resource.BinarySI), + }, Hosts: hostList, Weights: weights, } @@ -88,7 +83,7 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.SchedulerRequest opts KVMCommittedResourceReservationOpts expectedWeights map[string]float64 }{ @@ -171,9 +166,9 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { { name: "no resource group hint skips weigher", reservations: []*v1alpha1.Reservation{newCRReservation("res-1", "host1", "project-A", "group-2101", "qa-de-1a", "16Gi")}, - request: func() api.ExternalSchedulerRequest { + request: func() compute.SchedulerRequest { r := newCRRequest([]string{"host1", "host2"}) - r.Spec.Data.SchedulerHints = nil + r.ResourceGroup = "" return r }(), opts: KVMCommittedResourceReservationOpts{ReservationHostWeight: floatPtr(1.0), DefaultHostWeight: floatPtr(0.1)}, @@ -210,19 +205,15 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { reservations: []*v1alpha1.Reservation{ newCRReservation("res-1", "host1", "project-A", "group-2101", "qa-de-1a", "16Gi"), }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-A", - AvailabilityZone: "qa-de-1a", - Flavor: api.NovaObject[api.NovaFlavor]{Data: api.NovaFlavor{MemoryMB: 4096}}, - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForFailoverIntent), - api.HintKeyResourceGroup: "group-2101", - }, - }, + request: compute.SchedulerRequest{ + ProjectID: "project-A", + AvailabilityZone: "qa-de-1a", + Intent: api.ReserveForFailoverIntent, + ResourceGroup: "group-2101", + Flavor: compute.FlavorSpec{ + Memory: *resource.NewQuantity(4096*(1<<20), resource.BinarySI), }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, }, opts: KVMCommittedResourceReservationOpts{ReservationHostWeight: floatPtr(1.0), DefaultHostWeight: floatPtr(0.1)}, expectedWeights: map[string]float64{"host1": 0.0, "host2": 0.0}, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go index 3156c6699..f1a77cb0a 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go @@ -9,6 +9,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,22 +35,16 @@ func newCRMigrationSlotWeigher(t *testing.T, opts KVMCRMigrationSlotOpts, objs . } // migrationRequest builds a live-migration request from the given candidate hosts. -func migrationRequest(hosts ...string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func migrationRequest(hosts ...string) compute.SchedulerRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + hostList[i] = compute.Host{ComputeHost: h} } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: testInstanceUUID, - ProjectID: testProjectID, - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - }, - }, - }, - Hosts: hostList, + return compute.SchedulerRequest{ + InstanceUUID: testInstanceUUID, + ProjectID: testProjectID, + Intent: api.LiveMigrationIntent, + Hosts: hostList, } } @@ -135,7 +130,7 @@ func TestKVMCRMigrationSlotStep_Run(t *testing.T) { tests := []struct { name string objects []client.Object - request api.ExternalSchedulerRequest + request compute.SchedulerRequest opts KVMCRMigrationSlotOpts expectedWeights map[string]float64 }{ @@ -144,15 +139,11 @@ func TestKVMCRMigrationSlotStep_Run(t *testing.T) { objects: []client.Object{ confirmedSourceSlot("16Gi"), }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: testInstanceUUID, - ProjectID: testProjectID, - // no _nova_check_type → CreateIntent - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host-a"}, {ComputeHost: "host-b"}}, + request: compute.SchedulerRequest{ + InstanceUUID: testInstanceUUID, + ProjectID: testProjectID, + // no Intent → CreateIntent + Hosts: []compute.Host{{ComputeHost: "host-a"}, {ComputeHost: "host-b"}}, }, opts: defaultOpts, expectedWeights: map[string]float64{"host-a": 0.0, "host-b": 0.0}, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go index 16299100b..4165b3115 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go @@ -9,6 +9,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -112,48 +113,36 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { //nolint:unparam // instanceUUID varies in real usage - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newNovaRequest(instanceUUID string, evacuation bool, hosts []string) compute.SchedulerRequest { //nolint:unparam // instanceUUID varies in real usage + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + hostList[i] = compute.Host{ComputeHost: h} } - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", + req := compute.SchedulerRequest{ + ProjectID: "project-A", + InstanceUUID: instanceUUID, + NumInstances: 1, + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, + Flavor: compute.FlavorSpec{ + Name: "m1.large", + VCPUs: 4, + Memory: *resource.NewQuantity(int64(parseMemoryToMB("8Gi"))*(1<<20), resource.BinarySI), //nolint:gosec // test code + }, + Hosts: hostList, } - - var schedulerHints map[string]any if evacuation { - schedulerHints = map[string]any{ - "_nova_check_type": []any{"evacuate"}, - } - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: instanceUUID, - NumInstances: 1, - SchedulerHints: schedulerHints, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: 4, - MemoryMB: parseMemoryToMB("8Gi"), - ExtraSpecs: extraSpecs, - }, - }, + req.Intent = api.EvacuateIntent } weights := make(map[string]float64) for _, h := range hosts { weights[h] = 1.0 } + req.Weights = weights - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, - Hosts: hostList, - Weights: weights, - } + return req } // ============================================================================ @@ -166,7 +155,7 @@ func TestKVMFailoverEvacuationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.SchedulerRequest opts KVMFailoverEvacuationOpts expectedWeights map[string]float64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go index 44736e6a4..d53e40b6a 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go @@ -10,6 +10,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -50,30 +51,10 @@ func newFailoverReservationWithGroup(name, targetHost, resourceGroup string) *v1 } } -func newFailoverReservationRequest(resourceGroup string, hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newFailoverReservationRequest(resourceGroup string, hosts []string) compute.SchedulerRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: "test-instance", - NumInstances: 1, - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForFailoverIntent), - api.HintKeyResourceGroup: resourceGroup, - }, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: 4, - MemoryMB: 8192, - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "qemu", - }, - }, - }, + hostList[i] = compute.Host{ComputeHost: h} } weights := make(map[string]float64) @@ -81,8 +62,19 @@ func newFailoverReservationRequest(resourceGroup string, hosts []string) api.Ext weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.SchedulerRequest{ + ProjectID: "project-A", + InstanceUUID: "test-instance", + NumInstances: 1, + Intent: api.ReserveForFailoverIntent, + ResourceGroup: resourceGroup, + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, + Flavor: compute.FlavorSpec{ + Name: "m1.large", + VCPUs: 4, + Memory: *resource.NewQuantity(8192*(1<<20), resource.BinarySI), + }, Hosts: hostList, Weights: weights, } @@ -98,7 +90,7 @@ func TestKVMFailoverReservationConsolidationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.SchedulerRequest opts KVMFailoverReservationConsolidationOpts expectedWeights map[string]float64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go index b839388b2..0e0613493 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go @@ -7,8 +7,9 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -33,39 +34,32 @@ func newHypervisorWithInstances(name string, instanceIDs ...string) *hv1.Hypervi } } -func newInstanceGroupRequest(policy string, members, hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newInstanceGroupRequest(policy string, members, hosts []string) compute.SchedulerRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + hostList[i] = compute.Host{ComputeHost: h} } - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: "new-instance-uuid", - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: 4, - MemoryMB: 8192, - ExtraSpecs: extraSpecs, - }, + req := compute.SchedulerRequest{ + ProjectID: "project-A", + InstanceUUID: "new-instance-uuid", + NumInstances: 1, + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, + Flavor: compute.FlavorSpec{ + Name: "m1.large", + VCPUs: 4, + Memory: *resource.NewQuantity(8192*(1<<20), resource.BinarySI), }, + Hosts: hostList, } // Only set InstanceGroup if policy is provided if policy != "" { - spec.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - UUID: "instance-group-uuid", - Name: "test-instance-group", - Policy: policy, - Members: members, - }, + req.InstanceGroup = &compute.InstanceGroupSpec{ + UUID: "instance-group-uuid", + Policy: policy, + Members: members, } } @@ -73,12 +67,9 @@ func newInstanceGroupRequest(policy string, members, hosts []string) api.Externa for _, h := range hosts { weights[h] = 1.0 } + req.Weights = weights - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, - Hosts: hostList, - Weights: weights, - } + return req } func TestKVMInstanceGroupSoftAffinityStep_Run(t *testing.T) { @@ -87,7 +78,7 @@ func TestKVMInstanceGroupSoftAffinityStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request api.ExternalSchedulerRequest + request compute.SchedulerRequest expectedWeights map[string]float64 wantErr bool }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go index 1f5a07677..5a466c7e6 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,28 +30,10 @@ func newHypervisorWithCapacity(name, capacityCPU, capacityMem string) *hv1.Hyper } } -func newPreferSmallerHostsRequest(hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newPreferSmallerHostsRequest(hosts []string) compute.SchedulerRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} - } - - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: "instance-123", - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: 4, - MemoryMB: 8192, - ExtraSpecs: extraSpecs, - }, - }, + hostList[i] = compute.Host{ComputeHost: h} } weights := make(map[string]float64) @@ -59,8 +41,17 @@ func newPreferSmallerHostsRequest(hosts []string) api.ExternalSchedulerRequest { weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.SchedulerRequest{ + ProjectID: "project-A", + InstanceUUID: "instance-123", + NumInstances: 1, + HypervisorFamily: compute.HypervisorFamilyKVM, + HypervisorType: compute.HypervisorTypeQEMU, + Flavor: compute.FlavorSpec{ + Name: "m1.large", + VCPUs: 4, + Memory: *resource.NewQuantity(8192*(1<<20), resource.BinarySI), + }, Hosts: hostList, Weights: weights, } @@ -181,7 +172,7 @@ func TestKVMPreferSmallerHostsStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request api.ExternalSchedulerRequest + request compute.SchedulerRequest opts KVMPreferSmallerHostsStepOpts expectedWeights map[string]float64 wantErr bool diff --git a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go index 5005bae45..58770d271 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go @@ -9,9 +9,9 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -219,18 +219,14 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.SchedulerRequest downvotedHosts map[string]struct{} }{ { name: "Noisy project", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project1", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: sched.SchedulerRequest{ + ProjectID: "project1", + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -243,13 +239,9 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { }, { name: "Non-noisy project", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project2", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: sched.SchedulerRequest{ + ProjectID: "project2", + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -259,8 +251,8 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { }, { name: "No noisy project data", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.SchedulerRequest{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go index 701cef92e..138743750 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go @@ -9,9 +9,9 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -218,13 +218,13 @@ func TestVMwareAvoidLongTermContendedHostsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.SchedulerRequest expected map[string]float64 }{ { name: "Avoid contended hosts", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.SchedulerRequest{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -240,8 +240,8 @@ func TestVMwareAvoidLongTermContendedHostsStep_Run(t *testing.T) { }, { name: "Missing data", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.SchedulerRequest{ + Hosts: []sched.Host{ {ComputeHost: "host4"}, {ComputeHost: "host5"}, }, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go index 624ee7e14..517989707 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go @@ -9,9 +9,9 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -218,13 +218,13 @@ func TestVMwareAvoidShortTermContendedHostsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.SchedulerRequest expected map[string]float64 }{ { name: "Avoid contended hosts", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.SchedulerRequest{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -240,8 +240,8 @@ func TestVMwareAvoidShortTermContendedHostsStep_Run(t *testing.T) { }, { name: "Missing data", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.SchedulerRequest{ + Hosts: []sched.Host{ {ComputeHost: "host4"}, {ComputeHost: "host5"}, // No data for host5 }, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go b/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go index 404722f43..4a06b5852 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go @@ -7,10 +7,11 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -156,25 +157,19 @@ func TestVMwareBinpackStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.SchedulerRequest expectedHigherScore string expectedLowerScore string }{ { name: "binpack prefers hosts with higher utilization", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 1000, // 1GB - VCPUs: 1, - }, - }, - }, + request: sched.SchedulerRequest{ + NumInstances: 1, + Flavor: sched.FlavorSpec{ + Memory: *resource.NewQuantity(1000*(1<<20), resource.BinarySI), // 1GB + VCPUs: 1, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -187,19 +182,13 @@ func TestVMwareBinpackStep_Run(t *testing.T) { }, { name: "handles missing host data gracefully", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 1000, - VCPUs: 1, - }, - }, - }, + request: sched.SchedulerRequest{ + NumInstances: 1, + Flavor: sched.FlavorSpec{ + Memory: *resource.NewQuantity(1000*(1<<20), resource.BinarySI), + VCPUs: 1, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "unknown_host"}, // No utilization data }, @@ -292,23 +281,17 @@ func TestVMwareBinpackStep_CalcVMResources(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.SchedulerRequest expectedMemory int64 expectedCPU int64 }{ { name: "single instance", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 2048, - VCPUs: 4, - }, - }, - }, + request: sched.SchedulerRequest{ + NumInstances: 1, + Flavor: sched.FlavorSpec{ + Memory: *resource.NewQuantity(2048*(1<<20), resource.BinarySI), + VCPUs: 4, }, }, expectedMemory: 2048 * (1 << 20), // 2048 MiB in bytes @@ -316,17 +299,11 @@ func TestVMwareBinpackStep_CalcVMResources(t *testing.T) { }, { name: "multiple instances", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 3, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 1024, - VCPUs: 2, - }, - }, - }, + request: sched.SchedulerRequest{ + NumInstances: 3, + Flavor: sched.FlavorSpec{ + Memory: *resource.NewQuantity(1024*(1<<20), resource.BinarySI), + VCPUs: 2, }, }, expectedMemory: 3 * 1024 * (1 << 20), // 3 instances * 1024 MiB in bytes diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index fcbdaa7c2..bd1ae15fb 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -18,6 +18,7 @@ import ( novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" @@ -32,6 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +// passthroughGatherer is a nova.CandidateGatherer that leaves the request's +// preselected hosts unchanged (the tests seed their own eligible hosts). +type passthroughGatherer struct{} + +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *compute.SchedulerRequest) error { + return nil +} + // ============================================================================ // Integration Tests // ============================================================================ @@ -1142,12 +1151,13 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h Build() novaController := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, - Monitor: getSharedMonitor(), + Monitor: getSharedMonitor(), + Gatherer: passthroughGatherer{}, } // Register all pipelines needed for testing pipelines := []v1alpha1.Pipeline{ @@ -1272,6 +1282,14 @@ func (api *testHTTPAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req pipelineName = "test-filter-pipeline" } + // Resolve the scheduling intent from the request hints, mirroring the + // production handler (external_scheduler_api.go). The pipeline reads the + // resolved intent off the typed request, so it must be set on the decision. + intent, err := requestData.GetIntent() + if err != nil { + intent = novaapi.CreateIntent + } + decision := &v1alpha1.Decision{ TypeMeta: metav1.TypeMeta{ Kind: "Decision", @@ -1287,6 +1305,7 @@ func (api *testHTTPAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req }, ResourceID: requestData.Spec.Data.InstanceUUID, NovaRaw: &raw, + Intent: intent, }, } @@ -1342,12 +1361,13 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, Build() novaController := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, - Monitor: getSharedMonitor(), + Monitor: getSharedMonitor(), + Gatherer: passthroughGatherer{}, } pipelines := []v1alpha1.Pipeline{ { diff --git a/internal/scheduling/reservations/failoverhost/integration_test.go b/internal/scheduling/reservations/failoverhost/integration_test.go index 013d8e269..2fee2c5ea 100644 --- a/internal/scheduling/reservations/failoverhost/integration_test.go +++ b/internal/scheduling/reservations/failoverhost/integration_test.go @@ -15,6 +15,7 @@ import ( novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" @@ -26,6 +27,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// passthroughGatherer is a nova.CandidateGatherer that leaves the request's +// preselected hosts unchanged. The integration tests seed their own eligible +// hosts, so gathering is a no-op here. +type passthroughGatherer struct{} + +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { + return nil +} + // ============================================================================ // Integration Tests // @@ -262,12 +272,15 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors k8sClient := newTestClient(t, objects...) controller := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[sched.SchedulerRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: sharedPipelineMonitor(), + // KVM requests trigger candidate gathering; the test seeds its own + // eligible hosts, so use a passthrough gatherer that preserves them. + Gatherer: passthroughGatherer{}, } // Mirror the production KVM pipeline for the dimensions that shape a failover // partition: AZ, trait compatibility and whole-host capacity. @@ -448,6 +461,12 @@ func (a *integrationHTTPAPI) handle(w http.ResponseWriter, r *http.Request) { http.Error(w, "marshal", http.StatusInternalServerError) return } + // Resolve intent from the request hints, mirroring the production handler; + // the pipeline reads the resolved intent off the typed request. + intent, err := reqData.GetIntent() + if err != nil { + intent = novaapi.CreateIntent + } decision := &v1alpha1.Decision{ TypeMeta: metav1.TypeMeta{Kind: "Decision", APIVersion: "cortex.cloud/v1alpha1"}, ObjectMeta: metav1.ObjectMeta{GenerateName: "nova-"}, @@ -456,6 +475,7 @@ func (a *integrationHTTPAPI) handle(w http.ResponseWriter, r *http.Request) { PipelineRef: corev1.ObjectReference{Name: evacProbePipeline}, ResourceID: reqData.Spec.Data.InstanceUUID, NovaRaw: &runtime.RawExtension{Raw: rawBytes}, + Intent: intent, }, } if err := a.delegate.ProcessNewDecisionFromAPI(r.Context(), decision); err != nil { From cc349c508a8c8abb2b79393c6dc6f315cadde051 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 15:48:32 +0100 Subject: [PATCH 04/21] feat(nova): make compute.Scheduler the entrypoint for the external-scheduler API Phase 4a. The Nova external-scheduler HTTP handler now calls the shared compute.Scheduler directly instead of ferrying a Decision CRD through the controller's ProcessNewDecisionFromAPI. - external_scheduler_api.go: NewAPI takes a *compute.Scheduler, VirtualMachineHandler, crs.Recorder and FeatureGates. The handler maps the wire request to a compute.SchedulerRequest (exported ToSchedulerRequest) and runs the scheduler with a single post-hook that performs VM-resource upsert and committed-resource tracking under the scheduler lock. A `runner` interface seam keeps the handler unit-testable. - filter_weigher_pipeline_controller.go: drop the Decision Reconcile/watch, ProcessNewDecisionFromAPI, process(), peekReadOnly and the processMu lock. The controller now only (re)configures pipelines via its Pipeline/Knowledge watches and exposes GetPipeline + the exported Gatherer for the scheduler. Delete the HTTPAPIDelegate interface. - vm_handler.Upsert and crs.Recorder.Record* now take the typed result + intent instead of a *v1alpha1.Decision. - main.go: construct one shared compute.Scheduler over the Nova controller and inject it, the vm handler, CR recorder and feature gates into nova.NewAPI. The reservation/commitment/capacity/failover controllers still reach the pipeline via the HTTP SchedulerClient (now backed by compute.Scheduler through the same endpoint); they move in-process in the next step, after which SchedulerClient and SchedulerURL are removed. Decision CRD remains the cinder/manila/machines/pods carrier until Phases 5-6. go build ./..., go test ./..., go vet ./... and make lint are all clean. Signed-off-by: Markus Wieland --- cmd/manager/main.go | 21 +- internal/scheduling/nova/adapter.go | 4 +- internal/scheduling/nova/cr_metrics_test.go | 49 +- internal/scheduling/nova/crs/recorder.go | 16 +- internal/scheduling/nova/crs/recorder_test.go | 2 +- .../scheduling/nova/external_scheduler_api.go | 124 ++- .../nova/external_scheduler_api_test.go | 198 +++-- .../filter_weigher_pipeline_controller.go | 244 +---- ...filter_weigher_pipeline_controller_test.go | 840 ------------------ internal/scheduling/nova/integration_test.go | 14 +- internal/scheduling/nova/vm_handler.go | 28 +- internal/scheduling/nova/vm_handler_test.go | 21 +- .../reservations/failover/integration_test.go | 48 +- .../failoverhost/integration_test.go | 31 +- 14 files changed, 285 insertions(+), 1355 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 9ba24d770..6a242266c 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -54,6 +54,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/external" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" schedulinglib "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/machines" "github.com/cobaltcore-dev/cortex/internal/scheduling/manila" @@ -421,27 +422,31 @@ func main() { placementCounter := crs.NewPlacementCounter() // Filter-weigher pipeline controller setup. filterWeigherController := &nova.FilterWeigherPipelineController{ - Monitor: filterWeigherPipelineMonitor, - FeatureGates: featureGates, - CRRecorder: crs.Recorder{ - NoHostFoundCounter: noHostFoundCounter, - PlacementCounter: placementCounter, - }, + Monitor: filterWeigherPipelineMonitor, } metrics.Registry.MustRegister(noHostFoundCounter) metrics.Registry.MustRegister(placementCounter) // Inferred through the base controller. filterWeigherController.Client = multiclusterClient - filterWeigherController.CRRecorder.Client = multiclusterClient if err := filterWeigherController.SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "nova FilterWeigherPipelineController") os.Exit(1) } + // Shared in-process scheduler: owns the scheduling lock and runs the + // pipeline. The Nova HTTP API and the reservation controllers call it + // directly (no self-directed HTTP). + novaScheduler := compute.NewScheduler(filterWeigherController, filterWeigherController.Gatherer) + vmHandler := &nova.VirtualMachineHandler{Client: multiclusterClient} + crRecorder := crs.Recorder{ + Client: multiclusterClient, + NoHostFoundCounter: noHostFoundCounter, + PlacementCounter: placementCounter, + } novaAPIConfig := conf.GetConfigOrDie[nova.HTTPAPIConfig]() setupLog.Info("loaded nova API config", "evacuationShuffleK", novaAPIConfig.EvacuationShuffleK, "novaLimitHostsToRequest", novaAPIConfig.NovaLimitHostsToRequest) - nova.NewAPI(novaAPIConfig, filterWeigherController).Init(mux) + nova.NewAPI(novaAPIConfig, novaScheduler, vmHandler, crRecorder, featureGates).Init(mux) // Detector pipeline controller setup. novaClient := nova.NewNovaClient() diff --git a/internal/scheduling/nova/adapter.go b/internal/scheduling/nova/adapter.go index a70e03327..4a0d0bd77 100644 --- a/internal/scheduling/nova/adapter.go +++ b/internal/scheduling/nova/adapter.go @@ -23,14 +23,14 @@ var unsupportedCapabilityOps = []string{ "s==", "s!=", "s<", "s<=", "s>", "s>=", "", } -// toSchedulerRequest translates the Nova over-the-wire request into the typed, +// ToSchedulerRequest translates the Nova over-the-wire request into the typed, // Nova-agnostic compute.SchedulerRequest the pipeline consumes. This is the one // place that reads Nova's loosely-typed scheduler hints and flavor/image extra // specs; every downstream filter/weigher sees only resolved typed fields. // // intent and pipeline are resolved by the caller (the HTTP API layer) and passed // in. -func toSchedulerRequest( +func ToSchedulerRequest( wire api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent, pipeline string, diff --git a/internal/scheduling/nova/cr_metrics_test.go b/internal/scheduling/nova/cr_metrics_test.go index f5553808b..e58abb33a 100644 --- a/internal/scheduling/nova/cr_metrics_test.go +++ b/internal/scheduling/nova/cr_metrics_test.go @@ -4,14 +4,16 @@ package nova import ( + "bytes" "context" "encoding/json" "fmt" + "net/http" + "net/http/httptest" "testing" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -235,13 +237,10 @@ func TestLogNoHostFound(t *testing.T) { reg := prometheus.NewRegistry() reg.MustRegister(counter) - controller := &FilterWeigherPipelineController{ - CRRecorder: crs.Recorder{ - Client: fakeClient, - NoHostFoundCounter: counter, - }, + recorder := crs.Recorder{ + Client: fakeClient, + NoHostFoundCounter: counter, } - controller.Client = fakeClient requestFlavorName := flavorName if tt.payg { @@ -259,11 +258,7 @@ func TestLogNoHostFound(t *testing.T) { Context: api.NovaRequestContext{ProjectID: projectID}, Hosts: tt.requestHosts, } - decision := &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{Intent: api.CreateIntent}, - } - - controller.CRRecorder.RecordNoHostFound(context.Background(), decision, request) + recorder.RecordNoHostFound(context.Background(), api.CreateIntent, request) if tt.expectedCase == "" { total := testutil.ToFloat64(counter.WithLabelValues("no_cr", flavorGroup, string(api.CreateIntent))) + @@ -373,11 +368,7 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, - FeatureGates: FeatureGates{CommittedResourceTracking: enabled}, - CRRecorder: crs.Recorder{ - Client: fakeClient, - NoHostFoundCounter: counter, - }, + Gatherer: passthroughGatherer{}, } controller.PipelineConfigs["test-pipeline"] = pipelineConf initResult := controller.InitPipeline(context.Background(), pipelineConf) @@ -386,18 +377,20 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { } controller.Pipelines["test-pipeline"] = initResult.Pipeline - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{Name: "test-decision", Namespace: "default"}, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - NovaRaw: &runtime.RawExtension{Raw: novaRaw}, - }, - } + // Drive the committed-resource tracking path through the real HTTP API + // handler, where the feature gate now lives (as a post-hook). + scheduler := sched.NewScheduler(controller, controller.Gatherer) + crRecorder := crs.Recorder{Client: fakeClient, NoHostFoundCounter: counter} + api := newTestAPIWithScheduler( + HTTPAPIConfig{}, + scheduler, + crRecorder, + FeatureGates{CommittedResourceTracking: enabled}, + ) - if err := controller.ProcessNewDecisionFromAPI(context.Background(), decision); err != nil { - t.Fatalf("ProcessNewDecisionFromAPI: %v", err) - } + req := httptest.NewRequest(http.MethodPost, "/scheduler/nova/external", bytes.NewReader(novaRaw)) + w := httptest.NewRecorder() + api.NovaExternalScheduler(w, req) count := testutil.CollectAndCount(counter) if enabled && count == 0 { diff --git a/internal/scheduling/nova/crs/recorder.go b/internal/scheduling/nova/crs/recorder.go index 99cd0d8f4..71177213e 100644 --- a/internal/scheduling/nova/crs/recorder.go +++ b/internal/scheduling/nova/crs/recorder.go @@ -11,6 +11,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/prometheus/client_golang/prometheus" @@ -55,14 +56,14 @@ func (r *Recorder) resolveFlavorGroup(ctx context.Context, flavorName string) (s // RecordPlacement writes the placed VM UUID into the matching Reservation slot // and emits placement metrics. Called after a successful Nova placement. -func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decision, request api.ExternalSchedulerRequest) { +func (r *Recorder) RecordPlacement(ctx context.Context, result sched.SchedulerResult, intent v1alpha1.SchedulingIntent, request api.ExternalSchedulerRequest) { log := ctrl.LoggerFrom(ctx) instanceUUID := request.Spec.Data.InstanceUUID projectID := request.Context.ProjectID flavorName := request.Spec.Data.Flavor.Data.Name - selectedHost := *decision.Status.Result.TargetHost - intent := string(decision.Spec.Intent) + selectedHost := *result.TargetHost + intentStr := string(intent) flavorGroupName, flavorInGroup, err := r.resolveFlavorGroup(ctx, flavorName) if err != nil { @@ -72,7 +73,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis log.Error(err, "CR allocation: failed to resolve flavor group", "flavor", flavorName, "instanceUUID", instanceUUID) if r.PlacementCounter != nil { - r.PlacementCounter.WithLabelValues("unknown", intent, "error").Inc() + r.PlacementCounter.WithLabelValues("unknown", intentStr, "error").Inc() } } return @@ -97,7 +98,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis activeCRs = append(activeCRs, cr) } - candidateHosts := decision.Status.Result.OrderedHosts + candidateHosts := result.OrderedHosts crOutcome := ClassifyPlacement(evaluator, activeCRs, candidateHosts, projectID, flavorGroupName) log.V(1).Info("CR allocation: placement classified", @@ -108,7 +109,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis "flavorGroup", flavorGroupName, ) if r.PlacementCounter != nil { - r.PlacementCounter.WithLabelValues(flavorGroupName, intent, crOutcome).Inc() + r.PlacementCounter.WithLabelValues(flavorGroupName, intentStr, crOutcome).Inc() } if crOutcome != "slot_used" { @@ -170,13 +171,12 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis } // RecordNoHostFound classifies a no-host-found result and emits a log line and metric. -func (r *Recorder) RecordNoHostFound(ctx context.Context, decision *v1alpha1.Decision, request api.ExternalSchedulerRequest) { +func (r *Recorder) RecordNoHostFound(ctx context.Context, intent v1alpha1.SchedulingIntent, request api.ExternalSchedulerRequest) { log := ctrl.LoggerFrom(ctx) projectID := request.Context.ProjectID flavorName := request.Spec.Data.Flavor.Data.Name instanceUUID := request.Spec.Data.InstanceUUID - intent := decision.Spec.Intent flavorGroupName, flavorInGroup, err := r.resolveFlavorGroup(ctx, flavorName) if err != nil { diff --git a/internal/scheduling/nova/crs/recorder_test.go b/internal/scheduling/nova/crs/recorder_test.go index dd136784f..f9fbd3bd8 100644 --- a/internal/scheduling/nova/crs/recorder_test.go +++ b/internal/scheduling/nova/crs/recorder_test.go @@ -271,7 +271,7 @@ func TestRecordCRAllocation(t *testing.T) { PlacementCounter: counter, } - recorder.RecordPlacement(context.Background(), tt.decision, tt.request) + recorder.RecordPlacement(context.Background(), *tt.decision.Status.Result, tt.decision.Spec.Intent, tt.request) var res v1alpha1.Reservation if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: tt.checkSlot}, &res); err != nil { diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index 9e89f5f2a..8ea17c929 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -12,16 +12,14 @@ import ( "io" "math/rand" "net/http" + "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -46,27 +44,45 @@ func (c HTTPAPIConfig) forcedDestinationEnabled() bool { return c.ForcedDestinationEnabled == nil || *c.ForcedDestinationEnabled } -type HTTPAPIDelegate interface { - // Process the decision from the API. Should create and return the updated decision. - ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error -} - type HTTPAPI interface { // Bind the server handlers. Init(*http.ServeMux) } +// runner is the subset of compute.Scheduler the HTTP API depends on. It is an +// interface so the handler can be unit-tested with a mock. *compute.Scheduler +// satisfies it. +type runner interface { + Run(ctx context.Context, req compute.SchedulerRequest, pre compute.PreHook, post compute.PostHook) (compute.RunOutcome, error) +} + type httpAPI struct { - monitor scheduling.APIMonitor - delegate HTTPAPIDelegate - config HTTPAPIConfig + monitor scheduling.APIMonitor + scheduler runner + vmHandler *VirtualMachineHandler + crRecorder crs.Recorder + featureGates FeatureGates + config HTTPAPIConfig } -func NewAPI(config HTTPAPIConfig, delegate HTTPAPIDelegate) HTTPAPI { +// NewAPI builds the Nova external-scheduler HTTP API. It calls the shared +// compute.Scheduler directly; VM-resource creation and committed-resource +// tracking run as post-hooks after the pipeline, under the scheduler's lock. +func NewAPI( + config HTTPAPIConfig, + scheduler *compute.Scheduler, + vmHandler *VirtualMachineHandler, + crRecorder crs.Recorder, + featureGates FeatureGates, +) HTTPAPI { + return &httpAPI{ - monitor: scheduling.NewSchedulerMonitor(), - delegate: delegate, - config: config, + monitor: scheduling.NewSchedulerMonitor(), + scheduler: scheduler, + vmHandler: vmHandler, + crRecorder: crRecorder, + featureGates: featureGates, + config: config, } } @@ -202,7 +218,6 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req c.Respond(logr.Logger{}, http.StatusInternalServerError, err, "failed to read request body") return } - raw := runtime.RawExtension{Raw: body} var requestData api.ExternalSchedulerRequest // Copy the raw body to a io.Reader for json deserialization. cp := body @@ -257,43 +272,56 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req intent = api.CreateIntent } - decision := &v1alpha1.Decision{ - TypeMeta: metav1.TypeMeta{ - Kind: "Decision", - APIVersion: "cortex.cloud/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "nova-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: requestData.Pipeline, - }, - ResourceID: requestData.Spec.Data.InstanceUUID, - NovaRaw: &raw, - Intent: intent, - }, - } ctx := r.Context() - if err := httpAPI.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { - c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) - return - } - // Check if the decision contains status conditions indicating an error. - if meta.IsStatusConditionFalse(decision.Status.Conditions, v1alpha1.DecisionConditionReady) { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision contains error condition"), "decision failed") - return + + // Translate the Nova wire request into the typed pipeline request. + schedReq := ToSchedulerRequest(requestData, intent, requestData.Pipeline) + + // Side effects run as a single post-hook after the pipeline, under the + // scheduler lock. It operates on the Nova wire request (which carries the + // fields the VM resource and CR tracking need) plus the typed result. + postHook := func(ctx context.Context, _ compute.SchedulerRequest, out compute.RunOutcome) error { + // VM-resource creation. + switch { + case requestData.Options.ReadOnly: + logger.V(1).Info("skipping virtual machine creation, request is read-only", "instanceUUID", requestData.Spec.Data.InstanceUUID) + case !httpAPI.featureGates.VirtualMachineCreation: + logger.V(1).Info("skipping virtual machine creation, feature gate disabled", "instanceUUID", requestData.Spec.Data.InstanceUUID) + case !slices.Contains(api.NovaIntents, intent): + logger.V(1).Info("skipping virtual machine creation, intent not triggered by Nova", "instanceUUID", requestData.Spec.Data.InstanceUUID, "intent", intent) + case schedReq.HypervisorFamily != compute.HypervisorFamilyKVM: + logger.V(1).Info("skipping virtual machine creation, unsupported hypervisor type", "instanceUUID", requestData.Spec.Data.InstanceUUID) + default: + explanation := scheduling.GenerateExplanation(&out.Result, nil) + if upsertErr := httpAPI.vmHandler.Upsert(ctx, out.Result, explanation, requestData, intent); upsertErr != nil { + if httpAPI.featureGates.VirtualMachineCreationMandatory { + return fmt.Errorf("failed to upsert virtual machine resource: %w", upsertErr) + } + // Non-fatal: the placement decision itself succeeded. + logger.Error(upsertErr, "failed to upsert virtual machine resource", "instanceUUID", requestData.Spec.Data.InstanceUUID) + } + } + // Committed-resource tracking. + if httpAPI.featureGates.CommittedResourceTracking && !requestData.Options.SkipCommittedResourceTracking { + if out.Result.TargetHost != nil { + httpAPI.crRecorder.RecordPlacement(ctx, out.Result, intent, requestData) + } else { + httpAPI.crRecorder.RecordNoHostFound(ctx, intent, requestData) + } + } + return nil } - if decision.Status.Result == nil { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision didn't produce a result"), "decision failed") + + out, err := httpAPI.scheduler.Run(ctx, schedReq, nil, postHook) + if err != nil { + c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return } - hosts := decision.Status.Result.OrderedHosts + hosts := out.Result.OrderedHosts if httpAPI.config.NovaLimitHostsToRequest { hosts = limitHostsToRequest(requestData, hosts) logger.V(1).Info("limited hosts to request", - "hosts", hosts, "originalHosts", decision.Status.Result.OrderedHosts) + "hosts", hosts, "originalHosts", out.Result.OrderedHosts) } // This is a hack to address the problem that Nova only uses the first host in hosts for evacuation requests. // Only for evacuation we shuffle the first k hosts to ensure that we do not get stuck on a single host diff --git a/internal/scheduling/nova/external_scheduler_api_test.go b/internal/scheduling/nova/external_scheduler_api_test.go index f3dd2c82b..db1b5b97e 100644 --- a/internal/scheduling/nova/external_scheduler_api_test.go +++ b/internal/scheduling/nova/external_scheduler_api_test.go @@ -15,47 +15,74 @@ import ( novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type mockHTTPAPIDelegate struct { - processDecisionFunc func(ctx context.Context, decision *v1alpha1.Decision) error +// mockRunner is a test double for the scheduler the HTTP API depends on. runFunc +// receives the mapped request and may invoke the post-hook to exercise the +// handler's side-effect path; it returns the outcome/error the handler sees. +type mockRunner struct { + runFunc func(ctx context.Context, req compute.SchedulerRequest, post compute.PostHook) (compute.RunOutcome, error) } -func (m *mockHTTPAPIDelegate) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { - if m.processDecisionFunc != nil { - return m.processDecisionFunc(ctx, decision) +func (m *mockRunner) Run(ctx context.Context, req compute.SchedulerRequest, _ compute.PreHook, post compute.PostHook) (compute.RunOutcome, error) { + if m.runFunc != nil { + return m.runFunc(ctx, req, post) } - return nil + return compute.RunOutcome{}, nil } -func TestNewAPI(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - - api := NewAPI(HTTPAPIConfig{}, delegate) - - if api == nil { - t.Fatal("NewAPI returned nil") +// newTestAPI builds an *httpAPI with the given config and mock runner, leaving +// the side-effect handlers nil (the handler tests that need them set runFunc to +// not invoke the post-hook, or provide their own). +func newTestAPI(config HTTPAPIConfig, r runner) *httpAPI { + return &httpAPI{ + monitor: scheduling.NewSchedulerMonitor(), + scheduler: r, + config: config, } +} - httpAPI, ok := api.(*httpAPI) - if !ok { - t.Fatal("NewAPI did not return httpAPI type") +// newTestAPIWithScheduler builds an *httpAPI over a real compute.Scheduler plus +// the CR recorder and feature gates, for tests that exercise the committed- +// resource post-hook end-to-end. +func newTestAPIWithScheduler(config HTTPAPIConfig, scheduler runner, crRecorder crs.Recorder, featureGates FeatureGates) *httpAPI { + return &httpAPI{ + monitor: scheduling.NewSchedulerMonitor(), + scheduler: scheduler, + crRecorder: crRecorder, + featureGates: featureGates, + config: config, } +} - if httpAPI.delegate != delegate { - t.Error("Delegate not set correctly") - } +// passthroughGatherer is a CandidateGatherer that leaves the request's hosts +// unchanged (tests seed their own host sets). +type passthroughGatherer struct{} + +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *compute.SchedulerRequest) error { + return nil +} + +func TestNewAPI(t *testing.T) { + r := &mockRunner{} + httpAPI := newTestAPI(HTTPAPIConfig{}, r) + + if httpAPI.scheduler != r { + t.Error("scheduler (runner) not set correctly") + } if httpAPI.monitor.ApiRequestsTimer == nil { t.Error("Monitor not initialized") } } func TestHTTPAPI_Init(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(HTTPAPIConfig{}, delegate) + var api HTTPAPI = newTestAPI(HTTPAPIConfig{}, &mockRunner{}) mux := http.NewServeMux() api.Init(mux) @@ -72,8 +99,7 @@ func TestHTTPAPI_Init(t *testing.T) { } func TestHTTPAPI_canRunScheduler(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, &mockRunner{}) tests := []struct { name string @@ -263,20 +289,23 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { + r := &mockRunner{ + runFunc: func(_ context.Context, _ compute.SchedulerRequest, _ compute.PostHook) (compute.RunOutcome, error) { if tt.processDecisionErr != nil { - return tt.processDecisionErr + return compute.RunOutcome{}, tt.processDecisionErr } if tt.decisionResult != nil { - decision.Status = tt.decisionResult.Status - return nil + if tt.decisionResult.Status.Result != nil { + return compute.RunOutcome{Result: *tt.decisionResult.Status.Result}, nil + } + // A decisionResult without a Result models a failed run. + return compute.RunOutcome{}, errors.New("scheduling failed") } - return nil + return compute.RunOutcome{}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) var body *strings.Reader if tt.body != "" { @@ -318,15 +347,12 @@ func TestHTTPAPI_NovaExternalScheduler_EmptyHostsSerializedAsArray(t *testing.T) // When the pipeline filters out all hosts, OrderedHosts is nil. The // response must serialize hosts as an empty array, not null, since Nova's // schema requires an array. - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - decision.Status = v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{OrderedHosts: nil}, - } - return nil + r := &mockRunner{ + runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.RunOutcome, error) { + return compute.RunOutcome{Result: compute.SchedulerResult{OrderedHosts: nil}}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) req := novaapi.ExternalSchedulerRequest{ Spec: novaapi.NovaObject[novaapi.NovaSpec]{ @@ -357,20 +383,16 @@ func TestHTTPAPI_NovaExternalScheduler_EmptyHostsSerializedAsArray(t *testing.T) } } -func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) { - var capturedDecision *v1alpha1.Decision - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - capturedDecision = decision - // Set a successful result to avoid "decision didn't produce a result" error - decision.Status.Result = &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - } - return nil +func TestHTTPAPI_NovaExternalScheduler_RequestMapping(t *testing.T) { + var captured compute.SchedulerRequest + r := &mockRunner{ + runFunc: func(_ context.Context, req compute.SchedulerRequest, _ compute.PostHook) (compute.RunOutcome, error) { + captured = req + return compute.RunOutcome{Result: compute.SchedulerResult{OrderedHosts: []string{"host1"}}}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) requestData := novaapi.ExternalSchedulerRequest{ Spec: novaapi.NovaObject[novaapi.NovaSpec]{ @@ -400,29 +422,16 @@ func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) { t.Errorf("Expected status %d, got %d", http.StatusOK, w.Code) } - if capturedDecision == nil { - t.Fatal("Decision was not captured") - } - - // Verify decision fields - if capturedDecision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainNova { - t.Errorf("Expected scheduling domain %s, got %s", v1alpha1.SchedulingDomainNova, capturedDecision.Spec.SchedulingDomain) + // Verify the handler mapped the wire request into the typed scheduler request. + if captured.Pipeline != "test-pipeline" { + t.Errorf("Expected pipeline 'test-pipeline', got %s", captured.Pipeline) } - - if capturedDecision.Spec.PipelineRef.Name != "test-pipeline" { - t.Errorf("Expected pipeline 'test-pipeline', got %s", capturedDecision.Spec.PipelineRef.Name) - } - - if capturedDecision.Spec.ResourceID != "test-uuid-123" { - t.Errorf("Expected resource ID 'test-uuid-123', got %s", capturedDecision.Spec.ResourceID) + if captured.InstanceUUID != "test-uuid-123" { + t.Errorf("Expected instance UUID 'test-uuid-123', got %s", captured.InstanceUUID) } - - if capturedDecision.GenerateName != "nova-" { - t.Errorf("Expected generate name 'nova-', got %s", capturedDecision.GenerateName) - } - - if capturedDecision.Spec.NovaRaw == nil { - t.Error("NovaRaw should not be nil") + // No explicit _nova_check_type hint means the handler defaults to the create intent. + if captured.Intent != novaapi.CreateIntent { + t.Errorf("Expected intent %s, got %s", novaapi.CreateIntent, captured.Intent) } } @@ -445,14 +454,14 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestination(t *testing.T) { }, } - delegateCalled := false - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - delegateCalled = true - return nil + schedulerCalled := false + r := &mockRunner{ + runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.RunOutcome, error) { + schedulerCalled = true + return compute.RunOutcome{}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) body, err := json.Marshal(requestData) if err != nil { @@ -466,8 +475,8 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestination(t *testing.T) { if w.Code != http.StatusOK { t.Fatalf("Expected status %d, got %d", http.StatusOK, w.Code) } - if delegateCalled { - t.Error("delegate should not be called for forced destination requests") + if schedulerCalled { + t.Error("scheduler should not be called for forced destination requests") } var response novaapi.ExternalSchedulerResponse if err := json.NewDecoder(w.Body).Decode(&response); err != nil { @@ -495,14 +504,14 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationNoWeights(t *testing.T) // for forced-destination requests. } - delegateCalled := false - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - delegateCalled = true - return nil + schedulerCalled := false + r := &mockRunner{ + runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.RunOutcome, error) { + schedulerCalled = true + return compute.RunOutcome{}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) body, err := json.Marshal(requestData) if err != nil { @@ -516,8 +525,8 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationNoWeights(t *testing.T) if w.Code != http.StatusOK { t.Fatalf("Expected status %d, got %d (forced destination with no weights must not be rejected)", http.StatusOK, w.Code) } - if delegateCalled { - t.Error("delegate should not be called for forced destination requests") + if schedulerCalled { + t.Error("scheduler should not be called for forced destination requests") } var response novaapi.ExternalSchedulerResponse if err := json.NewDecoder(w.Body).Decode(&response); err != nil { @@ -548,15 +557,15 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationDisabled(t *testing.T) { Pipeline: "test-pipeline", } - delegateCalled := false - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - delegateCalled = true - return nil + schedulerCalled := false + r := &mockRunner{ + runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.RunOutcome, error) { + schedulerCalled = true + return compute.RunOutcome{}, nil }, } disabled := false - api := NewAPI(HTTPAPIConfig{ForcedDestinationEnabled: &disabled}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{ForcedDestinationEnabled: &disabled}, r) body, err := json.Marshal(requestData) if err != nil { @@ -567,8 +576,8 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationDisabled(t *testing.T) { api.NovaExternalScheduler(w, req) - if !delegateCalled { - t.Error("delegate should be called when forced destination is disabled") + if !schedulerCalled { + t.Error("scheduler should be called when forced destination is disabled") } } @@ -804,8 +813,7 @@ func TestShuffleTopHosts(t *testing.T) { } func TestHTTPAPI_inferPipelineName(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, &mockRunner{}) tests := []struct { name string diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index 93862ed44..b12dd436f 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -5,21 +5,11 @@ package nova import ( "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sync" - "time" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" - "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/weighers" "github.com/cobaltcore-dev/cortex/pkg/multicluster" @@ -31,32 +21,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) -// The decision pipeline controller takes decision resources containing a -// placement request spec and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. -// -// Additionally, the controller watches for pipeline and step changes to -// reconfigure the pipelines as needed. +// FilterWeigherPipelineController owns the Nova filter-weigher pipelines: it +// watches Pipeline and Knowledge resources to (re)configure the pipelines, and +// exposes them via GetPipeline for the compute.Scheduler to run. It also provides +// the candidate Gatherer. Placement requests no longer flow through this +// controller — the compute.Scheduler, driven by the external-scheduler HTTP API +// and the reservation controllers, is the entrypoint. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]] - // Mutex to only allow one process at a time. - // Read-only runs (opts.ReadOnly == true) acquire a read lock; write runs acquire the full lock. - processMu sync.RWMutex - - // FeatureGates holds feature flags for this controller. - FeatureGates FeatureGates // Monitor to pass down to all pipelines. Monitor lib.FilterWeigherPipelineMonitor - // Candidate gatherer to get all placement candidates if needed. + // Candidate gatherer to get all placement candidates if needed. Exported so + // it can be handed to the compute.Scheduler. Gatherer CandidateGatherer - - // vmHandler maps processed decisions into vm-operator VirtualMachine resources. - vmHandler *VirtualMachineHandler - - // CRRecorder receives scheduling events and updates CR reservations and metrics. - CRRecorder crs.Recorder } // The type of pipeline this controller manages. @@ -64,183 +43,6 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -// Callback executed when kubernetes asks to reconcile a decision resource. -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - // Peek at the decision before acquiring the lock so we can choose the right lock type. - // Read-only runs can proceed concurrently; write runs need the exclusive lock. - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - if c.peekReadOnly(decision) { - c.processMu.RLock() - defer c.processMu.RUnlock() - } else { - c.processMu.Lock() - defer c.processMu.Unlock() - // Re-fetch after acquiring the exclusive lock to see consistent state. - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - } - old := decision.DeepCopy() - if _, err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} - -// Process the decision from the API. Should create and return the updated decision. -func (c *FilterWeigherPipelineController) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { - // Read-only runs share the cached decision state; no re-fetch needed because they - // don't observe writes from concurrent exclusive-lock runs. - if c.peekReadOnly(decision) { - c.processMu.RLock() - defer c.processMu.RUnlock() - } else { - c.processMu.Lock() - defer c.processMu.Unlock() - } - - request, err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - if err == nil && decision.Status.Result != nil && request != nil && c.FeatureGates.CommittedResourceTracking { - if decision.Status.Result.TargetHost != nil { - if request.Options.SkipCommittedResourceTracking { - ctrl.LoggerFrom(ctx).V(1).Info("CR allocation tracking skipped (SkipCommittedResourceTracking=true)", - "instanceUUID", request.Spec.Data.InstanceUUID, - "targetHost", *decision.Status.Result.TargetHost, - "intent", decision.Spec.Intent, - ) - } else { - c.CRRecorder.RecordPlacement(ctx, decision, *request) - } - } - if decision.Status.Result.TargetHost == nil { - c.CRRecorder.RecordNoHostFound(ctx, decision, *request) - } - } - return err -} - -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) (*api.ExternalSchedulerRequest, error) { - log := ctrl.LoggerFrom(ctx) - startedAt := time.Now() // So we can measure sync duration. - - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) - if !ok { - log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return nil, fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) - } - if decision.Spec.NovaRaw == nil { - log.Error(nil, "skipping decision, no novaRaw spec defined") - return nil, errors.New("no novaRaw spec defined") - } - var wire api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.NovaRaw.Raw, &wire); err != nil { - log.Error(err, "failed to unmarshal novaRaw spec") - return nil, err - } - - // Translate the Nova wire request into the typed pipeline request. Intent was - // resolved by the API layer and carried on the decision spec. Whether to - // gather all candidates is derived from the hypervisor family inside the - // adapter (KVM gathers all; VMware uses the caller's preselection). - request := toSchedulerRequest(wire, decision.Spec.Intent, decision.Spec.PipelineRef.Name) - - // If necessary gather all placement candidates before filtering. This - // overrides the hosts and weights in the request. - if request.GatherAllCandidates { - if c.Gatherer == nil { - return &wire, errors.New("candidate gathering requested but no gatherer configured") - } - log.V(1).Info("gathering all placement candidates before filtering") - if err := c.Gatherer.MutateWithAllCandidates(ctx, &request); err != nil { - log.Error(err, "failed to gather all placement candidates") - return &wire, err - } - log.V(1).Info("gathered all placement candidates", "numHosts", len(request.Hosts)) - } - - result, err := pipeline.Run(request) - if err == nil { - decision.Status.Result = &result - } - // Record a human-readable explanation of the decision so it propagates to - // the VM CortexSpec.Explanation. On error the result is nil and the - // explanation carries the failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) - // We create a virtual machine resource based on the scheduling decision, - // logging the specific reason whenever we skip creation so it can be traced. - instanceUUID := request.InstanceUUID - switch { - case err != nil: - log.V(1).Info("skipping virtual machine creation, pipeline run failed", "instanceUUID", instanceUUID) - case request.Options.ReadOnly: - log.V(1).Info("skipping virtual machine creation, request is read-only", "instanceUUID", instanceUUID) - case !c.FeatureGates.VirtualMachineCreation: - log.V(1).Info("skipping virtual machine creation, feature gate disabled", "instanceUUID", instanceUUID) - case !slices.Contains(api.NovaIntents, decision.Spec.Intent): - log.V(1).Info("skipping virtual machine creation, intent not triggered by Nova", "instanceUUID", instanceUUID, "intent", decision.Spec.Intent) - case request.HypervisorFamily != compute.HypervisorFamilyKVM: - log.V(1).Info("skipping virtual machine creation, unsupported hypervisor type", "instanceUUID", instanceUUID) - default: - vmUpsertErr := c.vmHandler.Upsert(ctx, decision, wire) - if c.FeatureGates.VirtualMachineCreationMandatory && vmUpsertErr != nil { - log.Error(vmUpsertErr, "failed to upsert virtual machine resource and it is mandatory", "instanceUUID", instanceUUID) - return &wire, vmUpsertErr - } - if vmUpsertErr != nil { - // Non-fatal: the placement decision itself succeeded. - log.Error(vmUpsertErr, "failed to upsert virtual machine resource", "instanceUUID", instanceUUID) - } - } - - if err != nil { - log.Error(err, "failed to run pipeline") - return &wire, err - } - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return &wire, nil -} - -// peekReadOnly determines whether a decision should use a read lock instead of -// the exclusive write lock. Defaults to false (exclusive) on any parse error. -func (c *FilterWeigherPipelineController) peekReadOnly(decision *v1alpha1.Decision) bool { - if decision.Spec.NovaRaw == nil { - return false - } - var request api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.NovaRaw.Raw, &request); err != nil { - return false - } - return request.Options.ReadOnly -} - func (c *FilterWeigherPipelineController) InitPipeline( ctx context.Context, p v1alpha1.Pipeline, @@ -258,7 +60,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainNova c.Gatherer = &candidateGatherer{Client: mcl} - c.vmHandler = &VirtualMachineHandler{Client: mcl} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { return err } @@ -315,25 +116,14 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainNova { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } - return bldr.Named("cortex-nova-decisions"). + return bldr.Named("cortex-nova-pipelines"). Complete(c) } + +// Reconcile is a no-op: this controller reconfigures pipelines through its +// Pipeline/Knowledge watch handlers, and placement runs through the +// compute.Scheduler, not through object reconciliation. A Reconcile method is +// required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { + return ctrl.Result{}, nil +} diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go index 817bfa994..32079e9e5 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go @@ -5,21 +5,10 @@ package nova import ( "context" - "encoding/json" - "errors" - "strings" "testing" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - api "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" @@ -47,239 +36,6 @@ func (m *mockCandidateGatherer) MutateWithAllCandidates(ctx context.Context, req return nil } -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - novaRequest := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Name: "RequestSpec", - Namespace: "nova_object", - Version: "1.19", - Data: api.NovaSpec{ - ProjectID: "test-project", - UserID: "test-user", - InstanceUUID: "test-instance-uuid", - NumInstances: 1, - }, - }, - Context: api.NovaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: func() *string { s := "global-req-123"; return &s }(), - }, - Hosts: []api.ExternalSchedulerHost{ - {ComputeHost: "compute-1", HypervisorHostname: "hv-1"}, - {ComputeHost: "compute-2", HypervisorHostname: "hv-2"}, - }, - Weights: map[string]float64{"compute-1": 1.0, "compute-2": 0.5}, - Pipeline: "test-pipeline", - Options: scheduling.Options{}, - } - - novaRaw, err := json.Marshal(novaRequest) - if err != nil { - t.Fatalf("Failed to marshal nova request: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - pipeline *v1alpha1.Pipeline - expectError bool - expectResult bool - }{ - { - name: "successful nova decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "decision without novaRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: nil, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - { - name: "pipeline not found", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: nil, - expectError: true, - expectResult: false, - }, - { - name: "invalid novaRaw JSON", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-invalid-json", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: []byte("invalid json"), - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{tt.decision} - if tt.pipeline != nil { - objects = append(objects, tt.pipeline) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ - Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - - if tt.pipeline != nil { - initResult := controller.InitPipeline(t.Context(), v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.pipeline.Name, - }, - Spec: tt.pipeline.Spec, - }) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to initialize pipeline: filter errors: %v, weigher errors: %v", initResult.FilterErrors, initResult.WeigherErrors) - } - controller.Pipelines[tt.pipeline.Name] = initResult.Pipeline - controller.PipelineConfigs[tt.pipeline.Name] = *tt.pipeline - } - - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - Namespace: tt.decision.Namespace, - }, - } - - result, err := controller.Reconcile(context.Background(), req) - - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - - if result.RequeueAfter > 0 { - t.Error("Expected no requeue") - } - - var updatedDecision v1alpha1.Decision - if err := client.Get(context.Background(), req.NamespacedName, &updatedDecision); err != nil { - if !tt.expectError { - t.Fatalf("Failed to get updated decision: %v", err) - } - return - } - - if tt.expectResult && updatedDecision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - if !tt.expectResult && updatedDecision.Status.Result != nil { - t.Error("Expected result to be nil but was set") - } - }) - } -} - func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { controller := &FilterWeigherPipelineController{ Monitor: lib.FilterWeigherPipelineMonitor{}, @@ -356,599 +112,3 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { }) } } - -func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - novaRequest := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Name: "RequestSpec", - Namespace: "nova_object", - Version: "1.19", - Data: api.NovaSpec{ - ProjectID: "test-project", - UserID: "test-user", - InstanceUUID: "test-instance-uuid", - NumInstances: 1, - }, - }, - Context: api.NovaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: func() *string { s := "global-req-123"; return &s }(), - }, - Hosts: []api.ExternalSchedulerHost{ - {ComputeHost: "compute-1", HypervisorHostname: "hv-1"}, - {ComputeHost: "compute-2", HypervisorHostname: "hv-2"}, - }, - Weights: map[string]float64{"compute-1": 1.0, "compute-2": 0.5}, - Pipeline: "test-pipeline", - } - - novaRaw, err := json.Marshal(novaRequest) - if err != nil { - t.Fatalf("Failed to marshal nova request: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - pipeline *v1alpha1.Pipeline - pipelineConf *v1alpha1.Pipeline - setupPipelineConfigs bool - expectError bool - expectResult bool - expectUpdatedStatus bool - errorContains string - }{ - { - name: "successful processing with decision creation enabled", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-api", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "test-uuid-1", - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: false, - expectResult: true, - expectUpdatedStatus: true, - }, - { - name: "successful processing with decision creation disabled", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-create", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "test-uuid-2", - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline-no-create", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline-no-create", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline-no-create", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: false, - expectResult: true, - expectUpdatedStatus: false, - }, - { - name: "pipeline not configured", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-config", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: nil, - pipelineConf: nil, - setupPipelineConfigs: false, - expectError: true, - expectResult: false, - expectUpdatedStatus: false, - errorContains: "pipeline not found or not ready", - }, - { - name: "decision without novaRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw-api", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: nil, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: true, - expectResult: false, - expectUpdatedStatus: false, - errorContains: "no novaRaw spec defined", - }, - { - name: "processing fails after decision creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-process-fail", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: nil, // This will cause processing to fail after creation - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: true, - expectResult: false, - expectUpdatedStatus: false, - errorContains: "pipeline not found or not ready", - }, - { - name: "pipeline not found in runtime map", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-runtime-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "missing-runtime-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: nil, - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "missing-runtime-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: true, - expectResult: false, - expectUpdatedStatus: false, - errorContains: "pipeline not found or not ready", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{} - if tt.pipeline != nil { - objects = append(objects, tt.pipeline) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ - Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - - // Setup pipeline configurations if needed - if tt.setupPipelineConfigs && tt.pipelineConf != nil { - controller.PipelineConfigs[tt.pipelineConf.Name] = *tt.pipelineConf - } - - // Setup runtime pipeline if needed - if tt.pipeline != nil { - initResult := controller.InitPipeline(context.Background(), v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.pipeline.Name, - }, - Spec: tt.pipeline.Spec, - }) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to initialize pipeline: filter errors: %v, weigher errors: %v", initResult.FilterErrors, initResult.WeigherErrors) - } - controller.Pipelines[tt.pipeline.Name] = initResult.Pipeline - } - - if tt.decision.Spec.NovaRaw != nil { - req := novaRequest - req.Options = scheduling.Options{} - raw, marshalErr := json.Marshal(req) - if marshalErr != nil { - t.Fatalf("Failed to marshal request with options: %v", marshalErr) - } - tt.decision.Spec.NovaRaw = &runtime.RawExtension{Raw: raw} - } - - // Call the method under test - err := controller.ProcessNewDecisionFromAPI(context.Background(), tt.decision) - - // Validate error expectations - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - if tt.errorContains != "" && (err == nil || !strings.Contains(err.Error(), tt.errorContains)) { - t.Errorf("Expected error to contain %q, got: %v", tt.errorContains, err) - } - - // Validate result expectations - if tt.expectResult && tt.decision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - if !tt.expectResult && tt.decision.Status.Result != nil { - t.Error("Expected result to be nil but was set") - } - }) - } -} - -func TestFilterWeigherPipelineController_GathersCandidatesForKVM(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - // Build a Nova request with the given flavor hypervisor type. Whether the - // controller gathers all candidates is derived from the hypervisor family in - // the adapter (KVM gathers; VMware / unset do not), NOT from pipeline config. - makeRaw := func(hypervisorType string) []byte { - extraSpecs := map[string]string{} - if hypervisorType != "" { - extraSpecs["capabilities:hypervisor_type"] = hypervisorType - } - novaRequest := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "test-project", - UserID: "test-user", - InstanceUUID: "test-instance-uuid", - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{Data: api.NovaFlavor{ExtraSpecs: extraSpecs}}, - }, - }, - Context: api.NovaRequestContext{RequestID: "req-123"}, - Hosts: []api.ExternalSchedulerHost{ - {ComputeHost: "original-host-1", HypervisorHostname: "hv-1"}, - {ComputeHost: "original-host-2", HypervisorHostname: "hv-2"}, - }, - Weights: map[string]float64{"original-host-1": 1.0, "original-host-2": 0.5}, - Pipeline: "test-pipeline", - Options: scheduling.Options{}, - } - raw, err := json.Marshal(novaRequest) - if err != nil { - t.Fatalf("Failed to marshal nova request: %v", err) - } - return raw - } - - tests := []struct { - name string - hypervisorType string - gathererErr error - gatheredHosts []compute.Host - expectGathererCall bool - expectError bool - errorContains string - }{ - { - name: "VMware request - gatherer not called", - hypervisorType: "vmware vcenter server", - expectGathererCall: false, - }, - { - name: "unset hypervisor type - gatherer not called", - hypervisorType: "", - expectGathererCall: false, - }, - { - name: "KVM request - gatherer called and succeeds", - hypervisorType: "qemu", - gatheredHosts: []compute.Host{ - {ComputeHost: "gathered-host-1", HypervisorHostname: "gathered-host-1"}, - {ComputeHost: "gathered-host-2", HypervisorHostname: "gathered-host-2"}, - {ComputeHost: "gathered-host-3", HypervisorHostname: "gathered-host-3"}, - }, - expectGathererCall: true, - }, - { - name: "KVM request - gatherer returns error", - hypervisorType: "qemu", - gathererErr: errGathererFailed, - expectGathererCall: true, - expectError: true, - errorContains: "gatherer failed", - }, - { - name: "KVM request - gatherer returns empty hosts", - hypervisorType: "ch", - gatheredHosts: []compute.Host{}, - expectGathererCall: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockGatherer := &mockCandidateGatherer{ - err: tt.gathererErr, - gatheredHosts: tt.gatheredHosts, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ - Client: fakeClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - Gatherer: mockGatherer, - } - - pipelineConf := v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - } - controller.PipelineConfigs["test-pipeline"] = pipelineConf - - initResult := controller.InitPipeline(context.Background(), pipelineConf) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to initialize pipeline: filter errors: %v, weigher errors: %v", initResult.FilterErrors, initResult.WeigherErrors) - } - controller.Pipelines["test-pipeline"] = initResult.Pipeline - - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-preselection", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: makeRaw(tt.hypervisorType), - }, - }, - } - - err := controller.ProcessNewDecisionFromAPI(context.Background(), decision) - - if tt.expectGathererCall && !mockGatherer.called { - t.Error("Expected gatherer to be called but it was not") - } - if !tt.expectGathererCall && mockGatherer.called { - t.Error("Expected gatherer not to be called but it was") - } - - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - if tt.errorContains != "" && (err == nil || !strings.Contains(err.Error(), tt.errorContains)) { - t.Errorf("Expected error to contain %q, got: %v", tt.errorContains, err) - } - - if !tt.expectError && decision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - }) - } -} - -// Error variable for testing -var errGathererFailed = errors.New("gatherer failed") - -func TestFilterWeigherPipelineController_PeekReadOnly(t *testing.T) { - makeRaw := func(readOnly bool) []byte { - r := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: api.NovaSpec{NumInstances: 1}}, - Options: scheduling.Options{ReadOnly: readOnly}, - } - raw, err := json.Marshal(r) - if err != nil { - panic(err) - } - return raw - } - - c := &FilterWeigherPipelineController{} - - tests := []struct { - name string - decision *v1alpha1.Decision - want bool - }{ - { - name: "nil NovaRaw defaults to exclusive lock", - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - }, - }, - want: false, - }, - { - name: "invalid JSON defaults to exclusive lock", - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - NovaRaw: &runtime.RawExtension{Raw: []byte("not-json")}, - }, - }, - want: false, - }, - { - name: "ReadOnly=false uses exclusive lock", - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - NovaRaw: &runtime.RawExtension{Raw: makeRaw(false)}, - }, - }, - want: false, - }, - { - name: "ReadOnly=true uses read lock", - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - NovaRaw: &runtime.RawExtension{Raw: makeRaw(true)}, - }, - }, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := c.peekReadOnly(tt.decision) - if got != tt.want { - t.Errorf("expected peekReadOnly = %v, got %v", tt.want, got) - } - }) - } -} diff --git a/internal/scheduling/nova/integration_test.go b/internal/scheduling/nova/integration_test.go index 4606850f1..50b9ff49b 100644 --- a/internal/scheduling/nova/integration_test.go +++ b/internal/scheduling/nova/integration_test.go @@ -17,6 +17,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/weighers" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -286,12 +287,15 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec controller.Pipelines[testPipeline.Name] = result.Pipeline controller.PipelineConfigs[testPipeline.Name] = testPipeline - // Create the HTTP API with the controller as delegate - skip metrics registration - // Set EvacuationShuffleK=0 to disable shuffle for deterministic test results + // Create the HTTP API backed by a real compute.Scheduler over the controller. + // Set EvacuationShuffleK=0 to disable shuffle for deterministic test results. + scheduler := compute.NewScheduler(controller, controller.Gatherer) api := &httpAPI{ - config: HTTPAPIConfig{EvacuationShuffleK: 0}, - monitor: lib.NewSchedulerMonitor(), // Create new monitor but don't register - delegate: controller, + config: HTTPAPIConfig{EvacuationShuffleK: 0}, + monitor: lib.NewSchedulerMonitor(), // Create new monitor but don't register + scheduler: scheduler, + vmHandler: &VirtualMachineHandler{Client: k8sClient}, + crRecorder: crs.Recorder{Client: k8sClient}, } // Create test server diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index 3548122f3..0856b8495 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -12,6 +12,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -33,21 +34,22 @@ type VirtualMachineHandler struct { } // Upsert creates or patches the VirtualMachine resource described by the -// decision and request. Region and AZ are immutable on the CRD and only written -// on create; mutable fields (target host, cortex spec, labels) are patched on -// updates. -func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.Decision, request api.ExternalSchedulerRequest) error { +// scheduling result and request. Region and AZ are immutable on the CRD and only +// written on create; mutable fields (target host, cortex spec, labels) are +// patched on updates. The explanation and intent are recorded on the VM's cortex +// spec. +func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.SchedulerResult, explanation string, request api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent) error { log := ctrl.LoggerFrom(ctx) - if decision.Status.Result == nil || decision.Status.Result.TargetHost == nil { - log.V(1).Info("skipping virtual machine upsert, no target host in decision") + if result.TargetHost == nil { + log.V(1).Info("skipping virtual machine upsert, no target host in result") return nil } // We derive hierarchical location information (region and az) from the // target host (i.e. the first host in the list). This assumes that all // hosts in the list share the same region and availability zone. - targetHost := *decision.Status.Result.TargetHost + targetHost := *result.TargetHost targetHv := &hv1.Hypervisor{} if err := h.Get(ctx, client.ObjectKey{Name: targetHost}, targetHv); err != nil { return fmt.Errorf("failed to get hypervisor %q: %w", targetHost, err) @@ -136,20 +138,18 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D } } - // Read intent from the decision, which was already resolved by the API layer - // (including a graceful fallback to CreateIntent for initial placements). - intent := decision.Spec.Intent - - // Write the cortex spec (our scheduling result). + // Write the cortex spec (our scheduling result). The intent was resolved by + // the API layer (with a graceful fallback to CreateIntent for initial + // placements) and passed in. vm.Spec.Cortex = &vmv1alpha1.CortexSpec{} - for _, host := range decision.Status.Result.OrderedHosts { + for _, host := range result.OrderedHosts { vm.Spec.Cortex.HostCandidates = append(vm.Spec.Cortex.HostCandidates, vmv1alpha1.HostRef{ APIVersion: hv1.GroupVersion.String(), Kind: "Hypervisor", Name: host, }) } - vm.Spec.Cortex.Explanation = decision.Status.Explanation + vm.Spec.Cortex.Explanation = explanation vm.Spec.Cortex.Intent = string(intent) // Write the ConditionTypeScheduled condition to indicate that the VM has diff --git a/internal/scheduling/nova/vm_handler_test.go b/internal/scheduling/nova/vm_handler_test.go index 8c1499c07..bf6d488da 100644 --- a/internal/scheduling/nova/vm_handler_test.go +++ b/internal/scheduling/nova/vm_handler_test.go @@ -8,7 +8,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" vmopv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -56,15 +56,8 @@ func vmTestRequest() api.ExternalSchedulerRequest { return req } -func vmTestDecision(targetHost *string) *v1alpha1.Decision { - return &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - Intent: api.CreateIntent, - }, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{TargetHost: targetHost}, - }, - } +func vmTestResult(targetHost *string) compute.SchedulerResult { + return compute.SchedulerResult{TargetHost: targetHost} } func vmTestHypervisor(region, az string) *hv1.Hypervisor { @@ -88,7 +81,7 @@ func TestVirtualMachineHandlerCreate(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestDecision(&targetHost), vmTestRequest()); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), "", vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert: %v", err) } @@ -137,7 +130,7 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestDecision(&targetHost), vmTestRequest()); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), "", vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert: %v", err) } @@ -162,7 +155,7 @@ func TestVirtualMachineHandlerNoTargetHost(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() h := &VirtualMachineHandler{Client: fakeClient} - if err := h.Upsert(context.Background(), vmTestDecision(nil), vmTestRequest()); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(nil), "", vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert with nil target host should be a no-op, got: %v", err) } var vmList vmopv1alpha1.VirtualMachineList @@ -183,7 +176,7 @@ func TestVirtualMachineHandlerMissingRegion(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestDecision(&targetHost), vmTestRequest()); err == nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), "", vmTestRequest(), api.CreateIntent); err == nil { t.Fatal("expected error for missing region label, got nil") } vm := &vmopv1alpha1.VirtualMachine{} diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index bd1ae15fb..c46d1bd48 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -1233,7 +1233,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h novaController.PipelineConfigs[pipeline.Name] = pipeline } - api := &testHTTPAPI{delegate: novaController} + api := &testHTTPAPI{scheduler: compute.NewScheduler(novaController, novaController.Gatherer)} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) server := httptest.NewServer(mux) @@ -1252,7 +1252,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h // testHTTPAPI is a simplified HTTP API for testing that delegates to the controller. // It does NOT shuffle evacuation hosts, ensuring deterministic test results. type testHTTPAPI struct { - delegate nova.HTTPAPIDelegate + scheduler *compute.Scheduler } // NovaExternalScheduler handles the POST request from the Nova scheduler. @@ -1270,58 +1270,26 @@ func (api *testHTTPAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req return } - rawBytes, err := json.Marshal(requestData) - if err != nil { - http.Error(w, "failed to marshal request", http.StatusInternalServerError) - return - } - raw := runtime.RawExtension{Raw: rawBytes} - pipelineName := requestData.Pipeline if pipelineName == "" { pipelineName = "test-filter-pipeline" } // Resolve the scheduling intent from the request hints, mirroring the - // production handler (external_scheduler_api.go). The pipeline reads the - // resolved intent off the typed request, so it must be set on the decision. + // production handler (external_scheduler_api.go). intent, err := requestData.GetIntent() if err != nil { intent = novaapi.CreateIntent } - decision := &v1alpha1.Decision{ - TypeMeta: metav1.TypeMeta{ - Kind: "Decision", - APIVersion: "cortex.cloud/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "nova-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: pipelineName, - }, - ResourceID: requestData.Spec.Data.InstanceUUID, - NovaRaw: &raw, - Intent: intent, - }, - } - - ctx := r.Context() - if err := api.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { + req := nova.ToSchedulerRequest(requestData, intent, pipelineName) + out, err := api.scheduler.Run(r.Context(), req, nil, nil) + if err != nil { http.Error(w, "failed to process scheduling decision", http.StatusInternalServerError) return } - if decision.Status.Result == nil { - http.Error(w, "decision didn't produce a result", http.StatusInternalServerError) - return - } - - hosts := decision.Status.Result.OrderedHosts - response := novaapi.ExternalSchedulerResponse{Hosts: hosts} + response := novaapi.ExternalSchedulerResponse{Hosts: out.Result.OrderedHosts} w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(response); err != nil { http.Error(w, "failed to encode response", http.StatusInternalServerError) @@ -1444,7 +1412,7 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, novaController.PipelineConfigs[pipeline.Name] = pipeline } - api := &testHTTPAPI{delegate: novaController} + api := &testHTTPAPI{scheduler: compute.NewScheduler(novaController, novaController.Gatherer)} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) server := httptest.NewServer(mux) diff --git a/internal/scheduling/reservations/failoverhost/integration_test.go b/internal/scheduling/reservations/failoverhost/integration_test.go index 2fee2c5ea..c9243c6e6 100644 --- a/internal/scheduling/reservations/failoverhost/integration_test.go +++ b/internal/scheduling/reservations/failoverhost/integration_test.go @@ -23,7 +23,6 @@ import ( hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -307,7 +306,7 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors controller.Pipelines[pipeline.Name] = result.Pipeline controller.PipelineConfigs[pipeline.Name] = pipeline - api := &integrationHTTPAPI{delegate: controller} + api := &integrationHTTPAPI{scheduler: sched.NewScheduler(controller, controller.Gatherer)} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.handle) server := httptest.NewServer(mux) @@ -446,7 +445,7 @@ const evacProbePipeline = "kvm-general-purpose-load-balancing" // filter/weigher pipeline — the same shape the failover package's integration // tests use. It does NOT shuffle, so the probe answers are stable. type integrationHTTPAPI struct { - delegate nova.HTTPAPIDelegate + scheduler *sched.Scheduler } func (a *integrationHTTPAPI) handle(w http.ResponseWriter, r *http.Request) { @@ -456,38 +455,20 @@ func (a *integrationHTTPAPI) handle(w http.ResponseWriter, r *http.Request) { http.Error(w, "decode", http.StatusBadRequest) return } - rawBytes, err := json.Marshal(reqData) - if err != nil { - http.Error(w, "marshal", http.StatusInternalServerError) - return - } // Resolve intent from the request hints, mirroring the production handler; // the pipeline reads the resolved intent off the typed request. intent, err := reqData.GetIntent() if err != nil { intent = novaapi.CreateIntent } - decision := &v1alpha1.Decision{ - TypeMeta: metav1.TypeMeta{Kind: "Decision", APIVersion: "cortex.cloud/v1alpha1"}, - ObjectMeta: metav1.ObjectMeta{GenerateName: "nova-"}, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{Name: evacProbePipeline}, - ResourceID: reqData.Spec.Data.InstanceUUID, - NovaRaw: &runtime.RawExtension{Raw: rawBytes}, - Intent: intent, - }, - } - if err := a.delegate.ProcessNewDecisionFromAPI(r.Context(), decision); err != nil { + req := nova.ToSchedulerRequest(reqData, intent, evacProbePipeline) + out, err := a.scheduler.Run(r.Context(), req, nil, nil) + if err != nil { http.Error(w, "process", http.StatusInternalServerError) return } - if decision.Status.Result == nil { - http.Error(w, "no result", http.StatusInternalServerError) - return - } w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: decision.Status.Result.OrderedHosts}); err != nil { + if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: out.Result.OrderedHosts}); err != nil { http.Error(w, "encode", http.StatusInternalServerError) } } From 1a8f47a5bb8ac956a7a19a4cce68ece05e195eef Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 15:54:05 +0100 Subject: [PATCH 05/21] refactor(nova): move the candidate gatherer off the pipeline controller After the entrypoint swap the filter-weigher pipeline controller only (re)configures pipelines and exposes GetPipeline; the candidate gatherer is a scheduling-run concern owned by compute.Scheduler, not the controller. - Add nova.NewCandidateGatherer(client) and drop the controller's exported Gatherer field (and its assignment in SetupWithManager). - main.go builds the gatherer independently and passes it to compute.NewScheduler. - Tests construct the scheduler with an explicit gatherer instead of reading it off the controller. go build ./..., go test ./..., go vet ./... and make lint are all clean. Signed-off-by: Markus Wieland --- cmd/manager/main.go | 2 +- internal/scheduling/nova/candidate_gatherer.go | 7 +++++++ internal/scheduling/nova/cr_metrics_test.go | 3 +-- .../nova/filter_weigher_pipeline_controller.go | 12 ++++-------- internal/scheduling/nova/integration_test.go | 7 +++---- .../reservations/failover/integration_test.go | 10 ++++------ .../reservations/failoverhost/integration_test.go | 7 +++---- 7 files changed, 23 insertions(+), 25 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 6a242266c..57cb0ceeb 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -435,7 +435,7 @@ func main() { // Shared in-process scheduler: owns the scheduling lock and runs the // pipeline. The Nova HTTP API and the reservation controllers call it // directly (no self-directed HTTP). - novaScheduler := compute.NewScheduler(filterWeigherController, filterWeigherController.Gatherer) + novaScheduler := compute.NewScheduler(filterWeigherController, nova.NewCandidateGatherer(multiclusterClient)) vmHandler := &nova.VirtualMachineHandler{Client: multiclusterClient} crRecorder := crs.Recorder{ Client: multiclusterClient, diff --git a/internal/scheduling/nova/candidate_gatherer.go b/internal/scheduling/nova/candidate_gatherer.go index fee0abe51..8d20cc595 100644 --- a/internal/scheduling/nova/candidate_gatherer.go +++ b/internal/scheduling/nova/candidate_gatherer.go @@ -23,6 +23,13 @@ type CandidateGatherer interface { // for Nova scheduling requests. type candidateGatherer struct{ client.Client } +// NewCandidateGatherer builds the default Nova candidate gatherer backed by the +// given client. It is handed to the compute.Scheduler, which invokes it when a +// request opts into gathering all placement candidates. +func NewCandidateGatherer(c client.Client) CandidateGatherer { + return &candidateGatherer{Client: c} +} + // MutateWithAllCandidates gathers all placement candidates and mutates // the request accordingly. func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.SchedulerRequest) error { diff --git a/internal/scheduling/nova/cr_metrics_test.go b/internal/scheduling/nova/cr_metrics_test.go index e58abb33a..09f4ec125 100644 --- a/internal/scheduling/nova/cr_metrics_test.go +++ b/internal/scheduling/nova/cr_metrics_test.go @@ -368,7 +368,6 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, - Gatherer: passthroughGatherer{}, } controller.PipelineConfigs["test-pipeline"] = pipelineConf initResult := controller.InitPipeline(context.Background(), pipelineConf) @@ -379,7 +378,7 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { // Drive the committed-resource tracking path through the real HTTP API // handler, where the feature gate now lives (as a post-hook). - scheduler := sched.NewScheduler(controller, controller.Gatherer) + scheduler := sched.NewScheduler(controller, passthroughGatherer{}) crRecorder := crs.Recorder{Client: fakeClient, NoHostFoundCounter: counter} api := newTestAPIWithScheduler( HTTPAPIConfig{}, diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index b12dd436f..25bab98a6 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -23,19 +23,16 @@ import ( // FilterWeigherPipelineController owns the Nova filter-weigher pipelines: it // watches Pipeline and Knowledge resources to (re)configure the pipelines, and -// exposes them via GetPipeline for the compute.Scheduler to run. It also provides -// the candidate Gatherer. Placement requests no longer flow through this -// controller — the compute.Scheduler, driven by the external-scheduler HTTP API -// and the reservation controllers, is the entrypoint. +// exposes them via GetPipeline for the compute.Scheduler to run. Placement +// requests no longer flow through this controller — the compute.Scheduler, +// driven by the external-scheduler HTTP API and the reservation controllers, is +// the entrypoint. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]] // Monitor to pass down to all pipelines. Monitor lib.FilterWeigherPipelineMonitor - // Candidate gatherer to get all placement candidates if needed. Exported so - // it can be handed to the compute.Scheduler. - Gatherer CandidateGatherer } // The type of pipeline this controller manages. @@ -59,7 +56,6 @@ func (c *FilterWeigherPipelineController) InitPipeline( func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainNova - c.Gatherer = &candidateGatherer{Client: mcl} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { return err } diff --git a/internal/scheduling/nova/integration_test.go b/internal/scheduling/nova/integration_test.go index 50b9ff49b..882ccaf4f 100644 --- a/internal/scheduling/nova/integration_test.go +++ b/internal/scheduling/nova/integration_test.go @@ -256,9 +256,6 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: getSharedMonitor(), - // KVM requests trigger candidate gathering; use a passthrough gatherer - // (nil gatheredHosts) so the test's preselected hosts are preserved. - Gatherer: &mockCandidateGatherer{}, } // Create and register the test pipeline @@ -289,7 +286,9 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec // Create the HTTP API backed by a real compute.Scheduler over the controller. // Set EvacuationShuffleK=0 to disable shuffle for deterministic test results. - scheduler := compute.NewScheduler(controller, controller.Gatherer) + // KVM requests trigger candidate gathering; use a passthrough gatherer + // (nil gatheredHosts) so the test's preselected hosts are preserved. + scheduler := compute.NewScheduler(controller, &mockCandidateGatherer{}) api := &httpAPI{ config: HTTPAPIConfig{EvacuationShuffleK: 0}, monitor: lib.NewSchedulerMonitor(), // Create new monitor but don't register diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index c46d1bd48..bdece85dd 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -1156,8 +1156,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, - Monitor: getSharedMonitor(), - Gatherer: passthroughGatherer{}, + Monitor: getSharedMonitor(), } // Register all pipelines needed for testing pipelines := []v1alpha1.Pipeline{ @@ -1233,7 +1232,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h novaController.PipelineConfigs[pipeline.Name] = pipeline } - api := &testHTTPAPI{scheduler: compute.NewScheduler(novaController, novaController.Gatherer)} + api := &testHTTPAPI{scheduler: compute.NewScheduler(novaController, passthroughGatherer{})} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) server := httptest.NewServer(mux) @@ -1334,8 +1333,7 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, - Monitor: getSharedMonitor(), - Gatherer: passthroughGatherer{}, + Monitor: getSharedMonitor(), } pipelines := []v1alpha1.Pipeline{ { @@ -1412,7 +1410,7 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, novaController.PipelineConfigs[pipeline.Name] = pipeline } - api := &testHTTPAPI{scheduler: compute.NewScheduler(novaController, novaController.Gatherer)} + api := &testHTTPAPI{scheduler: compute.NewScheduler(novaController, passthroughGatherer{})} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) server := httptest.NewServer(mux) diff --git a/internal/scheduling/reservations/failoverhost/integration_test.go b/internal/scheduling/reservations/failoverhost/integration_test.go index c9243c6e6..814bcad7a 100644 --- a/internal/scheduling/reservations/failoverhost/integration_test.go +++ b/internal/scheduling/reservations/failoverhost/integration_test.go @@ -277,9 +277,6 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: sharedPipelineMonitor(), - // KVM requests trigger candidate gathering; the test seeds its own - // eligible hosts, so use a passthrough gatherer that preserves them. - Gatherer: passthroughGatherer{}, } // Mirror the production KVM pipeline for the dimensions that shape a failover // partition: AZ, trait compatibility and whole-host capacity. @@ -306,7 +303,9 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors controller.Pipelines[pipeline.Name] = result.Pipeline controller.PipelineConfigs[pipeline.Name] = pipeline - api := &integrationHTTPAPI{scheduler: sched.NewScheduler(controller, controller.Gatherer)} + // KVM requests trigger candidate gathering; the test seeds its own eligible + // hosts, so use a passthrough gatherer that preserves them. + api := &integrationHTTPAPI{scheduler: sched.NewScheduler(controller, passthroughGatherer{})} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.handle) server := httptest.NewServer(mux) From a4cb411a668a6e30a4f235dacf780e005a9d15f8 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 16:52:56 +0100 Subject: [PATCH 06/21] feat(reservations): call the in-process compute.Scheduler, drop the HTTP client Phase 4b. The committed-resource, capacity and failover reservation controllers now build a typed compute.SchedulerRequest and call the shared in-process compute.Scheduler directly (Run for write placements, RunWithoutLock for read-only capacity probes and failover reuse/evacuate checks) instead of POSTing to the Nova external-scheduler endpoint over HTTP. - Delete reservations.SchedulerClient and its ScheduleReservationRequest/Response DTOs (they only existed to build the HTTP wire body). - Remove the SchedulerURL config field from the commitments, capacity and failover configs and the related main.go log lines. - main.go hoists the shared novaScheduler and injects it into all three reservation controllers, guarding that nova-pipeline-controllers (the in-process scheduler) is enabled. - Reservation requests set GatherAllCandidates + HypervisorFamilyKVM, matching the previous behavior where the KVM pipelines gathered all candidates (ignorePreselection=true), so the eligible-host list is a starting set. - Migrate the reservation controller tests off the httptest mock-scheduler servers to an in-process mock compute.Scheduler returning canned hosts. Note: failover reservation_scheduling.go now also resolves required/forbidden traits from the flavor extra specs onto the typed request (traitsFromExtraSpecs), preserving trait filtering for HANA/trait reservations. Out of scope: the failoverhost controller keeps its own separate HTTP path-finder (ExternalSchedulerURL); it is migrated in a later follow-up. go build ./..., go test ./..., go vet ./... and make lint are all clean. Signed-off-by: Markus Wieland --- cmd/manager/main.go | 37 ++- .../reservations/capacity/config.go | 7 - .../reservations/capacity/controller.go | 68 +++-- .../reservations/capacity/controller_test.go | 248 +++++++++--------- .../api/change_commitments_e2e_test.go | 91 +++---- .../reservations/commitments/config.go | 2 - .../commitments/integration_test.go | 101 +++---- .../commitments/reservation_controller.go | 85 +++--- .../reservation_controller_test.go | 140 +++++----- .../reservations/failover/config.go | 8 - .../reservations/failover/controller.go | 26 +- .../reservations/failover/integration_test.go | 6 +- .../failover/reservation_scheduling.go | 169 +++++++----- .../failover/reservation_scheduling_test.go | 62 +++-- .../reservations/scheduler_client.go | 231 ---------------- 15 files changed, 558 insertions(+), 723 deletions(-) delete mode 100644 internal/scheduling/reservations/scheduler_client.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 57cb0ceeb..ab5d00b04 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -416,6 +416,12 @@ func main() { commitmentsAPI.Init(mux, metrics.Registry, ctrl.Log.WithName("commitments-api")) } + // novaScheduler is the shared in-process scheduler. Built when the + // nova-pipeline-controllers are enabled and consumed in-process by the Nova + // HTTP API and the reservation/commitment/capacity/failover controllers (which + // must run in the same process — see the cortex-nova-scheduling deployment). + var novaScheduler *compute.Scheduler + if slices.Contains(mainConfig.EnabledControllers, "nova-pipeline-controllers") { featureGates := conf.GetConfigOrDie[nova.FeatureGates]() noHostFoundCounter := crs.NewNoHostFoundCounter() @@ -435,7 +441,7 @@ func main() { // Shared in-process scheduler: owns the scheduling lock and runs the // pipeline. The Nova HTTP API and the reservation controllers call it // directly (no self-directed HTTP). - novaScheduler := compute.NewScheduler(filterWeigherController, nova.NewCandidateGatherer(multiclusterClient)) + novaScheduler = compute.NewScheduler(filterWeigherController, nova.NewCandidateGatherer(multiclusterClient)) vmHandler := &nova.VirtualMachineHandler{Client: multiclusterClient} crRecorder := crs.Recorder{ Client: multiclusterClient, @@ -609,10 +615,15 @@ func main() { if slices.Contains(mainConfig.EnabledControllers, "committed-resource-reservations-controller") { setupLog.Info("enabling controller", "controller", "committed-resource-reservations-controller") + if novaScheduler == nil { + setupLog.Error(nil, "committed-resource-reservations-controller requires nova-pipeline-controllers (in-process scheduler) to be enabled") + os.Exit(1) + } if err := (&commitments.CommitmentReservationController{ - Client: multiclusterClient, - Scheme: mgr.GetScheme(), - Conf: commitmentsConfig.ReservationController, + Client: multiclusterClient, + Scheme: mgr.GetScheme(), + Conf: commitmentsConfig.ReservationController, + Scheduler: novaScheduler, }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CommitmentReservation") os.Exit(1) @@ -730,9 +741,10 @@ func main() { setupLog.Error(nil, "failover-reservations-controller requires datasourceName to be configured") os.Exit(1) } - - // The scheduler client calls the nova external scheduler API to get placement decisions - schedulerClient := reservations.NewSchedulerClient(failoverConfig.SchedulerURL) + if novaScheduler == nil { + setupLog.Error(nil, "failover-reservations-controller requires nova-pipeline-controllers (in-process scheduler) to be enabled") + os.Exit(1) + } failoverMonitor := failover.NewFailoverMonitor() if err := metrics.Registry.Register(failoverMonitor); err != nil { @@ -764,7 +776,7 @@ func main() { multiclusterClient, vmSource, failoverConfig, - schedulerClient, + novaScheduler, failoverMonitor, ) @@ -776,7 +788,6 @@ func main() { setupLog.Info("failover-reservations-controller starting", "datasourceName", failoverConfig.DatasourceName, - "schedulerURL", failoverConfig.SchedulerURL, "reconcileInterval", failoverConfig.ReconcileInterval, "revalidationInterval", failoverConfig.RevalidationInterval) @@ -788,7 +799,6 @@ func main() { } setupLog.Info("failover-reservations-controller registered", "datasourceName", failoverConfig.DatasourceName, - "schedulerURL", failoverConfig.SchedulerURL, "reconcileInterval", failoverConfig.ReconcileInterval, "revalidationInterval", failoverConfig.RevalidationInterval, "trustHypervisorLocation", failoverConfig.TrustHypervisorLocation, @@ -828,18 +838,21 @@ func main() { os.Exit(1) } + if novaScheduler == nil { + setupLog.Error(nil, "capacity-controller requires nova-pipeline-controllers (in-process scheduler) to be enabled") + os.Exit(1) + } capacityMonitor := capacity.NewMonitor(multiclusterClient) if err := metrics.Registry.Register(&capacityMonitor); err != nil { setupLog.Error(err, "failed to register capacity monitor metrics, continuing without metrics") } - if err := capacity.NewController(multiclusterClient, capacityConfig, commitmentsVMSource). + if err := capacity.NewController(multiclusterClient, capacityConfig, commitmentsVMSource, novaScheduler). SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "capacity") os.Exit(1) } setupLog.Info("capacity-controller registered", - "schedulerURL", capacityConfig.SchedulerURL, "reconcileInterval", capacityConfig.ReconcileInterval, "minReconcileInterval", capacityConfig.MinReconcileInterval, "totalPipeline", capacityConfig.TotalPipeline, diff --git a/internal/scheduling/reservations/capacity/config.go b/internal/scheduling/reservations/capacity/config.go index 4b9292042..beab45642 100644 --- a/internal/scheduling/reservations/capacity/config.go +++ b/internal/scheduling/reservations/capacity/config.go @@ -29,9 +29,6 @@ type Config struct { // PlaceablePipeline is the scheduler pipeline used for the current-state probe. // This pipeline considers current VM allocations to determine remaining placement capacity. PlaceablePipeline string `json:"capacityPlaceablePipeline"` - - // SchedulerURL is the endpoint of the nova external scheduler. - SchedulerURL string `json:"schedulerURL"` } // ApplyDefaults fills in any unset values with defaults. @@ -49,9 +46,6 @@ func (c *Config) ApplyDefaults() { if c.PlaceablePipeline == "" { c.PlaceablePipeline = defaults.PlaceablePipeline } - if c.SchedulerURL == "" { - c.SchedulerURL = defaults.SchedulerURL - } } // Validate checks that the config is internally consistent after defaults are applied. @@ -69,6 +63,5 @@ func DefaultConfig() Config { MinReconcileInterval: metav1.Duration{Duration: 30 * time.Second}, TotalPipeline: "kvm-general-purpose-load-balancing", PlaceablePipeline: "kvm-general-purpose-load-balancing", - SchedulerURL: "http://localhost:8080/scheduler/nova/external", } } diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index caeab9556..3bb68c99c 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -29,6 +29,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/go-logr/logr" @@ -102,20 +103,20 @@ func capacityMapsEqual(a, b map[hv1.ResourceName]resource.Quantity) bool { // periodic floor timer. All watch events are coalesced into a single reconcile key so // rapid changes produce at most one queued reconcile. type Reconciler struct { - client client.Client - vmSource reservations.VMSource - schedulerClient *reservations.SchedulerClient - config Config + client client.Client + vmSource reservations.VMSource + scheduler *sched.Scheduler + config Config lastReconcileAt time.Time } -func NewController(c client.Client, config Config, vmSource reservations.VMSource) *Reconciler { +func NewController(c client.Client, config Config, vmSource reservations.VMSource, scheduler *sched.Scheduler) *Reconciler { return &Reconciler{ - client: c, - vmSource: vmSource, - schedulerClient: reservations.NewSchedulerClient(config.SchedulerURL), - config: config, + client: c, + vmSource: vmSource, + scheduler: scheduler, + config: config, } } @@ -814,29 +815,40 @@ func (c *Reconciler) probeScheduler( } } - resp, err := c.schedulerClient.ScheduleReservation(ctx, reservations.ScheduleReservationRequest{ - InstanceUUID: "capacity-" + flavor.Name, - ProjectID: "cortex-capacity-probe", - FlavorName: flavor.Name, - MemoryMB: flavor.MemoryMB, - VCPUs: flavor.VCPUs, - FlavorExtraSpecs: flavor.ExtraSpecs, - AvailabilityZone: az, - Pipeline: pipeline, - EligibleHosts: eligibleHosts, - SchedulerHints: map[string]any{"_nova_check_type": string(schedulerapi.CapacityProbeIntent)}, - }, scheduling.Options{ - ReadOnly: true, - AssumeEmptyHosts: ignoreAllocations, - IgnoredReservationTypes: ignoredReservationTypes, - SkipInflight: true, - SkipCommittedResourceTracking: true, - }) + probeHosts := make([]sched.Host, 0, len(eligibleHosts)) + probeWeights := make(map[string]float64, len(eligibleHosts)) + for _, h := range eligibleHosts { + probeHosts = append(probeHosts, sched.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) + probeWeights[h.ComputeHost] = 0.0 + } + out, err := c.scheduler.RunWithoutLock(ctx, sched.SchedulerRequest{ + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: sched.HypervisorFamilyKVM, + Hosts: probeHosts, + Weights: probeWeights, + Intent: schedulerapi.CapacityProbeIntent, + InstanceUUID: "capacity-" + flavor.Name, + ProjectID: "cortex-capacity-probe", + AvailabilityZone: az, + Flavor: sched.FlavorSpec{ + Name: flavor.Name, + VCPUs: flavor.VCPUs, + Memory: *resource.NewQuantity(int64(flavor.MemoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // flavor memory bounded by specs + }, + Options: scheduling.Options{ + ReadOnly: true, + AssumeEmptyHosts: ignoreAllocations, + IgnoredReservationTypes: ignoredReservationTypes, + SkipInflight: true, + SkipCommittedResourceTracking: true, + }, + }, nil, nil) if err != nil { return 0, 0, nil, fmt.Errorf("scheduler call failed (pipeline=%s): %w", pipeline, err) } - for _, hostName := range resp.Hosts { + for _, hostName := range out.Result.OrderedHosts { hv, ok := hvByName[hostName] if !ok { continue diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 1894c859e..6ccafdbab 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -5,9 +5,7 @@ package capacity import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" + "errors" "regexp" "sort" "testing" @@ -25,6 +23,8 @@ import ( schedulerapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) @@ -99,38 +99,98 @@ func newHypervisor(name, az string, memoryBytes int64) *hv1.Hypervisor { return hv } -// newMockSchedulerServer creates an httptest server that always returns the given host list. -func newMockSchedulerServer(t *testing.T, hosts []string) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - resp := schedulerapi.ExternalSchedulerResponse{Hosts: hosts} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Errorf("mock scheduler: failed to encode response: %v", err) - } - })) +// fakePipeline returns a fixed ordered-host list (modelling a scheduler run). +type fakePipeline struct{ hosts []string } + +func (p fakePipeline) Run(sched.SchedulerRequest) (sched.SchedulerResult, error) { + return sched.SchedulerResult{OrderedHosts: p.hosts}, nil } -// newMockSchedulerServerPerPipeline returns different candidate hosts depending on the request's -// pipeline, so a test can model a host that passes the total (empty-datacenter) probe but drops -// out of the placeable probe because it is fully occupied. -func newMockSchedulerServerPerPipeline(t *testing.T, hostsByPipeline map[string][]string) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req schedulerapi.ExternalSchedulerRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("mock scheduler: failed to decode request: %v", err) - } - resp := schedulerapi.ExternalSchedulerResponse{Hosts: hostsByPipeline[req.Pipeline]} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Errorf("mock scheduler: failed to encode response: %v", err) - } - })) +// fakePipelineProvider serves a fakePipeline per pipeline name, returning the +// per-pipeline host list configured for the test. When allPipelines is set it +// returns that host list for every pipeline name regardless of hostsByPipeline. +type fakePipelineProvider struct { + hostsByPipeline map[string][]string + allPipelines []string + useAll bool +} + +func (f fakePipelineProvider) GetPipeline(name string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { + if f.useAll { + return fakePipeline{hosts: f.allPipelines}, true + } + return fakePipeline{hosts: f.hostsByPipeline[name]}, true +} + +// fakeGatherer is a no-op gatherer (tests preselect their own hosts). +type fakeGatherer struct{} + +func (fakeGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { + return nil +} + +// newMockScheduler builds an in-process compute.Scheduler whose pipelines return +// the given per-pipeline host lists. Replaces the former httptest mock server. +func newMockScheduler(hostsByPipeline map[string][]string) *sched.Scheduler { + return sched.NewScheduler(fakePipelineProvider{hostsByPipeline: hostsByPipeline}, fakeGatherer{}) +} + +// newMockSchedulerHosts is a convenience for a scheduler that returns the same +// host list regardless of pipeline. +func newMockSchedulerHosts(hosts []string) *sched.Scheduler { + return sched.NewScheduler(fakePipelineProvider{allPipelines: hosts, useAll: true}, fakeGatherer{}) +} + +// fakeErrPipeline models a scheduler run that fails (replaces the former +// HTTP 500 mock server). +type fakeErrPipeline struct{} + +func (fakeErrPipeline) Run(sched.SchedulerRequest) (sched.SchedulerResult, error) { + return sched.SchedulerResult{}, errors.New("boom") +} + +type fakeErrPipelineProvider struct{} + +func (fakeErrPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { + return fakeErrPipeline{}, true +} + +// newErrScheduler builds a scheduler whose pipeline always returns an error. +func newErrScheduler() *sched.Scheduler { + return sched.NewScheduler(fakeErrPipelineProvider{}, fakeGatherer{}) +} + +// capturePipeline records the request it is run with and returns a fixed host list. +type capturePipeline struct { + captured *sched.SchedulerRequest + hosts []string +} + +func (p capturePipeline) Run(req sched.SchedulerRequest) (sched.SchedulerResult, error) { + *p.captured = req + return sched.SchedulerResult{OrderedHosts: p.hosts}, nil +} + +type capturePipelineProvider struct { + captured *sched.SchedulerRequest + hosts []string +} + +func (f capturePipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { + return capturePipeline(f), true } -// newController is a test helper that creates a Reconciler with a nil VMSource. -func newController(t *testing.T, c client.Client, cfg Config) *Reconciler { +// captureScheduler builds a scheduler that records the request it is run with +// into captured and returns the given hosts. +func captureScheduler(captured *sched.SchedulerRequest, hosts []string) *sched.Scheduler { + return sched.NewScheduler(capturePipelineProvider{captured: captured, hosts: hosts}, fakeGatherer{}) +} + +// newController is a test helper that creates a Reconciler with a nil VMSource +// and the given scheduler. +func newController(t *testing.T, c client.Client, cfg Config, scheduler *sched.Scheduler) *Reconciler { t.Helper() - return NewController(c, cfg, nil) + return NewController(c, cfg, nil, scheduler) } // --- unit tests for pure helper functions --- @@ -224,14 +284,12 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { Build() // Both probes return host-1 → total capacity = floor(4GiB/4GiB) = 1, placeable = 1. - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) ctrl := newController(t, fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -311,17 +369,15 @@ func TestReconcileAZ_RawCapacityIncludesOccupiedHosts(t *testing.T) { // Total probe sees both hosts (empty-datacenter); placeable probe sees only host-1 because // host-2 is fully occupied and the smallest flavor no longer fits. - schedulerServer := newMockSchedulerServerPerPipeline(t, map[string][]string{ + scheduler := newMockScheduler(map[string][]string{ totalPipe: {"host-1", "host-2"}, placeablePipe: {"host-1"}, }) - defer schedulerServer.Close() ctrl := newController(t, fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: totalPipe, PlaceablePipeline: placeablePipe, - }) + }, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -370,14 +426,12 @@ func TestReconcileAZ_WritesReservedCapacity(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) ctrl := newController(t, fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: flavorCPUs} groupData := compute.FlavorGroupFeature{SmallestFlavor: smallFlavor, Flavors: []compute.FlavorInGroup{smallFlavor}} @@ -495,7 +549,7 @@ func TestReservedResourcesByGroupAZ(t *testing.T) { WithObjects(committedA, failoverB, failoverAByFlavor, pending, inflight). Build() - c := newController(t, fakeClient, Config{}) + c := newController(t, fakeClient, Config{}, newMockSchedulerHosts(nil)) flavorA := compute.FlavorInGroup{Name: "group-a-small", MemoryMB: 4096, VCPUs: flavorCPUs} flavorB := compute.FlavorInGroup{Name: "group-b-small", MemoryMB: 4096, VCPUs: flavorCPUs} @@ -546,17 +600,11 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - // Scheduler returns 500 to simulate error. - failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer failServer.Close() - + // Scheduler returns an error to simulate a failed probe. ctrl := newController(t, fakeClient, Config{ - SchedulerURL: failServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, newErrScheduler()) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -616,17 +664,11 @@ func TestReconcileAZ_MarksExistingCRDNotReadyOnSchedulerError(t *testing.T) { t.Fatalf("failed to set Ready=True: %v", err) } - // Scheduler returns 500 to simulate error. - failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer failServer.Close() - + // Scheduler returns an error to simulate a failed probe. ctrl := newController(t, fakeClient, Config{ - SchedulerURL: failServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, newErrScheduler()) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -685,14 +727,12 @@ func TestReconcileAZ_IdempotentUpdate(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) ctrl := newController(t, fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -731,10 +771,9 @@ func TestReconcileAll_SkipsGroupsWithNoAZs(t *testing.T) { Build() ctrl := newController(t, fakeClient, Config{ - SchedulerURL: "http://localhost:9999", // unreachable; not called TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, newMockSchedulerHosts(nil)) // scheduler not called (no AZs) if err := ctrl.reconcileAll(context.Background()); err != nil { t.Errorf("reconcileAll with no hypervisors returned error: %v", err) @@ -760,10 +799,9 @@ func TestProbeScheduler_CapacityCalculation(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() // Scheduler returns both hosts. - srv := newMockSchedulerServer(t, []string{"host-1", "host-2"}) - defer srv.Close() + scheduler := newMockSchedulerHosts([]string{"host-1", "host-2"}) - c := newController(t, fakeClient, Config{SchedulerURL: srv.URL}) + c := newController(t, fakeClient, Config{}, scheduler) hvByName := map[string]hv1.Hypervisor{ "host-1": *hv1Obj, "host-2": *hv2Obj, @@ -810,10 +848,9 @@ func TestProbeScheduler_SubtractsAllocationsWhenNotIgnored(t *testing.T) { } fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() - srv := newMockSchedulerServer(t, []string{"host-1"}) - defer srv.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) - c := newController(t, fakeClient, Config{SchedulerURL: srv.URL}) + c := newController(t, fakeClient, Config{}, scheduler) hvByName := map[string]hv1.Hypervisor{"host-1": *hv} flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: memMB} @@ -853,14 +890,12 @@ func TestReconcileAll_MultipleGroupsAndAZs(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - srv := newMockSchedulerServer(t, []string{}) - defer srv.Close() + scheduler := newMockSchedulerHosts([]string{}) c := newController(t, fakeClient, Config{ - SchedulerURL: srv.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, scheduler) if err := c.reconcileAll(context.Background()); err != nil { t.Fatalf("reconcileAll failed: %v", err) @@ -908,10 +943,9 @@ func TestReconcileAll_FlavorGroupsKnowledgeNotReady(t *testing.T) { Build() c := newController(t, fakeClient, Config{ - SchedulerURL: "http://localhost:9999", TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, newMockSchedulerHosts(nil)) // scheduler not called (knowledge not ready) // Should return an error when knowledge is not ready. if err := c.reconcileAll(context.Background()); err == nil { @@ -925,7 +959,7 @@ func TestReconcileAZ_ZeroMemoryFlavorSkipped(t *testing.T) { WithScheme(scheme). WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}). Build() - c := newController(t, fakeClient, Config{}) + c := newController(t, fakeClient, Config{}, newMockSchedulerHosts(nil)) groupData := compute.FlavorGroupFeature{ SmallestFlavor: compute.FlavorInGroup{Name: "bad-flavor", MemoryMB: 0}, @@ -1075,12 +1109,6 @@ func TestComputeTotalCapacity(t *testing.T) { } } -// Verify that the module-level log variable from reservations package doesn't -// collide with the one in this package. -func TestPackageLogVar(t *testing.T) { - _ = reservations.NewSchedulerClient("http://localhost") -} - func TestSumCommittedCapacity(t *testing.T) { const ( groupName = "hana-v2" @@ -1131,7 +1159,7 @@ func TestSumCommittedCapacity(t *testing.T) { WithObjects(objects...). Build() - c := newController(t, fakeClient, Config{}) + c := newController(t, fakeClient, Config{}, newMockSchedulerHosts(nil)) // smallestFlavorBytes = 4GiB → cr1 = 8GiB/4GiB = 2 slots, cr2 = 4GiB/4GiB = 1 slot → total = 3. got, err := c.sumCommittedCapacity(context.Background(), groupName, az, memBytes) if err != nil { @@ -1217,17 +1245,15 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { } // Scheduler returns host-1 so probes succeed. - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) // VMSource returns empty list (all VMs removed). vmSource := &mockVMSource{vms: []reservations.VM{}} ctrl := NewController(fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }, vmSource) + }, vmSource, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -1284,10 +1310,9 @@ func TestProbeScheduler_SubtractsReservationBlocksWhenNotIgnored(t *testing.T) { } fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() - srv := newMockSchedulerServer(t, []string{"host-1"}) - defer srv.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) - c := newController(t, fakeClient, Config{SchedulerURL: srv.URL}) + c := newController(t, fakeClient, Config{}, scheduler) hvByName := map[string]hv1.Hypervisor{"host-1": *hv} flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: memMB} @@ -1317,32 +1342,23 @@ func TestProbeScheduler_SetsCapacityProbeIntent(t *testing.T) { scheme := newTestScheme(t) hv := newHypervisor("host-1", "az-a", 4096*1024*1024) - var capturedReq schedulerapi.ExternalSchedulerRequest - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - json.NewEncoder(w).Encode(schedulerapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}}) //nolint:errcheck - })) - defer srv.Close() + // Capture the request the scheduler receives so we can assert the probe + // intent and project are set. + var capturedReq sched.SchedulerRequest + captured := captureScheduler(&capturedReq, []string{"host-1"}) - c := NewController(fake.NewClientBuilder().WithScheme(scheme).Build(), Config{SchedulerURL: srv.URL}, nil) + c := NewController(fake.NewClientBuilder().WithScheme(scheme).Build(), Config{}, nil, captured) hvByName := map[string]hv1.Hypervisor{"host-1": *hv} flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: 4096} if _, _, _, err := c.probeScheduler(context.Background(), flavor, "az-a", "test-pipeline", hvByName, true, nil); err != nil { t.Fatalf("probeScheduler failed: %v", err) } - hint, err := capturedReq.Spec.Data.GetSchedulerHintStr("_nova_check_type") - if err != nil { - t.Fatalf("failed to get _nova_check_type hint: %v", err) - } - if hint != string(schedulerapi.CapacityProbeIntent) { - t.Errorf("capacity probe must set _nova_check_type=%q, got %q", schedulerapi.CapacityProbeIntent, hint) + if capturedReq.Intent != schedulerapi.CapacityProbeIntent { + t.Errorf("capacity probe must set Intent=%q, got %q", schedulerapi.CapacityProbeIntent, capturedReq.Intent) } - if capturedReq.Spec.Data.ProjectID != "cortex-capacity-probe" { - t.Errorf("capacity probe must send ProjectID cortex-capacity-probe, got %q", capturedReq.Spec.Data.ProjectID) + if capturedReq.ProjectID != "cortex-capacity-probe" { + t.Errorf("capacity probe must send ProjectID cortex-capacity-probe, got %q", capturedReq.ProjectID) } } @@ -1367,16 +1383,14 @@ func TestReconcile_ReactsToKnowledgeChange(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) r := NewController(fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, MinReconcileInterval: metav1.Duration{Duration: 30 * time.Second}, - }, nil) + }, nil, scheduler) req := ctrl.Request{NamespacedName: types.NamespacedName{Name: coalescedKey}} result, err := r.Reconcile(context.Background(), req) @@ -1418,17 +1432,15 @@ func TestReconcile_MinIntervalEarlyReturn(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) minInterval := 30 * time.Second r := NewController(fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, MinReconcileInterval: metav1.Duration{Duration: minInterval}, - }, nil) + }, nil, scheduler) req := ctrl.Request{NamespacedName: types.NamespacedName{Name: coalescedKey}} diff --git a/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go b/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go index b9c1978aa..fb1f31a52 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go @@ -12,7 +12,6 @@ package api import ( "context" - "encoding/json" "net/http" "net/http/httptest" "os" @@ -20,8 +19,9 @@ import ( "testing" "time" - schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/prometheus/client_golang/prometheus" @@ -46,19 +46,18 @@ const ( // e2eEnv is a full end-to-end test environment: real controllers, fake k8s client, // mock scheduler, and a background reconcile driver goroutine. type e2eEnv struct { - t *testing.T - k8sClient client.Client - httpServer *httptest.Server - schedulerSrv *httptest.Server - crCtrl *commitments.CommittedResourceController - resCtrl *commitments.CommitmentReservationController - cancelBg context.CancelFunc - bgDone chan struct{} + t *testing.T + k8sClient client.Client + httpServer *httptest.Server + crCtrl *commitments.CommittedResourceController + resCtrl *commitments.CommitmentReservationController + cancelBg context.CancelFunc + bgDone chan struct{} } // newE2EEnv creates an e2eEnv with the given flavors and scheduler handler. // The scheduler handler controls what the mock Nova scheduler returns. -func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, schedulerHandler http.HandlerFunc) *e2eEnv { +func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler *sched.Scheduler) *e2eEnv { t.Helper() log.SetLogger(zap.New(zap.WriteTo(os.Stderr), zap.UseDevMode(true))) @@ -98,8 +97,6 @@ func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler }). Build() - schedulerSrv := httptest.NewServer(schedulerHandler) - crCtrl := &commitments.CommittedResourceController{ Client: k8sClient, Scheme: scheme, @@ -107,10 +104,10 @@ func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler } resCtrl := &commitments.CommitmentReservationController{ - Client: k8sClient, - Scheme: scheme, + Client: k8sClient, + Scheme: scheme, + Scheduler: scheduler, Conf: commitments.ReservationControllerConfig{ - SchedulerURL: schedulerSrv.URL, AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}, RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, RequeueIntervalRetry: metav1.Duration{Duration: 100 * time.Millisecond}, @@ -134,14 +131,13 @@ func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler ctx, cancel := context.WithCancel(context.Background()) env := &e2eEnv{ - t: t, - k8sClient: k8sClient, - httpServer: httpServer, - schedulerSrv: schedulerSrv, - crCtrl: crCtrl, - resCtrl: resCtrl, - cancelBg: cancel, - bgDone: make(chan struct{}), + t: t, + k8sClient: k8sClient, + httpServer: httpServer, + crCtrl: crCtrl, + resCtrl: resCtrl, + cancelBg: cancel, + bgDone: make(chan struct{}), } go env.driveReconciles(ctx) return env @@ -151,7 +147,6 @@ func (e *e2eEnv) close() { e.cancelBg() <-e.bgDone e.httpServer.Close() - e.schedulerSrv.Close() } // asCRTestEnv wraps e2eEnv as a CRTestEnv to reuse its HTTP-call and assertion helpers. @@ -274,27 +269,35 @@ func e2eIsTerminalReservation(res v1alpha1.Reservation) bool { // Scheduler handlers // ============================================================================ -func e2eAcceptScheduler(t *testing.T) http.HandlerFunc { - t.Helper() - return func(w http.ResponseWriter, r *http.Request) { - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Errorf("scheduler encode: %v", err) - } - } +// e2eFakePipeline models a scheduler run returning a fixed host list. +type e2eFakePipeline struct{ hosts []string } + +func (p e2eFakePipeline) Run(sched.SchedulerRequest) (sched.SchedulerResult, error) { + return sched.SchedulerResult{OrderedHosts: p.hosts}, nil } -func e2eRejectScheduler(t *testing.T) http.HandlerFunc { - t.Helper() - return func(w http.ResponseWriter, r *http.Request) { - // Return an empty hosts list — the reservation controller treats this as NoHostsFound. - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{}} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Errorf("scheduler encode: %v", err) - } - } +type e2eFakeProvider struct{ hosts []string } + +func (f e2eFakeProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { + return e2eFakePipeline(f), true +} + +type e2eFakeGatherer struct{} + +func (e2eFakeGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { + return nil } +func e2eScheduler(hosts []string) *sched.Scheduler { + return sched.NewScheduler(e2eFakeProvider{hosts: hosts}, e2eFakeGatherer{}) +} + +// e2eAcceptScheduler returns a scheduler that always places on host-1. +func e2eAcceptScheduler() *sched.Scheduler { return e2eScheduler([]string{"host-1"}) } + +// e2eRejectScheduler returns a scheduler that finds no host (NoHostsFound). +func e2eRejectScheduler() *sched.Scheduler { return e2eScheduler(nil) } + // assertNoDryRunProbes verifies that all probe CommittedResource CRDs and their child // Reservations have been cleaned up. It only checks objects associated with dry-run probes // (name prefix "commitment-dryrun-") rather than asserting zero total objects, so the @@ -334,7 +337,7 @@ var e2eFlavor = &TestFlavor{Name: "m1.small", Group: "hana_1", MemoryMB: 1024, V func TestE2EChangeCommitments(t *testing.T) { testCases := []struct { Name string - Scheduler func(*testing.T) http.HandlerFunc + Scheduler func() *sched.Scheduler ReqJSON string WantResp APIResponseExpectation WantAbsent []string @@ -522,7 +525,7 @@ func TestE2EChangeCommitments(t *testing.T) { for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { - env := newE2EEnv(t, []*TestFlavor{e2eFlavor}, e2eInfoVersion, tc.Scheduler(t)) + env := newE2EEnv(t, []*TestFlavor{e2eFlavor}, e2eInfoVersion, tc.Scheduler()) defer env.close() te := env.asCRTestEnv() diff --git a/internal/scheduling/reservations/commitments/config.go b/internal/scheduling/reservations/commitments/config.go index d82740b53..7ea1d0ef4 100644 --- a/internal/scheduling/reservations/commitments/config.go +++ b/internal/scheduling/reservations/commitments/config.go @@ -79,8 +79,6 @@ type ReservationControllerConfig struct { // When false, violations are detected and exposed via metrics but no eviction is performed. // Has no effect when EnableOversubscriptionCheck is false. EnableOversubscriptionReservationEviction bool `json:"enableOversubscriptionReservationEviction,omitempty"` - // SchedulerURL is the endpoint of the nova external scheduler. - SchedulerURL string `json:"schedulerURL"` // PipelineDefault is the fallback pipeline when no FlavorGroupPipelines entry matches. PipelineDefault string `json:"pipelineDefault"` // FlavorGroupPipelines maps flavor group IDs to pipeline names; "*" acts as catch-all. diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go index 154d96137..5e0bab167 100644 --- a/internal/scheduling/reservations/commitments/integration_test.go +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -19,10 +19,7 @@ package commitments import ( "context" - "encoding/json" "fmt" - "net/http" - "net/http/httptest" "strings" "sync/atomic" "testing" @@ -30,6 +27,8 @@ import ( schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -405,10 +404,30 @@ type intgEnv struct { crController *CommittedResourceController resController *CommitmentReservationController hvController *HostOversubscriptionController - schedulerSrv *httptest.Server } -func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.HandlerFunc, vmSource reservations.VMSource) *intgEnv { +// intgSchedulerFn models a scheduler run: given the request it returns the ordered +// hosts (an empty list models NoHostsFound). It replaces the former httptest mock +// Nova scheduler handler. +type intgSchedulerFn func(sched.SchedulerRequest) sched.SchedulerResult + +type intgPipeline struct{ fn intgSchedulerFn } + +func (p intgPipeline) Run(req sched.SchedulerRequest) (sched.SchedulerResult, error) { + return p.fn(req), nil +} + +type intgPipelineProvider struct{ fn intgSchedulerFn } + +func (p intgPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { + return intgPipeline(p), true +} + +func newIntgScheduler(fn intgSchedulerFn) *sched.Scheduler { + return sched.NewScheduler(intgPipelineProvider{fn: fn}, fakeGatherer{}) +} + +func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn intgSchedulerFn, vmSource reservations.VMSource) *intgEnv { t.Helper() scheme := newCRTestScheme(t) @@ -465,8 +484,6 @@ func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.H }). Build() - schedulerSrv := httptest.NewServer(schedulerFn) - crCtrl := &CommittedResourceController{ Client: k8sClient, Scheme: scheme, @@ -478,10 +495,10 @@ func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.H } monitor := NewReservationControllerMonitor() resCtrl := &CommitmentReservationController{ - Client: k8sClient, - Scheme: scheme, + Client: k8sClient, + Scheme: scheme, + Scheduler: newIntgScheduler(schedulerFn), Conf: ReservationControllerConfig{ - SchedulerURL: schedulerSrv.URL, AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}, RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, EnableOversubscriptionCheck: true, @@ -496,10 +513,10 @@ func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.H Monitor: &monitor, Conf: resCtrl.Conf, } - return &intgEnv{k8sClient: k8sClient, crController: crCtrl, resController: resCtrl, hvController: hvCtrl, schedulerSrv: schedulerSrv} + return &intgEnv{k8sClient: k8sClient, crController: crCtrl, resController: resCtrl, hvController: hvCtrl} } -func (e *intgEnv) close() { e.schedulerSrv.Close() } +func (e *intgEnv) close() {} func newDefaultIntgEnv(t *testing.T) *intgEnv { t.Helper() @@ -698,42 +715,38 @@ func intgAssertCRCondition(t *testing.T, k8sClient client.Client, crNames []stri } // ============================================================================ -// Scheduler handlers +// Scheduler functions // ============================================================================ -func intgAcceptScheduler(w http.ResponseWriter, r *http.Request) { - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} - json.NewEncoder(w).Encode(resp) //nolint:errcheck +func intgAcceptScheduler(sched.SchedulerRequest) sched.SchedulerResult { + return sched.SchedulerResult{OrderedHosts: []string{"host-1"}} } -func intgRejectScheduler(w http.ResponseWriter, r *http.Request) { - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{}} - json.NewEncoder(w).Encode(resp) //nolint:errcheck +func intgRejectScheduler(sched.SchedulerRequest) sched.SchedulerResult { + return sched.SchedulerResult{OrderedHosts: []string{}} } -// intgAcceptFirstScheduler returns a handler that accepts the first count placement calls +// intgAcceptFirstScheduler returns a scheduler fn that accepts the first count placement calls // and rejects all subsequent ones. Uses an atomic counter so concurrent calls are safe. -func intgAcceptFirstScheduler(count int) http.HandlerFunc { +func intgAcceptFirstScheduler(count int) intgSchedulerFn { var calls atomic.Int32 - return func(w http.ResponseWriter, r *http.Request) { + return func(req sched.SchedulerRequest) sched.SchedulerResult { if int(calls.Add(1)) <= count { - intgAcceptScheduler(w, r) - } else { - intgRejectScheduler(w, r) + return intgAcceptScheduler(req) } + return intgRejectScheduler(req) } } -// intgRejectFirstScheduler returns a handler that rejects the first count placement calls +// intgRejectFirstScheduler returns a scheduler fn that rejects the first count placement calls // and accepts all subsequent ones. Used to test AllowRejection=false retry-until-success paths. -func intgRejectFirstScheduler(count int) http.HandlerFunc { +func intgRejectFirstScheduler(count int) intgSchedulerFn { var calls atomic.Int32 - return func(w http.ResponseWriter, r *http.Request) { + return func(req sched.SchedulerRequest) sched.SchedulerResult { if int(calls.Add(1)) <= count { - intgRejectScheduler(w, r) - } else { - intgAcceptScheduler(w, r) + return intgRejectScheduler(req) } + return intgAcceptScheduler(req) } } @@ -1237,17 +1250,13 @@ func TestCRScheduling_SetsReserveForCommittedResourceIntent(t *testing.T) { // CR slot scheduling has a real project ID and must run tenant-context filters // (filter_allowed_projects, filter_aggregate_metadata, etc.). Verify the intent is // reserve_for_committed_resource so filters can identify the call type correctly. - var capturedReq schedulerdelegationapi.ExternalSchedulerRequest + var capturedReq sched.SchedulerRequest var schedulerCalled bool - captureScheduler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + captureScheduler := func(req sched.SchedulerRequest) sched.SchedulerResult { schedulerCalled = true - if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} - json.NewEncoder(w).Encode(resp) //nolint:errcheck - }) + capturedReq = req + return sched.SchedulerResult{OrderedHosts: []string{"host-1"}} + } env := newIntgEnv(t, []client.Object{newTestFlavorKnowledge(), intgHypervisor("host-1")}, captureScheduler, nil) defer env.close() @@ -1262,12 +1271,8 @@ func TestCRScheduling_SetsReserveForCommittedResourceIntent(t *testing.T) { if !schedulerCalled { t.Fatal("scheduler was never called — test did not exercise the scheduling path") } - hint, err := capturedReq.Spec.Data.GetSchedulerHintStr("_nova_check_type") - if err != nil { - t.Fatalf("failed to get _nova_check_type hint: %v", err) - } - if hint != string(schedulerdelegationapi.ReserveForCommittedResourceIntent) { - t.Errorf("CR slot scheduling must set _nova_check_type=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, hint) + if capturedReq.Intent != schedulerdelegationapi.ReserveForCommittedResourceIntent { + t.Errorf("CR slot scheduling must set Intent=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, capturedReq.Intent) } } @@ -1339,9 +1344,9 @@ func TestCROversubscriptionRemediation(t *testing.T) { } var schedulerCalls atomic.Int32 - schedulerFn := func(w http.ResponseWriter, r *http.Request) { + schedulerFn := func(req sched.SchedulerRequest) sched.SchedulerResult { schedulerCalls.Add(1) - intgRejectScheduler(w, r) // reject so evicted slots don't land back on host-1 + return intgRejectScheduler(req) // reject so evicted slots don't land back on host-1 } env := newIntgEnv(t, append(objects, newTestFlavorKnowledge()), schedulerFn, nil) defer env.close() diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 249d10e47..7b5bb513a 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -11,6 +11,7 @@ import ( "time" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -27,6 +28,7 @@ import ( schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/cobaltcore-dev/cortex/pkg/multicluster" @@ -44,8 +46,8 @@ type CommitmentReservationController struct { Scheme *runtime.Scheme // Configuration for the controller. Conf ReservationControllerConfig - // SchedulerClient for making scheduler API calls. - SchedulerClient *reservations.SchedulerClient + // Scheduler is the in-process scheduler used to place reservations. + Scheduler *compute.Scheduler // DomainResolver resolves OpenStack domain IDs to domain names so the // domain_name scheduler hint can be populated for filter_external_customer. // Nil when KeystoneSecretRef is not configured; hint is omitted in that case. @@ -266,16 +268,17 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // KeystoneSecretRef) or the lookup fails, we log and proceed without the hint. // filter_external_customer already handles a missing hint by skipping the filter, // so omitting it degrades gracefully rather than blocking scheduling. - schedulerHints := map[string]any{ - "_nova_check_type": string(schedulerdelegationapi.ReserveForCommittedResourceIntent), - } + // Resolve the customer domain name for the typed request. If unresolved or + // the resolver is unset we proceed without it; filter_external_customer skips + // gracefully on an empty domain. + var domainName string if r.DomainResolver != nil && res.Spec.CommittedResourceReservation != nil && res.Spec.CommittedResourceReservation.DomainID != "" { - domainName, err := r.DomainResolver.ResolveDomainName(ctx, res.Spec.CommittedResourceReservation.DomainID) + resolved, err := r.DomainResolver.ResolveDomainName(ctx, res.Spec.CommittedResourceReservation.DomainID) if err != nil { logger.Error(err, "failed to resolve domain name for scheduler hint, proceeding without it", "domainID", res.Spec.CommittedResourceReservation.DomainID) } else { - schedulerHints["domain_name"] = domainName + domainName = resolved } } @@ -343,36 +346,46 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr return "initial" }()) - // Use the SchedulerClient to schedule the reservation - scheduleReq := reservations.ScheduleReservationRequest{ - InstanceUUID: res.Name, - ProjectID: projectID, - FlavorName: flavorDetails.Name, - FlavorExtraSpecs: flavorDetails.ExtraSpecs, - MemoryMB: flavorDetails.MemoryMB, - VCPUs: flavorDetails.VCPUs, - EligibleHosts: eligibleHosts, - Pipeline: pipelineName, - AvailabilityZone: availabilityZone, - SchedulerHints: schedulerHints, - } - scheduleOpts := scheduling.Options{ - ReadOnly: false, // mutates state (reservation placement) - LockReservations: true, // don't unlock CR reservations; finding a slot, not placing a VM - AssumeEmptyHosts: false, - IgnoredReservationTypes: nil, - MaxCandidates: 1, - SkipInflight: false, // TODO pessimistic blocking needed, will be addressed in follow up ticket - SkipCommittedResourceTracking: true, // CR slot scheduling, not a VM placement - } - - scheduleResp, err := r.SchedulerClient.ScheduleReservation(ctx, scheduleReq, scheduleOpts) + // Build the typed scheduler request directly (in-process; no HTTP). KVM + // reservations gather all candidates, so the eligible-host list is a starting + // set; GatherAllCandidates mirrors the previous ignore-preselection behavior. + hosts := make([]compute.Host, 0, len(eligibleHosts)) + weights := make(map[string]float64, len(eligibleHosts)) + for _, h := range eligibleHosts { + hosts = append(hosts, compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) + weights[h.ComputeHost] = 0.0 + } + schedReq := compute.SchedulerRequest{ + Pipeline: pipelineName, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: hosts, + Weights: weights, + Intent: schedulerdelegationapi.ReserveForCommittedResourceIntent, + InstanceUUID: res.Name, + ProjectID: projectID, + AvailabilityZone: availabilityZone, + DomainName: domainName, + Flavor: compute.FlavorSpec{ + Name: flavorDetails.Name, + VCPUs: flavorDetails.VCPUs, + Memory: *resource.NewQuantity(int64(flavorDetails.MemoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // flavor memory bounded by specs + }, + Options: scheduling.Options{ + ReadOnly: false, // mutates state (reservation placement) + LockReservations: true, // don't unlock CR reservations; finding a slot, not placing a VM + MaxCandidates: 1, + SkipCommittedResourceTracking: true, // CR slot scheduling, not a VM placement + }, + } + + out, err := r.Scheduler.Run(ctx, schedReq, nil, nil) if err != nil { logger.Error(err, "failed to schedule reservation") return ctrl.Result{}, err } - if len(scheduleResp.Hosts) == 0 { + if len(out.Result.OrderedHosts) == 0 { logger.Info("no hosts found for reservation, will retry", "reservation", res.Name, "flavorName", resourceName) old := res.DeepCopy() meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ @@ -398,7 +411,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // Update the reservation Spec with the found host (idx 0) // Only update Spec here - the Status will be synced in the next reconcile cycle // This avoids race conditions from doing two patches in one reconcile - host := scheduleResp.Hosts[0] + host := out.Result.OrderedHosts[0] logger.Info("found host for reservation", "host", host, "flavorName", resourceName) old := res.DeepCopy() @@ -782,11 +795,9 @@ func (r *CommitmentReservationController) hypervisorToReservations(ctx context.C return requests } -// Init initializes the reconciler with required clients and DB connection. +// Init initializes the reconciler with required clients and DB connection. The +// scheduler is injected by main.go; this only wires auxiliary clients. func (r *CommitmentReservationController) Init(ctx context.Context, conf ReservationControllerConfig) error { - r.SchedulerClient = reservations.NewSchedulerClient(conf.SchedulerURL) - logf.FromContext(ctx).Info("scheduler client initialized for commitment reservation controller", "url", conf.SchedulerURL) - if conf.KeystoneSecretRef.Name != "" { var authenticatedHTTP *http.Client if conf.SSOSecretRef != nil { diff --git a/internal/scheduling/reservations/commitments/reservation_controller_test.go b/internal/scheduling/reservations/commitments/reservation_controller_test.go index 98e4cb867..f8d9c5eb5 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller_test.go +++ b/internal/scheduling/reservations/commitments/reservation_controller_test.go @@ -5,11 +5,8 @@ package commitments import ( "context" - "encoding/json" "errors" "fmt" - "net/http" - "net/http/httptest" "strconv" "testing" "time" @@ -25,9 +22,38 @@ import ( schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) +// fakePipeline returns a fixed ordered-host list (modelling a scheduler run). +type fakePipeline struct{ hosts []string } + +func (p fakePipeline) Run(sched.SchedulerRequest) (sched.SchedulerResult, error) { + return sched.SchedulerResult{OrderedHosts: p.hosts}, nil +} + +// fakePipelineProvider serves a fakePipeline for every pipeline name, returning +// the configured host list. +type fakePipelineProvider struct{ hosts []string } + +func (f fakePipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { + return fakePipeline(f), true +} + +// fakeGatherer is a no-op gatherer (tests supply their own hosts). +type fakeGatherer struct{} + +func (fakeGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { + return nil +} + +// newMockScheduler builds an in-process compute.Scheduler whose pipeline returns +// the given host list. Replaces the former httptest mock scheduler server. +func newMockScheduler(hosts []string) *sched.Scheduler { + return sched.NewScheduler(fakePipelineProvider{hosts: hosts}, fakeGatherer{}) +} + // mockDomainResolver is a test double for DomainResolver. type mockDomainResolver struct { // names maps domainID → name returned on success. @@ -780,44 +806,11 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t k8sClient := newCRTestClient(scheme, reservation, newTestFlavorKnowledge(), hypervisor1, hypervisor2) - // Create a mock server that returns a successful response - mockResponse := &schedulerdelegationapi.ExternalSchedulerResponse{ - Hosts: []string{"test-host-1", "test-host-2"}, - } - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Verify the request body - var req schedulerdelegationapi.ExternalSchedulerRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("Failed to decode request: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - - // Verify request structure - if req.Spec.Data.NumInstances != 1 { - t.Errorf("Expected NumInstances to be 1, got %d", req.Spec.Data.NumInstances) - } - - if err := json.NewEncoder(w).Encode(mockResponse); err != nil { - t.Fatalf("failed to write response: %v", err) - } - })) - defer server.Close() - - config := ReservationControllerConfig{ - SchedulerURL: server.URL, - } - reconciler := &CommitmentReservationController{ - Client: k8sClient, - Scheme: scheme, - Conf: config, - } - - // Initialize the reconciler (this sets up SchedulerClient) - if err := reconciler.Init(context.Background(), config); err != nil { - t.Fatalf("Failed to initialize reconciler: %v", err) + Client: k8sClient, + Scheme: scheme, + Conf: ReservationControllerConfig{}, + Scheduler: newMockScheduler([]string{"test-host-1", "test-host-2"}), } req := ctrl.Request{ @@ -876,20 +869,19 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t // Test: domain_name scheduler hint // ============================================================================ -// newTestSchedulerServer captures the decoded request and returns a single host. -func newTestSchedulerServer(t *testing.T, captured *schedulerdelegationapi.ExternalSchedulerRequest) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(captured); err != nil { - t.Errorf("failed to decode scheduler request: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - resp := schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"test-host-1"}} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Fatalf("failed to write scheduler response: %v", err) - } - })) +// capturingPipeline records the SchedulerRequest it was run with and returns a +// single host, so tests can assert on the request the reconciler built. +type capturingPipeline struct{ captured *sched.SchedulerRequest } + +func (p capturingPipeline) Run(req sched.SchedulerRequest) (sched.SchedulerResult, error) { + *p.captured = req + return sched.SchedulerResult{OrderedHosts: []string{"test-host-1"}}, nil +} + +type capturingPipelineProvider struct{ captured *sched.SchedulerRequest } + +func (f capturingPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { + return capturingPipeline(f), true } func TestCommitmentReservationController_DomainNameHint(t *testing.T) { @@ -938,16 +930,13 @@ func TestCommitmentReservationController_DomainNameHint(t *testing.T) { k8sClient := newCRTestClient(scheme, reservation, newTestFlavorKnowledge(), &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "test-host-1"}}) - var captured schedulerdelegationapi.ExternalSchedulerRequest - server := newTestSchedulerServer(t, &captured) - defer server.Close() - + var captured sched.SchedulerRequest reconciler := &CommitmentReservationController{ - Client: k8sClient, - Scheme: scheme, - Conf: ReservationControllerConfig{SchedulerURL: server.URL}, - SchedulerClient: reservations.NewSchedulerClient(server.URL), - DomainResolver: tt.resolver, + Client: k8sClient, + Scheme: scheme, + Conf: ReservationControllerConfig{}, + Scheduler: sched.NewScheduler(capturingPipelineProvider{captured: &captured}, fakeGatherer{}), + DomainResolver: tt.resolver, } if _, err := reconciler.Reconcile(context.Background(), ctrl.Request{ @@ -964,25 +953,14 @@ func TestCommitmentReservationController_DomainNameHint(t *testing.T) { } } - // Verify domain_name hint. - gotDomain, err := captured.Spec.Data.GetSchedulerHintStr("domain_name") - if tt.wantDomainName == "" { - if err == nil { - t.Errorf("expected domain_name hint absent, got %q", gotDomain) - } - } else { - if err != nil { - t.Fatalf("expected domain_name hint %q, got error: %v", tt.wantDomainName, err) - } - if gotDomain != tt.wantDomainName { - t.Errorf("expected domain_name hint %q, got %q", tt.wantDomainName, gotDomain) - } + // Verify domain_name hint (now the typed request field). + if captured.DomainName != tt.wantDomainName { + t.Errorf("expected domain name %q, got %q", tt.wantDomainName, captured.DomainName) } - // _nova_check_type must always be present regardless of resolver outcome. - gotIntent, err := captured.Spec.Data.GetSchedulerHintStr("_nova_check_type") - if err != nil || gotIntent != string(schedulerdelegationapi.ReserveForCommittedResourceIntent) { - t.Errorf("expected _nova_check_type=%q, got %q (err=%v)", schedulerdelegationapi.ReserveForCommittedResourceIntent, gotIntent, err) + // Intent must always be set regardless of resolver outcome. + if captured.Intent != schedulerdelegationapi.ReserveForCommittedResourceIntent { + t.Errorf("expected Intent=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, captured.Intent) } }) } diff --git a/internal/scheduling/reservations/failover/config.go b/internal/scheduling/reservations/failover/config.go index b8bde49a6..3c2f9d3f1 100644 --- a/internal/scheduling/reservations/failover/config.go +++ b/internal/scheduling/reservations/failover/config.go @@ -27,10 +27,6 @@ type FailoverConfig struct { // This is used to read VM data from the Nova database. DatasourceName string `json:"datasourceName"` - // SchedulerURL is the URL of the nova external scheduler API. - // Example: "http://localhost:8080/scheduler/nova/external" - SchedulerURL string `json:"schedulerURL"` - // MaxVMsToProcess limits the number of VMs to process per reconciliation cycle. // Set to negative to process all VMs (default behavior). // Useful for debugging and testing with large VM counts. @@ -99,9 +95,6 @@ func (c *FailoverConfig) ApplyDefaults() { if c.DatasourceName == "" { c.DatasourceName = defaults.DatasourceName } - if c.SchedulerURL == "" { - c.SchedulerURL = defaults.SchedulerURL - } if c.ReconcileInterval.Duration == 0 { c.ReconcileInterval = defaults.ReconcileInterval } @@ -142,7 +135,6 @@ func DefaultConfig() FailoverConfig { MaxVMsToProcess: 30, Creator: "cortex-failover-controller", DatasourceName: "nova-servers", // we have the server and flavor data source (both store in same postgres and same secret but still) - SchedulerURL: "http://localhost:8080/scheduler/nova/external", TrustHypervisorLocation: false, RevalidationInterval: metav1.Duration{Duration: 30 * time.Minute}, LimitOneNewReservationPerHypervisor: true, diff --git a/internal/scheduling/reservations/failover/controller.go b/internal/scheduling/reservations/failover/controller.go index 213708245..4de28cca8 100644 --- a/internal/scheduling/reservations/failover/controller.go +++ b/internal/scheduling/reservations/failover/controller.go @@ -14,6 +14,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/pkg/multicluster" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -40,21 +41,21 @@ var log = ctrl.Log.WithName("failover-reservation-controller").WithValues("modul // 2. Watch-based per-reservation reconciliation (Reconcile) - handles acknowledgment and validation of individual reservations type FailoverReservationController struct { client.Client - VMSource reservations.VMSource - Config FailoverConfig - SchedulerClient *reservations.SchedulerClient - Recorder events.EventRecorder // Event recorder for emitting Kubernetes events - Monitor *FailoverMonitor - reconcileCount int64 // Track reconciliation count for rotating VM selection + VMSource reservations.VMSource + Config FailoverConfig + Scheduler *sched.Scheduler + Recorder events.EventRecorder // Event recorder for emitting Kubernetes events + Monitor *FailoverMonitor + reconcileCount int64 // Track reconciliation count for rotating VM selection } -func NewFailoverReservationController(c client.Client, vmSource reservations.VMSource, config FailoverConfig, schedulerClient *reservations.SchedulerClient, monitor *FailoverMonitor) *FailoverReservationController { +func NewFailoverReservationController(c client.Client, vmSource reservations.VMSource, config FailoverConfig, scheduler *sched.Scheduler, monitor *FailoverMonitor) *FailoverReservationController { return &FailoverReservationController{ - Client: c, - VMSource: vmSource, - Config: config, - SchedulerClient: schedulerClient, - Monitor: monitor, + Client: c, + VMSource: vmSource, + Config: config, + Scheduler: scheduler, + Monitor: monitor, } } @@ -881,7 +882,6 @@ func (c *FailoverReservationController) Start(ctx context.Context) error { "maxFailuresForShortInterval", c.Config.MaxFailuresForShortInterval, "creator", c.Config.Creator, "datasourceName", c.Config.DatasourceName, - "schedulerURL", c.Config.SchedulerURL, "flavorFailoverRequirements", c.Config.FlavorFailoverRequirements, "maxVMsToProcess", c.Config.MaxVMsToProcess, "trustHypervisorLocation", c.Config.TrustHypervisorLocation, diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index bdece85dd..27b63ab40 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -704,7 +704,9 @@ func (env *IntegrationTestEnv) LogStateSummary() { func (env *IntegrationTestEnv) TriggerFailoverReconcile(flavorRequirements map[string]int) error { env.T.Helper() - schedulerClient := reservations.NewSchedulerClient(env.SchedulerBaseURL + "/scheduler/nova/external") + // Drive the same in-process pipeline the testHTTPAPI uses, so the failover + // reservation flow exercises the real scheduler over the env's nova controller. + scheduler := compute.NewScheduler(env.NovaController, passthroughGatherer{}) config := FailoverConfig{ ReconcileInterval: metav1.Duration{Duration: time.Minute}, @@ -717,7 +719,7 @@ func (env *IntegrationTestEnv) TriggerFailoverReconcile(flavorRequirements map[s env.K8sClient, env.VMSource, config, - schedulerClient, + scheduler, nil, ) diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 2cae7105a..46ea5d961 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -13,7 +13,9 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "k8s.io/apimachinery/pkg/api/resource" ) // Pipeline names for failover reservation scheduling @@ -39,6 +41,25 @@ func inferFailoverPipeline(extraSpecs map[string]string) string { return "kvm-general-purpose-load-balancing" } +// traitsFromExtraSpecs splits the flavor extra specs' "trait:" entries into +// required and forbidden trait names, mirroring the Nova spec adapter so failover +// reservation scheduling honors the same trait constraints as regular placement. +func traitsFromExtraSpecs(extraSpecs map[string]string) (required, forbidden []string) { + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "trait:") + if !ok { + continue + } + switch value { + case "required": + required = append(required, name) + case "forbidden": + forbidden = append(forbidden, name) + } + } + return required, forbidden +} + // imagePropertiesFromFlavor synthesizes Glance-style image properties from a flavor's // capabilities:hypervisor_type extra spec. The failover controller does not have access // to the real Glance image metadata of the VM, but for reservation scheduling the VM @@ -93,38 +114,38 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex ignoreHypervisors := []string{vm.CurrentHypervisor} - // Build flavor extra specs from VM's extra specs - // Start with the VM's actual extra specs, then ensure required defaults are set - flavorExtraSpecs := make(map[string]string) - for k, v := range vm.FlavorExtraSpecs { - flavorExtraSpecs[k] = v - } - // Ensure hypervisor_type is set for KVM scheduling if not already present - if _, ok := flavorExtraSpecs["capabilities:hypervisor_type"]; !ok { - flavorExtraSpecs["capabilities:hypervisor_type"] = "qemu" + // Build the typed scheduler request directly (in-process; no HTTP). KVM + // failover reservations gather all candidates; the eligible list is a + // starting set. The filter_correct_az filter excludes hosts outside the VM's + // AZ. ImageHypervisorType is irrelevant here: filter_image_properties skips + // the failover/reuse reservation intents. + hosts := make([]compute.Host, 0, len(eligibleHypervisors)) + weights := make(map[string]float64, len(eligibleHypervisors)) + for _, h := range eligibleHypervisors { + hosts = append(hosts, compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) + weights[h.ComputeHost] = 0.0 } - - // Schedule the reservation using the SchedulerClient. - // Note: We pass all hypervisors (from all AZs) in EligibleHosts. The scheduler pipeline's - // filter_correct_az filter will exclude hosts that are not in the VM's availability zone. - // Use resSpec.FlavorName and reservation spec resources so the scheduler checks capacity for the - // correct flavor size (which may be the LargestFlavor from the flavor group). - scheduleReq := reservations.ScheduleReservationRequest{ - InstanceUUID: vm.UUID, - ProjectID: vm.ProjectID, - FlavorName: resSpec.FlavorName, - FlavorExtraSpecs: flavorExtraSpecs, - MemoryMB: resSpec.MemoryMB, - VCPUs: resSpec.VCPUs, - EligibleHosts: eligibleHypervisors, - IgnoreHosts: ignoreHypervisors, - Pipeline: pipeline, - AvailabilityZone: vm.AvailabilityZone, - SchedulerHints: map[string]any{ - "_nova_check_type": string(intent), - api.HintKeyResourceGroup: resSpec.ResourceGroup(vm.FlavorName), + requiredTraits, forbiddenTraits := traitsFromExtraSpecs(vm.FlavorExtraSpecs) + schedReq := compute.SchedulerRequest{ + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: hosts, + Weights: weights, + Options: opts, + Intent: intent, + InstanceUUID: vm.UUID, + ProjectID: vm.ProjectID, + AvailabilityZone: vm.AvailabilityZone, + IgnoreHosts: ignoreHypervisors, + ResourceGroup: resSpec.ResourceGroup(vm.FlavorName), + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + Flavor: compute.FlavorSpec{ + Name: resSpec.FlavorName, + VCPUs: resSpec.VCPUs, + Memory: *resource.NewQuantity(int64(resSpec.MemoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // flavor memory bounded by specs }, - ImageProperties: imagePropertiesFromFlavor(flavorExtraSpecs), } logger.V(1).Info("scheduling failover reservation", @@ -133,7 +154,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex "eligibleHypervisors", len(eligibleHypervisors), "ignoreHypervisors", ignoreHypervisors) - scheduleResp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, opts) + out, err := c.runScheduler(ctx, schedReq) if err != nil { logger.Error(err, "failed to schedule failover reservation", "vmUUID", vm.UUID, "pipeline", pipeline) return nil, fmt.Errorf("failed to schedule failover reservation: %w", err) @@ -144,9 +165,19 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex "pipeline", pipeline, "eligibleHypervisors", len(eligibleHypervisors), "ignoreHypervisors", ignoreHypervisors, - "returnedHypervisors", scheduleResp.Hosts) + "returnedHypervisors", out.Result.OrderedHosts) - return scheduleResp.Hosts, nil + return out.Result.OrderedHosts, nil +} + +// runScheduler runs the shared scheduler, choosing the lock-free path for +// read-only runs (e.g. reuse/evacuate compatibility checks) and the locked path +// for runs that mutate shared reservation state. +func (c *FailoverReservationController) runScheduler(ctx context.Context, req compute.SchedulerRequest) (compute.RunOutcome, error) { + if req.Options.ReadOnly { + return c.Scheduler.RunWithoutLock(ctx, req, nil, nil) + } + return c.Scheduler.Run(ctx, req, nil, nil) } // tryReuseExistingReservation finds an existing reservation that can be reused for a VM. @@ -232,7 +263,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( vcpus = uint64(vcpusRes.Value()) //nolint:gosec // vcpus values won't overflow } - // Build flavor extra specs from VM's extra specs + // Infer the pipeline from the VM's flavor extra specs (HANA vs general purpose). flavorExtraSpecs := make(map[string]string) for k, v := range vm.FlavorExtraSpecs { flavorExtraSpecs[k] = v @@ -241,57 +272,59 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( flavorExtraSpecs["capabilities:hypervisor_type"] = "qemu" } - // Build a single-host request to validate the VM can use the reservation host - // Use vm.CurrentHypervisor directly instead of a separate parameter to avoid stale data - // Set _nova_check_type to "evacuate" so that filter_has_enough_capacity unlocks - // the failover reservation's resources for this VM (avoiding self-blocking). - // Use the actual VM UUID (not prefixed) so the filter can match it in the reservation's allocations. - scheduleReq := reservations.ScheduleReservationRequest{ - InstanceUUID: vm.UUID, - ProjectID: vm.ProjectID, - FlavorName: vm.FlavorName, - FlavorExtraSpecs: flavorExtraSpecs, - MemoryMB: memoryMB, - VCPUs: vcpus, - EligibleHosts: []api.ExternalSchedulerHost{{ComputeHost: reservationHost}}, - IgnoreHosts: []string{vm.CurrentHypervisor}, - Pipeline: inferFailoverPipeline(flavorExtraSpecs), - AvailabilityZone: vm.AvailabilityZone, - SchedulerHints: map[string]any{"_nova_check_type": string(api.EvacuateIntent)}, - ImageProperties: imagePropertiesFromFlavor(flavorExtraSpecs), + // Validate the VM can use the reservation host. Set the Evacuate intent so + // filter_has_enough_capacity unlocks the failover reservation's resources for + // this VM (avoiding self-blocking). Use the actual VM UUID so the filter can + // match it in the reservation's allocations. KVM gathers all candidates; the + // reservation host must come back ranked for the VM to be compatible. + pipeline := inferFailoverPipeline(flavorExtraSpecs) + requiredTraits, forbiddenTraits := traitsFromExtraSpecs(flavorExtraSpecs) + evacuateOpts := DefaultFailoverOptions + evacuateOpts.ReadOnly = true + evacuateOpts.LockReservations = true + schedReq := compute.SchedulerRequest{ + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: []compute.Host{{ComputeHost: reservationHost, HypervisorHostname: reservationHost}}, + Weights: map[string]float64{reservationHost: 0.0}, + Options: evacuateOpts, + Intent: api.EvacuateIntent, + InstanceUUID: vm.UUID, + ProjectID: vm.ProjectID, + AvailabilityZone: vm.AvailabilityZone, + IgnoreHosts: []string{vm.CurrentHypervisor}, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + Flavor: compute.FlavorSpec{ + Name: vm.FlavorName, + VCPUs: vcpus, + Memory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI), + }, } logger.V(1).Info("validating VM via scheduler evacuation", "vmUUID", vm.UUID, "reservationHost", reservationHost, "vmCurrentHost", vm.CurrentHypervisor, - "pipeline", scheduleReq.Pipeline) + "pipeline", pipeline) - evacuateOpts := DefaultFailoverOptions - evacuateOpts.ReadOnly = true - evacuateOpts.LockReservations = true - resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, evacuateOpts) + out, err := c.runScheduler(ctx, schedReq) if err != nil { logger.Error(err, "failed to validate VM for reservation host", "vmUUID", vm.UUID, "reservationHost", reservationHost) return false, fmt.Errorf("failed to validate VM for reservation host: %w", err) } + hosts := out.Result.OrderedHosts // Handle empty response - no hosts returned - if len(resp.Hosts) < 1 { + if len(hosts) < 1 { logger.V(1).Info("scheduler returned no hosts for VM validation", "vmUUID", vm.UUID, "reservationHost", reservationHost) return false, nil } - // Log unexpected scheduler responses - if len(resp.Hosts) > 1 || resp.Hosts[0] != reservationHost { - logger.Error(nil, "scheduler returned unexpected hosts for single-host validation request", - "vmUUID", vm.UUID, - "reservationHost", reservationHost, - "returnedHosts", resp.Hosts) - } - - // If the reservation host is returned, the VM can use it - return resp.Hosts[0] == reservationHost, nil + // The reservation host must be among the returned hosts for the VM to be + // compatible with it. + return slices.Contains(hosts, reservationHost), nil } // scheduleAndBuildNewFailoverReservation schedules a failover reservation for a VM. diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go index 506b6033b..aef2b584a 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling_test.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -5,14 +5,12 @@ package failover import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" "testing" - novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -477,22 +475,40 @@ func buildSchedulingTestVMWithResources(uuid, hypervisor string, memoryMB, vcpus } } -// captureSchedulerRequest spins up a test HTTP server that captures one scheduler request -// and returns a single host. The captured request is written into *out. -func captureSchedulerRequest(t *testing.T, host string, out *novaapi.ExternalSchedulerRequest) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(out); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - resp := novaapi.ExternalSchedulerResponse{Hosts: []string{host}} - _ = json.NewEncoder(w).Encode(resp) //nolint:errcheck - })) +// capturingPipeline records the last SchedulerRequest it ran and returns a +// single fixed host, so tests can assert the options the controller built. +type capturingPipeline struct { + host string + captured *sched.SchedulerRequest +} + +func (p *capturingPipeline) Run(req sched.SchedulerRequest) (sched.SchedulerResult, error) { + *p.captured = req + return sched.SchedulerResult{OrderedHosts: []string{p.host}}, nil +} + +// capturingProvider serves the same capturingPipeline for every pipeline name. +type capturingProvider struct{ pipeline *capturingPipeline } + +func (f capturingProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { + return f.pipeline, true +} + +// capturingGatherer is a no-op gatherer (the tests preselect their own hosts). +type capturingGatherer struct{} + +func (capturingGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { + return nil +} + +// newCapturingScheduler builds an in-process scheduler whose pipeline returns +// the given host and records every request it ran into *out. +func newCapturingScheduler(host string, out *sched.SchedulerRequest) *sched.Scheduler { + return sched.NewScheduler(capturingProvider{pipeline: &capturingPipeline{host: host, captured: out}}, capturingGatherer{}) } -// buildMinimalController returns a FailoverReservationController wired to a scheduler at the given URL. -func buildMinimalController(t *testing.T, schedulerURL string) *FailoverReservationController { +// buildMinimalController returns a FailoverReservationController wired to the given scheduler. +func buildMinimalController(t *testing.T, scheduler *sched.Scheduler) *FailoverReservationController { t.Helper() scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { @@ -502,10 +518,9 @@ func buildMinimalController(t *testing.T, schedulerURL string) *FailoverReservat t.Fatalf("add hv1 to scheme: %v", err) } k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - schedulerClient := reservations.NewSchedulerClient(schedulerURL + "/scheduler/nova/external") config := FailoverConfig{} config.ApplyDefaults() - return NewFailoverReservationController(k8sClient, nil, config, schedulerClient, nil) + return NewFailoverReservationController(k8sClient, nil, config, scheduler, nil) } // ============================================================================ @@ -561,11 +576,10 @@ func TestFailoverSchedulerOptions(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var capturedReq novaapi.ExternalSchedulerRequest - server := captureSchedulerRequest(t, "host-2", &capturedReq) - defer server.Close() + var capturedReq sched.SchedulerRequest + scheduler := newCapturingScheduler("host-2", &capturedReq) - tt.call(buildMinimalController(t, server.URL), context.Background()) + tt.call(buildMinimalController(t, scheduler), context.Background()) if capturedReq.Options.ReadOnly != tt.wantReadOnly { t.Errorf("ReadOnly = %v, want %v", capturedReq.Options.ReadOnly, tt.wantReadOnly) diff --git a/internal/scheduling/reservations/scheduler_client.go b/internal/scheduling/reservations/scheduler_client.go deleted file mode 100644 index e5ee2c3ee..000000000 --- a/internal/scheduling/reservations/scheduler_client.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package reservations - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - api "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/scheduling" - "github.com/go-logr/logr" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -// DefaultHTTPTimeout is the default timeout for HTTP requests to the scheduler API. -const DefaultHTTPTimeout = 30 * time.Second - -var log = logf.Log.WithName("scheduler-client").WithValues("module", "reservations") - -// loggerFromContext returns a logger with greq and req values from the context. -func loggerFromContext(ctx context.Context) logr.Logger { - return log.WithValues( - "greq", GlobalRequestIDFromContext(ctx), - "req", RequestIDFromContext(ctx), - ) -} - -// NOTE+FIXME: we should not send ourselves REST API calls. This needs to be replaced by direct Go call (if possible) or communication via CRDs - -// SchedulerClient is a client for the external scheduler API. -// It can be used by both the ReservationReconciler and FailoverReservationController. -type SchedulerClient struct { - // URL of the external scheduler API. - URL string - // HTTP client to use for requests. - HTTPClient *http.Client -} - -// NewSchedulerClient creates a new SchedulerClient with a default timeout. -func NewSchedulerClient(url string) *SchedulerClient { - return &SchedulerClient{ - URL: url, - HTTPClient: &http.Client{ - Timeout: DefaultHTTPTimeout, - }, - } -} - -// ScheduleReservationRequest contains the parameters for scheduling a reservation. -type ScheduleReservationRequest struct { - // InstanceUUID is the unique identifier for the reservation (usually the reservation name). - InstanceUUID string - // ProjectID is the OpenStack project ID. - ProjectID string - // FlavorName is the name of the flavor. - FlavorName string - // FlavorExtraSpecs are extra specifications for the flavor. - FlavorExtraSpecs map[string]string - // MemoryMB is the memory in MB. - MemoryMB uint64 - // VCPUs is the number of virtual CPUs. - VCPUs uint64 - // EligibleHosts is the list of hosts that can be considered for placement. - EligibleHosts []api.ExternalSchedulerHost - // IgnoreHosts is a list of hosts to ignore during scheduling. - // This is used for failover reservations to avoid placing on the same host as the VMs. - IgnoreHosts []string - // Pipeline is the name of the pipeline to execute. - // If empty, the default pipeline will be used. - Pipeline string - // AvailabilityZone is the availability zone to schedule in. - // This is used by the filter_correct_az filter to ensure hosts are in the correct AZ. - AvailabilityZone string - // SchedulerHints are hints passed to the scheduler pipeline. - // Used to set _nova_check_type for evacuation intent detection. - SchedulerHints map[string]any - // ImageProperties are Glance-style image properties (e.g., "img_hv_type", "hypervisor_type") - // forwarded on Spec.Data.Image.Data.Properties.Data. Optional; when empty the outgoing - // request leaves the image field at its zero value. Used by FilterImagePropertiesStep. - ImageProperties map[string]any -} - -// ScheduleReservationResponse contains the result of scheduling a reservation. -type ScheduleReservationResponse struct { - // Hosts is the ordered list of hosts that the reservation can be placed on. - // The first host is the best choice. - Hosts []string -} - -// ScheduleReservation calls the external scheduler API to find a host for a reservation. -// The context should contain GlobalRequestID and RequestID for logging (use WithGlobalRequestID/WithRequestID). -func (c *SchedulerClient) ScheduleReservation(ctx context.Context, req ScheduleReservationRequest, opts scheduling.Options) (*ScheduleReservationResponse, error) { - logger := loggerFromContext(ctx) - - if err := opts.Validate(); err != nil { - return nil, fmt.Errorf("invalid scheduling options: %w", err) - } - - // Build weights map (all zero for reservations) - weights := make(map[string]float64, len(req.EligibleHosts)) - for _, host := range req.EligibleHosts { - weights[host.ComputeHost] = 0.0 - } - - // Build ignore hosts pointer - var ignoreHosts *[]string - if len(req.IgnoreHosts) > 0 { - ignoreHosts = &req.IgnoreHosts - } - - // Build the context with request IDs - var globalReqID *string - if greq := GlobalRequestIDFromContext(ctx); greq != "" { - globalReqID = &greq - } - - // Build the external scheduler request - externalSchedulerRequest := api.ExternalSchedulerRequest{ - Pipeline: req.Pipeline, - Hosts: req.EligibleHosts, - Weights: weights, - Options: opts, - Context: api.NovaRequestContext{ - RequestID: RequestIDFromContext(ctx), - GlobalRequestID: globalReqID, - ProjectID: req.ProjectID, - }, - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: req.InstanceUUID, - NumInstances: 1, // One for each reservation. - ProjectID: req.ProjectID, - AvailabilityZone: req.AvailabilityZone, - IgnoreHosts: ignoreHosts, - SchedulerHints: req.getSchedulerHints(), - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: req.FlavorName, - ExtraSpecs: req.FlavorExtraSpecs, - MemoryMB: req.MemoryMB, - VCPUs: req.VCPUs, - // Disk is currently not considered. - }, - }, - Image: api.NovaObject[api.NovaImageMeta]{ - Data: api.NovaImageMeta{ - Properties: api.NovaObject[map[string]any]{ - Data: req.ImageProperties, - }, - }, - }, - }, - }, - } - - logger.V(1).Info("sending external scheduler request", - "url", c.URL, - "pipeline", req.Pipeline, - "instanceUUID", req.InstanceUUID, - "flavorName", req.FlavorName, - "flavorExtraSpecs", req.FlavorExtraSpecs, - "eligibleHostsCount", len(req.EligibleHosts), - "ignoreHostsCount", len(req.IgnoreHosts), - "hasSchedulerHints", len(req.SchedulerHints) > 0, - "availabilityZone", req.AvailabilityZone) - - // Marshal the request - reqBody, err := json.Marshal(externalSchedulerRequest) - if err != nil { - logger.Error(err, "failed to marshal external scheduler request") - return nil, fmt.Errorf("failed to marshal external scheduler request: %w", err) - } - - // Create HTTP request with context - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL, bytes.NewReader(reqBody)) - if err != nil { - logger.Error(err, "failed to create HTTP request") - return nil, fmt.Errorf("failed to create HTTP request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") - - // Send the request and measure duration - startTime := time.Now() - response, err := c.HTTPClient.Do(httpReq) - duration := time.Since(startTime) - if err != nil { - logger.Error(err, "failed to send external scheduler request") - return nil, fmt.Errorf("failed to send external scheduler request: %w", err) - } - defer response.Body.Close() - - // Check response status - if response.StatusCode != http.StatusOK { - body, err := io.ReadAll(io.LimitReader(response.Body, 1024)) - if err != nil { - logger.Error(err, "failed to read error response body", "statusCode", response.StatusCode) - return nil, fmt.Errorf("external scheduler returned status %d", response.StatusCode) - } - logger.Error(nil, "external scheduler returned non-OK status", "statusCode", response.StatusCode, "body", string(body)) - return nil, fmt.Errorf("external scheduler returned status %d: %s", response.StatusCode, string(body)) - } - - // Decode the response - var externalSchedulerResponse api.ExternalSchedulerResponse - if err := json.NewDecoder(response.Body).Decode(&externalSchedulerResponse); err != nil { - logger.Error(err, "failed to decode external scheduler response") - return nil, fmt.Errorf("failed to decode external scheduler response: %w", err) - } - - logger.V(1).Info("received external scheduler response", - "hostsCount", len(externalSchedulerResponse.Hosts), - "duration", duration.Round(time.Millisecond)) - - return &ScheduleReservationResponse{ - Hosts: externalSchedulerResponse.Hosts, - }, nil -} - -// getSchedulerHints returns the scheduler hints, or an empty map if nil. -func (req ScheduleReservationRequest) getSchedulerHints() map[string]any { - if req.SchedulerHints == nil { - return make(map[string]any) - } - return req.SchedulerHints -} From 429acd436f1345a92d09de98589407c3f5c7ea82 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 17:46:18 +0100 Subject: [PATCH 07/21] Refactor scheduling components to replace Decision with SchedulerResult Signed-off-by: Markus Wieland --- api/v1alpha1/decision_types.go | 160 ------ api/v1alpha1/scheduling_intent.go | 14 + api/v1alpha1/zz_generated.deepcopy.go | 216 --------- .../cortex-cinder/templates/alerts.yaml | 40 -- .../bundles/cortex-cinder/templates/kpis.yaml | 12 - .../bundles/cortex-manila/templates/kpis.yaml | 12 - .../bundles/cortex-nova/templates/alerts.yaml | 40 -- helm/bundles/cortex-nova/templates/kpis.yaml | 12 - .../cortex-shim/templates/clusterrole.yaml | 3 - .../files/crds/cortex.cloud_decisions.yaml | 412 ---------------- helm/library/cortex/templates/rbac/role.yaml | 3 - .../kpis/plugins/deployment/decision_state.go | 97 ---- .../plugins/deployment/decision_state_test.go | 327 ------------- .../cinder/external_scheduler_api.go | 54 +-- .../cinder/external_scheduler_api_test.go | 214 +++----- .../filter_weigher_pipeline_controller.go | 113 +---- ...filter_weigher_pipeline_controller_test.go | 372 +------------- internal/scheduling/compute/result.go | 11 +- internal/scheduling/lib/explanation.go | 11 +- internal/scheduling/lib/explanation_test.go | 32 +- .../scheduling/lib/filter_weigher_pipeline.go | 14 +- internal/scheduling/lib/pipeline_result.go | 31 ++ internal/scheduling/lib/weighing_explainer.go | 12 +- .../scheduling/lib/weighing_explainer_test.go | 76 ++- .../filter_weigher_pipeline_controller.go | 110 +---- ...filter_weigher_pipeline_controller_test.go | 219 ++------- .../manila/external_scheduler_api.go | 54 +-- .../manila/external_scheduler_api_test.go | 214 +++----- .../filter_weigher_pipeline_controller.go | 113 +---- ...filter_weigher_pipeline_controller_test.go | 458 ++---------------- internal/scheduling/nova/cr_metrics_test.go | 1 - internal/scheduling/nova/crs/recorder_test.go | 34 +- .../nova/external_scheduler_api_test.go | 35 +- .../filter_weigher_pipeline_controller.go | 115 +---- ...filter_weigher_pipeline_controller_test.go | 164 +------ pkg/multicluster/client_test.go | 6 +- 36 files changed, 464 insertions(+), 3347 deletions(-) delete mode 100644 api/v1alpha1/decision_types.go create mode 100644 api/v1alpha1/scheduling_intent.go delete mode 100644 helm/library/cortex/files/crds/cortex.cloud_decisions.yaml delete mode 100644 internal/knowledge/kpis/plugins/deployment/decision_state.go delete mode 100644 internal/knowledge/kpis/plugins/deployment/decision_state_test.go create mode 100644 internal/scheduling/lib/pipeline_result.go diff --git a/api/v1alpha1/decision_types.go b/api/v1alpha1/decision_types.go deleted file mode 100644 index fb1e07ca9..000000000 --- a/api/v1alpha1/decision_types.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// SchedulingIntent defines the intent of a scheduling decision. -type SchedulingIntent string - -// Other intents can be defined by the operators. -const ( - // Used as default intent if the operator does not specify one. - SchedulingIntentUnknown SchedulingIntent = "Unknown" -) - -type DecisionSpec struct { - // SchedulingDomain defines in which scheduling domain this decision - // was or is processed (e.g., nova, cinder, manila). - SchedulingDomain SchedulingDomain `json:"schedulingDomain"` - - // A reference to the pipeline that should be used for this decision. - // This reference can be used to look up the pipeline definition and its - // scheduler step configuration for additional context. - PipelineRef corev1.ObjectReference `json:"pipelineRef"` - - // An identifier for the underlying resource to be scheduled. - // For example, this can be the UUID of a nova instance or cinder volume. - // This can be used to correlate multiple decisions for the same resource. - ResourceID string `json:"resourceID"` - - // If the type is "nova", this field contains the raw nova decision request. - // +kubebuilder:validation:Optional - NovaRaw *runtime.RawExtension `json:"novaRaw,omitempty"` - // If the type is "cinder", this field contains the raw cinder decision request. - // +kubebuilder:validation:Optional - CinderRaw *runtime.RawExtension `json:"cinderRaw,omitempty"` - // If the type is "manila", this field contains the raw manila decision request. - // +kubebuilder:validation:Optional - ManilaRaw *runtime.RawExtension `json:"manilaRaw,omitempty"` - // If the type is "machine", this field contains the machine reference. - // +kubebuilder:validation:Optional - MachineRef *corev1.ObjectReference `json:"machineRef,omitempty"` - // If the type is "pod", this field contains the pod reference. - // +kubebuilder:validation:Optional - PodRef *corev1.ObjectReference `json:"podRef,omitempty"` - - // The intent of the scheduling decision (e.g., initial scheduling, rescheduling, etc.). - // +kubebuilder:validation:Optional - Intent SchedulingIntent `json:"intent,omitempty"` -} - -type StepResult struct { - // object reference to the scheduler step. - StepName string `json:"stepName"` - // Activations of the step for each host. - Activations map[string]float64 `json:"activations"` -} - -type DecisionResult struct { - // Raw input weights to the pipeline. - // +kubebuilder:validation:Optional - RawInWeights map[string]float64 `json:"rawInWeights"` - // Normalized input weights to the pipeline. - // +kubebuilder:validation:Optional - NormalizedInWeights map[string]float64 `json:"normalizedInWeights"` - // Outputs of the decision pipeline including the activations used - // to make the final ordering of compute hosts. - // +kubebuilder:validation:Optional - StepResults []StepResult `json:"stepResults,omitempty"` - // Aggregated output weights from the pipeline. - // +kubebuilder:validation:Optional - AggregatedOutWeights map[string]float64 `json:"aggregatedOutWeights"` - // Final ordered list of hosts from most preferred to least preferred. - // +kubebuilder:validation:Optional - OrderedHosts []string `json:"orderedHosts,omitempty"` - // The first element of the ordered hosts is considered the target host. - // +kubebuilder:validation:Optional - TargetHost *string `json:"targetHost,omitempty"` -} - -const ( - // The decision was successfully processed. - DecisionConditionReady = "Ready" -) - -type DecisionStatus struct { - // The result of this decision. - // +kubebuilder:validation:Optional - Result *DecisionResult `json:"result,omitempty"` - - // If there were previous decisions for the underlying resource, they can - // be resolved here to provide historical context for the decision. - // +kubebuilder:validation:Optional - History *[]corev1.ObjectReference `json:"history,omitempty"` - - // The number of decisions that preceded this one for the same resource. - // +kubebuilder:validation:Optional - Precedence *int `json:"precedence,omitempty"` - - // A human-readable explanation of the decision result. - // +kubebuilder:validation:Optional - Explanation string `json:"explanation,omitempty"` - - // The current status conditions of the decision. - // +kubebuilder:validation:Optional - Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster -// +kubebuilder:printcolumn:name="Domain",type="string",JSONPath=".spec.schedulingDomain" -// +kubebuilder:printcolumn:name="Resource ID",type="string",JSONPath=".spec.resourceID" -// +kubebuilder:printcolumn:name="#",type="string",JSONPath=".status.precedence" -// +kubebuilder:printcolumn:name="Created",type="date",JSONPath=".metadata.creationTimestamp" -// +kubebuilder:printcolumn:name="Pipeline",type="string",JSONPath=".spec.pipelineRef.name" -// +kubebuilder:printcolumn:name="TargetHost",type="string",JSONPath=".status.result.targetHost" -// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" -// +kubebuilder:selectablefield:JSONPath=".spec.resourceID" - -// Currently the Decision CRD is an in-memory scheduling object used by the external scheduler API -// and filter-weigher pipelines to compute a placement result. It is currently NOT persisted -// to etcd — the scheduling outcome (including a human-readable explanation) is written onto the -// scheduled resource instead, e.g. the nova VM's spec.cortex fields. -// The long-term shape of this CRD is still under discussion; until that is -// settled the Decision serves only as a transient carrier within a single -// scheduling run. -type Decision struct { - metav1.TypeMeta `json:",inline"` - - // metadata is a standard object metadata - // +optional - metav1.ObjectMeta `json:"metadata,omitempty,omitzero"` - - // spec defines the desired state of Decision - // +required - Spec DecisionSpec `json:"spec"` - - // status defines the observed state of Decision - // +optional - Status DecisionStatus `json:"status,omitempty,omitzero"` -} - -// +kubebuilder:object:root=true - -// DecisionList contains a list of Decision -type DecisionList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Decision `json:"items"` -} - -func init() { - SchemeBuilder.Register(&Decision{}, &DecisionList{}) -} diff --git a/api/v1alpha1/scheduling_intent.go b/api/v1alpha1/scheduling_intent.go new file mode 100644 index 000000000..03eff7f86 --- /dev/null +++ b/api/v1alpha1/scheduling_intent.go @@ -0,0 +1,14 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +// SchedulingIntent defines the intent of a scheduling decision (e.g. initial +// scheduling, rescheduling, failover). It is the base type for the Nova intent +// constants in api/external/nova and is carried on Reservation.Spec.Intent. +type SchedulingIntent string + +const ( + // SchedulingIntentUnknown is the default intent when none is specified. + SchedulingIntentUnknown SchedulingIntent = "Unknown" +) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a60d5a1f4..f5534ddbf 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -484,200 +484,6 @@ func (in *DatasourceStatus) DeepCopy() *DatasourceStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Decision) DeepCopyInto(out *Decision) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Decision. -func (in *Decision) DeepCopy() *Decision { - if in == nil { - return nil - } - out := new(Decision) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Decision) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DecisionList) DeepCopyInto(out *DecisionList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Decision, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecisionList. -func (in *DecisionList) DeepCopy() *DecisionList { - if in == nil { - return nil - } - out := new(DecisionList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *DecisionList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DecisionResult) DeepCopyInto(out *DecisionResult) { - *out = *in - if in.RawInWeights != nil { - in, out := &in.RawInWeights, &out.RawInWeights - *out = make(map[string]float64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.NormalizedInWeights != nil { - in, out := &in.NormalizedInWeights, &out.NormalizedInWeights - *out = make(map[string]float64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.StepResults != nil { - in, out := &in.StepResults, &out.StepResults - *out = make([]StepResult, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.AggregatedOutWeights != nil { - in, out := &in.AggregatedOutWeights, &out.AggregatedOutWeights - *out = make(map[string]float64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.OrderedHosts != nil { - in, out := &in.OrderedHosts, &out.OrderedHosts - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.TargetHost != nil { - in, out := &in.TargetHost, &out.TargetHost - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecisionResult. -func (in *DecisionResult) DeepCopy() *DecisionResult { - if in == nil { - return nil - } - out := new(DecisionResult) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DecisionSpec) DeepCopyInto(out *DecisionSpec) { - *out = *in - out.PipelineRef = in.PipelineRef - if in.NovaRaw != nil { - in, out := &in.NovaRaw, &out.NovaRaw - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } - if in.CinderRaw != nil { - in, out := &in.CinderRaw, &out.CinderRaw - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } - if in.ManilaRaw != nil { - in, out := &in.ManilaRaw, &out.ManilaRaw - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } - if in.MachineRef != nil { - in, out := &in.MachineRef, &out.MachineRef - *out = new(v1.ObjectReference) - **out = **in - } - if in.PodRef != nil { - in, out := &in.PodRef, &out.PodRef - *out = new(v1.ObjectReference) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecisionSpec. -func (in *DecisionSpec) DeepCopy() *DecisionSpec { - if in == nil { - return nil - } - out := new(DecisionSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DecisionStatus) DeepCopyInto(out *DecisionStatus) { - *out = *in - if in.Result != nil { - in, out := &in.Result, &out.Result - *out = new(DecisionResult) - (*in).DeepCopyInto(*out) - } - if in.History != nil { - in, out := &in.History, &out.History - *out = new([]v1.ObjectReference) - if **in != nil { - in, out := *in, *out - *out = make([]v1.ObjectReference, len(*in)) - copy(*out, *in) - } - } - if in.Precedence != nil { - in, out := &in.Precedence, &out.Precedence - *out = new(int) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]metav1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecisionStatus. -func (in *DecisionStatus) DeepCopy() *DecisionStatus { - if in == nil { - return nil - } - out := new(DecisionStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Descheduling) DeepCopyInto(out *Descheduling) { *out = *in @@ -1951,28 +1757,6 @@ func (in *ReservationStatus) DeepCopy() *ReservationStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StepResult) DeepCopyInto(out *StepResult) { - *out = *in - if in.Activations != nil { - in, out := &in.Activations, &out.Activations - *out = make(map[string]float64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StepResult. -func (in *StepResult) DeepCopy() *StepResult { - if in == nil { - return nil - } - out := new(StepResult) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WeigherSpec) DeepCopyInto(out *WeigherSpec) { *out = *in diff --git a/helm/bundles/cortex-cinder/templates/alerts.yaml b/helm/bundles/cortex-cinder/templates/alerts.yaml index f15cfb755..b0c6c6557 100644 --- a/helm/bundles/cortex-cinder/templates/alerts.yaml +++ b/helm/bundles/cortex-cinder/templates/alerts.yaml @@ -252,46 +252,6 @@ spec: configuration. It is recommended to investigate the knowledge status and logs for more details. - - alert: CortexCinderDecisionsWithErrors - expr: cortex_decision_state{domain="cinder",state="error"} > 0 - for: 5m - labels: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status" -{{- end }} - context: decisions - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Some decisions are in error state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" - description: > - The cortex scheduling pipeline generated decisions that are in error state. - This may indicate issues with the decision logic or the underlying infrastructure. - It is recommended to investigate the decision logs and the state of the - VMs being processed. - - - alert: CortexCinderTooManyDecisionsWaiting - expr: cortex_decision_state{domain="cinder",state="waiting"} > 10 - for: 5m - labels: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status" -{{- end }} - context: decisions - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Too many decisions are in waiting state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" - description: > - The cortex scheduling pipeline has a high number of decisions for which - no target host has been assigned yet. - - This may indicate a backlog in processing or issues with the decision logic. - It is recommended to investigate the decision logs and the state of the - VMs being processed. - - alert: CortexCinderKPIUnready expr: | cortex_kpi_state{domain="cinder",state!="ready"} != 0 diff --git a/helm/bundles/cortex-cinder/templates/kpis.yaml b/helm/bundles/cortex-cinder/templates/kpis.yaml index fb3958c08..78fd25dd9 100644 --- a/helm/bundles/cortex-cinder/templates/kpis.yaml +++ b/helm/bundles/cortex-cinder/templates/kpis.yaml @@ -25,18 +25,6 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI -metadata: - name: cortex-cinder-decision-state -spec: - schedulingDomain: cinder - impl: decision_state_kpi - opts: - decisionSchedulingDomain: cinder - description: | - This aggregated KPI tracks the state of decision resources managed by cortex. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: KPI metadata: name: cortex-cinder-kpi-state spec: diff --git a/helm/bundles/cortex-manila/templates/kpis.yaml b/helm/bundles/cortex-manila/templates/kpis.yaml index d05910f3b..6289d0262 100644 --- a/helm/bundles/cortex-manila/templates/kpis.yaml +++ b/helm/bundles/cortex-manila/templates/kpis.yaml @@ -38,18 +38,6 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI -metadata: - name: cortex-manila-decision-state -spec: - schedulingDomain: manila - impl: decision_state_kpi - opts: - decisionSchedulingDomain: manila - description: | - This aggregated KPI tracks the state of decision resources managed by cortex. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: KPI metadata: name: cortex-manila-kpi-state spec: diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index bb82e889a..18322417b 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -278,46 +278,6 @@ spec: configuration. It is recommended to investigate the knowledge status and logs for more details. - - alert: CortexNovaDecisionsWithErrors - expr: cortex_decision_state{domain="nova",state="error"} > 0 - for: 5m - labels: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics" -{{- end }} - context: decisions - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Some decisions are in error state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" - description: > - The cortex scheduling pipeline generated decisions that are in error state. - This may indicate issues with the decision logic or the underlying infrastructure. - It is recommended to investigate the decision logs and the state of the - VMs being processed. - - - alert: CortexNovaTooManyDecisionsWaiting - expr: cortex_decision_state{domain="nova",state="waiting"} > 10 - for: 5m - labels: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics" -{{- end }} - context: decisions - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Too many decisions are in waiting state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" - description: > - The cortex scheduling pipeline has a high number of decisions for which - no target host has been assigned yet. - - This may indicate a backlog in processing or issues with the decision logic. - It is recommended to investigate the decision logs and the state of the - VMs being processed. - - alert: CortexNovaKPIUnready expr: | cortex_kpi_state{domain="nova",state!="ready"} != 0 diff --git a/helm/bundles/cortex-nova/templates/kpis.yaml b/helm/bundles/cortex-nova/templates/kpis.yaml index 4c8dee11a..d52b6099e 100644 --- a/helm/bundles/cortex-nova/templates/kpis.yaml +++ b/helm/bundles/cortex-nova/templates/kpis.yaml @@ -110,18 +110,6 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI -metadata: - name: cortex-nova-decision-state -spec: - schedulingDomain: nova - impl: decision_state_kpi - opts: - decisionSchedulingDomain: nova - description: | - This aggregated KPI tracks the state of decision resources managed by cortex. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: KPI metadata: name: cortex-nova-kpi-state spec: diff --git a/helm/library/cortex-shim/templates/clusterrole.yaml b/helm/library/cortex-shim/templates/clusterrole.yaml index 74f8e7ad4..867b45ede 100644 --- a/helm/library/cortex-shim/templates/clusterrole.yaml +++ b/helm/library/cortex-shim/templates/clusterrole.yaml @@ -14,7 +14,6 @@ rules: - knowledges - datasources - reservations - - decisions - deschedulings - pipelines - kpis @@ -33,7 +32,6 @@ rules: - knowledges/finalizers - datasources/finalizers - reservations/finalizers - - decisions/finalizers - deschedulings/finalizers - pipelines/finalizers - kpis/finalizers @@ -46,7 +44,6 @@ rules: - knowledges/status - datasources/status - reservations/status - - decisions/status - deschedulings/status - pipelines/status - kpis/status diff --git a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml b/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml deleted file mode 100644 index ec4c99649..000000000 --- a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml +++ /dev/null @@ -1,412 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.23.0 - name: decisions.cortex.cloud -spec: - group: cortex.cloud - names: - kind: Decision - listKind: DecisionList - plural: decisions - singular: decision - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.schedulingDomain - name: Domain - type: string - - jsonPath: .spec.resourceID - name: Resource ID - type: string - - jsonPath: .status.precedence - name: '#' - type: string - - jsonPath: .metadata.creationTimestamp - name: Created - type: date - - jsonPath: .spec.pipelineRef.name - name: Pipeline - type: string - - jsonPath: .status.result.targetHost - name: TargetHost - type: string - - jsonPath: .status.conditions[?(@.type=='Ready')].status - name: Ready - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - Currently the Decision CRD is an in-memory scheduling object used by the external scheduler API - and filter-weigher pipelines to compute a placement result. It is currently NOT persisted - to etcd — the scheduling outcome (including a human-readable explanation) is written onto the - scheduled resource instead, e.g. the nova VM's spec.cortex fields. - The long-term shape of this CRD is still under discussion; until that is - settled the Decision serves only as a transient carrier within a single - scheduling run. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: spec defines the desired state of Decision - properties: - cinderRaw: - description: If the type is "cinder", this field contains the raw - cinder decision request. - type: object - x-kubernetes-preserve-unknown-fields: true - intent: - description: The intent of the scheduling decision (e.g., initial - scheduling, rescheduling, etc.). - type: string - machineRef: - description: If the type is "machine", this field contains the machine - reference. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - manilaRaw: - description: If the type is "manila", this field contains the raw - manila decision request. - type: object - x-kubernetes-preserve-unknown-fields: true - novaRaw: - description: If the type is "nova", this field contains the raw nova - decision request. - type: object - x-kubernetes-preserve-unknown-fields: true - pipelineRef: - description: |- - A reference to the pipeline that should be used for this decision. - This reference can be used to look up the pipeline definition and its - scheduler step configuration for additional context. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - podRef: - description: If the type is "pod", this field contains the pod reference. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - resourceID: - description: |- - An identifier for the underlying resource to be scheduled. - For example, this can be the UUID of a nova instance or cinder volume. - This can be used to correlate multiple decisions for the same resource. - type: string - schedulingDomain: - description: |- - SchedulingDomain defines in which scheduling domain this decision - was or is processed (e.g., nova, cinder, manila). - type: string - required: - - pipelineRef - - resourceID - - schedulingDomain - type: object - status: - description: status defines the observed state of Decision - properties: - conditions: - description: The current status conditions of the decision. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - explanation: - description: A human-readable explanation of the decision result. - type: string - history: - description: |- - If there were previous decisions for the underlying resource, they can - be resolved here to provide historical context for the decision. - items: - description: ObjectReference contains enough information to let - you inspect or modify the referred object. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - type: array - precedence: - description: The number of decisions that preceded this one for the - same resource. - type: integer - result: - description: The result of this decision. - properties: - aggregatedOutWeights: - additionalProperties: - type: number - description: Aggregated output weights from the pipeline. - type: object - normalizedInWeights: - additionalProperties: - type: number - description: Normalized input weights to the pipeline. - type: object - orderedHosts: - description: Final ordered list of hosts from most preferred to - least preferred. - items: - type: string - type: array - rawInWeights: - additionalProperties: - type: number - description: Raw input weights to the pipeline. - type: object - stepResults: - description: |- - Outputs of the decision pipeline including the activations used - to make the final ordering of compute hosts. - items: - properties: - activations: - additionalProperties: - type: number - description: Activations of the step for each host. - type: object - stepName: - description: object reference to the scheduler step. - type: string - required: - - activations - - stepName - type: object - type: array - targetHost: - description: The first element of the ordered hosts is considered - the target host. - type: string - type: object - type: object - required: - - spec - type: object - selectableFields: - - jsonPath: .spec.resourceID - served: true - storage: true - subresources: - status: {} diff --git a/helm/library/cortex/templates/rbac/role.yaml b/helm/library/cortex/templates/rbac/role.yaml index 54991d0b4..16e6b9012 100644 --- a/helm/library/cortex/templates/rbac/role.yaml +++ b/helm/library/cortex/templates/rbac/role.yaml @@ -17,7 +17,6 @@ rules: - committedresourcechanges - projectquotas - flavorgroupcapacities - - decisions - deschedulings - pipelines - kpis @@ -40,7 +39,6 @@ rules: - committedresourcechanges/finalizers - projectquotas/finalizers - flavorgroupcapacities/finalizers - - decisions/finalizers - deschedulings/finalizers - pipelines/finalizers - kpis/finalizers @@ -57,7 +55,6 @@ rules: - committedresourcechanges/status - projectquotas/status - flavorgroupcapacities/status - - decisions/status - deschedulings/status - pipelines/status - kpis/status diff --git a/internal/knowledge/kpis/plugins/deployment/decision_state.go b/internal/knowledge/kpis/plugins/deployment/decision_state.go deleted file mode 100644 index 1d07ac7f9..000000000 --- a/internal/knowledge/kpis/plugins/deployment/decision_state.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package deployment - -import ( - "context" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/db" - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" - "github.com/cobaltcore-dev/cortex/pkg/conf" - "github.com/prometheus/client_golang/prometheus" - "k8s.io/apimachinery/pkg/api/meta" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type DecisionStateKPIOpts struct { - // The scheduling domain to filter decisions by. - DecisionSchedulingDomain v1alpha1.SchedulingDomain `json:"decisionSchedulingDomain"` -} - -// KPI observing the state of decision resources managed by cortex. -type DecisionStateKPI struct { - // Common base for all KPIs that provides standard functionality. - plugins.BaseKPI[DecisionStateKPIOpts] - - // Prometheus descriptor for the decision state metric. - counter *prometheus.Desc -} - -func (DecisionStateKPI) GetName() string { return "decision_state_kpi" } - -// Initialize the KPI. -func (k *DecisionStateKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error { - if err := k.BaseKPI.Init(db, client, opts); err != nil { - return err - } - k.counter = prometheus.NewDesc( - "cortex_decision_state", - "State of cortex managed decisions (aggregated)", - []string{"domain", "state"}, - nil, - ) - return nil -} - -// Conform to the prometheus collector interface by providing the descriptor. -func (k *DecisionStateKPI) Describe(ch chan<- *prometheus.Desc) { ch <- k.counter } - -// Collect the decision state metrics. -func (k *DecisionStateKPI) Collect(ch chan<- prometheus.Metric) { - // Get all decisions with the specified decision operator. - decisionList := &v1alpha1.DecisionList{} - if err := k.Client.List(context.Background(), decisionList); err != nil { - return - } - var decisions []v1alpha1.Decision - for _, d := range decisionList.Items { - if d.Spec.SchedulingDomain != k.Options.DecisionSchedulingDomain { - continue - } - decisions = append(decisions, d) - } - // For each decision, categorize by state: error, waiting, or success - var errorCount, waitingCount, successCount float64 - for _, d := range decisions { - switch { - // Error state: decision has a false Ready condition - case meta.IsStatusConditionFalse(d.Status.Conditions, v1alpha1.DecisionConditionReady): - errorCount++ - // Waiting state: decision has a target host set (waiting for migration/placement) - case d.Status.Result != nil && d.Status.Result.TargetHost != nil: - waitingCount++ - // Success state: decision is complete (has result with ordered hosts or no result needed) - default: - successCount++ - } - } - // Emit metrics for all three states - ch <- prometheus.MustNewConstMetric( - k.counter, prometheus.GaugeValue, errorCount, - string(k.Options.DecisionSchedulingDomain), "error", - ) - ch <- prometheus.MustNewConstMetric( - k.counter, prometheus.GaugeValue, waitingCount, - string(k.Options.DecisionSchedulingDomain), "waiting", - ) - ch <- prometheus.MustNewConstMetric( - k.counter, prometheus.GaugeValue, successCount, - string(k.Options.DecisionSchedulingDomain), "success", - ) -} - -func init() { - plugins.Index["decision_state_kpi"] = func() plugins.KPI { return &DecisionStateKPI{} } -} diff --git a/internal/knowledge/kpis/plugins/deployment/decision_state_test.go b/internal/knowledge/kpis/plugins/deployment/decision_state_test.go deleted file mode 100644 index 0d6bd4568..000000000 --- a/internal/knowledge/kpis/plugins/deployment/decision_state_test.go +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package deployment - -import ( - "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/pkg/conf" - "github.com/prometheus/client_golang/prometheus" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestDecisionStateKPI_Init(t *testing.T) { - kpi := &DecisionStateKPI{} - if err := kpi.Init(nil, nil, conf.NewRawOpts(`{"decisionSchedulingDomain": "test-operator"}`)); err != nil { - t.Fatalf("expected no error, got %v", err) - } -} - -func TestDecisionStateKPI_Collect(t *testing.T) { - scheme, err := v1alpha1.SchemeBuilder.Build() - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - targetHost := "host1" - - tests := []struct { - name string - decisions []v1alpha1.Decision - operator string - expectedCount int - description string - expectedError int - expectedWaiting int - expectedSuccess int - }{ - { - name: "no decisions", - decisions: []v1alpha1.Decision{}, - operator: "test-operator", - expectedCount: 3, // always emits 3 metrics: error, waiting, success - description: "should collect metrics with zero counts when no decisions exist", - expectedError: 0, - expectedWaiting: 0, - expectedSuccess: 0, - }, - { - name: "single decision in error state", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec1"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count decision in error state", - expectedError: 1, - expectedWaiting: 0, - expectedSuccess: 0, - }, - { - name: "single decision in waiting state", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec2"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &targetHost, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count decision with target host as waiting", - expectedError: 0, - expectedWaiting: 1, - expectedSuccess: 0, - }, - { - name: "single decision in success state", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec3"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1", "host2"}, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count decision without target host and no error as success", - expectedError: 0, - expectedWaiting: 0, - expectedSuccess: 1, - }, - { - name: "multiple decisions in different states", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-error"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - { - ObjectMeta: v1.ObjectMeta{Name: "dec-waiting"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &targetHost, - }, - }, - }, - { - ObjectMeta: v1.ObjectMeta{Name: "dec-success"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should correctly count decisions across all states", - expectedError: 1, - expectedWaiting: 1, - expectedSuccess: 1, - }, - { - name: "filter by operator", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-correct-operator"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - { - ObjectMeta: v1.ObjectMeta{Name: "dec-wrong-operator"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "other-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should only count decisions with matching operator", - expectedError: 1, - expectedWaiting: 0, - expectedSuccess: 0, - }, - { - name: "multiple decisions same state", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-error-1"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - { - ObjectMeta: v1.ObjectMeta{Name: "dec-error-2"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should correctly aggregate multiple decisions in same state", - expectedError: 2, - expectedWaiting: 0, - expectedSuccess: 0, - }, - { - name: "decision with no result", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-no-result"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{}, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count decision with no result as success", - expectedError: 0, - expectedWaiting: 0, - expectedSuccess: 1, - }, - { - name: "error condition takes precedence", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-error-with-target"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &targetHost, - }, - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count as error even if target host is present", - expectedError: 1, - expectedWaiting: 0, - expectedSuccess: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := make([]v1alpha1.Decision, len(tt.decisions)) - copy(objects, tt.decisions) - - clientBuilder := fake.NewClientBuilder().WithScheme(scheme) - for i := range objects { - clientBuilder = clientBuilder.WithObjects(&objects[i]) - } - client := clientBuilder.Build() - - kpi := &DecisionStateKPI{} - if err := kpi.Init(nil, client, conf.NewRawOpts(`{"decisionSchedulingDomain": "`+tt.operator+`"}`)); err != nil { - t.Fatalf("expected no error, got %v", err) - } - - ch := make(chan prometheus.Metric, 10) - kpi.Collect(ch) - close(ch) - - metricsCount := 0 - for range ch { - metricsCount++ - } - - if metricsCount != tt.expectedCount { - t.Errorf("%s: expected %d metrics, got %d", tt.description, tt.expectedCount, metricsCount) - } - }) - } -} - -func TestDecisionStateKPI_GetName(t *testing.T) { - kpi := &DecisionStateKPI{} - expectedName := "decision_state_kpi" - if name := kpi.GetName(); name != expectedName { - t.Errorf("expected name %q, got %q", expectedName, name) - } -} - -func TestDecisionStateKPI_Describe(t *testing.T) { - kpi := &DecisionStateKPI{} - if err := kpi.Init(nil, nil, conf.NewRawOpts(`{"decisionSchedulingDomain": "test-operator"}`)); err != nil { - t.Fatalf("expected no error, got %v", err) - } - - ch := make(chan *prometheus.Desc, 1) - kpi.Describe(ch) - close(ch) - - descCount := 0 - for range ch { - descCount++ - } - - if descCount != 1 { - t.Errorf("expected 1 descriptor, got %d", descCount) - } -} diff --git a/internal/scheduling/cinder/external_scheduler_api.go b/internal/scheduling/cinder/external_scheduler_api.go index 96bea5590..c717a8966 100644 --- a/internal/scheduling/cinder/external_scheduler_api.go +++ b/internal/scheduling/cinder/external_scheduler_api.go @@ -7,29 +7,24 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "net/http" api "github.com/cobaltcore-dev/cortex/api/external/cinder" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/metrics" ) var log = logf.Log.WithName("cinder-scheduler-api") -type HTTPAPIDelegate interface { - // Process the decision from the API. Should create and return the updated decision. - ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error +// Scheduler runs a cinder pipeline for a request and returns the result and a +// human-readable explanation. *FilterWeigherPipelineController satisfies it. +type Scheduler interface { + Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (scheduling.PipelineResult, string, error) } type HTTPAPI interface { @@ -38,14 +33,14 @@ type HTTPAPI interface { } type httpAPI struct { - monitor scheduling.APIMonitor - delegate HTTPAPIDelegate + monitor scheduling.APIMonitor + scheduler Scheduler } -func NewAPI(delegate HTTPAPIDelegate) HTTPAPI { +func NewAPI(scheduler Scheduler) HTTPAPI { return &httpAPI{ - monitor: scheduling.NewSchedulerMonitor(), - delegate: delegate, + monitor: scheduling.NewSchedulerMonitor(), + scheduler: scheduler, } } @@ -109,7 +104,6 @@ func (httpAPI *httpAPI) CinderExternalScheduler(w http.ResponseWriter, r *http.R c.Respond(logr.Logger{}, http.StatusInternalServerError, err, "failed to read request body") return } - raw := runtime.RawExtension{Raw: body} var requestData api.ExternalSchedulerRequest // Copy the raw body to a io.Reader for json deserialization. cp := body @@ -138,36 +132,12 @@ func (httpAPI *httpAPI) CinderExternalScheduler(w http.ResponseWriter, r *http.R logger.V(1).Info("inferred pipeline name", "pipeline", requestData.Pipeline) } - // Create the decision object in kubernetes. - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "cinder-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: requestData.Pipeline, - }, - ResourceID: "", // TODO - CinderRaw: &raw, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - } - ctx := r.Context() - if err := httpAPI.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { + result, _, err := httpAPI.scheduler.Schedule(r.Context(), requestData.Pipeline, requestData) + if err != nil { c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return } - // Check if the decision contains status conditions indicating an error. - if meta.IsStatusConditionFalse(decision.Status.Conditions, v1alpha1.DecisionConditionReady) { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision contains error condition"), "decision failed") - return - } - if decision.Status.Result == nil { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision didn't produce a result"), "decision failed") - return - } - hosts := decision.Status.Result.OrderedHosts + hosts := result.OrderedHosts // Ensure we always return an array, never null. The schema requires hosts // to be an array, and a nil slice would serialize to null. if hosts == nil { diff --git a/internal/scheduling/cinder/external_scheduler_api_test.go b/internal/scheduling/cinder/external_scheduler_api_test.go index d5f7f394f..fb81e99ff 100644 --- a/internal/scheduling/cinder/external_scheduler_api_test.go +++ b/internal/scheduling/cinder/external_scheduler_api_test.go @@ -14,26 +14,24 @@ import ( "testing" cinderapi "github.com/cobaltcore-dev/cortex/api/external/cinder" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type mockHTTPAPIDelegate struct { - processDecisionFunc func(ctx context.Context, decision *v1alpha1.Decision) error +type mockScheduler struct { + scheduleFunc func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) } -func (m *mockHTTPAPIDelegate) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { - if m.processDecisionFunc != nil { - return m.processDecisionFunc(ctx, decision) +func (m *mockScheduler) Schedule(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { + if m.scheduleFunc != nil { + return m.scheduleFunc(ctx, pipelineName, request) } - return nil + return lib.PipelineResult{}, "", nil } func TestNewAPI(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} + scheduler := &mockScheduler{} - api := NewAPI(delegate) + api := NewAPI(scheduler) if api == nil { t.Fatal("NewAPI returned nil") @@ -44,8 +42,8 @@ func TestNewAPI(t *testing.T) { t.Fatal("NewAPI did not return httpAPI type") } - if httpAPI.delegate != delegate { - t.Error("Delegate not set correctly") + if httpAPI.scheduler != scheduler { + t.Error("Scheduler not set correctly") } if httpAPI.monitor.ApiRequestsTimer == nil { @@ -54,8 +52,7 @@ func TestNewAPI(t *testing.T) { } func TestHTTPAPI_Init(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate) + api := NewAPI(&mockScheduler{}) mux := http.NewServeMux() api.Init(mux) @@ -72,8 +69,7 @@ func TestHTTPAPI_Init(t *testing.T) { } func TestHTTPAPI_canRunScheduler(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(&mockScheduler{}).(*httpAPI) tests := []struct { name string @@ -141,14 +137,27 @@ func TestHTTPAPI_canRunScheduler(t *testing.T) { } func TestHTTPAPI_CinderExternalScheduler(t *testing.T) { + validBody := func(t *testing.T) string { + req := cinderapi.ExternalSchedulerRequest{ + Hosts: []cinderapi.ExternalSchedulerHost{{VolumeHost: "host1"}}, + Weights: map[string]float64{"host1": 1.0}, + Pipeline: "test-pipeline", + } + data, err := json.Marshal(req) + if err != nil { + t.Fatalf("Failed to marshal request data: %v", err) + } + return string(data) + } + tests := []struct { - name string - method string - body string - processDecisionErr error - decisionResult *v1alpha1.Decision - expectedStatus int - expectedHosts []string + name string + method string + body string + scheduleResult lib.PipelineResult + scheduleErr error + expectedStatus int + expectedHosts []string }{ { name: "invalid method", @@ -162,106 +171,31 @@ func TestHTTPAPI_CinderExternalScheduler(t *testing.T) { expectedStatus: http.StatusBadRequest, }, { - name: "valid request with successful processing", - method: http.MethodPost, - body: func() string { - req := cinderapi.ExternalSchedulerRequest{ - Hosts: []cinderapi.ExternalSchedulerHost{ - {VolumeHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - }, - }, - }, + name: "valid request with successful processing", + method: http.MethodPost, + body: validBody(t), + scheduleResult: lib.PipelineResult{OrderedHosts: []string{"host1"}}, expectedStatus: http.StatusOK, expectedHosts: []string{"host1"}, }, { - name: "processing error", - method: http.MethodPost, - body: func() string { - req := cinderapi.ExternalSchedulerRequest{ - Hosts: []cinderapi.ExternalSchedulerHost{ - {VolumeHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - processDecisionErr: errors.New("processing failed"), - expectedStatus: http.StatusInternalServerError, - }, - { - name: "decision failed", - method: http.MethodPost, - body: func() string { - req := cinderapi.ExternalSchedulerRequest{ - Hosts: []cinderapi.ExternalSchedulerHost{ - {VolumeHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "SchedulingError", - }, - }, - }, - }, + name: "processing error", + method: http.MethodPost, + body: validBody(t), + scheduleErr: errors.New("processing failed"), expectedStatus: http.StatusInternalServerError, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - if tt.processDecisionErr != nil { - return tt.processDecisionErr - } - if tt.decisionResult != nil { - decision.Status = tt.decisionResult.Status - return nil - } - return nil + scheduler := &mockScheduler{ + scheduleFunc: func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { + return tt.scheduleResult, "", tt.scheduleErr }, } - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(scheduler).(*httpAPI) var body *strings.Reader if tt.body != "" { @@ -300,8 +234,7 @@ func TestHTTPAPI_CinderExternalScheduler(t *testing.T) { } func TestHTTPAPI_inferPipelineName(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(&mockScheduler{}).(*httpAPI) tests := []struct { name string @@ -347,31 +280,25 @@ func TestHTTPAPI_inferPipelineName(t *testing.T) { } } -func TestHTTPAPI_CinderExternalScheduler_DecisionCreation(t *testing.T) { - var capturedDecision *v1alpha1.Decision - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - capturedDecision = decision - // Set a successful result to avoid "decision didn't produce a result" error - decision.Status.Result = &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - } - return nil +// When the request carries no pipeline, the handler infers the default name and +// passes it to the scheduler. +func TestHTTPAPI_CinderExternalScheduler_InfersPipeline(t *testing.T) { + var capturedPipeline string + var capturedRequest cinderapi.ExternalSchedulerRequest + scheduler := &mockScheduler{ + scheduleFunc: func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { + capturedPipeline = pipelineName + capturedRequest = request + return lib.PipelineResult{OrderedHosts: []string{"host1"}}, "", nil }, } - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(scheduler).(*httpAPI) requestData := cinderapi.ExternalSchedulerRequest{ - Hosts: []cinderapi.ExternalSchedulerHost{ - {VolumeHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", + Hosts: []cinderapi.ExternalSchedulerHost{{VolumeHost: "host1"}}, + Weights: map[string]float64{"host1": 1.0}, } - body, err := json.Marshal(requestData) if err != nil { t.Fatalf("Failed to marshal request data: %v", err) @@ -384,25 +311,10 @@ func TestHTTPAPI_CinderExternalScheduler_DecisionCreation(t *testing.T) { if w.Code != http.StatusOK { t.Errorf("Expected status %d, got %d", http.StatusOK, w.Code) } - - if capturedDecision == nil { - t.Fatal("Decision was not captured") + if capturedPipeline != "cinder-external-scheduler" { + t.Errorf("Expected inferred pipeline 'cinder-external-scheduler', got %q", capturedPipeline) } - - // Verify decision fields - if capturedDecision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainCinder { - t.Errorf("Expected scheduling domain %s, got %s", v1alpha1.SchedulingDomainCinder, capturedDecision.Spec.SchedulingDomain) - } - - if capturedDecision.Spec.PipelineRef.Name != "test-pipeline" { - t.Errorf("Expected pipeline 'test-pipeline', got %s", capturedDecision.Spec.PipelineRef.Name) - } - - if capturedDecision.GenerateName != "cinder-" { - t.Errorf("Expected generate name 'cinder-', got %s", capturedDecision.GenerateName) - } - - if capturedDecision.Spec.CinderRaw == nil { - t.Error("CinderRaw should not be nil") + if len(capturedRequest.Hosts) != 1 || capturedRequest.Hosts[0].VolumeHost != "host1" { + t.Errorf("Request not passed through to scheduler: %+v", capturedRequest) } } diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go index 6d2593147..bff7d0394 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go @@ -5,16 +5,12 @@ package cinder import ( "context" - "encoding/json" - "errors" "fmt" "sync" "time" api "github.com/cobaltcore-dev/cortex/api/external/cinder" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cobaltcore-dev/cortex/internal/scheduling/cinder/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/cinder/plugins/weighers" @@ -27,17 +23,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) -// The decision pipeline controller takes decision resources containing a -// placement request spec and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. -// -// Additionally, the controller watches for pipeline and step changes to -// reconfigure the pipelines as needed. +// FilterWeigherPipelineController owns the cinder filter-weigher pipelines: it +// watches Pipeline and Knowledge resources to (re)configure the pipelines and +// runs a pipeline on demand via Schedule, called directly by the external +// scheduler HTTP API. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]] - // Mutex to only allow one process at a time + // Mutex to only allow one scheduling run at a time. processMu sync.Mutex // Monitor to pass down to all pipelines. @@ -49,83 +43,41 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -// Callback executed when kubernetes asks to reconcile a decision resource. -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - c.processMu.Lock() - defer c.processMu.Unlock() - - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - old := decision.DeepCopy() - if err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } +// Reconcile is a no-op: this controller reconfigures pipelines through its +// Pipeline/Knowledge watch handlers and runs placements via Schedule. A +// Reconcile method is required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } -// Process the decision from the API. Should create and return the updated decision. -func (c *FilterWeigherPipelineController) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { +// Schedule runs the named pipeline for the given request and returns the result +// plus a human-readable explanation. It serializes runs with the process lock. +func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { c.processMu.Lock() defer c.processMu.Unlock() - - err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - return err -} - -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) error { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) + pipeline, ok := c.GetPipeline(pipelineName) if !ok { - log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) - } - if decision.Spec.CinderRaw == nil { - log.Error(nil, "skipping decision, no cinderRaw spec defined") - return errors.New("no cinderRaw spec defined") - } - var request api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.CinderRaw.Raw, &request); err != nil { - log.Error(err, "failed to unmarshal cinderRaw spec") - return err + log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) + return lib.PipelineResult{}, "", fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } result, err := pipeline.Run(request) + // A human-readable explanation of the decision for downstream consumers. On + // error the result is zero and the explanation carries the failure reason. + var resultPtr *lib.PipelineResult if err == nil { - decision.Status.Result = &result + resultPtr = &result } - // Record a human-readable explanation of the decision for downstream - // consumers. On error the result is nil and the explanation carries the - // failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) + explanation := lib.GenerateExplanation(resultPtr, err) if err != nil { log.Error(err, "failed to run pipeline") - return err + return lib.PipelineResult{}, explanation, err } log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return nil + return result, explanation, nil } // The base controller will delegate the pipeline creation down to this method. @@ -186,25 +138,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainCinder { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } - return bldr.Named("cortex-cinder-decisions"). + return bldr.Named("cortex-cinder-pipelines"). Complete(c) } diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go index 8ec56c40f..725c854ff 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go @@ -4,15 +4,8 @@ package cinder import ( - "context" - "encoding/json" "testing" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" api "github.com/cobaltcore-dev/cortex/api/external/cinder" @@ -21,25 +14,16 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { +func TestFilterWeigherPipelineController_Schedule(t *testing.T) { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { t.Fatalf("Failed to add v1alpha1 scheme: %v", err) } - cinderRequest := api.ExternalSchedulerRequest{ - Spec: map[string]any{ - "volume_id": "test-volume-id", - "size": 10, - }, - Context: api.CinderRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: "global-req-123", - }, + request := api.ExternalSchedulerRequest{ Hosts: []api.ExternalSchedulerHost{ {VolumeHost: "cinder-volume-1"}, {VolumeHost: "cinder-volume-2"}, @@ -49,127 +33,41 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { Options: scheduling.Options{}, } - cinderRaw, err := json.Marshal(cinderRequest) - if err != nil { - t.Fatalf("Failed to marshal cinder request: %v", err) - } - tests := []struct { name string - decision *v1alpha1.Decision - pipeline *v1alpha1.Pipeline + pipelineName string + configured bool expectError bool - expectResult bool }{ { - name: "successful cinder decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, + name: "successful scheduling run", + pipelineName: "test-pipeline", + configured: true, expectError: false, - expectResult: true, }, { - name: "decision without cinderRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: nil, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, + name: "pipeline not found", + pipelineName: "nonexistent-pipeline", + configured: false, expectError: true, - expectResult: false, - }, - { - name: "pipeline not found", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipeline: nil, - expectError: true, - expectResult: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{tt.decision} - if tt.pipeline != nil { - objects = append(objects, tt.pipeline) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() + cl := fake.NewClientBuilder().WithScheme(scheme).Build() controller := &FilterWeigherPipelineController{ BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, + Client: cl, Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), }, Monitor: lib.FilterWeigherPipelineMonitor{}, } - if tt.pipeline != nil { + if tt.configured { initResult := controller.InitPipeline(t.Context(), v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, + ObjectMeta: metav1.ObjectMeta{Name: "test-pipeline"}, Spec: v1alpha1.PipelineSpec{ Type: v1alpha1.PipelineTypeFilterWeigher, SchedulingDomain: v1alpha1.SchedulingDomainCinder, @@ -180,17 +78,10 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { t.Fatalf("Failed to init pipeline: %v", initResult) } - controller.Pipelines[tt.pipeline.Name] = initResult.Pipeline + controller.Pipelines["test-pipeline"] = initResult.Pipeline } - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - Namespace: tt.decision.Namespace, - }, - } - - result, err := controller.Reconcile(context.Background(), req) + result, explanation, err := controller.Schedule(t.Context(), tt.pipelineName, request) if tt.expectError && err == nil { t.Error("Expected error but got none") @@ -198,231 +89,12 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { if !tt.expectError && err != nil { t.Errorf("Expected no error but got: %v", err) } - - if result.RequeueAfter > 0 { - t.Error("Expected no requeue") - } - - var updatedDecision v1alpha1.Decision - if err := client.Get(context.Background(), req.NamespacedName, &updatedDecision); err != nil { - t.Fatalf("Failed to get updated decision: %v", err) - } - - if tt.expectResult && updatedDecision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - if !tt.expectResult && updatedDecision.Status.Result != nil { - t.Error("Expected result to be nil but was set") - } - }) - } -} - -func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - cinderRequest := api.ExternalSchedulerRequest{ - Spec: map[string]any{ - "volume_id": "test-volume-id", - "size": 10, - }, - Context: api.CinderRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: "global-req-123", - }, - Hosts: []api.ExternalSchedulerHost{ - {VolumeHost: "cinder-volume-1"}, - {VolumeHost: "cinder-volume-2"}, - }, - Weights: map[string]float64{"cinder-volume-1": 1.0, "cinder-volume-2": 0.5}, - Pipeline: "test-pipeline", - } - - cinderRaw, err := json.Marshal(cinderRequest) - if err != nil { - t.Fatalf("Failed to marshal cinder request: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - pipelineConfig *v1alpha1.Pipeline - expectError bool - expectResult bool - }{ - { - name: "successful decision processing with creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-decision-", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "test-uuid-1", - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "successful decision processing without creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-create", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "pipeline not configured", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipelineConfig: nil, - expectError: true, - expectResult: false, - }, - { - name: "decision without cinderRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: nil, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{} - if tt.pipelineConfig != nil { - objects = append(objects, tt.pipelineConfig) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - - if tt.pipelineConfig != nil { - controller.PipelineConfigs[tt.pipelineConfig.Name] = *tt.pipelineConfig - initResult := controller.InitPipeline(t.Context(), *tt.pipelineConfig) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to init pipeline: %v", initResult) - } - controller.Pipelines[tt.pipelineConfig.Name] = initResult.Pipeline - } - - if tt.decision.Spec.CinderRaw != nil { - req := cinderRequest - req.Options = scheduling.Options{} - raw, marshalErr := json.Marshal(req) - if marshalErr != nil { - t.Fatalf("Failed to marshal request with options: %v", marshalErr) - } - tt.decision.Spec.CinderRaw = &runtime.RawExtension{Raw: raw} - } - - err := controller.ProcessNewDecisionFromAPI(context.Background(), tt.decision) - - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - if !tt.expectError { - if tt.expectResult && tt.decision.Status.Result == nil { - t.Error("expected decision result to be set in original decision object") + if len(result.OrderedHosts) == 0 { + t.Error("Expected ordered hosts but got none") + } + if explanation == "" { + t.Error("Expected a non-empty explanation") } } }) diff --git a/internal/scheduling/compute/result.go b/internal/scheduling/compute/result.go index c05b8d409..ae9d009c7 100644 --- a/internal/scheduling/compute/result.go +++ b/internal/scheduling/compute/result.go @@ -5,17 +5,14 @@ package compute import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) // SchedulerResult is the well-defined output of a scheduling pipeline run: // ordered hosts, the target host, and the input/output weights plus per-step -// activations used to rank them. -// -// It is currently an alias of v1alpha1.DecisionResult, the struct the generic -// pipeline returns today. Once the Decision CRD is removed the underlying type -// moves to lib.PipelineResult and this alias is re-pointed there; callers that -// read compute.SchedulerResult are unaffected by that move. -type SchedulerResult = v1alpha1.DecisionResult +// activations used to rank them. It is an alias of lib.PipelineResult, the +// domain-neutral type the generic pipeline returns. +type SchedulerResult = lib.PipelineResult // MatchedReservation records a reservation that matched a candidate host during // the run, for observability and downstream consumers. diff --git a/internal/scheduling/lib/explanation.go b/internal/scheduling/lib/explanation.go index c1deeff0f..4eff7017a 100644 --- a/internal/scheduling/lib/explanation.go +++ b/internal/scheduling/lib/explanation.go @@ -7,8 +7,6 @@ import ( "fmt" "sort" "strings" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) // maxHostsInExplanation caps how many host names are listed per line in a @@ -24,13 +22,12 @@ func joinHostsCapped(hosts []string, maxHosts int) string { return fmt.Sprintf("%s (and %d more)", strings.Join(hosts[:maxHosts], ", "), len(hosts)-maxHosts) } -// GenerateExplanation produces a human-readable explanation from a decision +// GenerateExplanation produces a human-readable explanation from a pipeline // result. On failure it includes the error. On success it describes which // pipeline steps filtered out which hosts and, for the surviving hosts, how the -// weighers ranked them (via ExplainWeighing). The result is written to -// Decision.Status.Explanation so it propagates to the scheduled resource (e.g. -// the VM CortexSpec.Explanation). -func GenerateExplanation(result *v1alpha1.DecisionResult, pipelineErr error) string { +// weighers ranked them (via ExplainWeighing). Consumers propagate the result to +// the scheduled resource (e.g. the Nova VM's CortexSpec.Explanation). +func GenerateExplanation(result *PipelineResult, pipelineErr error) string { if pipelineErr != nil { return fmt.Sprintf("Pipeline run failed: %s.", pipelineErr.Error()) } diff --git a/internal/scheduling/lib/explanation_test.go b/internal/scheduling/lib/explanation_test.go index 2c020f396..8ddcd2af4 100644 --- a/internal/scheduling/lib/explanation_test.go +++ b/internal/scheduling/lib/explanation_test.go @@ -7,8 +7,6 @@ import ( "errors" "fmt" "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) func ptr[T any](v T) *T { return &v } @@ -16,7 +14,7 @@ func ptr[T any](v T) *T { return &v } func TestGenerateExplanation(t *testing.T) { tests := []struct { name string - result *v1alpha1.DecisionResult + result *PipelineResult err error expected string }{ @@ -34,20 +32,20 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "result with target host only no steps", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ TargetHost: ptr("host-1"), }, expected: "Selected host: host-1.", }, { name: "result with RawInWeights and filtering steps", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ RawInWeights: map[string]float64{ "host-a": 1.0, "host-b": 0.5, "host-c": 0.3, }, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ { StepName: "filter_capacity", Activations: map[string]float64{ @@ -72,12 +70,12 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "uses NormalizedInWeights when RawInWeights empty", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ NormalizedInWeights: map[string]float64{ "host-x": 0.6, "host-y": 0.4, }, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ { StepName: "weigher_cpu", Activations: map[string]float64{ @@ -92,8 +90,8 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "no weights no target", - result: &v1alpha1.DecisionResult{ - StepResults: []v1alpha1.StepResult{ + result: &PipelineResult{ + StepResults: []StepResult{ {StepName: "some-step", Activations: map[string]float64{}}, }, }, @@ -101,9 +99,9 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "no weights with target", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ TargetHost: ptr("host-1"), - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "some-step", Activations: map[string]float64{}}, }, }, @@ -111,12 +109,12 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "all hosts survive all steps", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ RawInWeights: map[string]float64{ "host-a": 1.0, "host-b": 0.5, }, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ { StepName: "noop", Activations: map[string]float64{ @@ -131,7 +129,7 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "large cluster caps host lists", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { weights := make(map[string]float64, 50) for i := range 50 { weights[fmt.Sprintf("host-%03d", i)] = 1.0 @@ -141,9 +139,9 @@ func TestGenerateExplanation(t *testing.T) { for i := range 30 { surviving[fmt.Sprintf("host-%03d", i)] = 1.0 } - return &v1alpha1.DecisionResult{ + return &PipelineResult{ RawInWeights: weights, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ { StepName: "filter_big", Activations: surviving, diff --git a/internal/scheduling/lib/filter_weigher_pipeline.go b/internal/scheduling/lib/filter_weigher_pipeline.go index 57ab4e321..d29ff2449 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline.go +++ b/internal/scheduling/lib/filter_weigher_pipeline.go @@ -23,7 +23,7 @@ var pipelineLog = logf.Log.WithName("scheduler-pipeline") type FilterWeigherPipeline[RequestType FilterWeigherPipelineRequest] interface { // Run the scheduling pipeline with the given request. // Call-time options are read from request.GetOptions(). - Run(request RequestType) (v1alpha1.DecisionResult, error) + Run(request RequestType) (PipelineResult, error) } // Pipeline of scheduler steps. @@ -142,7 +142,7 @@ func InitNewFilterWeigherPipeline[RequestType FilterWeigherPipelineRequest]( func (p *filterWeigherPipeline[RequestType]) runFilters( traceLog logr.Logger, request RequestType, -) (filteredRequest RequestType, stepResults []v1alpha1.StepResult) { +) (filteredRequest RequestType, stepResults []StepResult) { filteredRequest = request for _, filterName := range p.filtersOrder { @@ -159,7 +159,7 @@ func (p *filterWeigherPipeline[RequestType]) runFilters( continue } stepLog.V(1).Info("scheduler: finished filter") - stepResults = append(stepResults, v1alpha1.StepResult{ + stepResults = append(stepResults, StepResult{ StepName: filterName, Activations: result.Activations, }) @@ -269,10 +269,10 @@ func (s *filterWeigherPipeline[RequestType]) sortHostsByWeights(weights map[stri } // Evaluate the pipeline and return a list of hosts in order of preference. -func (p *filterWeigherPipeline[RequestType]) Run(request RequestType) (v1alpha1.DecisionResult, error) { +func (p *filterWeigherPipeline[RequestType]) Run(request RequestType) (PipelineResult, error) { opts := request.GetOptions() if err := opts.Validate(); err != nil { - return v1alpha1.DecisionResult{}, err + return PipelineResult{}, err } traceLog := pipelineLog.WithValues(request.GetTraceLogArgs()...) @@ -341,13 +341,13 @@ func (p *filterWeigherPipeline[RequestType]) Run(request RequestType) (v1alpha1. if !ok { continue } - stepResults = append(stepResults, v1alpha1.StepResult{ + stepResults = append(stepResults, StepResult{ StepName: weigherName, Activations: activations, }) } - result := v1alpha1.DecisionResult{ + result := PipelineResult{ RawInWeights: request.GetWeights(), NormalizedInWeights: inWeights, StepResults: stepResults, diff --git a/internal/scheduling/lib/pipeline_result.go b/internal/scheduling/lib/pipeline_result.go new file mode 100644 index 000000000..156558317 --- /dev/null +++ b/internal/scheduling/lib/pipeline_result.go @@ -0,0 +1,31 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package lib + +// StepResult holds the per-host activations a single pipeline step produced. +type StepResult struct { + // StepName is the name of the filter or weigher that produced this result. + StepName string + // Activations maps each host to the activation the step assigned it. + Activations map[string]float64 +} + +// PipelineResult is the domain-neutral output of a filter-weigher pipeline run: +// the ordered hosts, the target host, and the input/output weights plus per-step +// activations used to rank them. Domain-facing aliases (e.g. +// compute.SchedulerResult) point at this type. +type PipelineResult struct { + // RawInWeights are the input weights to the pipeline, before normalization. + RawInWeights map[string]float64 + // NormalizedInWeights are the input weights after normalization. + NormalizedInWeights map[string]float64 + // StepResults are the activations each pipeline step produced, in order. + StepResults []StepResult + // AggregatedOutWeights are the final weights after applying all weighers. + AggregatedOutWeights map[string]float64 + // OrderedHosts is the final ordering from most to least preferred. + OrderedHosts []string + // TargetHost is the first element of OrderedHosts, if any. + TargetHost *string +} diff --git a/internal/scheduling/lib/weighing_explainer.go b/internal/scheduling/lib/weighing_explainer.go index 2e66bce41..9584afb6f 100644 --- a/internal/scheduling/lib/weighing_explainer.go +++ b/internal/scheduling/lib/weighing_explainer.go @@ -8,8 +8,6 @@ import ( "math" "sort" "strings" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) const ( @@ -55,7 +53,7 @@ type weigherContribution struct { // // Returns an empty string when the result is nil, has fewer than 2 ordered // hosts, or contains no weigher steps. -func ExplainWeighing(result *v1alpha1.DecisionResult) string { +func ExplainWeighing(result *PipelineResult) string { if result == nil || len(result.OrderedHosts) < 2 { return "" } @@ -199,7 +197,7 @@ func ExplainWeighing(result *v1alpha1.DecisionResult) string { // recovery is not possible (e.g., all activations are zero, or the system is // under-determined). It reports only the initial weight bias and raw activation // differentials without exact contribution magnitudes. -func explainWithoutMultipliers(result *v1alpha1.DecisionResult, topHosts []string, _ []v1alpha1.StepResult) string { +func explainWithoutMultipliers(result *PipelineResult, topHosts []string, _ []StepResult) string { topN := len(topHosts) var sb strings.Builder @@ -239,13 +237,13 @@ func explainWithoutMultipliers(result *v1alpha1.DecisionResult, topHosts []strin // weigher (scoring) steps rather than filter steps. A weigher step is one // whose activation map contains entries for ALL hosts in OrderedHosts — // filters reduce the host set while weighers score all remaining hosts. -func identifyWeigherSteps(result *v1alpha1.DecisionResult) []v1alpha1.StepResult { +func identifyWeigherSteps(result *PipelineResult) []StepResult { orderedHostSet := make(map[string]struct{}, len(result.OrderedHosts)) for _, h := range result.OrderedHosts { orderedHostSet[h] = struct{}{} } - var weigherSteps []v1alpha1.StepResult + var weigherSteps []StepResult for _, step := range result.StepResults { if len(step.Activations) == 0 { continue @@ -281,7 +279,7 @@ func identifyWeigherSteps(result *v1alpha1.DecisionResult) []v1alpha1.StepResult // Returns the recovered multipliers and true on success, or nil and false if // the system is under-determined or ill-conditioned. func recoverMultipliers( - weigherSteps []v1alpha1.StepResult, + weigherSteps []StepResult, orderedHosts []string, normalizedIn map[string]float64, aggregatedOut map[string]float64, diff --git a/internal/scheduling/lib/weighing_explainer_test.go b/internal/scheduling/lib/weighing_explainer_test.go index 3e63508d7..f333ecb8b 100644 --- a/internal/scheduling/lib/weighing_explainer_test.go +++ b/internal/scheduling/lib/weighing_explainer_test.go @@ -8,14 +8,12 @@ import ( "sort" "strings" "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) func TestExplainWeighing(t *testing.T) { tests := []struct { name string - result *v1alpha1.DecisionResult + result *PipelineResult contains []string // substrings that must appear in output excludes []string // substrings that must NOT appear in output empty bool // expect empty string @@ -27,7 +25,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "single host returns empty", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ OrderedHosts: []string{"host-a"}, NormalizedInWeights: map[string]float64{"host-a": 0.5}, AggregatedOutWeights: map[string]float64{"host-a": 0.8}, @@ -36,11 +34,11 @@ func TestExplainWeighing(t *testing.T) { }, { name: "only filter steps returns empty", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ NormalizedInWeights: map[string]float64{"host-a": 0.5, "host-b": 0.3, "host-c": 0.1}, AggregatedOutWeights: map[string]float64{"host-a": 0.5, "host-b": 0.3}, OrderedHosts: []string{"host-a", "host-b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ // A true filter: activations do NOT include host-b (it was filtered after this step). {StepName: "filter_az", Activations: map[string]float64{"host-a": 1.0}}, }, @@ -49,7 +47,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "two hosts single weigher positive multiplier", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // Simulate: multiplier = 2.0, act[a] = 1.0, act[b] = -0.5 // contrib[a] = 2.0 * tanh(1.0) = 1.5231 // contrib[b] = 2.0 * tanh(-0.5) = -0.9242 @@ -58,11 +56,11 @@ func TestExplainWeighing(t *testing.T) { inA, inB := 0.3, 0.3 outA := inA + mult*math.Tanh(actA) outB := inB + mult*math.Tanh(actB) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: map[string]float64{"host-a": inA, "host-b": inB}, AggregatedOutWeights: map[string]float64{"host-a": outA, "host-b": outB}, OrderedHosts: []string{"host-a", "host-b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_cpu", Activations: map[string]float64{"host-a": actA, "host-b": actB}}, }, } @@ -71,7 +69,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "two hosts single weigher negative multiplier", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // Simulate: multiplier = -1.0 (inverts behavior) // A host with HIGH raw activation is penalized. // act[a] = -2.0 (high penalty activates less due to neg mult -> boosts) @@ -83,11 +81,11 @@ func TestExplainWeighing(t *testing.T) { outB := inB + mult*math.Tanh(actB) // outA = 0 + (-1)*tanh(-2) = 0 + (-1)*(-0.964) = +0.964 // outB = 0 + (-1)*tanh(3) = 0 + (-1)*(0.995) = -0.995 - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: map[string]float64{"host-a": inA, "host-b": inB}, AggregatedOutWeights: map[string]float64{"host-a": outA, "host-b": outB}, OrderedHosts: []string{"host-a", "host-b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "kvm_binpack", Activations: map[string]float64{"host-a": actA, "host-b": actB}}, }, } @@ -96,7 +94,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "three hosts two weighers with counterfactual", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // weigher_cpu (mult=3.0) strongly favors host-a // weigher_mem (mult=1.0) slightly favors host-b // Without weigher_cpu, host-b should be #1. @@ -111,11 +109,11 @@ func TestExplainWeighing(t *testing.T) { // Sort to determine OrderedHosts. hosts := []string{"h1", "h2", "h3"} sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_cpu", Activations: actCPU}, {StepName: "weigher_mem", Activations: actMem}, }, @@ -125,7 +123,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "opposing weigher reported", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // weigher_boost (mult=2.0) pushes host-a up // weigher_penalty (mult=-0.5) opposes host-a // Net: host-a still wins due to weigher_boost dominance. @@ -139,11 +137,11 @@ func TestExplainWeighing(t *testing.T) { } hosts := []string{"x", "y"} sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_boost", Activations: actBoost}, {StepName: "weigher_penalty", Activations: actPenalty}, }, @@ -153,7 +151,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "negligible impact weigher reported", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // weigher_big (mult=5.0) dominates // weigher_tiny (mult=0.001) is negligible multBig, multTiny := 5.0, 0.001 @@ -166,11 +164,11 @@ func TestExplainWeighing(t *testing.T) { } hosts := []string{"a", "b"} sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_big", Activations: actBig}, {StepName: "weigher_tiny", Activations: actTiny}, }, @@ -180,11 +178,11 @@ func TestExplainWeighing(t *testing.T) { }, { name: "initial weight bias dominates when activations are zero", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ NormalizedInWeights: map[string]float64{"fast": 0.8, "slow": 0.2}, AggregatedOutWeights: map[string]float64{"fast": 0.8, "slow": 0.2}, OrderedHosts: []string{"fast", "slow"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_noop", Activations: map[string]float64{"fast": 0.0, "slow": 0.0}}, }, }, @@ -193,7 +191,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "mixed filter and weigher steps ignores filters", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { mult := 1.5 actW := map[string]float64{"a": 1.0, "b": -0.5} in := map[string]float64{"a": 0.0, "b": 0.0, "c": 0.0} @@ -203,11 +201,11 @@ func TestExplainWeighing(t *testing.T) { } hosts := []string{"a", "b"} sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: map[string]float64{"a": 0.0, "b": 0.0}, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ // Filter step: removed host c (activations don't include all OrderedHosts... wait, they do include a and b). // Actually a filter that removed c would have activations for a and b only. // Since OrderedHosts = [a, b], this LOOKS like a weigher. @@ -224,11 +222,11 @@ func TestExplainWeighing(t *testing.T) { }, { name: "under-determined system falls back to gap-only explanation", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ NormalizedInWeights: map[string]float64{"a": 0.0, "b": 0.0}, AggregatedOutWeights: map[string]float64{"a": 1.0, "b": 0.5}, OrderedHosts: []string{"a", "b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "w1", Activations: map[string]float64{"a": 1.0, "b": 0.5}}, {StepName: "w2", Activations: map[string]float64{"a": 0.8, "b": 0.3}}, {StepName: "w3", Activations: map[string]float64{"a": 0.6, "b": 0.1}}, @@ -239,7 +237,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "multiplier recovery accuracy", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // Known multipliers: [2.5, -1.0, 0.7] mults := []float64{2.5, -1.0, 0.7} acts := []map[string]float64{ @@ -257,11 +255,11 @@ func TestExplainWeighing(t *testing.T) { } } sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "w_big", Activations: acts[0]}, {StepName: "w_neg", Activations: acts[1]}, {StepName: "w_small", Activations: acts[2]}, @@ -272,7 +270,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "counterfactual not reported when removal does not change #1", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // Two weighers both favor host-a. Removing either still leaves a as #1. mult1, mult2 := 2.0, 2.0 act1 := map[string]float64{"a": 1.0, "b": -1.0} @@ -282,11 +280,11 @@ func TestExplainWeighing(t *testing.T) { for _, h := range []string{"a", "b"} { out[h] = in[h] + mult1*math.Tanh(act1[h]) + mult2*math.Tanh(act2[h]) } - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: []string{"a", "b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "w1", Activations: act1}, {StepName: "w2", Activations: act2}, }, @@ -340,7 +338,7 @@ func TestRecoverMultipliers(t *testing.T) { } } - steps := []v1alpha1.StepResult{ + steps := []StepResult{ {StepName: "w1", Activations: acts[0]}, {StepName: "w2", Activations: acts[1]}, {StepName: "w3", Activations: acts[2]}, @@ -382,9 +380,9 @@ func TestSolveLinearSystem(t *testing.T) { } func TestIdentifyWeigherSteps(t *testing.T) { - result := &v1alpha1.DecisionResult{ + result := &PipelineResult{ OrderedHosts: []string{"a", "b", "c"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ // Filter: missing host c {StepName: "filter_x", Activations: map[string]float64{"a": 1.0, "b": 1.0}}, // Weigher: has all ordered hosts @@ -482,12 +480,12 @@ func TestExplainWeighingDemo(t *testing.T) { // Sort hosts by aggregated weight to get OrderedHosts. sortByWeight(hosts, aggregatedOut) - result := &v1alpha1.DecisionResult{ + result := &PipelineResult{ NormalizedInWeights: normalizedIn, AggregatedOutWeights: aggregatedOut, OrderedHosts: hosts, TargetHost: &hosts[0], - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "kvm_binpack", Activations: actBinpack}, {StepName: "kvm_failover_evacuation", Activations: actFailover}, {StepName: "kvm_prefer_smaller_hosts", Activations: actSmaller}, diff --git a/internal/scheduling/machines/filter_weigher_pipeline_controller.go b/internal/scheduling/machines/filter_weigher_pipeline_controller.go index 3c542487b..aa708e22a 100644 --- a/internal/scheduling/machines/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/machines/filter_weigher_pipeline_controller.go @@ -19,8 +19,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/machines/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/machines/plugins/weighers" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,9 +29,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -// The decision pipeline controller takes decision resources containing a -// machine ref and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. +// The machine pipeline controller watches for new machines and runs the +// scheduling pipeline to assign each a machine pool. The result is written back +// directly to the machine's MachinePoolRef. // // Additionally, the controller watches for pipeline and step changes to // reconfigure the pipelines as needed. @@ -53,75 +51,28 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - c.processMu.Lock() - defer c.processMu.Unlock() - - // Determine if this is a decision or machine reconciliation. - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - old := decision.DeepCopy() - if err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } +// Reconcile is a no-op: this controller schedules machines through its Machine +// watch handler and reconfigures pipelines through its Pipeline watch handler. +// A Reconcile method is required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } func (c *FilterWeigherPipelineController) ProcessNewMachine(ctx context.Context, machine *ironcorev1alpha1.Machine) error { c.processMu.Lock() defer c.processMu.Unlock() - - // Create a decision resource to schedule the machine. - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "machine-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainMachines, - ResourceID: machine.Name, - PipelineRef: corev1.ObjectReference{ - Name: "machines-scheduler", - }, - MachineRef: &corev1.ObjectReference{ - Name: machine.Name, - Namespace: machine.Namespace, - }, - }, - } - - err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - return err + return c.process(ctx, machine) } -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) error { +func (c *FilterWeigherPipelineController) process(ctx context.Context, machine *ironcorev1alpha1.Machine) error { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) + const pipelineName = "machines-scheduler" + pipeline, ok := c.GetPipeline(pipelineName) if !ok { - log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) + log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) + return fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } // Find all available machine pools. @@ -136,28 +87,12 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision // Execute the scheduling pipeline. request := ironcore.MachinePipelineRequest{Pools: pools.Items} result, err := pipeline.Run(request) - if err == nil { - decision.Status.Result = &result - } - // Record a human-readable explanation of the decision for downstream - // consumers. On error the result is nil and the explanation carries the - // failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.V(1).Error(err, "failed to run scheduler pipeline") return errors.New("failed to run scheduler pipeline") } log.Info("decision processed successfully", "duration", time.Since(startedAt)) - // Set the machine pool ref on the machine. - machine := &ironcorev1alpha1.Machine{} - if err := c.Get(ctx, client.ObjectKey{ - Name: decision.Spec.MachineRef.Name, - Namespace: decision.Spec.MachineRef.Namespace, - }, machine); err != nil { - log.Error(err, "failed to fetch machine for decision") - return err - } // Assign the first machine pool returned by the pipeline. old := machine.DeepCopy() machine.Spec.MachinePoolRef = &corev1.LocalObjectReference{Name: *result.TargetHost} @@ -254,25 +189,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainMachines { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } return bldr.Named("cortex-machine-scheduler"). Complete(c) } diff --git a/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go b/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go index af85f6382..e31032bca 100644 --- a/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go @@ -12,194 +12,12 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheduling scheme: %v", err) - } - if err := ironcorev1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add ironcore scheme: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - machinePools []ironcorev1alpha1.MachinePool - machine *ironcorev1alpha1.Machine - expectError bool - expectDecision bool - expectTargetHost string - expectMachinePool string - }{ - { - name: "successful machine decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainMachines, - ResourceID: "test-machine", - PipelineRef: corev1.ObjectReference{ - Name: "machines-scheduler", - }, - MachineRef: &corev1.ObjectReference{ - Name: "test-machine", - Namespace: "default", - }, - }, - }, - machinePools: []ironcorev1alpha1.MachinePool{ - { - ObjectMeta: metav1.ObjectMeta{Name: "pool1"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "pool2"}, - }, - }, - machine: &ironcorev1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-machine", - Namespace: "default", - }, - Spec: ironcorev1alpha1.MachineSpec{ - Scheduler: "", - }, - }, - expectError: false, - expectDecision: true, - expectTargetHost: "pool1", // NoopFilter returns first pool - expectMachinePool: "pool1", - }, - { - name: "no machine pools available", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pools", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainMachines, - ResourceID: "test-machine", - PipelineRef: corev1.ObjectReference{ - Name: "machines-scheduler", - }, - MachineRef: &corev1.ObjectReference{ - Name: "test-machine", - Namespace: "default", - }, - }, - }, - machinePools: []ironcorev1alpha1.MachinePool{}, - expectError: true, - expectDecision: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []runtime.Object{tt.decision} - for i := range tt.machinePools { - objects = append(objects, &tt.machinePools[i]) - } - if tt.machine != nil { - objects = append(objects, tt.machine) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[ironcore.MachinePipelineRequest]]{ - Pipelines: map[string]lib.FilterWeigherPipeline[ironcore.MachinePipelineRequest]{ - "machines-scheduler": createMockPipeline(), - }, - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - controller.Client = client - - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - }, - } - - result, err := controller.Reconcile(context.Background(), req) - - if tt.expectError && err == nil { - t.Error("expected error but got none") - return - } - - if !tt.expectError && err != nil { - t.Errorf("expected no error, got: %v", err) - return - } - - if result.RequeueAfter > 0 { - t.Errorf("unexpected requeue: %v", result.RequeueAfter) - } - - // Verify decision status if expected - if tt.expectDecision { - var updatedDecision v1alpha1.Decision - err := client.Get(context.Background(), req.NamespacedName, &updatedDecision) - if err != nil { - t.Errorf("Failed to get updated decision: %v", err) - return - } - - if updatedDecision.Status.Result == nil { - t.Error("expected decision result to be set") - return - } - - if updatedDecision.Status.Result.TargetHost == nil { - t.Error("expected target host to be set") - return - } - - if *updatedDecision.Status.Result.TargetHost != tt.expectTargetHost { - t.Errorf("expected target host %q, got %q", tt.expectTargetHost, *updatedDecision.Status.Result.TargetHost) - } - - // Verify machine was updated with machine pool ref - if tt.machine != nil { - var updatedMachine ironcorev1alpha1.Machine - err := client.Get(context.Background(), types.NamespacedName{ - Name: tt.machine.Name, - Namespace: tt.machine.Namespace, - }, &updatedMachine) - if err != nil { - t.Errorf("Failed to get updated machine: %v", err) - return - } - - if updatedMachine.Spec.MachinePoolRef == nil { - t.Error("expected machine pool ref to be set") - return - } - - if updatedMachine.Spec.MachinePoolRef.Name != tt.expectMachinePool { - t.Errorf("expected machine pool %q, got %q", tt.expectMachinePool, updatedMachine.Spec.MachinePoolRef.Name) - } - } - } - }) - } -} - func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { controller := &FilterWeigherPipelineController{ Monitor: lib.FilterWeigherPipelineMonitor{}, @@ -293,6 +111,36 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { expectMachinePoolAssigned bool expectTargetHost string }{ + { + name: "successful machine scheduling", + machine: &ironcorev1alpha1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-machine", + Namespace: "default", + }, + Spec: ironcorev1alpha1.MachineSpec{ + Scheduler: "", + }, + }, + machinePools: []ironcorev1alpha1.MachinePool{ + {ObjectMeta: metav1.ObjectMeta{Name: "pool1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "pool2"}}, + }, + pipelineConfig: &v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machines-scheduler", + }, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + SchedulingDomain: v1alpha1.SchedulingDomainMachines, + Filters: []v1alpha1.FilterSpec{}, + Weighers: []v1alpha1.WeigherSpec{}, + }, + }, + expectError: false, + expectMachinePoolAssigned: true, + expectTargetHost: "pool1", // mock pipeline returns first pool + }, { name: "pipeline not configured", machine: &ironcorev1alpha1.Machine{ @@ -350,7 +198,6 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { client := fake.NewClientBuilder(). WithScheme(scheme). WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ @@ -411,14 +258,14 @@ func createMockPipeline() lib.FilterWeigherPipeline[ironcore.MachinePipelineRequ type mockMachinePipeline struct{} -func (m *mockMachinePipeline) Run(request ironcore.MachinePipelineRequest) (v1alpha1.DecisionResult, error) { +func (m *mockMachinePipeline) Run(request ironcore.MachinePipelineRequest) (lib.PipelineResult, error) { if len(request.Pools) == 0 { - return v1alpha1.DecisionResult{}, nil + return lib.PipelineResult{}, nil } // Return the first pool as the target host targetHost := request.Pools[0].Name - return v1alpha1.DecisionResult{ + return lib.PipelineResult{ TargetHost: &targetHost, }, nil } diff --git a/internal/scheduling/manila/external_scheduler_api.go b/internal/scheduling/manila/external_scheduler_api.go index 2dc18d13a..f341f4be5 100644 --- a/internal/scheduling/manila/external_scheduler_api.go +++ b/internal/scheduling/manila/external_scheduler_api.go @@ -7,29 +7,24 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "net/http" api "github.com/cobaltcore-dev/cortex/api/external/manila" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/metrics" ) var log = logf.Log.WithName("manila-scheduler-api") -type HTTPAPIDelegate interface { - // Process the decision from the API. Should create and return the updated decision. - ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error +// Scheduler runs a manila pipeline for a request and returns the result and a +// human-readable explanation. *FilterWeigherPipelineController satisfies it. +type Scheduler interface { + Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (scheduling.PipelineResult, string, error) } type HTTPAPI interface { @@ -38,14 +33,14 @@ type HTTPAPI interface { } type httpAPI struct { - monitor scheduling.APIMonitor - delegate HTTPAPIDelegate + monitor scheduling.APIMonitor + scheduler Scheduler } -func NewAPI(delegate HTTPAPIDelegate) HTTPAPI { +func NewAPI(scheduler Scheduler) HTTPAPI { return &httpAPI{ - monitor: scheduling.NewSchedulerMonitor(), - delegate: delegate, + monitor: scheduling.NewSchedulerMonitor(), + scheduler: scheduler, } } @@ -109,7 +104,6 @@ func (httpAPI *httpAPI) ManilaExternalScheduler(w http.ResponseWriter, r *http.R c.Respond(logr.Logger{}, http.StatusInternalServerError, err, "failed to read request body") return } - raw := runtime.RawExtension{Raw: body} var requestData api.ExternalSchedulerRequest // Copy the raw body to a io.Reader for json deserialization. cp := body @@ -138,36 +132,12 @@ func (httpAPI *httpAPI) ManilaExternalScheduler(w http.ResponseWriter, r *http.R logger.V(1).Info("inferred pipeline name", "pipeline", requestData.Pipeline) } - // Create the decision object in kubernetes. - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "manila-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: requestData.Pipeline, - }, - ResourceID: "", // TODO model out the spec. - ManilaRaw: &raw, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - } - ctx := r.Context() - if err := httpAPI.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { + result, _, err := httpAPI.scheduler.Schedule(r.Context(), requestData.Pipeline, requestData) + if err != nil { c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return } - // Check if the decision contains status conditions indicating an error. - if meta.IsStatusConditionFalse(decision.Status.Conditions, v1alpha1.DecisionConditionReady) { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision contains error condition"), "decision failed") - return - } - if decision.Status.Result == nil { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision didn't produce a result"), "decision failed") - return - } - hosts := decision.Status.Result.OrderedHosts + hosts := result.OrderedHosts // Ensure we always return an array, never null. The schema requires hosts // to be an array, and a nil slice would serialize to null. if hosts == nil { diff --git a/internal/scheduling/manila/external_scheduler_api_test.go b/internal/scheduling/manila/external_scheduler_api_test.go index dfdb3f534..0bd83467b 100644 --- a/internal/scheduling/manila/external_scheduler_api_test.go +++ b/internal/scheduling/manila/external_scheduler_api_test.go @@ -14,26 +14,24 @@ import ( "testing" manilaapi "github.com/cobaltcore-dev/cortex/api/external/manila" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type mockHTTPAPIDelegate struct { - processDecisionFunc func(ctx context.Context, decision *v1alpha1.Decision) error +type mockScheduler struct { + scheduleFunc func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) } -func (m *mockHTTPAPIDelegate) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { - if m.processDecisionFunc != nil { - return m.processDecisionFunc(ctx, decision) +func (m *mockScheduler) Schedule(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { + if m.scheduleFunc != nil { + return m.scheduleFunc(ctx, pipelineName, request) } - return nil + return lib.PipelineResult{}, "", nil } func TestNewAPI(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} + scheduler := &mockScheduler{} - api := NewAPI(delegate) + api := NewAPI(scheduler) if api == nil { t.Fatal("NewAPI returned nil") @@ -44,8 +42,8 @@ func TestNewAPI(t *testing.T) { t.Fatal("NewAPI did not return httpAPI type") } - if httpAPI.delegate != delegate { - t.Error("Delegate not set correctly") + if httpAPI.scheduler != scheduler { + t.Error("Scheduler not set correctly") } if httpAPI.monitor.ApiRequestsTimer == nil { @@ -54,8 +52,7 @@ func TestNewAPI(t *testing.T) { } func TestHTTPAPI_Init(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate) + api := NewAPI(&mockScheduler{}) mux := http.NewServeMux() api.Init(mux) @@ -72,8 +69,7 @@ func TestHTTPAPI_Init(t *testing.T) { } func TestHTTPAPI_canRunScheduler(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(&mockScheduler{}).(*httpAPI) tests := []struct { name string @@ -141,14 +137,27 @@ func TestHTTPAPI_canRunScheduler(t *testing.T) { } func TestHTTPAPI_ManilaExternalScheduler(t *testing.T) { + validBody := func(t *testing.T) string { + req := manilaapi.ExternalSchedulerRequest{ + Hosts: []manilaapi.ExternalSchedulerHost{{ShareHost: "host1"}}, + Weights: map[string]float64{"host1": 1.0}, + Pipeline: "test-pipeline", + } + data, err := json.Marshal(req) + if err != nil { + t.Fatalf("Failed to marshal request data: %v", err) + } + return string(data) + } + tests := []struct { - name string - method string - body string - processDecisionErr error - decisionResult *v1alpha1.Decision - expectedStatus int - expectedHosts []string + name string + method string + body string + scheduleResult lib.PipelineResult + scheduleErr error + expectedStatus int + expectedHosts []string }{ { name: "invalid method", @@ -162,106 +171,31 @@ func TestHTTPAPI_ManilaExternalScheduler(t *testing.T) { expectedStatus: http.StatusBadRequest, }, { - name: "valid request with successful processing", - method: http.MethodPost, - body: func() string { - req := manilaapi.ExternalSchedulerRequest{ - Hosts: []manilaapi.ExternalSchedulerHost{ - {ShareHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - }, - }, - }, + name: "valid request with successful processing", + method: http.MethodPost, + body: validBody(t), + scheduleResult: lib.PipelineResult{OrderedHosts: []string{"host1"}}, expectedStatus: http.StatusOK, expectedHosts: []string{"host1"}, }, { - name: "processing error", - method: http.MethodPost, - body: func() string { - req := manilaapi.ExternalSchedulerRequest{ - Hosts: []manilaapi.ExternalSchedulerHost{ - {ShareHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - processDecisionErr: errors.New("processing failed"), - expectedStatus: http.StatusInternalServerError, - }, - { - name: "decision failed", - method: http.MethodPost, - body: func() string { - req := manilaapi.ExternalSchedulerRequest{ - Hosts: []manilaapi.ExternalSchedulerHost{ - {ShareHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "SchedulingError", - }, - }, - }, - }, + name: "processing error", + method: http.MethodPost, + body: validBody(t), + scheduleErr: errors.New("processing failed"), expectedStatus: http.StatusInternalServerError, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - if tt.processDecisionErr != nil { - return tt.processDecisionErr - } - if tt.decisionResult != nil { - decision.Status = tt.decisionResult.Status - return nil - } - return nil + scheduler := &mockScheduler{ + scheduleFunc: func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { + return tt.scheduleResult, "", tt.scheduleErr }, } - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(scheduler).(*httpAPI) var body *strings.Reader if tt.body != "" { @@ -300,8 +234,7 @@ func TestHTTPAPI_ManilaExternalScheduler(t *testing.T) { } func TestHTTPAPI_inferPipelineName(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(&mockScheduler{}).(*httpAPI) tests := []struct { name string @@ -347,31 +280,25 @@ func TestHTTPAPI_inferPipelineName(t *testing.T) { } } -func TestHTTPAPI_ManilaExternalScheduler_DecisionCreation(t *testing.T) { - var capturedDecision *v1alpha1.Decision - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - capturedDecision = decision - // Set a successful result to avoid "decision didn't produce a result" error - decision.Status.Result = &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - } - return nil +// When the request carries no pipeline, the handler infers the default name and +// passes it to the scheduler. +func TestHTTPAPI_ManilaExternalScheduler_InfersPipeline(t *testing.T) { + var capturedPipeline string + var capturedRequest manilaapi.ExternalSchedulerRequest + scheduler := &mockScheduler{ + scheduleFunc: func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { + capturedPipeline = pipelineName + capturedRequest = request + return lib.PipelineResult{OrderedHosts: []string{"host1"}}, "", nil }, } - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(scheduler).(*httpAPI) requestData := manilaapi.ExternalSchedulerRequest{ - Hosts: []manilaapi.ExternalSchedulerHost{ - {ShareHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", + Hosts: []manilaapi.ExternalSchedulerHost{{ShareHost: "host1"}}, + Weights: map[string]float64{"host1": 1.0}, } - body, err := json.Marshal(requestData) if err != nil { t.Fatalf("Failed to marshal request data: %v", err) @@ -384,25 +311,10 @@ func TestHTTPAPI_ManilaExternalScheduler_DecisionCreation(t *testing.T) { if w.Code != http.StatusOK { t.Errorf("Expected status %d, got %d", http.StatusOK, w.Code) } - - if capturedDecision == nil { - t.Fatal("Decision was not captured") + if capturedPipeline != "manila-external-scheduler" { + t.Errorf("Expected inferred pipeline 'manila-external-scheduler', got %q", capturedPipeline) } - - // Verify decision fields - if capturedDecision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainManila { - t.Errorf("Expected scheduling domain %s, got %s", v1alpha1.SchedulingDomainManila, capturedDecision.Spec.SchedulingDomain) - } - - if capturedDecision.Spec.PipelineRef.Name != "test-pipeline" { - t.Errorf("Expected pipeline 'test-pipeline', got %s", capturedDecision.Spec.PipelineRef.Name) - } - - if capturedDecision.GenerateName != "manila-" { - t.Errorf("Expected generate name 'manila-', got %s", capturedDecision.GenerateName) - } - - if capturedDecision.Spec.ManilaRaw == nil { - t.Error("ManilaRaw should not be nil") + if len(capturedRequest.Hosts) != 1 || capturedRequest.Hosts[0].ShareHost != "host1" { + t.Errorf("Request not passed through to scheduler: %+v", capturedRequest) } } diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller.go b/internal/scheduling/manila/filter_weigher_pipeline_controller.go index f6442f697..00fedc444 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller.go @@ -5,16 +5,12 @@ package manila import ( "context" - "encoding/json" - "errors" "fmt" "sync" "time" api "github.com/cobaltcore-dev/cortex/api/external/manila" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/manila/plugins/filters" @@ -27,17 +23,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) -// The decision pipeline controller takes decision resources containing a -// placement request spec and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. -// -// Additionally, the controller watches for pipeline and step changes to -// reconfigure the pipelines as needed. +// FilterWeigherPipelineController owns the manila filter-weigher pipelines: it +// watches Pipeline and Knowledge resources to (re)configure the pipelines and +// runs a pipeline on demand via Schedule, called directly by the external +// scheduler HTTP API. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]] - // Mutex to only allow one process at a time + // Mutex to only allow one scheduling run at a time. processMu sync.Mutex // Monitor to pass down to all pipelines. @@ -49,83 +43,41 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -// Callback executed when kubernetes asks to reconcile a decision resource. -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - c.processMu.Lock() - defer c.processMu.Unlock() - - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - old := decision.DeepCopy() - if err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } +// Reconcile is a no-op: this controller reconfigures pipelines through its +// Pipeline/Knowledge watch handlers and runs placements via Schedule. A +// Reconcile method is required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } -// Process the decision from the API. Should create and return the updated decision. -func (c *FilterWeigherPipelineController) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { +// Schedule runs the named pipeline for the given request and returns the result +// plus a human-readable explanation. It serializes runs with the process lock. +func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { c.processMu.Lock() defer c.processMu.Unlock() - - err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - return err -} - -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) error { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) + pipeline, ok := c.GetPipeline(pipelineName) if !ok { - log.Error(nil, "skipping decision, pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) - } - if decision.Spec.ManilaRaw == nil { - log.Error(nil, "skipping decision, no manilaRaw spec defined") - return errors.New("no manilaRaw spec defined") - } - var request api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.ManilaRaw.Raw, &request); err != nil { - log.Error(err, "failed to unmarshal manilaRaw spec") - return err + log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) + return lib.PipelineResult{}, "", fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } result, err := pipeline.Run(request) + // A human-readable explanation of the decision for downstream consumers. On + // error the result is zero and the explanation carries the failure reason. + var resultPtr *lib.PipelineResult if err == nil { - decision.Status.Result = &result + resultPtr = &result } - // Record a human-readable explanation of the decision for downstream - // consumers. On error the result is nil and the explanation carries the - // failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) + explanation := lib.GenerateExplanation(resultPtr, err) if err != nil { log.Error(err, "failed to run pipeline") - return err + return lib.PipelineResult{}, explanation, err } log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return nil + return result, explanation, nil } // The base controller will delegate the pipeline creation down to this method. @@ -186,25 +138,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainManila { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } - return bldr.Named("cortex-manila-decisions"). + return bldr.Named("cortex-manila-pipelines"). Complete(c) } diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go b/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go index 76dbd6f93..185aa9ee2 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go @@ -4,44 +4,26 @@ package manila import ( - "context" - "encoding/json" "testing" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" api "github.com/cobaltcore-dev/cortex/api/external/manila" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/sapcc/go-bits/must" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/storage" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { +func TestFilterWeigherPipelineController_Schedule(t *testing.T) { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { t.Fatalf("Failed to add v1alpha1 scheme: %v", err) } - manilaRequest := api.ExternalSchedulerRequest{ - Spec: map[string]any{ - "share_id": "test-share-id", - "size": 10, - }, - Context: api.ManilaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: "global-req-123", - }, + request := api.ExternalSchedulerRequest{ Hosts: []api.ExternalSchedulerHost{ {ShareHost: "manila-share-1@backend1"}, {ShareHost: "manila-share-2@backend2"}, @@ -51,143 +33,55 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { Options: scheduling.Options{}, } - manilaRaw, err := json.Marshal(manilaRequest) - if err != nil { - t.Fatalf("Failed to marshal manila request: %v", err) - } - tests := []struct { name string - decision *v1alpha1.Decision - pipeline *v1alpha1.Pipeline + pipelineName string + configured bool expectError bool - expectResult bool }{ { - name: "successful manila decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, + name: "successful scheduling run", + pipelineName: "test-pipeline", + configured: true, expectError: false, - expectResult: true, }, { - name: "decision without manilaRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: nil, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, + name: "pipeline not found", + pipelineName: "nonexistent-pipeline", + configured: false, expectError: true, - expectResult: false, - }, - { - name: "pipeline not found", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipeline: nil, - expectError: true, - expectResult: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{tt.decision} - if tt.pipeline != nil { - objects = append(objects, tt.pipeline) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() + cl := fake.NewClientBuilder().WithScheme(scheme).Build() controller := &FilterWeigherPipelineController{ BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, + Client: cl, Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), }, Monitor: lib.FilterWeigherPipelineMonitor{}, } - if tt.pipeline != nil { + if tt.configured { initResult := controller.InitPipeline(t.Context(), v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.pipeline.Name, + ObjectMeta: metav1.ObjectMeta{Name: "test-pipeline"}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + SchedulingDomain: v1alpha1.SchedulingDomainManila, + Filters: []v1alpha1.FilterSpec{}, + Weighers: []v1alpha1.WeigherSpec{}, }, - Spec: tt.pipeline.Spec, }) - if err != nil { - t.Fatalf("Failed to init pipeline: %v", err) + if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { + t.Fatalf("Failed to init pipeline: %v", initResult) } - controller.Pipelines[tt.pipeline.Name] = initResult.Pipeline + controller.Pipelines["test-pipeline"] = initResult.Pipeline } - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - Namespace: tt.decision.Namespace, - }, - } - - result, err := controller.Reconcile(context.Background(), req) + result, explanation, err := controller.Schedule(t.Context(), tt.pipelineName, request) if tt.expectError && err == nil { t.Error("Expected error but got none") @@ -195,227 +89,12 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { if !tt.expectError && err != nil { t.Errorf("Expected no error but got: %v", err) } - - if result.RequeueAfter > 0 { - t.Error("Expected no requeue") - } - - var updatedDecision v1alpha1.Decision - if err := client.Get(context.Background(), req.NamespacedName, &updatedDecision); err != nil { - t.Fatalf("Failed to get updated decision: %v", err) - } - - if tt.expectResult && updatedDecision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - if !tt.expectResult && updatedDecision.Status.Result != nil { - t.Error("Expected result to be nil but was set") - } - }) - } -} - -func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - manilaRequest := api.ExternalSchedulerRequest{ - Spec: map[string]any{ - "share_id": "test-share-id", - "size": 10, - }, - Context: api.ManilaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: "global-req-123", - }, - Hosts: []api.ExternalSchedulerHost{ - {ShareHost: "manila-share-1@backend1"}, - {ShareHost: "manila-share-2@backend2"}, - }, - Weights: map[string]float64{"manila-share-1@backend1": 1.0, "manila-share-2@backend2": 0.5}, - Pipeline: "test-pipeline", - } - - manilaRaw, err := json.Marshal(manilaRequest) - if err != nil { - t.Fatalf("Failed to marshal manila request: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - pipelineConfig *v1alpha1.Pipeline - expectError bool - expectResult bool - }{ - { - name: "successful decision processing with creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-decision-", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - ResourceID: "test-uuid-1", - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "successful decision processing without creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-create", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "pipeline not configured", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipelineConfig: nil, - expectError: true, - expectResult: false, - }, - { - name: "decision without manilaRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: nil, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{} - if tt.pipelineConfig != nil { - objects = append(objects, tt.pipelineConfig) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - - if tt.pipelineConfig != nil { - controller.PipelineConfigs[tt.pipelineConfig.Name] = *tt.pipelineConfig - initResult := controller.InitPipeline(t.Context(), *tt.pipelineConfig) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to init pipeline: %v", initResult) - } - controller.Pipelines[tt.pipelineConfig.Name] = initResult.Pipeline - } - - if tt.decision.Spec.ManilaRaw != nil { - req := manilaRequest - req.Options = scheduling.Options{} - raw, marshalErr := json.Marshal(req) - if marshalErr != nil { - t.Fatalf("Failed to marshal request with options: %v", marshalErr) - } - tt.decision.Spec.ManilaRaw = &runtime.RawExtension{Raw: raw} - } - - err := controller.ProcessNewDecisionFromAPI(context.Background(), tt.decision) - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - if !tt.expectError { - if tt.expectResult && tt.decision.Status.Result == nil { - t.Error("expected decision result to be set in original decision object") + if len(result.OrderedHosts) == 0 { + t.Error("Expected ordered hosts but got none") + } + if explanation == "" { + t.Error("Expected a non-empty explanation") } } }) @@ -433,16 +112,14 @@ func TestFilterWeigherPipelineController_PipelineType(t *testing.T) { } func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) + controller := &FilterWeigherPipelineController{ + Monitor: lib.FilterWeigherPipelineMonitor{}, } tests := []struct { name string filters []v1alpha1.FilterSpec weighers []v1alpha1.WeigherSpec - knowledges []client.Object expectNonCriticalError bool expectCriticalError bool expectUnknownFilter bool @@ -452,53 +129,6 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { name: "empty steps", filters: []v1alpha1.FilterSpec{}, weighers: []v1alpha1.WeigherSpec{}, - knowledges: []client.Object{}, - expectNonCriticalError: false, - expectCriticalError: false, - expectUnknownFilter: false, - expectUnknownWeigher: false, - }, - { - name: "supported netapp step", - weighers: []v1alpha1.WeigherSpec{ - { - Name: "netapp_cpu_usage_balancing", - Params: []v1alpha1.Parameter{ - {Key: "AvgCPUUsageLowerBound", FloatValue: new(0.0)}, - {Key: "AvgCPUUsageUpperBound", FloatValue: new(90.0)}, - {Key: "MaxCPUUsageLowerBound", FloatValue: new(0.0)}, - {Key: "MaxCPUUsageUpperBound", FloatValue: new(100.0)}, - }, - }, - }, - knowledges: []client.Object{ - &v1alpha1.Knowledge{ - ObjectMeta: metav1.ObjectMeta{ - Name: "netapp-storage-pool-cpu-usage-manila", - }, - Status: v1alpha1.KnowledgeStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.KnowledgeConditionReady, - Status: metav1.ConditionTrue, - }, - }, - Raw: must.Return(v1alpha1.BoxFeatureList([]storage.StoragePoolCPUUsage{ - { - StoragePoolName: "manila-share-1@backend1", - AvgCPUUsagePct: 50, - MaxCPUUsagePct: 80, - }, - { - StoragePoolName: "manila-share-2@backend2", - AvgCPUUsagePct: 20, - MaxCPUUsagePct: 40, - }, - })), - RawLength: 2, - }, - }, - }, expectNonCriticalError: false, expectCriticalError: false, expectUnknownFilter: false, @@ -508,7 +138,7 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { name: "unsupported step", filters: []v1alpha1.FilterSpec{ { - Name: "unsupported-plugin", + Name: "test-plugin", }, }, expectNonCriticalError: false, @@ -520,16 +150,6 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(tt.knowledges...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - controller := &FilterWeigherPipelineController{ - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - controller.Client = client // Through basepipelinecontroller - initResult := controller.InitPipeline(t.Context(), v1alpha1.Pipeline{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pipeline", @@ -542,19 +162,19 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { }, }) - if !tt.expectCriticalError && len(initResult.FilterErrors) > 0 { - t.Errorf("Expected no critical error but got: %v", initResult.FilterErrors) - } if tt.expectCriticalError && len(initResult.FilterErrors) == 0 { - t.Error("Expected critical error but got none") + t.Error("Expected error but got none") } - - if !tt.expectNonCriticalError && len(initResult.WeigherErrors) > 0 { - t.Errorf("Expected no non-critical error but got: %v", initResult.WeigherErrors) + if !tt.expectCriticalError && len(initResult.FilterErrors) > 0 { + t.Errorf("Expected no error but got: %v", initResult.FilterErrors) } + if tt.expectNonCriticalError && len(initResult.WeigherErrors) == 0 { t.Error("Expected non-critical error but got none") } + if !tt.expectNonCriticalError && len(initResult.WeigherErrors) > 0 { + t.Errorf("Expected no non-critical error but got: %v", initResult.WeigherErrors) + } }) } } diff --git a/internal/scheduling/nova/cr_metrics_test.go b/internal/scheduling/nova/cr_metrics_test.go index 09f4ec125..a7417e382 100644 --- a/internal/scheduling/nova/cr_metrics_test.go +++ b/internal/scheduling/nova/cr_metrics_test.go @@ -355,7 +355,6 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(flavorKnowledge). - WithStatusSubresource(&v1alpha1.Decision{}). Build() counter := crs.NewNoHostFoundCounter() diff --git a/internal/scheduling/nova/crs/recorder_test.go b/internal/scheduling/nova/crs/recorder_test.go index f9fbd3bd8..29fee664d 100644 --- a/internal/scheduling/nova/crs/recorder_test.go +++ b/internal/scheduling/nova/crs/recorder_test.go @@ -18,6 +18,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -104,15 +105,11 @@ func TestRecordCRAllocation(t *testing.T) { } } - makeDecision := func(host string, candidates ...string) *v1alpha1.Decision { + makeResult := func(host string, candidates ...string) sched.SchedulerResult { h := host - return &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &h, - OrderedHosts: candidates, - }, - }, + return sched.SchedulerResult{ + TargetHost: &h, + OrderedHosts: candidates, } } @@ -154,7 +151,8 @@ func TestRecordCRAllocation(t *testing.T) { name string objects []client.Object request api.ExternalSchedulerRequest - decision *v1alpha1.Decision + result sched.SchedulerResult + intent v1alpha1.SchedulingIntent checkSlot string expectAllocation bool expectedCRSlot string // if non-empty, asserts PlacementCounter cr_slot label @@ -167,7 +165,7 @@ func TestRecordCRAllocation(t *testing.T) { setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), checkSlot: "slot-1", expectAllocation: true, }, @@ -179,7 +177,7 @@ func TestRecordCRAllocation(t *testing.T) { setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, vmAlloc())), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), checkSlot: "slot-1", expectAllocation: true, }, @@ -190,7 +188,7 @@ func TestRecordCRAllocation(t *testing.T) { setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, request: makeRequest(instanceUUID, projectID, "unknown-flavor"), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), checkSlot: "slot-1", expectAllocation: false, }, @@ -202,7 +200,7 @@ func TestRecordCRAllocation(t *testing.T) { setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, "other-host", nil)), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), checkSlot: "slot-1", expectAllocation: false, }, @@ -222,7 +220,7 @@ func TestRecordCRAllocation(t *testing.T) { })), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), checkSlot: "slot-full", expectAllocation: false, }, @@ -238,7 +236,7 @@ func TestRecordCRAllocation(t *testing.T) { setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), checkSlot: "slot-1", expectAllocation: false, }, @@ -248,7 +246,7 @@ func TestRecordCRAllocation(t *testing.T) { makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), checkSlot: "slot-1", expectAllocation: false, expectedCRSlot: "error", @@ -271,7 +269,7 @@ func TestRecordCRAllocation(t *testing.T) { PlacementCounter: counter, } - recorder.RecordPlacement(context.Background(), *tt.decision.Status.Result, tt.decision.Spec.Intent, tt.request) + recorder.RecordPlacement(context.Background(), tt.result, tt.intent, tt.request) var res v1alpha1.Reservation if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: tt.checkSlot}, &res); err != nil { @@ -285,7 +283,7 @@ func TestRecordCRAllocation(t *testing.T) { t.Errorf("expected no allocation for UUID %q but one was written", instanceUUID) } if tt.expectedCRSlot != "" { - intent := string(tt.decision.Spec.Intent) + intent := string(tt.intent) got := testutil.ToFloat64(counter.WithLabelValues("unknown", intent, tt.expectedCRSlot)) if got != 1 { t.Errorf("PlacementCounter[flavor_group=unknown, intent=%q, cr_slot=%q] = %.0f, want 1", diff --git a/internal/scheduling/nova/external_scheduler_api_test.go b/internal/scheduling/nova/external_scheduler_api_test.go index db1b5b97e..5885bfc80 100644 --- a/internal/scheduling/nova/external_scheduler_api_test.go +++ b/internal/scheduling/nova/external_scheduler_api_test.go @@ -14,12 +14,9 @@ import ( "testing" novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // mockRunner is a test double for the scheduler the HTTP API depends on. runFunc @@ -172,7 +169,7 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { method string body string processDecisionErr error - decisionResult *v1alpha1.Decision + result *compute.SchedulerResult expectedStatus int expectedHosts []string }{ @@ -211,12 +208,8 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { } return string(data) }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - }, - }, + result: &compute.SchedulerResult{ + OrderedHosts: []string{"host1"}, }, expectedStatus: http.StatusOK, expectedHosts: []string{"host1"}, @@ -272,18 +265,8 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { } return string(data) }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "SchedulingError", - }, - }, - }, - }, - expectedStatus: http.StatusInternalServerError, + processDecisionErr: errors.New("scheduling failed"), + expectedStatus: http.StatusInternalServerError, }, } @@ -294,12 +277,8 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { if tt.processDecisionErr != nil { return compute.RunOutcome{}, tt.processDecisionErr } - if tt.decisionResult != nil { - if tt.decisionResult.Status.Result != nil { - return compute.RunOutcome{Result: *tt.decisionResult.Status.Result}, nil - } - // A decisionResult without a Result models a failed run. - return compute.RunOutcome{}, errors.New("scheduling failed") + if tt.result != nil { + return compute.RunOutcome{Result: *tt.result}, nil } return compute.RunOutcome{}, nil }, diff --git a/internal/scheduling/pods/filter_weigher_pipeline_controller.go b/internal/scheduling/pods/filter_weigher_pipeline_controller.go index 50afde197..ee140526c 100644 --- a/internal/scheduling/pods/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/pods/filter_weigher_pipeline_controller.go @@ -18,7 +18,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/pods/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/pods/plugins/weighers" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" @@ -30,9 +29,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -// The decision pipeline controller takes decision resources containing a -// pod ref and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. +// The pod pipeline controller watches for new pods and runs the scheduling +// pipeline to assign each a node. The result is written back directly via a +// Binding. // // Additionally, the controller watches for pipeline and step changes to // reconfigure the pipelines as needed. @@ -52,87 +51,30 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - c.processMu.Lock() - defer c.processMu.Unlock() - - // Determine if this is a decision or pod reconciliation. - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - old := decision.DeepCopy() - if err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } +// Reconcile is a no-op: this controller schedules pods through its Pod watch +// handler and reconfigures pipelines through its Pipeline watch handler. A +// Reconcile method is required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } func (c *FilterWeigherPipelineController) ProcessNewPod(ctx context.Context, pod *corev1.Pod) error { c.processMu.Lock() defer c.processMu.Unlock() - - // Create a decision resource to schedule the pod. - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pod-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainPods, - ResourceID: pod.Namespace + "--" + pod.Name, - PipelineRef: corev1.ObjectReference{ - Name: "pods-scheduler", - }, - PodRef: &corev1.ObjectReference{ - Name: pod.Name, - Namespace: pod.Namespace, - }, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - } - - err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - return err + return c.process(ctx, pod) } -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) error { +func (c *FilterWeigherPipelineController) process(ctx context.Context, pod *corev1.Pod) error { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) + const pipelineName = "pods-scheduler" + pipeline, ok := c.GetPipeline(pipelineName) if !ok { - log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) + log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) + return fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } - // Check if the pod is already assigned to a node. - pod := &corev1.Pod{} - if err := c.Get(ctx, client.ObjectKey{ - Name: decision.Spec.PodRef.Name, - Namespace: decision.Spec.PodRef.Namespace, - }, pod); err != nil { - log.Error(err, "failed to fetch pod for decision") - return err - } if pod.Spec.NodeName != "" { log.V(1).Info("pod is already assigned to a node", "node", pod.Spec.NodeName) return nil @@ -150,14 +92,6 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision // Execute the scheduling pipeline. request := pods.PodPipelineRequest{Nodes: nodes.Items, Pod: *pod} result, err := pipeline.Run(request) - - if err == nil { - decision.Status.Result = &result - } - // Record a human-readable explanation of the decision for downstream - // consumers. On error the result is nil and the explanation carries the - // failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.V(1).Error(err, "failed to run scheduler pipeline") return errors.New("failed to run scheduler pipeline") @@ -167,8 +101,8 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision // Assign the first node returned by the pipeline using a Binding. binding := &corev1.Binding{ ObjectMeta: metav1.ObjectMeta{ - Name: decision.Spec.PodRef.Name, - Namespace: decision.Spec.PodRef.Namespace, + Name: pod.Name, + Namespace: pod.Namespace, }, Target: corev1.ObjectReference{ Kind: "Node", @@ -263,25 +197,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainPods { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } return bldr.Named("cortex-pod-scheduler"). Complete(c) } diff --git a/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go b/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go index 0c9ad6fb8..e9a8f93c3 100644 --- a/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go @@ -15,166 +15,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheduling scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add corev1 scheme: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - nodes []corev1.Node - pod *corev1.Pod - expectError bool - expectDecision bool - expectTargetHost string - }{ - { - name: "successful pod decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainPods, - ResourceID: "test-pod", - PipelineRef: corev1.ObjectReference{ - Name: "pods-scheduler", - }, - PodRef: &corev1.ObjectReference{ - Name: "test-pod", - Namespace: "default", - }, - }, - }, - nodes: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "node2"}, - }, - }, - pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Spec: corev1.PodSpec{ - SchedulerName: "", - }, - }, - expectError: false, - expectDecision: true, - expectTargetHost: "node1", // NoopFilter returns first node - }, - { - name: "no nodes available", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-nodes", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainPods, - ResourceID: "test-pod", - PipelineRef: corev1.ObjectReference{ - Name: "pods-scheduler", - }, - PodRef: &corev1.ObjectReference{ - Name: "test-pod", - Namespace: "default", - }, - }, - }, - nodes: []corev1.Node{}, - expectError: true, - expectDecision: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []runtime.Object{tt.decision} - for i := range tt.nodes { - objects = append(objects, &tt.nodes[i]) - } - if tt.pod != nil { - objects = append(objects, tt.pod) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[pods.PodPipelineRequest]]{ - Pipelines: map[string]lib.FilterWeigherPipeline[pods.PodPipelineRequest]{ - "pods-scheduler": createMockPodPipeline(), - }, - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - controller.Client = client - - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - }, - } - - result, err := controller.Reconcile(context.Background(), req) - - if tt.expectError && err == nil { - t.Error("expected error but got none") - return - } - - if !tt.expectError && err != nil { - t.Errorf("expected no error, got: %v", err) - return - } - - if result.RequeueAfter > 0 { - t.Errorf("unexpected requeue: %v", result.RequeueAfter) - } - - // Verify decision status if expected - if tt.expectDecision { - var updatedDecision v1alpha1.Decision - err := client.Get(context.Background(), req.NamespacedName, &updatedDecision) - if err != nil { - t.Errorf("Failed to get updated decision: %v", err) - return - } - - if updatedDecision.Status.Result == nil { - t.Error("expected decision result to be set") - return - } - - if updatedDecision.Status.Result.TargetHost == nil { - t.Error("expected target host to be set") - return - } - - if *updatedDecision.Status.Result.TargetHost != tt.expectTargetHost { - t.Errorf("expected target host %q, got %q", tt.expectTargetHost, *updatedDecision.Status.Result.TargetHost) - } - } - }) - } -} - func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { controller := &FilterWeigherPipelineController{ Monitor: lib.FilterWeigherPipelineMonitor{}, @@ -393,7 +236,6 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { client := fake.NewClientBuilder(). WithScheme(scheme). WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ @@ -452,14 +294,14 @@ func createMockPodPipeline() lib.FilterWeigherPipeline[pods.PodPipelineRequest] type mockPodPipeline struct{} -func (m *mockPodPipeline) Run(request pods.PodPipelineRequest) (v1alpha1.DecisionResult, error) { +func (m *mockPodPipeline) Run(request pods.PodPipelineRequest) (lib.PipelineResult, error) { if len(request.Nodes) == 0 { - return v1alpha1.DecisionResult{}, nil + return lib.PipelineResult{}, nil } // Return the first node as the target host targetHost := request.Nodes[0].Name - return v1alpha1.DecisionResult{ + return lib.PipelineResult{ TargetHost: &targetHost, }, nil } diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index f31b5e6d3..2d0b7b06a 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -563,8 +563,8 @@ func TestGVKFromHomeScheme_Success(t *testing.T) { }{ {"ConfigMap", &corev1.ConfigMap{}, configMapGVK}, {"ConfigMapList", &corev1.ConfigMapList{}, configMapListGVK}, - {"Decision", &v1alpha1.Decision{}, schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Decision"}}, - {"DecisionList", &v1alpha1.DecisionList{}, schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "DecisionList"}}, + {"Pipeline", &v1alpha1.Pipeline{}, schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Pipeline"}}, + {"PipelineList", &v1alpha1.PipelineList{}, schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "PipelineList"}}, } for _, tt := range tests { @@ -1778,7 +1778,7 @@ func TestClient_InitFromConf_GVKFormatting(t *testing.T) { expectedStr string }{ {"core ConfigMap", configMapGVK, "v1/ConfigMap"}, - {"cortex Decision", schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Decision"}, "cortex.cloud/v1alpha1/Decision"}, + {"cortex Pipeline", schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Pipeline"}, "cortex.cloud/v1alpha1/Pipeline"}, } for _, tt := range tests { From e9c315bec022c897829aa6e5f7803dc368487297 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 18:05:40 +0100 Subject: [PATCH 08/21] refactor(scheduling): remove explanation return value from Scheduler interface and related methods Signed-off-by: Markus Wieland --- .../cinder/external_scheduler_api.go | 8 ++++---- .../cinder/external_scheduler_api_test.go | 14 +++++++------- .../filter_weigher_pipeline_controller.go | 19 ++++++------------- ...filter_weigher_pipeline_controller_test.go | 5 +---- .../manila/external_scheduler_api.go | 8 ++++---- .../manila/external_scheduler_api_test.go | 14 +++++++------- .../filter_weigher_pipeline_controller.go | 19 ++++++------------- ...filter_weigher_pipeline_controller_test.go | 5 +---- .../scheduling/nova/external_scheduler_api.go | 3 +-- internal/scheduling/nova/vm_handler.go | 11 ++++++++--- internal/scheduling/nova/vm_handler_test.go | 8 ++++---- 11 files changed, 49 insertions(+), 65 deletions(-) diff --git a/internal/scheduling/cinder/external_scheduler_api.go b/internal/scheduling/cinder/external_scheduler_api.go index c717a8966..8ae12e3de 100644 --- a/internal/scheduling/cinder/external_scheduler_api.go +++ b/internal/scheduling/cinder/external_scheduler_api.go @@ -21,10 +21,10 @@ import ( var log = logf.Log.WithName("cinder-scheduler-api") -// Scheduler runs a cinder pipeline for a request and returns the result and a -// human-readable explanation. *FilterWeigherPipelineController satisfies it. +// Scheduler runs a cinder pipeline for a request and returns the result. +// *FilterWeigherPipelineController satisfies it. type Scheduler interface { - Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (scheduling.PipelineResult, string, error) + Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (scheduling.PipelineResult, error) } type HTTPAPI interface { @@ -132,7 +132,7 @@ func (httpAPI *httpAPI) CinderExternalScheduler(w http.ResponseWriter, r *http.R logger.V(1).Info("inferred pipeline name", "pipeline", requestData.Pipeline) } - result, _, err := httpAPI.scheduler.Schedule(r.Context(), requestData.Pipeline, requestData) + result, err := httpAPI.scheduler.Schedule(r.Context(), requestData.Pipeline, requestData) if err != nil { c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return diff --git a/internal/scheduling/cinder/external_scheduler_api_test.go b/internal/scheduling/cinder/external_scheduler_api_test.go index fb81e99ff..ec8c34feb 100644 --- a/internal/scheduling/cinder/external_scheduler_api_test.go +++ b/internal/scheduling/cinder/external_scheduler_api_test.go @@ -18,14 +18,14 @@ import ( ) type mockScheduler struct { - scheduleFunc func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) + scheduleFunc func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, error) } -func (m *mockScheduler) Schedule(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { +func (m *mockScheduler) Schedule(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { if m.scheduleFunc != nil { return m.scheduleFunc(ctx, pipelineName, request) } - return lib.PipelineResult{}, "", nil + return lib.PipelineResult{}, nil } func TestNewAPI(t *testing.T) { @@ -190,8 +190,8 @@ func TestHTTPAPI_CinderExternalScheduler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scheduler := &mockScheduler{ - scheduleFunc: func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { - return tt.scheduleResult, "", tt.scheduleErr + scheduleFunc: func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { + return tt.scheduleResult, tt.scheduleErr }, } @@ -286,10 +286,10 @@ func TestHTTPAPI_CinderExternalScheduler_InfersPipeline(t *testing.T) { var capturedPipeline string var capturedRequest cinderapi.ExternalSchedulerRequest scheduler := &mockScheduler{ - scheduleFunc: func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { + scheduleFunc: func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { capturedPipeline = pipelineName capturedRequest = request - return lib.PipelineResult{OrderedHosts: []string{"host1"}}, "", nil + return lib.PipelineResult{OrderedHosts: []string{"host1"}}, nil }, } diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go index bff7d0394..815571e4c 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go @@ -50,9 +50,9 @@ func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Reques return ctrl.Result{}, nil } -// Schedule runs the named pipeline for the given request and returns the result -// plus a human-readable explanation. It serializes runs with the process lock. -func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { +// Schedule runs the named pipeline for the given request and returns the +// result. It serializes runs with the process lock. +func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (lib.PipelineResult, error) { c.processMu.Lock() defer c.processMu.Unlock() log := ctrl.LoggerFrom(ctx) @@ -61,23 +61,16 @@ func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipeline pipeline, ok := c.GetPipeline(pipelineName) if !ok { log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) - return lib.PipelineResult{}, "", fmt.Errorf("pipeline not found or not ready: %q", pipelineName) + return lib.PipelineResult{}, fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } result, err := pipeline.Run(request) - // A human-readable explanation of the decision for downstream consumers. On - // error the result is zero and the explanation carries the failure reason. - var resultPtr *lib.PipelineResult - if err == nil { - resultPtr = &result - } - explanation := lib.GenerateExplanation(resultPtr, err) if err != nil { log.Error(err, "failed to run pipeline") - return lib.PipelineResult{}, explanation, err + return lib.PipelineResult{}, err } log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return result, explanation, nil + return result, nil } // The base controller will delegate the pipeline creation down to this method. diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go index 725c854ff..4daf8fee0 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go @@ -81,7 +81,7 @@ func TestFilterWeigherPipelineController_Schedule(t *testing.T) { controller.Pipelines["test-pipeline"] = initResult.Pipeline } - result, explanation, err := controller.Schedule(t.Context(), tt.pipelineName, request) + result, err := controller.Schedule(t.Context(), tt.pipelineName, request) if tt.expectError && err == nil { t.Error("Expected error but got none") @@ -93,9 +93,6 @@ func TestFilterWeigherPipelineController_Schedule(t *testing.T) { if len(result.OrderedHosts) == 0 { t.Error("Expected ordered hosts but got none") } - if explanation == "" { - t.Error("Expected a non-empty explanation") - } } }) } diff --git a/internal/scheduling/manila/external_scheduler_api.go b/internal/scheduling/manila/external_scheduler_api.go index f341f4be5..273c282dd 100644 --- a/internal/scheduling/manila/external_scheduler_api.go +++ b/internal/scheduling/manila/external_scheduler_api.go @@ -21,10 +21,10 @@ import ( var log = logf.Log.WithName("manila-scheduler-api") -// Scheduler runs a manila pipeline for a request and returns the result and a -// human-readable explanation. *FilterWeigherPipelineController satisfies it. +// Scheduler runs a manila pipeline for a request and returns the result. +// *FilterWeigherPipelineController satisfies it. type Scheduler interface { - Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (scheduling.PipelineResult, string, error) + Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (scheduling.PipelineResult, error) } type HTTPAPI interface { @@ -132,7 +132,7 @@ func (httpAPI *httpAPI) ManilaExternalScheduler(w http.ResponseWriter, r *http.R logger.V(1).Info("inferred pipeline name", "pipeline", requestData.Pipeline) } - result, _, err := httpAPI.scheduler.Schedule(r.Context(), requestData.Pipeline, requestData) + result, err := httpAPI.scheduler.Schedule(r.Context(), requestData.Pipeline, requestData) if err != nil { c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return diff --git a/internal/scheduling/manila/external_scheduler_api_test.go b/internal/scheduling/manila/external_scheduler_api_test.go index 0bd83467b..7012984ea 100644 --- a/internal/scheduling/manila/external_scheduler_api_test.go +++ b/internal/scheduling/manila/external_scheduler_api_test.go @@ -18,14 +18,14 @@ import ( ) type mockScheduler struct { - scheduleFunc func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) + scheduleFunc func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, error) } -func (m *mockScheduler) Schedule(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { +func (m *mockScheduler) Schedule(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { if m.scheduleFunc != nil { return m.scheduleFunc(ctx, pipelineName, request) } - return lib.PipelineResult{}, "", nil + return lib.PipelineResult{}, nil } func TestNewAPI(t *testing.T) { @@ -190,8 +190,8 @@ func TestHTTPAPI_ManilaExternalScheduler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scheduler := &mockScheduler{ - scheduleFunc: func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { - return tt.scheduleResult, "", tt.scheduleErr + scheduleFunc: func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { + return tt.scheduleResult, tt.scheduleErr }, } @@ -286,10 +286,10 @@ func TestHTTPAPI_ManilaExternalScheduler_InfersPipeline(t *testing.T) { var capturedPipeline string var capturedRequest manilaapi.ExternalSchedulerRequest scheduler := &mockScheduler{ - scheduleFunc: func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { + scheduleFunc: func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { capturedPipeline = pipelineName capturedRequest = request - return lib.PipelineResult{OrderedHosts: []string{"host1"}}, "", nil + return lib.PipelineResult{OrderedHosts: []string{"host1"}}, nil }, } diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller.go b/internal/scheduling/manila/filter_weigher_pipeline_controller.go index 00fedc444..cc7ced78e 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller.go @@ -50,9 +50,9 @@ func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Reques return ctrl.Result{}, nil } -// Schedule runs the named pipeline for the given request and returns the result -// plus a human-readable explanation. It serializes runs with the process lock. -func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (lib.PipelineResult, string, error) { +// Schedule runs the named pipeline for the given request and returns the +// result. It serializes runs with the process lock. +func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (lib.PipelineResult, error) { c.processMu.Lock() defer c.processMu.Unlock() log := ctrl.LoggerFrom(ctx) @@ -61,23 +61,16 @@ func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipeline pipeline, ok := c.GetPipeline(pipelineName) if !ok { log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) - return lib.PipelineResult{}, "", fmt.Errorf("pipeline not found or not ready: %q", pipelineName) + return lib.PipelineResult{}, fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } result, err := pipeline.Run(request) - // A human-readable explanation of the decision for downstream consumers. On - // error the result is zero and the explanation carries the failure reason. - var resultPtr *lib.PipelineResult - if err == nil { - resultPtr = &result - } - explanation := lib.GenerateExplanation(resultPtr, err) if err != nil { log.Error(err, "failed to run pipeline") - return lib.PipelineResult{}, explanation, err + return lib.PipelineResult{}, err } log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return result, explanation, nil + return result, nil } // The base controller will delegate the pipeline creation down to this method. diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go b/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go index 185aa9ee2..418c9297d 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go @@ -81,7 +81,7 @@ func TestFilterWeigherPipelineController_Schedule(t *testing.T) { controller.Pipelines["test-pipeline"] = initResult.Pipeline } - result, explanation, err := controller.Schedule(t.Context(), tt.pipelineName, request) + result, err := controller.Schedule(t.Context(), tt.pipelineName, request) if tt.expectError && err == nil { t.Error("Expected error but got none") @@ -93,9 +93,6 @@ func TestFilterWeigherPipelineController_Schedule(t *testing.T) { if len(result.OrderedHosts) == 0 { t.Error("Expected ordered hosts but got none") } - if explanation == "" { - t.Error("Expected a non-empty explanation") - } } }) } diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index 8ea17c929..2a4778b24 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -292,8 +292,7 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req case schedReq.HypervisorFamily != compute.HypervisorFamilyKVM: logger.V(1).Info("skipping virtual machine creation, unsupported hypervisor type", "instanceUUID", requestData.Spec.Data.InstanceUUID) default: - explanation := scheduling.GenerateExplanation(&out.Result, nil) - if upsertErr := httpAPI.vmHandler.Upsert(ctx, out.Result, explanation, requestData, intent); upsertErr != nil { + if upsertErr := httpAPI.vmHandler.Upsert(ctx, out.Result, requestData, intent); upsertErr != nil { if httpAPI.featureGates.VirtualMachineCreationMandatory { return fmt.Errorf("failed to upsert virtual machine resource: %w", upsertErr) } diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index 0856b8495..11a2c05ad 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -13,6 +13,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -36,9 +37,9 @@ type VirtualMachineHandler struct { // Upsert creates or patches the VirtualMachine resource described by the // scheduling result and request. Region and AZ are immutable on the CRD and only // written on create; mutable fields (target host, cortex spec, labels) are -// patched on updates. The explanation and intent are recorded on the VM's cortex -// spec. -func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.SchedulerResult, explanation string, request api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent) error { +// patched on updates. A human-readable explanation of the result and the intent +// are recorded on the VM's cortex spec. +func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.SchedulerResult, request api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent) error { log := ctrl.LoggerFrom(ctx) if result.TargetHost == nil { @@ -46,6 +47,10 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Sched return nil } + // The explanation is a VM-creation concern, so it is generated here from the + // pipeline result and recorded on the VM's cortex spec. + explanation := lib.GenerateExplanation(&result, nil) + // We derive hierarchical location information (region and az) from the // target host (i.e. the first host in the list). This assumes that all // hosts in the list share the same region and availability zone. diff --git a/internal/scheduling/nova/vm_handler_test.go b/internal/scheduling/nova/vm_handler_test.go index bf6d488da..3c8ff2cac 100644 --- a/internal/scheduling/nova/vm_handler_test.go +++ b/internal/scheduling/nova/vm_handler_test.go @@ -81,7 +81,7 @@ func TestVirtualMachineHandlerCreate(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestResult(&targetHost), "", vmTestRequest(), api.CreateIntent); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert: %v", err) } @@ -130,7 +130,7 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestResult(&targetHost), "", vmTestRequest(), api.CreateIntent); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert: %v", err) } @@ -155,7 +155,7 @@ func TestVirtualMachineHandlerNoTargetHost(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() h := &VirtualMachineHandler{Client: fakeClient} - if err := h.Upsert(context.Background(), vmTestResult(nil), "", vmTestRequest(), api.CreateIntent); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(nil), vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert with nil target host should be a no-op, got: %v", err) } var vmList vmopv1alpha1.VirtualMachineList @@ -176,7 +176,7 @@ func TestVirtualMachineHandlerMissingRegion(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestResult(&targetHost), "", vmTestRequest(), api.CreateIntent); err == nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.CreateIntent); err == nil { t.Fatal("expected error for missing region label, got nil") } vm := &vmopv1alpha1.VirtualMachine{} From a434f6bd3af21d2d6af696d8723ea5af5939e0a2 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 2 Nov 2026 18:27:34 +0100 Subject: [PATCH 09/21] feat(reservations): add NumInstances field to scheduler probes and reservations for capacity management Signed-off-by: Markus Wieland --- internal/scheduling/reservations/capacity/controller.go | 1 + .../reservations/commitments/reservation_controller.go | 1 + .../scheduling/reservations/failover/reservation_scheduling.go | 2 ++ 3 files changed, 4 insertions(+) diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 3bb68c99c..5c30fbc4f 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -829,6 +829,7 @@ func (c *Reconciler) probeScheduler( Weights: probeWeights, Intent: schedulerapi.CapacityProbeIntent, InstanceUUID: "capacity-" + flavor.Name, + NumInstances: 1, // probe a single flavor slot; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free ProjectID: "cortex-capacity-probe", AvailabilityZone: az, Flavor: sched.FlavorSpec{ diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 7b5bb513a..a4732568e 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -363,6 +363,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr Weights: weights, Intent: schedulerdelegationapi.ReserveForCommittedResourceIntent, InstanceUUID: res.Name, + NumInstances: 1, // one reservation slot; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free ProjectID: projectID, AvailabilityZone: availabilityZone, DomainName: domainName, diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 46ea5d961..4bbb7567f 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -135,6 +135,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex Options: opts, Intent: intent, InstanceUUID: vm.UUID, + NumInstances: 1, // one reservation per VM; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free ProjectID: vm.ProjectID, AvailabilityZone: vm.AvailabilityZone, IgnoreHosts: ignoreHypervisors, @@ -291,6 +292,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( Options: evacuateOpts, Intent: api.EvacuateIntent, InstanceUUID: vm.UUID, + NumInstances: 1, // validating one VM; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free ProjectID: vm.ProjectID, AvailabilityZone: vm.AvailabilityZone, IgnoreHosts: []string{vm.CurrentHypervisor}, From 16fb4cbb72d6db5d31f770dc6ccfaa345bd48039 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Tue, 3 Nov 2026 12:56:32 +0100 Subject: [PATCH 10/21] refactor(scheduling): replace resource.Quantity with uint64 for memory in FlavorSpec Signed-off-by: Markus Wieland --- internal/scheduling/compute/request.go | 8 +++----- internal/scheduling/nova/adapter.go | 6 ++---- .../filters/filter_has_enough_capacity.go | 14 ++++++++------ .../filters/filter_has_enough_capacity_test.go | 6 +++--- .../plugins/filters/filter_quota_enforcement.go | 8 ++++---- .../filters/filter_quota_enforcement_test.go | 4 ++-- .../nova/plugins/weighers/kvm_binpack.go | 3 ++- .../nova/plugins/weighers/kvm_binpack_test.go | 6 +++--- .../kvm_committed_resource_reservation.go | 2 +- .../kvm_committed_resource_reservation_test.go | 4 ++-- .../weighers/kvm_failover_evacuation_test.go | 6 +++--- ...m_failover_reservation_consolidation_test.go | 6 +++--- .../kvm_instance_group_soft_affinity_test.go | 7 +++---- .../weighers/kvm_prefer_smaller_hosts_test.go | 6 +++--- .../nova/plugins/weighers/vmware_binpack.go | 3 ++- .../plugins/weighers/vmware_binpack_test.go | 17 ++++++++--------- .../reservations/capacity/controller.go | 6 +++--- .../commitments/reservation_controller.go | 7 +++---- .../failover/reservation_scheduling.go | 13 ++++++------- 19 files changed, 64 insertions(+), 68 deletions(-) diff --git a/internal/scheduling/compute/request.go b/internal/scheduling/compute/request.go index c8d9e7940..b5fd99577 100644 --- a/internal/scheduling/compute/request.go +++ b/internal/scheduling/compute/request.go @@ -23,7 +23,6 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" - "k8s.io/apimachinery/pkg/api/resource" ) // HypervisorFamily is the coarse hypervisor family a request targets. It is the @@ -111,10 +110,9 @@ type FlavorSpec struct { // FlavorID is the flavor ID. Empty only affects the VM resource written by // side effects. FlavorID string - // Memory is the requested memory per instance, as a resource.Quantity in - // bytes. The zero value is a zero-memory instance to capacity/bin-pack steps - // (fits anywhere), so the adapter sets it (from Nova's memory_mb * 2^20). - Memory resource.Quantity + // MemoryMB is the requested memory per instance. 0 means a zero-memory + // instance to capacity/bin-pack steps (fits anywhere), so the adapter sets it. + MemoryMB uint64 // VCPUs is the requested vCPUs per instance. 0 means a zero-CPU instance to // capacity/bin-pack steps (fits anywhere), so the adapter sets it. VCPUs uint64 diff --git a/internal/scheduling/nova/adapter.go b/internal/scheduling/nova/adapter.go index 4a0d0bd77..9554abe85 100644 --- a/internal/scheduling/nova/adapter.go +++ b/internal/scheduling/nova/adapter.go @@ -9,7 +9,6 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" - "k8s.io/apimachinery/pkg/api/resource" ) // unsupportedCapabilityOps are the Nova extra-spec comparison operators the @@ -70,9 +69,8 @@ func ToSchedulerRequest( Name: spec.Flavor.Data.Name, FlavorID: spec.Flavor.Data.FlavorID, VCPUs: spec.Flavor.Data.VCPUs, - // Nova's memory_mb is in mebibytes (2^20 bytes). - Memory: *resource.NewQuantity(int64(spec.Flavor.Data.MemoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // flavor memory bounded by Nova - Type: toFlavorType(wire), + MemoryMB: spec.Flavor.Data.MemoryMB, + Type: toFlavorType(wire), } // Hypervisor family + specific type. diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index b590ab495..d1d80b7d8 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -307,7 +307,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.Sche } // Check memory capacity. - if request.Flavor.Memory.IsZero() { + if request.Flavor.MemoryMB == 0 { return nil, errors.New("flavor has 0 memory") } freeMemory, ok := free["memory"] @@ -320,14 +320,16 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.Sche continue } // Calculate how many instances can fit on this host, based on memory. - // Flavor.Memory is already in bytes (resource.Quantity). + // Nova's memory_mb field is named "megabytes" in the API docs, but the values + // are in mebibytes (1 MiB = 2^20 bytes). Multiply by 1<<20 to convert to bytes. //nolint:gosec // We're checking for underflows above (< 0). - memorySlots := uint64(freeMemory.Value()) / uint64(request.Flavor.Memory.Value()) + memorySlots := uint64(freeMemory.Value()) / + (request.Flavor.MemoryMB * (1 << 20)) if memorySlots < request.NumInstances { traceLog.V(1).Info( "filtering host due to insufficient RAM capacity", - "host", host, "requested_memory", request.Flavor.Memory.String(), - "available_memory", freeMemory.String(), + "host", host, "requested_mb", request.Flavor.MemoryMB, + "available_mb", freeMemory.String(), ) delete(result.Activations, host) continue @@ -336,7 +338,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.Sche "host has enough capacity", "host", host, "requested_cpus", request.Flavor.VCPUs, "available_cpus", freeCPU.String(), - "requested_memory", request.Flavor.Memory.String(), + "requested_memory_mb", request.Flavor.MemoryMB, "available_memory", freeMemory.String(), ) } diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go index 0ed265cea..afe8b969d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go @@ -287,9 +287,9 @@ func newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup s HypervisorFamily: compute.HypervisorFamilyKVM, HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: flavorName, - VCPUs: uint64(vcpus), //nolint:gosec // test code - Memory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // test code + Name: flavorName, + VCPUs: uint64(vcpus), //nolint:gosec // test code + MemoryMB: memoryMB, }, Hosts: hostList, Weights: weights, diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 7fe3df57e..ae73b26ba 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -6,6 +6,7 @@ package filters import ( "context" "fmt" + "math" "time" "github.com/go-logr/logr" @@ -153,11 +154,10 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request compute.Sched if numInstances == 0 { numInstances = 1 } + memoryMB := request.Flavor.MemoryMB vcpus := request.Flavor.VCPUs - // Total requested RAM across all instances, converted to GiB (ceiling, so we - // never under-count headroom demand). Flavor.Memory is bytes per instance. - totalMemBytes := request.Flavor.Memory.Value() * int64(numInstances) - requestRAM := (totalMemBytes + (1 << 30) - 1) / (1 << 30) + // MB to GiB: using 1024 as divisor is slightly conservative (overestimates GiB needed). + requestRAM := int64(math.Ceil(float64(memoryMB*numInstances) / 1024.0)) requestCores := int64(vcpus * numInstances) //nolint:gosec // VCPUs from flavor specs, realistically bounded requestInstances := int64(numInstances) diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 039fa0c48..eddf38c88 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -53,8 +53,8 @@ func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numI Intent: intentFromHint(hints), HardwareVersion: hwVersion, Flavor: compute.FlavorSpec{ - VCPUs: 4, - Memory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // test code + VCPUs: 4, + MemoryMB: memoryMB, }, Hosts: []compute.Host{ {ComputeHost: "host1"}, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go index 85d1b5d12..a1b9932c7 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go @@ -143,7 +143,8 @@ func (s *KVMBinpackStep) Run(traceLog logr.Logger, request compute.SchedulerRequ // calcVMResources calculates the total resource requests for the VM to be scheduled. func (s *KVMBinpackStep) calcVMResources(req compute.SchedulerRequest) map[hv1.ResourceName]resource.Quantity { resources := make(map[hv1.ResourceName]resource.Quantity) - resourcesMemBytes := req.Flavor.Memory.Value() * int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesMemBytes := int64(req.Flavor.MemoryMB) * (1024 * 1024) //nolint:gosec // memory values are bounded by Nova + resourcesMemBytes *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[hv1.ResourceMemory] = *resource. NewQuantity(resourcesMemBytes, resource.BinarySI) resourcesCPU := int64(req.Flavor.VCPUs) //nolint:gosec // vCPU values are bounded by Nova diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go index 0435fbc60..5ef1d2af8 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go @@ -52,9 +52,9 @@ func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) com HypervisorFamily: compute.HypervisorFamilyKVM, HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", - VCPUs: vcpus, - Memory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // test code + Name: "m1.large", + VCPUs: vcpus, + MemoryMB: memoryMB, }, Hosts: hostList, Weights: weights, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go index 4c3314840..cd1b739b9 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go @@ -72,7 +72,7 @@ func (s *KVMCommittedResourceReservationStep) Run(traceLog logr.Logger, request } // Flavor memory in bytes for free-capacity check. - flavorMemoryBytes := request.Flavor.Memory.Value() + flavorMemoryBytes := int64(request.Flavor.MemoryMB) * 1024 * 1024 //nolint:gosec var reservations v1alpha1.ReservationList if err := s.Client.List(context.Background(), &reservations); err != nil { diff --git a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go index d80f8bb0f..b4a3a8bf7 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go @@ -68,7 +68,7 @@ func newCRRequest(hosts []string) compute.SchedulerRequest { AvailabilityZone: "qa-de-1a", ResourceGroup: "group-2101", Flavor: compute.FlavorSpec{ - Memory: *resource.NewQuantity(8*1024*(1<<20), resource.BinarySI), + MemoryMB: 8 * 1024, }, Hosts: hostList, Weights: weights, @@ -211,7 +211,7 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { Intent: api.ReserveForFailoverIntent, ResourceGroup: "group-2101", Flavor: compute.FlavorSpec{ - Memory: *resource.NewQuantity(4096*(1<<20), resource.BinarySI), + MemoryMB: 4096, }, Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, }, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go index 4165b3115..5f1548227 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go @@ -126,9 +126,9 @@ func newNovaRequest(instanceUUID string, evacuation bool, hosts []string) comput HypervisorFamily: compute.HypervisorFamilyKVM, HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", - VCPUs: 4, - Memory: *resource.NewQuantity(int64(parseMemoryToMB("8Gi"))*(1<<20), resource.BinarySI), //nolint:gosec // test code + Name: "m1.large", + VCPUs: 4, + MemoryMB: parseMemoryToMB("8Gi"), }, Hosts: hostList, } diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go index d53e40b6a..1c41e796c 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go @@ -71,9 +71,9 @@ func newFailoverReservationRequest(resourceGroup string, hosts []string) compute HypervisorFamily: compute.HypervisorFamilyKVM, HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", - VCPUs: 4, - Memory: *resource.NewQuantity(8192*(1<<20), resource.BinarySI), + Name: "m1.large", + VCPUs: 4, + MemoryMB: 8192, }, Hosts: hostList, Weights: weights, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go index 0e0613493..1ad1f0dc3 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go @@ -9,7 +9,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -47,9 +46,9 @@ func newInstanceGroupRequest(policy string, members, hosts []string) compute.Sch HypervisorFamily: compute.HypervisorFamilyKVM, HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", - VCPUs: 4, - Memory: *resource.NewQuantity(8192*(1<<20), resource.BinarySI), + Name: "m1.large", + VCPUs: 4, + MemoryMB: 8192, }, Hosts: hostList, } diff --git a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go index 5a466c7e6..9b4d642b6 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go @@ -48,9 +48,9 @@ func newPreferSmallerHostsRequest(hosts []string) compute.SchedulerRequest { HypervisorFamily: compute.HypervisorFamilyKVM, HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", - VCPUs: 4, - Memory: *resource.NewQuantity(8192*(1<<20), resource.BinarySI), + Name: "m1.large", + VCPUs: 4, + MemoryMB: 8192, }, Hosts: hostList, Weights: weights, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_binpack.go b/internal/scheduling/nova/plugins/weighers/vmware_binpack.go index d554588c7..686a3eb59 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_binpack.go @@ -187,7 +187,8 @@ func (s *VMwareBinpackStep) calcHostAllocation(hostUtilization compute.HostUtili // calcVMResources calculates the total resource requests for the VM to be scheduled. func (s *VMwareBinpackStep) calcVMResources(req sched.SchedulerRequest) map[corev1.ResourceName]resource.Quantity { resources := make(map[corev1.ResourceName]resource.Quantity) - resourcesMemBytes := req.Flavor.Memory.Value() * int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesMemBytes := int64(req.Flavor.MemoryMB) * (1 << 20) //nolint:gosec // memory values are bounded by Nova + resourcesMemBytes *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[corev1.ResourceMemory] = *resource. NewQuantity(resourcesMemBytes, resource.BinarySI) resourcesCPU := int64(req.Flavor.VCPUs) //nolint:gosec // vCPU values are bounded by Nova diff --git a/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go b/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go index 4a06b5852..682d354cb 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go @@ -11,7 +11,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -166,8 +165,8 @@ func TestVMwareBinpackStep_Run(t *testing.T) { request: sched.SchedulerRequest{ NumInstances: 1, Flavor: sched.FlavorSpec{ - Memory: *resource.NewQuantity(1000*(1<<20), resource.BinarySI), // 1GB - VCPUs: 1, + MemoryMB: 1000, // 1GB + VCPUs: 1, }, Hosts: []sched.Host{ {ComputeHost: "host1"}, @@ -185,8 +184,8 @@ func TestVMwareBinpackStep_Run(t *testing.T) { request: sched.SchedulerRequest{ NumInstances: 1, Flavor: sched.FlavorSpec{ - Memory: *resource.NewQuantity(1000*(1<<20), resource.BinarySI), - VCPUs: 1, + MemoryMB: 1000, + VCPUs: 1, }, Hosts: []sched.Host{ {ComputeHost: "host1"}, @@ -290,8 +289,8 @@ func TestVMwareBinpackStep_CalcVMResources(t *testing.T) { request: sched.SchedulerRequest{ NumInstances: 1, Flavor: sched.FlavorSpec{ - Memory: *resource.NewQuantity(2048*(1<<20), resource.BinarySI), - VCPUs: 4, + MemoryMB: 2048, + VCPUs: 4, }, }, expectedMemory: 2048 * (1 << 20), // 2048 MiB in bytes @@ -302,8 +301,8 @@ func TestVMwareBinpackStep_CalcVMResources(t *testing.T) { request: sched.SchedulerRequest{ NumInstances: 3, Flavor: sched.FlavorSpec{ - Memory: *resource.NewQuantity(1024*(1<<20), resource.BinarySI), - VCPUs: 2, + MemoryMB: 1024, + VCPUs: 2, }, }, expectedMemory: 3 * 1024 * (1 << 20), // 3 instances * 1024 MiB in bytes diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 5c30fbc4f..e7614879f 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -833,9 +833,9 @@ func (c *Reconciler) probeScheduler( ProjectID: "cortex-capacity-probe", AvailabilityZone: az, Flavor: sched.FlavorSpec{ - Name: flavor.Name, - VCPUs: flavor.VCPUs, - Memory: *resource.NewQuantity(int64(flavor.MemoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // flavor memory bounded by specs + Name: flavor.Name, + VCPUs: flavor.VCPUs, + MemoryMB: flavor.MemoryMB, }, Options: scheduling.Options{ ReadOnly: true, diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index a4732568e..456a9ee9e 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -11,7 +11,6 @@ import ( "time" "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -368,9 +367,9 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr AvailabilityZone: availabilityZone, DomainName: domainName, Flavor: compute.FlavorSpec{ - Name: flavorDetails.Name, - VCPUs: flavorDetails.VCPUs, - Memory: *resource.NewQuantity(int64(flavorDetails.MemoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // flavor memory bounded by specs + Name: flavorDetails.Name, + VCPUs: flavorDetails.VCPUs, + MemoryMB: flavorDetails.MemoryMB, }, Options: scheduling.Options{ ReadOnly: false, // mutates state (reservation placement) diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 4bbb7567f..f676f97bd 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -15,7 +15,6 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" - "k8s.io/apimachinery/pkg/api/resource" ) // Pipeline names for failover reservation scheduling @@ -143,9 +142,9 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, Flavor: compute.FlavorSpec{ - Name: resSpec.FlavorName, - VCPUs: resSpec.VCPUs, - Memory: *resource.NewQuantity(int64(resSpec.MemoryMB)*(1<<20), resource.BinarySI), //nolint:gosec // flavor memory bounded by specs + Name: resSpec.FlavorName, + VCPUs: resSpec.VCPUs, + MemoryMB: resSpec.MemoryMB, }, } @@ -299,9 +298,9 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, Flavor: compute.FlavorSpec{ - Name: vm.FlavorName, - VCPUs: vcpus, - Memory: *resource.NewQuantity(int64(memoryMB)*(1<<20), resource.BinarySI), + Name: vm.FlavorName, + VCPUs: vcpus, + MemoryMB: memoryMB, }, } From 96846ee4baf087a18917030cf1c0f8f25bb77a44 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Tue, 3 Nov 2026 13:33:29 +0100 Subject: [PATCH 11/21] refactor(scheduling): replace RunOutcome with SchedulerResult in scheduling components Signed-off-by: Markus Wieland --- internal/scheduling/compute/result.go | 29 +++------------ internal/scheduling/compute/scheduler.go | 29 ++++++++------- internal/scheduling/compute/scheduler_test.go | 10 +++--- .../scheduling/nova/external_scheduler_api.go | 16 ++++----- .../nova/external_scheduler_api_test.go | 36 +++++++++---------- .../reservations/capacity/controller.go | 2 +- .../commitments/reservation_controller.go | 4 +-- .../reservations/failover/integration_test.go | 2 +- .../failover/reservation_scheduling.go | 8 ++--- .../failoverhost/integration_test.go | 2 +- 10 files changed, 58 insertions(+), 80 deletions(-) diff --git a/internal/scheduling/compute/result.go b/internal/scheduling/compute/result.go index ae9d009c7..5b1ddbe7f 100644 --- a/internal/scheduling/compute/result.go +++ b/internal/scheduling/compute/result.go @@ -4,35 +4,14 @@ package compute import ( - "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) // SchedulerResult is the well-defined output of a scheduling pipeline run: // ordered hosts, the target host, and the input/output weights plus per-step // activations used to rank them. It is an alias of lib.PipelineResult, the -// domain-neutral type the generic pipeline returns. +// domain-neutral type the generic pipeline returns. The human-readable +// explanation is NOT part of the result — it is a concern of writing the +// decision onto the scheduled resource (e.g. the Nova VM's CortexSpec), so the +// relevant side effect generates it from the result via lib.GenerateExplanation. type SchedulerResult = lib.PipelineResult - -// MatchedReservation records a reservation that matched a candidate host during -// the run, for observability and downstream consumers. -type MatchedReservation struct { - // Host is the compute host the reservation applies to. - Host string - // ReservationName is the name of the matched Reservation resource. - ReservationName string - // Type is the reservation type (e.g. inflight, committed-resource, failover). - Type v1alpha1.ReservationType -} - -// RunOutcome bundles everything the pipeline produces for a scheduling run: the -// pipeline result and any matched reservations. The human-readable explanation -// is NOT part of the outcome — it is a concern of writing the decision onto the -// scheduled resource (e.g. the Nova VM's CortexSpec), so the relevant side -// effect generates it from the Result via lib.GenerateExplanation. -type RunOutcome struct { - // Result is the pipeline output (ordered hosts, target host, weights, steps). - Result SchedulerResult - // MatchedReservations lists reservations that matched candidate hosts. - MatchedReservations []MatchedReservation -} diff --git a/internal/scheduling/compute/scheduler.go b/internal/scheduling/compute/scheduler.go index decfd7136..4d9842e51 100644 --- a/internal/scheduling/compute/scheduler.go +++ b/internal/scheduling/compute/scheduler.go @@ -36,10 +36,10 @@ type PreHook func(ctx context.Context, req *SchedulerRequest) error // PostHook is a caller-provided side effect executed after the pipeline run, // under the scheduling lock (e.g. upserting the VM resource, recording the -// decision). Returning an error aborts the run; non-fatal side effects should -// log and return nil so the placement decision still stands. A nil PostHook is a -// no-op. -type PostHook func(ctx context.Context, req SchedulerRequest, out RunOutcome) error +// decision). It receives the typed pipeline result. Returning an error aborts +// the run; non-fatal side effects should log and return nil so the placement +// decision still stands. A nil PostHook is a no-op. +type PostHook func(ctx context.Context, req SchedulerRequest, result SchedulerResult) error // Scheduler is the single in-process owner of Nova placement decisions. It holds // the one shared scheduling lock and runs the pipeline plus a caller-provided @@ -71,7 +71,7 @@ func (s *Scheduler) Run( req SchedulerRequest, pre PreHook, post PostHook, -) (RunOutcome, error) { +) (SchedulerResult, error) { s.mu.Lock() defer s.mu.Unlock() @@ -87,7 +87,7 @@ func (s *Scheduler) RunWithoutLock( req SchedulerRequest, pre PreHook, post PostHook, -) (RunOutcome, error) { +) (SchedulerResult, error) { return s.run(ctx, req, pre, post) } @@ -99,19 +99,19 @@ func (s *Scheduler) run( req SchedulerRequest, pre PreHook, post PostHook, -) (RunOutcome, error) { +) (SchedulerResult, error) { log := ctrl.LoggerFrom(ctx) pipeline, ok := s.pipelines.GetPipeline(req.Pipeline) if !ok { - return RunOutcome{}, fmt.Errorf("pipeline not found or not ready: %q", req.Pipeline) + return SchedulerResult{}, fmt.Errorf("pipeline not found or not ready: %q", req.Pipeline) } // The pre-hook runs before the pipeline and may mutate the request. if pre != nil { if err := pre(ctx, &req); err != nil { - return RunOutcome{}, err + return SchedulerResult{}, err } } @@ -120,22 +120,21 @@ func (s *Scheduler) run( if req.GatherAllCandidates { log.V(1).Info("gathering all placement candidates before filtering") if err := s.gatherer.MutateWithAllCandidates(ctx, &req); err != nil { - return RunOutcome{}, fmt.Errorf("failed to gather all placement candidates: %w", err) + return SchedulerResult{}, fmt.Errorf("failed to gather all placement candidates: %w", err) } log.V(1).Info("gathered all placement candidates", "numHosts", len(req.Hosts)) } result, runErr := pipeline.Run(req) if runErr != nil { - return RunOutcome{}, runErr + return SchedulerResult{}, runErr } - out := RunOutcome{Result: result} // The post-hook runs after a successful pipeline run, under the lock. if post != nil { - if err := post(ctx, req, out); err != nil { - return out, err + if err := post(ctx, req, result); err != nil { + return result, err } } - return out, nil + return result, nil } diff --git a/internal/scheduling/compute/scheduler_test.go b/internal/scheduling/compute/scheduler_test.go index 10193ab3f..74d941f84 100644 --- a/internal/scheduling/compute/scheduler_test.go +++ b/internal/scheduling/compute/scheduler_test.go @@ -80,7 +80,7 @@ func TestRunPreThenPost(t *testing.T) { order = append(order, "pre") return nil } - post := func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { + post := func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { order = append(order, "post") return nil } @@ -102,7 +102,7 @@ func TestRunNilHooks(t *testing.T) { func TestRunPostHookErrorAborts(t *testing.T) { s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) boom := errors.New("boom") - post := func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { return boom } + post := func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { return boom } _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, post) if !errors.Is(err, boom) { t.Fatalf("expected boom error, got %v", err) @@ -113,7 +113,7 @@ func TestRunPipelineErrorSkipsPostHook(t *testing.T) { boom := errors.New("pipeline failed") s := newTestScheduler(&fakePipeline{err: boom}, &fakeGatherer{}) var ran bool - post := func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { ran = true; return nil } + post := func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { ran = true; return nil } _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, post) if !errors.Is(err, boom) { t.Fatalf("expected pipeline error, got %v", err) @@ -195,7 +195,7 @@ func TestRunHoldsLockAcrossPostHook(t *testing.T) { releaseFirst := make(chan struct{}) secondDone := make(chan struct{}) - var slowPost PostHook = func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { + var slowPost PostHook = func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { close(firstInPost) <-releaseFirst return nil @@ -238,7 +238,7 @@ func TestRunWithoutLockDoesNotBlock(t *testing.T) { firstInPost := make(chan struct{}) releaseFirst := make(chan struct{}) - var slowPost PostHook = func(_ context.Context, _ SchedulerRequest, _ RunOutcome) error { + var slowPost PostHook = func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { close(firstInPost) <-releaseFirst return nil diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index 2a4778b24..e7836370e 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -53,7 +53,7 @@ type HTTPAPI interface { // interface so the handler can be unit-tested with a mock. *compute.Scheduler // satisfies it. type runner interface { - Run(ctx context.Context, req compute.SchedulerRequest, pre compute.PreHook, post compute.PostHook) (compute.RunOutcome, error) + Run(ctx context.Context, req compute.SchedulerRequest, pre compute.PreHook, post compute.PostHook) (compute.SchedulerResult, error) } type httpAPI struct { @@ -280,7 +280,7 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req // Side effects run as a single post-hook after the pipeline, under the // scheduler lock. It operates on the Nova wire request (which carries the // fields the VM resource and CR tracking need) plus the typed result. - postHook := func(ctx context.Context, _ compute.SchedulerRequest, out compute.RunOutcome) error { + postHook := func(ctx context.Context, _ compute.SchedulerRequest, result compute.SchedulerResult) error { // VM-resource creation. switch { case requestData.Options.ReadOnly: @@ -292,7 +292,7 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req case schedReq.HypervisorFamily != compute.HypervisorFamilyKVM: logger.V(1).Info("skipping virtual machine creation, unsupported hypervisor type", "instanceUUID", requestData.Spec.Data.InstanceUUID) default: - if upsertErr := httpAPI.vmHandler.Upsert(ctx, out.Result, requestData, intent); upsertErr != nil { + if upsertErr := httpAPI.vmHandler.Upsert(ctx, result, requestData, intent); upsertErr != nil { if httpAPI.featureGates.VirtualMachineCreationMandatory { return fmt.Errorf("failed to upsert virtual machine resource: %w", upsertErr) } @@ -302,8 +302,8 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req } // Committed-resource tracking. if httpAPI.featureGates.CommittedResourceTracking && !requestData.Options.SkipCommittedResourceTracking { - if out.Result.TargetHost != nil { - httpAPI.crRecorder.RecordPlacement(ctx, out.Result, intent, requestData) + if result.TargetHost != nil { + httpAPI.crRecorder.RecordPlacement(ctx, result, intent, requestData) } else { httpAPI.crRecorder.RecordNoHostFound(ctx, intent, requestData) } @@ -311,16 +311,16 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req return nil } - out, err := httpAPI.scheduler.Run(ctx, schedReq, nil, postHook) + result, err := httpAPI.scheduler.Run(ctx, schedReq, nil, postHook) if err != nil { c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return } - hosts := out.Result.OrderedHosts + hosts := result.OrderedHosts if httpAPI.config.NovaLimitHostsToRequest { hosts = limitHostsToRequest(requestData, hosts) logger.V(1).Info("limited hosts to request", - "hosts", hosts, "originalHosts", out.Result.OrderedHosts) + "hosts", hosts, "originalHosts", result.OrderedHosts) } // This is a hack to address the problem that Nova only uses the first host in hosts for evacuation requests. // Only for evacuation we shuffle the first k hosts to ensure that we do not get stuck on a single host diff --git a/internal/scheduling/nova/external_scheduler_api_test.go b/internal/scheduling/nova/external_scheduler_api_test.go index 5885bfc80..0e21f0538 100644 --- a/internal/scheduling/nova/external_scheduler_api_test.go +++ b/internal/scheduling/nova/external_scheduler_api_test.go @@ -21,16 +21,16 @@ import ( // mockRunner is a test double for the scheduler the HTTP API depends on. runFunc // receives the mapped request and may invoke the post-hook to exercise the -// handler's side-effect path; it returns the outcome/error the handler sees. +// handler's side-effect path; it returns the result/error the handler sees. type mockRunner struct { - runFunc func(ctx context.Context, req compute.SchedulerRequest, post compute.PostHook) (compute.RunOutcome, error) + runFunc func(ctx context.Context, req compute.SchedulerRequest, post compute.PostHook) (compute.SchedulerResult, error) } -func (m *mockRunner) Run(ctx context.Context, req compute.SchedulerRequest, _ compute.PreHook, post compute.PostHook) (compute.RunOutcome, error) { +func (m *mockRunner) Run(ctx context.Context, req compute.SchedulerRequest, _ compute.PreHook, post compute.PostHook) (compute.SchedulerResult, error) { if m.runFunc != nil { return m.runFunc(ctx, req, post) } - return compute.RunOutcome{}, nil + return compute.SchedulerResult{}, nil } // newTestAPI builds an *httpAPI with the given config and mock runner, leaving @@ -273,14 +273,14 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r := &mockRunner{ - runFunc: func(_ context.Context, _ compute.SchedulerRequest, _ compute.PostHook) (compute.RunOutcome, error) { + runFunc: func(_ context.Context, _ compute.SchedulerRequest, _ compute.PostHook) (compute.SchedulerResult, error) { if tt.processDecisionErr != nil { - return compute.RunOutcome{}, tt.processDecisionErr + return compute.SchedulerResult{}, tt.processDecisionErr } if tt.result != nil { - return compute.RunOutcome{Result: *tt.result}, nil + return *tt.result, nil } - return compute.RunOutcome{}, nil + return compute.SchedulerResult{}, nil }, } @@ -327,8 +327,8 @@ func TestHTTPAPI_NovaExternalScheduler_EmptyHostsSerializedAsArray(t *testing.T) // response must serialize hosts as an empty array, not null, since Nova's // schema requires an array. r := &mockRunner{ - runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.RunOutcome, error) { - return compute.RunOutcome{Result: compute.SchedulerResult{OrderedHosts: nil}}, nil + runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.SchedulerResult, error) { + return compute.SchedulerResult{OrderedHosts: nil}, nil }, } api := newTestAPI(HTTPAPIConfig{}, r) @@ -365,9 +365,9 @@ func TestHTTPAPI_NovaExternalScheduler_EmptyHostsSerializedAsArray(t *testing.T) func TestHTTPAPI_NovaExternalScheduler_RequestMapping(t *testing.T) { var captured compute.SchedulerRequest r := &mockRunner{ - runFunc: func(_ context.Context, req compute.SchedulerRequest, _ compute.PostHook) (compute.RunOutcome, error) { + runFunc: func(_ context.Context, req compute.SchedulerRequest, _ compute.PostHook) (compute.SchedulerResult, error) { captured = req - return compute.RunOutcome{Result: compute.SchedulerResult{OrderedHosts: []string{"host1"}}}, nil + return compute.SchedulerResult{OrderedHosts: []string{"host1"}}, nil }, } @@ -435,9 +435,9 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestination(t *testing.T) { schedulerCalled := false r := &mockRunner{ - runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.RunOutcome, error) { + runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.SchedulerResult, error) { schedulerCalled = true - return compute.RunOutcome{}, nil + return compute.SchedulerResult{}, nil }, } api := newTestAPI(HTTPAPIConfig{}, r) @@ -485,9 +485,9 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationNoWeights(t *testing.T) schedulerCalled := false r := &mockRunner{ - runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.RunOutcome, error) { + runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.SchedulerResult, error) { schedulerCalled = true - return compute.RunOutcome{}, nil + return compute.SchedulerResult{}, nil }, } api := newTestAPI(HTTPAPIConfig{}, r) @@ -538,9 +538,9 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationDisabled(t *testing.T) { schedulerCalled := false r := &mockRunner{ - runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.RunOutcome, error) { + runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.SchedulerResult, error) { schedulerCalled = true - return compute.RunOutcome{}, nil + return compute.SchedulerResult{}, nil }, } disabled := false diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index e7614879f..5e76f4449 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -849,7 +849,7 @@ func (c *Reconciler) probeScheduler( return 0, 0, nil, fmt.Errorf("scheduler call failed (pipeline=%s): %w", pipeline, err) } - for _, hostName := range out.Result.OrderedHosts { + for _, hostName := range out.OrderedHosts { hv, ok := hvByName[hostName] if !ok { continue diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 456a9ee9e..25c22e302 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -385,7 +385,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, err } - if len(out.Result.OrderedHosts) == 0 { + if len(out.OrderedHosts) == 0 { logger.Info("no hosts found for reservation, will retry", "reservation", res.Name, "flavorName", resourceName) old := res.DeepCopy() meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ @@ -411,7 +411,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // Update the reservation Spec with the found host (idx 0) // Only update Spec here - the Status will be synced in the next reconcile cycle // This avoids race conditions from doing two patches in one reconcile - host := out.Result.OrderedHosts[0] + host := out.OrderedHosts[0] logger.Info("found host for reservation", "host", host, "flavorName", resourceName) old := res.DeepCopy() diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index 27b63ab40..26da3f930 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -1290,7 +1290,7 @@ func (api *testHTTPAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req return } - response := novaapi.ExternalSchedulerResponse{Hosts: out.Result.OrderedHosts} + response := novaapi.ExternalSchedulerResponse{Hosts: out.OrderedHosts} w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(response); err != nil { http.Error(w, "failed to encode response", http.StatusInternalServerError) diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index f676f97bd..8efcc9425 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -165,15 +165,15 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex "pipeline", pipeline, "eligibleHypervisors", len(eligibleHypervisors), "ignoreHypervisors", ignoreHypervisors, - "returnedHypervisors", out.Result.OrderedHosts) + "returnedHypervisors", out.OrderedHosts) - return out.Result.OrderedHosts, nil + return out.OrderedHosts, nil } // runScheduler runs the shared scheduler, choosing the lock-free path for // read-only runs (e.g. reuse/evacuate compatibility checks) and the locked path // for runs that mutate shared reservation state. -func (c *FailoverReservationController) runScheduler(ctx context.Context, req compute.SchedulerRequest) (compute.RunOutcome, error) { +func (c *FailoverReservationController) runScheduler(ctx context.Context, req compute.SchedulerRequest) (compute.SchedulerResult, error) { if req.Options.ReadOnly { return c.Scheduler.RunWithoutLock(ctx, req, nil, nil) } @@ -315,7 +315,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( logger.Error(err, "failed to validate VM for reservation host", "vmUUID", vm.UUID, "reservationHost", reservationHost) return false, fmt.Errorf("failed to validate VM for reservation host: %w", err) } - hosts := out.Result.OrderedHosts + hosts := out.OrderedHosts // Handle empty response - no hosts returned if len(hosts) < 1 { diff --git a/internal/scheduling/reservations/failoverhost/integration_test.go b/internal/scheduling/reservations/failoverhost/integration_test.go index 814bcad7a..9449b184d 100644 --- a/internal/scheduling/reservations/failoverhost/integration_test.go +++ b/internal/scheduling/reservations/failoverhost/integration_test.go @@ -467,7 +467,7 @@ func (a *integrationHTTPAPI) handle(w http.ResponseWriter, r *http.Request) { return } w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: out.Result.OrderedHosts}); err != nil { + if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: out.OrderedHosts}); err != nil { http.Error(w, "encode", http.StatusInternalServerError) } } From c9c7ce5fce3cfd6b73fdca76b3bace9181bd32db Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Tue, 3 Nov 2026 14:01:14 +0100 Subject: [PATCH 12/21] refactor(failover): integrate in-process scheduler and remove external scheduler dependencies Signed-off-by: Markus Wieland --- cmd/manager/main.go | 14 +- helm/bundles/cortex-nova/values.yaml | 4 - .../failover/reservation_scheduling.go | 8 +- .../reservations/failoverhost/controller.go | 20 +-- .../failoverhost/generation_test.go | 9 +- .../failoverhost/integration_test.go | 73 ++------- .../reservations/failoverhost/pathfinding.go | 132 ++++++----------- .../failoverhost/pathfinding_test.go | 139 ++++++++++-------- 8 files changed, 162 insertions(+), 237 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index ab5d00b04..5dc7cd3f3 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -813,10 +813,15 @@ func main() { setupLog.Error(err, "failed to register failover host monitor metrics, continuing without metrics") failoverHostMonitor = nil } + if novaScheduler == nil { + setupLog.Error(nil, "failover-host-reservations-controller requires nova-pipeline-controllers (in-process scheduler) to be enabled") + os.Exit(1) + } controller := &failoverhost.Controller{ - Client: multiclusterClient, - Config: failoverHostConfig, - Monitor: failoverHostMonitor, + Client: multiclusterClient, + Config: failoverHostConfig, + Scheduler: novaScheduler, + Monitor: failoverHostMonitor, } if err := controller.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", @@ -826,8 +831,7 @@ func main() { setupLog.Info("failover-host-reservations-controller registered", "ratio", failoverHostConfig.Ratio, "minimum", failoverHostConfig.Minimum, - "generationInterval", failoverHostConfig.GenerationInterval, - "externalSchedulerURL", failoverHostConfig.ExternalSchedulerURL) + "generationInterval", failoverHostConfig.GenerationInterval) } if slices.Contains(mainConfig.EnabledControllers, "capacity-controller") { setupLog.Info("enabling controller", "controller", "capacity-controller") diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 755bb8f42..12d9af450 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -392,10 +392,6 @@ cortex-scheduling-controllers: # How often the failover partitions are recalculated and the reservations # updated to match. Must be at least 1m. generationInterval: 5m - # URL of the nova external scheduler API, used to discover which hosts a - # given host's workload could be evacuated onto. Typically the same - # scheduler endpoint the flavor-based failover controller uses. - externalSchedulerURL: "http://localhost:8080/scheduler/nova/external" cortex-knowledge-controllers: <<: *cortex diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 8efcc9425..22c65ab7d 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -31,9 +31,9 @@ var DefaultFailoverOptions = scheduling.Options{ SkipCommittedResourceTracking: true, } -// inferFailoverPipeline returns the standard pipeline for a failover scheduling call based on +// InferFailoverPipeline returns the standard pipeline for a failover scheduling call based on // the VM's flavor extra specs — the same HANA vs general-purpose split used by Nova placement. -func inferFailoverPipeline(extraSpecs map[string]string) string { +func InferFailoverPipeline(extraSpecs map[string]string) string { if strings.ToLower(extraSpecs["trait:CUSTOM_HANA_EXCLUSIVE_HOST"]) == "required" { return "kvm-hana-bin-packing" } @@ -196,7 +196,7 @@ func (c *FailoverReservationController) tryReuseExistingReservation( reuseOpts := DefaultFailoverOptions reuseOpts.ReadOnly = true - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, inferFailoverPipeline(vm.FlavorExtraSpecs), resSpec, api.ReuseFailoverReservationIntent, reuseOpts) + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, InferFailoverPipeline(vm.FlavorExtraSpecs), resSpec, api.ReuseFailoverReservationIntent, reuseOpts) if err != nil { logger.Error(err, "failed to get potential hypervisors for VM", "vmUUID", vm.UUID) return nil @@ -277,7 +277,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( // this VM (avoiding self-blocking). Use the actual VM UUID so the filter can // match it in the reservation's allocations. KVM gathers all candidates; the // reservation host must come back ranked for the VM to be compatible. - pipeline := inferFailoverPipeline(flavorExtraSpecs) + pipeline := InferFailoverPipeline(flavorExtraSpecs) requiredTraits, forbiddenTraits := traitsFromExtraSpecs(flavorExtraSpecs) evacuateOpts := DefaultFailoverOptions evacuateOpts.ReadOnly = true diff --git a/internal/scheduling/reservations/failoverhost/controller.go b/internal/scheduling/reservations/failoverhost/controller.go index 8f001304f..0693c60ca 100644 --- a/internal/scheduling/reservations/failoverhost/controller.go +++ b/internal/scheduling/reservations/failoverhost/controller.go @@ -59,11 +59,11 @@ import ( "context" "errors" "maps" - "net/http" "sync" "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -107,10 +107,6 @@ type ControllerConfig struct { // GenerationInterval is the interval within which failover partitions are // recalculated and failover host reservations are updated accordingly. GenerationInterval metav1.Duration `json:"generationInterval"` - - // ExternalSchedulerURL is the URL of the external scheduler service that - // will be used to request failover host placements. - ExternalSchedulerURL string `json:"externalSchedulerURL"` } // Controller keeps the cluster's failover host reservations in sync with the @@ -129,6 +125,11 @@ type Controller struct { Config ControllerConfig + // Scheduler is the shared in-process scheduler the path finder probes for + // placement candidates. Required (the controller cannot compute failover + // partitions without it). + Scheduler *compute.Scheduler + // Monitor records Prometheus metrics for the calculation loop and the // reconciler. It may be nil (e.g. in tests); every call site nil-guards it. Monitor *Monitor @@ -336,15 +337,16 @@ func (c *Controller) SetupWithManager(mgr ctrl.Manager) error { // Default the generator. Tests may set it before calling Setup to substitute // a fake; here we wire the production pipeline stages. if c.generator == nil { + if c.Scheduler == nil { + return errors.New("scheduler cannot be nil") + } c.generator = &pipelineGenerator{ Client: c.Client, Monitor: c.Monitor, seeder: &seeder{k8s: c.Client}, pathFinder: &evacuationPathFinder{ - // Timeout so a hung scheduler can't stall the calculation loop. - httpClient: &http.Client{Timeout: 30 * time.Second}, - schedulerURL: c.Config.ExternalSchedulerURL, - monitor: c.Monitor, + scheduler: c.Scheduler, + monitor: c.Monitor, }, partitionBuilder: cliquePartitionBuilder{}, reservationPlanner: ratioReservationPlanner{ratio: c.Config.Ratio, minimum: c.Config.Minimum}, diff --git a/internal/scheduling/reservations/failoverhost/generation_test.go b/internal/scheduling/reservations/failoverhost/generation_test.go index 074ea1a98..1d6191040 100644 --- a/internal/scheduling/reservations/failoverhost/generation_test.go +++ b/internal/scheduling/reservations/failoverhost/generation_test.go @@ -38,10 +38,9 @@ func TestGenerateProducesReservations(t *testing.T) { hvA := buildHV("hvA", 64, 32) hvB := buildHV("hvB", 64, 32) - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"hvA", "hvB"}, }, nil) - defer server.Close() g := &pipelineGenerator{ Client: newTestClient(t, hvA, hvB), @@ -84,10 +83,9 @@ func TestGenerateDeduplicatesSharedHost(t *testing.T) { // The path finder's edges are irrelevant here: the fake builder ignores them // and hands Generate the overlapping cliques directly. The stub still needs an // answer so FindPaths succeeds; one seed accepted on all three suffices. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"hvA", "hvB", "hvC"}, }, nil) - defer server.Close() g := &pipelineGenerator{ Client: newTestClient(t, hvA, hvB, hvC), @@ -131,10 +129,9 @@ func TestGenerateSharedHostKeptInSmallestPartition(t *testing.T) { hvC := buildHV("hvC", 64, 32) hvD := buildHV("hvD", 64, 32) - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"hvA", "hvB", "hvC", "hvD"}, }, nil) - defer server.Close() g := &pipelineGenerator{ Client: newTestClient(t, hvA, hvB, hvC, hvD), diff --git a/internal/scheduling/reservations/failoverhost/integration_test.go b/internal/scheduling/reservations/failoverhost/integration_test.go index 9449b184d..31d9b5dc9 100644 --- a/internal/scheduling/reservations/failoverhost/integration_test.go +++ b/internal/scheduling/reservations/failoverhost/integration_test.go @@ -5,14 +5,10 @@ package failoverhost import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" "slices" "sync" "testing" - novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" @@ -208,7 +204,6 @@ func TestIntegration(t *testing.T) { for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { env := newIntegrationTestEnv(t, tc.Hypervisors, tc.Flavors) - defer env.Close() ratio, minimum := tc.Ratio, tc.Minimum if ratio == 0 { @@ -243,19 +238,17 @@ func TestIntegration(t *testing.T) { // Test Environment // ============================================================================ -// integrationTestEnv wires a real Nova filter/weigher pipeline (served over -// HTTP) on top of a fake k8s client holding the fleet and the flavor-groups -// Knowledge CRD, so the generation pipeline can be driven end to end. +// integrationTestEnv wires a real Nova filter/weigher pipeline behind the shared +// in-process scheduler on top of a fake k8s client holding the fleet and the +// flavor-groups Knowledge CRD, so the generation pipeline can be driven end to +// end. type integrationTestEnv struct { - k8sClient client.Client - server *httptest.Server - schedulerURL string - flavors []compute.FlavorInGroup - hypervisors map[string]hv1.Hypervisor + k8sClient client.Client + scheduler *sched.Scheduler + flavors []compute.FlavorInGroup + hypervisors map[string]hv1.Hypervisor } -func (env *integrationTestEnv) Close() { env.server.Close() } - // newIntegrationTestEnv builds the fake cluster and the real scheduler pipeline. func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors []compute.FlavorInGroup) *integrationTestEnv { t.Helper() @@ -305,17 +298,13 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors // KVM requests trigger candidate gathering; the test seeds its own eligible // hosts, so use a passthrough gatherer that preserves them. - api := &integrationHTTPAPI{scheduler: sched.NewScheduler(controller, passthroughGatherer{})} - mux := http.NewServeMux() - mux.HandleFunc("/scheduler/nova/external", api.handle) - server := httptest.NewServer(mux) + scheduler := sched.NewScheduler(controller, passthroughGatherer{}) return &integrationTestEnv{ - k8sClient: k8sClient, - server: server, - schedulerURL: server.URL + "/scheduler/nova/external", - flavors: flavors, - hypervisors: hvsByName, + k8sClient: k8sClient, + scheduler: scheduler, + flavors: flavors, + hypervisors: hvsByName, } } @@ -330,7 +319,7 @@ func (env *integrationTestEnv) Generate(t *testing.T, ratio float64, minimum int g := &pipelineGenerator{ Client: env.k8sClient, seeder: &seeder{k8s: env.k8sClient}, - pathFinder: &evacuationPathFinder{httpClient: http.DefaultClient, schedulerURL: env.schedulerURL}, + pathFinder: &evacuationPathFinder{scheduler: env.scheduler}, partitionBuilder: cliquePartitionBuilder{}, reservationPlanner: ratioReservationPlanner{ratio: ratio, minimum: minimum}, } @@ -353,7 +342,7 @@ func (env *integrationTestEnv) AcceptsByHost(t *testing.T) map[string]map[string for _, hv := range env.hypervisors { zones[hv.Labels[corev1.LabelTopologyZone]] = true } - finder := &evacuationPathFinder{httpClient: http.DefaultClient, schedulerURL: env.schedulerURL} + finder := &evacuationPathFinder{scheduler: env.scheduler} accepts := map[string]map[string]bool{} for _, flavor := range env.flavors { for zone := range zones { @@ -440,38 +429,6 @@ func (env *integrationTestEnv) VerifySiblingSources(t *testing.T, desired map[co // probe exercises. const evacProbePipeline = "kvm-general-purpose-load-balancing" -// integrationHTTPAPI is a thin, deterministic HTTP front-end for the real Nova -// filter/weigher pipeline — the same shape the failover package's integration -// tests use. It does NOT shuffle, so the probe answers are stable. -type integrationHTTPAPI struct { - scheduler *sched.Scheduler -} - -func (a *integrationHTTPAPI) handle(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - var reqData novaapi.ExternalSchedulerRequest - if err := json.NewDecoder(r.Body).Decode(&reqData); err != nil { - http.Error(w, "decode", http.StatusBadRequest) - return - } - // Resolve intent from the request hints, mirroring the production handler; - // the pipeline reads the resolved intent off the typed request. - intent, err := reqData.GetIntent() - if err != nil { - intent = novaapi.CreateIntent - } - req := nova.ToSchedulerRequest(reqData, intent, evacProbePipeline) - out, err := a.scheduler.Run(r.Context(), req, nil, nil) - if err != nil { - http.Error(w, "process", http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: out.OrderedHosts}); err != nil { - http.Error(w, "encode", http.StatusInternalServerError) - } -} - // ============================================================================ // Object Creation Functions // ============================================================================ diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go index 5ca983e97..ecd4f5a01 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding.go @@ -4,20 +4,18 @@ package failoverhost import ( - "bytes" "context" - "encoding/json" "fmt" "maps" - "net/http" "slices" "sort" - "strconv" "time" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -71,21 +69,19 @@ type PathFinder interface { FindPaths(ctx context.Context, topo topology, seeds []Seed) ([]Path, error) } -// evacuationPathFinder finds failover paths by asking the external cortex -// scheduler, per seed, which hosts would accept that flavor, then pairing up the -// co-accepting hosts subject to gang-level fit. +// evacuationPathFinder finds failover paths by probing the shared in-process +// scheduler, per seed, for which hosts would accept that flavor, then pairing up +// the co-accepting hosts subject to gang-level fit. // // Asking the real scheduler (rather than reimplementing placement here) is // deliberate: it means failover paths are computed with exactly the same // filters, weighers and constraints that govern ordinary placement, so a // partition can never contain a target the scheduler would actually reject. type evacuationPathFinder struct { - // httpClient issues the scheduler requests. Give it a timeout so a hung - // scheduler cannot stall the whole calculation loop. Separately injected so - // tests can point it at a stub server. - httpClient *http.Client - // schedulerURL is the external cortex scheduler endpoint to POST to. - schedulerURL string + // scheduler is the shared in-process scheduler used to probe placement. The + // pathfinder issues read-only runs via RunWithoutLock. Separately injected so + // tests can point it at a fake pipeline. + scheduler *compute.Scheduler // monitor records scheduler request metrics. May be nil (e.g. in tests). monitor *Monitor } @@ -242,25 +238,42 @@ func hvCapacity(hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity { func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed Seed, hvsByName map[string]hv1.Hypervisor) ([]hv1.Hypervisor, error) { log := logf.FromContext(ctx) - // Build valid image properties. We set the image hypervisor type explicitly so - // the filter that checks flavor/image compatibility can evaluate it. Since we - // only handle KVM virtual machines, we set it to KVM. - imageProps := map[string]any{ - "img_hv_type": string(api.NovaImageMetaHVTypeKVM), - } // Offer every known host as a candidate. The scheduler filters/weighs over // exactly this host set — it does not gather the cluster itself — so leaving // Hosts empty would make it return no candidates. Each host also needs a // weight entry, which the scheduler requires to accompany the host set. - hosts := make([]api.ExternalSchedulerHost, 0, len(hvsByName)) + hosts := make([]compute.Host, 0, len(hvsByName)) weights := make(map[string]float64, len(hvsByName)) for name := range hvsByName { - hosts = append(hosts, api.ExternalSchedulerHost{ComputeHost: name}) + hosts = append(hosts, compute.Host{ComputeHost: name}) weights[name] = 0 } - requestObj := api.ExternalSchedulerRequest{ - Hosts: hosts, - Weights: weights, + req := compute.SchedulerRequest{ + // Same HANA vs general-purpose split as regular failover placement. + Pipeline: failover.InferFailoverPipeline(seed.Flavor.ExtraSpecs), + // We offer the host set ourselves; the scheduler must not gather the + // cluster itself and overwrite our candidates. + GatherAllCandidates: false, + HypervisorFamily: compute.HypervisorFamilyKVM, + // We only handle KVM virtual machines, so the flavor/image compatibility + // filter must see a KVM image hypervisor type. + ImageHypervisorType: compute.ImageHypervisorTypeKVM, + Hosts: hosts, + Weights: weights, + // Probe with evacuation intent. A failover cold-migrates (evacuates) the + // workload, and evacuation does not run the live-migratability feature + // check, so no source host is needed. This is deliberately weaker than a + // live-migration probe — see the PathFinder doc. + Intent: api.EvacuateIntent, + // Probe a single flavor slot; NumInstances must be >= 1 or capacity/ + // bin-pack math treats the host as free. + NumInstances: 1, + AvailabilityZone: seed.AvailabilityZone, + Flavor: compute.FlavorSpec{ + Name: seed.Flavor.Name, + VCPUs: seed.Flavor.VCPUs, + MemoryMB: seed.Flavor.MemoryMB, + }, Options: scheduling.Options{ // Read-only probe: don't write any resources. ReadOnly additionally // requires SkipInflight and SkipCommittedResourceTracking, else the @@ -280,80 +293,23 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See v1alpha1.ReservationTypeFailoverHost, }, }, - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 1, - AvailabilityZone: seed.AvailabilityZone, - // Probe with evacuation intent. A failover cold-migrates (evacuates) - // the workload, and evacuation does not run the live-migratability - // feature check, so no source_host is needed. The check type must be - // the literal "evacuate" nova uses, which maps to EvacuateIntent in - // GetIntent. See the PathFinder doc for why this is deliberately - // weaker than a live-migration probe. - SchedulerHints: map[string]any{ - "_nova_check_type": "evacuate", - }, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: seed.Flavor.Name, - ExtraSpecs: seed.Flavor.ExtraSpecs, - MemoryMB: seed.Flavor.MemoryMB, - VCPUs: seed.Flavor.VCPUs, - }, - }, - Image: api.NovaObject[api.NovaImageMeta]{ - Data: api.NovaImageMeta{ - Properties: api.NovaObject[map[string]any]{ - Data: imageProps, - }, - }, - }, - }, - }, } - log.V(1).Info("external scheduler request prepared", "request", requestObj) - - reqBody, err := json.Marshal(requestObj) - if err != nil { - log.Error(err, "failed to marshal external scheduler request") - return nil, fmt.Errorf("failed to marshal external scheduler request: %w", err) - } - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, - f.schedulerURL, bytes.NewReader(reqBody)) - if err != nil { - log.Error(err, "failed to create HTTP request") - return nil, fmt.Errorf("failed to create HTTP request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") + log.V(1).Info("scheduler probe prepared", "request", req) startTime := time.Now() - response, err := f.httpClient.Do(httpReq) + result, err := f.scheduler.RunWithoutLock(ctx, req, nil, nil) duration := time.Since(startTime) if err != nil { f.monitor.RecordSchedulerRequest("error", duration) - log.Error(err, "failed to send external scheduler request") - return nil, fmt.Errorf("failed to send external scheduler request: %w", err) - } - defer response.Body.Close() - f.monitor.RecordSchedulerRequest(strconv.Itoa(response.StatusCode), duration) - log.V(1).Info("external scheduler request sent", - "duration", duration, "status", response.StatusCode) - - if response.StatusCode != http.StatusOK { - log.Error(nil, "external scheduler request failed", "status", response.StatusCode) - return nil, fmt.Errorf("external scheduler request failed with status: %d", response.StatusCode) - } - - var responseObj api.ExternalSchedulerResponse - if err := json.NewDecoder(response.Body).Decode(&responseObj); err != nil { - log.Error(err, "failed to decode external scheduler response") - return nil, fmt.Errorf("failed to decode external scheduler response: %w", err) + log.Error(err, "scheduler probe failed") + return nil, fmt.Errorf("scheduler probe failed: %w", err) } - log.V(1).Info("external scheduler response received", "response", responseObj) + f.monitor.RecordSchedulerRequest("200", duration) + log.V(1).Info("scheduler probe completed", "duration", duration, "hosts", result.OrderedHosts) candidates := []hv1.Hypervisor{} for _, hv := range hvsByName { - if slices.Contains(responseObj.Hosts, hv.Name) { + if slices.Contains(result.OrderedHosts, hv.Name) { candidates = append(candidates, hv) } } diff --git a/internal/scheduling/reservations/failoverhost/pathfinding_test.go b/internal/scheduling/reservations/failoverhost/pathfinding_test.go index e489e0ba4..7c91042f8 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding_test.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding_test.go @@ -5,66 +5,85 @@ package failoverhost import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" + "fmt" "slices" "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) -// stubScheduler returns a path finder wired to an httptest server whose +// stubScheduler returns a path finder wired to an in-process scheduler whose // per-flavor answers come from candidatesByFlavor (keyed by the seed's flavor -// name); a flavor in errFlavors gets an HTTP 500. The handler asserts the request -// carries valid options, a non-empty host set, a kvm img_hv_type, and evacuation -// intent with no source_host — mirroring what the new seeded probe sends, so a -// regression that drops any of them is caught here. The caller closes the server. -func stubScheduler(t *testing.T, candidatesByFlavor map[string][]string, errFlavors map[string]bool) (*evacuationPathFinder, *httptest.Server) { +// name); a flavor in errFlavors makes the probe fail. The fake pipeline asserts +// the request carries valid options, a non-empty host set, AssumeEmptyHosts, a +// KVM image hypervisor type, and evacuation intent with no source host — +// mirroring what the seeded probe sends, so a regression that drops any of them +// is caught here. +func stubScheduler(t *testing.T, candidatesByFlavor map[string][]string, errFlavors map[string]bool) *evacuationPathFinder { t.Helper() - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req api.ExternalSchedulerRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("decode request: %v", err) - } - if err := req.Options.Validate(); err != nil { - t.Errorf("invalid scheduling options: %v", err) - } - if !req.Options.AssumeEmptyHosts { - // A failover target is reserved empty, so candidates must be judged by - // total capacity. Dropping this would orphan every busy host. - t.Errorf("request for flavor %q must set AssumeEmptyHosts", req.Spec.Data.Flavor.Data.Name) - } - if len(req.Hosts) == 0 { - t.Errorf("request for flavor %q has no candidate hosts", req.Spec.Data.Flavor.Data.Name) - } - if got := req.Spec.Data.Image.Data.Properties.Data["img_hv_type"]; got != string(api.NovaImageMetaHVTypeKVM) { - // The hypervisor-type filter needs img_hv_type to judge flavor/image - // compatibility; dropping it would make that filter misbehave. - t.Errorf("request for flavor %q must set img_hv_type=%q, got %q", req.Spec.Data.Flavor.Data.Name, api.NovaImageMetaHVTypeKVM, got) - } - // The probe uses evacuation intent (cold migration) and sets no source_host: - // a failover evacuates rather than live-migrates, so the live-migratability - // feature check is deliberately skipped. Catch a regression to the stricter - // live-migration probe here. - if intent, err := req.GetIntent(); err != nil || intent != api.EvacuateIntent { - t.Errorf("request for flavor %q must use evacuate intent, got %v (err=%v)", req.Spec.Data.Flavor.Data.Name, intent, err) - } - if _, ok := req.Spec.Data.SchedulerHints["source_host"]; ok { - t.Errorf("request for flavor %q must not set source_host", req.Spec.Data.Flavor.Data.Name) - } - flavor := req.Spec.Data.Flavor.Data.Name - if errFlavors[flavor] { - w.WriteHeader(http.StatusInternalServerError) - return - } - if err := json.NewEncoder(w).Encode(api.ExternalSchedulerResponse{Hosts: candidatesByFlavor[flavor]}); err != nil { - t.Errorf("encode response: %v", err) - } - })) - finder := &evacuationPathFinder{httpClient: server.Client(), schedulerURL: server.URL} - return finder, server + provider := &stubPipelineProvider{pipeline: &stubPipeline{ + t: t, + candidatesByFlavor: candidatesByFlavor, + errFlavors: errFlavors, + }} + scheduler := compute.NewScheduler(provider, passthroughGatherer{}) + return &evacuationPathFinder{scheduler: scheduler} +} + +// stubPipelineProvider serves the same stub pipeline for every pipeline name the +// probe infers (always a KVM pipeline here). +type stubPipelineProvider struct { + pipeline *stubPipeline +} + +func (p *stubPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[compute.SchedulerRequest], bool) { + return p.pipeline, true +} + +// stubPipeline is a fake filter/weigher pipeline that validates the probe's +// request shape and returns the preconfigured candidate hosts for the request's +// flavor, mirroring the old httptest stub. +type stubPipeline struct { + t *testing.T + candidatesByFlavor map[string][]string + errFlavors map[string]bool +} + +func (s *stubPipeline) Run(req compute.SchedulerRequest) (compute.SchedulerResult, error) { + flavor := req.Flavor.Name + if err := req.Options.Validate(); err != nil { + s.t.Errorf("invalid scheduling options: %v", err) + } + if !req.Options.AssumeEmptyHosts { + // A failover target is reserved empty, so candidates must be judged by + // total capacity. Dropping this would orphan every busy host. + s.t.Errorf("request for flavor %q must set AssumeEmptyHosts", flavor) + } + if len(req.Hosts) == 0 { + s.t.Errorf("request for flavor %q has no candidate hosts", flavor) + } + if req.ImageHypervisorType != compute.ImageHypervisorTypeKVM { + // The hypervisor-type filter needs the image hypervisor type to judge + // flavor/image compatibility; dropping it would make that filter misbehave. + s.t.Errorf("request for flavor %q must set ImageHypervisorType=%q, got %q", flavor, compute.ImageHypervisorTypeKVM, req.ImageHypervisorType) + } + // The probe uses evacuation intent (cold migration) and sets no source host: + // a failover evacuates rather than live-migrates, so the live-migratability + // feature check is deliberately skipped. Catch a regression to the stricter + // live-migration probe here. + if req.Intent != api.EvacuateIntent { + s.t.Errorf("request for flavor %q must use evacuate intent, got %v", flavor, req.Intent) + } + if req.SourceHost != "" { + s.t.Errorf("request for flavor %q must not set a source host", flavor) + } + if s.errFlavors[flavor] { + return compute.SchedulerResult{}, fmt.Errorf("probe failed for flavor %q", flavor) + } + return compute.SchedulerResult{OrderedHosts: s.candidatesByFlavor[flavor]}, nil } // pathNames collapses failover edges into sorted "source->target" strings, for @@ -100,11 +119,10 @@ func TestFindPathsErrorsOnLookupFailure(t *testing.T) { // return an error to abort the pass rather than silently dropping paths — // otherwise the caller would publish an empty desired state and prune valid // reservations. - finder, server := stubScheduler(t, + finder := stubScheduler(t, map[string][]string{"good": {"a", "b"}}, map[string]bool{"bad": true}, ) - defer server.Close() // "bad" is probed and errors; the pass must abort regardless of "good". topo := hostTopo(map[string]int64{"a": 32, "b": 32}) @@ -116,10 +134,9 @@ func TestFindPathsErrorsOnLookupFailure(t *testing.T) { func TestFindPathsPairsCoAcceptingHosts(t *testing.T) { // A single seed accepted on a, b and c makes all three mutually compatible. // Equal-sized hosts yield edges both ways between every pair. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"a", "b", "c"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"a": 32, "b": 32, "c": 32}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) @@ -136,10 +153,9 @@ func TestFindPathsDropsTargetSmallerThanSource(t *testing.T) { // A seed is accepted on "big" (64Gi), "small" (16Gi) and "equal" (64Gi). The // gang-fit constraint keeps an edge only toward a target at least as large as // the source, so no edge points at "small", and big<->equal are mutual. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"big", "small", "equal"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"big": 64, "small": 16, "equal": 64}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) @@ -157,11 +173,10 @@ func TestFindPathsDropsTargetSmallerThanSource(t *testing.T) { func TestFindPathsDeduplicatesAcrossSeeds(t *testing.T) { // Two seeds both accepted on the same pair of equal hosts. Each directed edge // must appear once, not once per seed. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavorA": {"hvA", "hvB"}, "flavorB": {"hvA", "hvB"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"hvA": 64, "hvB": 64}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavorA"), seed("flavorB")}) @@ -175,10 +190,9 @@ func TestFindPathsDeduplicatesAcrossSeeds(t *testing.T) { func TestFindPathsSingleCandidateYieldsNoEdge(t *testing.T) { // A seed accepted on only one host can form no pair, so contributes no edge. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"only"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"only": 64, "other": 64}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) @@ -194,11 +208,10 @@ func TestFindPathsRecordsSupportingFlavors(t *testing.T) { // flavorA is accepted on both hosts, flavorB only on hvA. The edge hvA<->hvB is // produced solely by flavorA, so each directed edge must carry exactly // ["flavorA"] — flavorB co-accepts on no pair and leaves no trace. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavorA": {"hvA", "hvB"}, "flavorB": {"hvA"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"hvA": 64, "hvB": 64}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavorA"), seed("flavorB")}) From e04d7795dd27419d6cbbbd6cad071926ce9a920b Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Tue, 3 Nov 2026 15:12:37 +0100 Subject: [PATCH 13/21] refactor(reservations): enhance trait and hypervisor-type handling in scheduler requests Signed-off-by: Markus Wieland --- .../reservations/capacity/controller.go | 29 ++++--- .../commitments/reservation_controller.go | 31 ++++--- .../scheduling/reservations/extra_specs.go | 49 +++++++++++ .../reservations/extra_specs_test.go | 66 +++++++++++++++ .../failover/reservation_scheduling.go | 83 ++++++++----------- .../failover/reservation_scheduling_test.go | 48 +++++++++++ .../reservations/failoverhost/pathfinding.go | 12 +++ 7 files changed, 247 insertions(+), 71 deletions(-) create mode 100644 internal/scheduling/reservations/extra_specs.go create mode 100644 internal/scheduling/reservations/extra_specs_test.go diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 5e76f4449..a1f01279f 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -821,17 +821,26 @@ func (c *Reconciler) probeScheduler( probeHosts = append(probeHosts, sched.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) probeWeights[h.ComputeHost] = 0.0 } + // Honor the flavor's trait and hypervisor-type constraints so the probe only + // counts compatible hosts (the typed request is built directly, so + // filter_has_requested_traits/filter_capabilities would otherwise see empty + // slices and become no-ops). No hypervisor-type default — only match when the + // flavor names one. + requiredTraits, forbiddenTraits := reservations.ExtractTraits(flavor.ExtraSpecs) out, err := c.scheduler.RunWithoutLock(ctx, sched.SchedulerRequest{ - Pipeline: pipeline, - GatherAllCandidates: true, - HypervisorFamily: sched.HypervisorFamilyKVM, - Hosts: probeHosts, - Weights: probeWeights, - Intent: schedulerapi.CapacityProbeIntent, - InstanceUUID: "capacity-" + flavor.Name, - NumInstances: 1, // probe a single flavor slot; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free - ProjectID: "cortex-capacity-probe", - AvailabilityZone: az, + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: sched.HypervisorFamilyKVM, + Hosts: probeHosts, + Weights: probeWeights, + Intent: schedulerapi.CapacityProbeIntent, + InstanceUUID: "capacity-" + flavor.Name, + NumInstances: 1, // probe a single flavor slot; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free + ProjectID: "cortex-capacity-probe", + AvailabilityZone: az, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + RequestedCapabilities: reservations.HypervisorTypeCapability(flavor.ExtraSpecs, false), Flavor: sched.FlavorSpec{ Name: flavor.Name, VCPUs: flavor.VCPUs, diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 25c22e302..932fdd859 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -354,18 +354,27 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr hosts = append(hosts, compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) weights[h.ComputeHost] = 0.0 } + // Honor the flavor's trait and hypervisor-type constraints so a + // committed-resource reservation lands on a compatible host (the typed + // request is built directly, so filter_has_requested_traits/filter_capabilities + // would otherwise see empty slices and become no-ops). No hypervisor-type + // default here — only match when the flavor names one. + requiredTraits, forbiddenTraits := reservations.ExtractTraits(flavorDetails.ExtraSpecs) schedReq := compute.SchedulerRequest{ - Pipeline: pipelineName, - GatherAllCandidates: true, - HypervisorFamily: compute.HypervisorFamilyKVM, - Hosts: hosts, - Weights: weights, - Intent: schedulerdelegationapi.ReserveForCommittedResourceIntent, - InstanceUUID: res.Name, - NumInstances: 1, // one reservation slot; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free - ProjectID: projectID, - AvailabilityZone: availabilityZone, - DomainName: domainName, + Pipeline: pipelineName, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: hosts, + Weights: weights, + Intent: schedulerdelegationapi.ReserveForCommittedResourceIntent, + InstanceUUID: res.Name, + NumInstances: 1, // one reservation slot; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free + ProjectID: projectID, + AvailabilityZone: availabilityZone, + DomainName: domainName, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + RequestedCapabilities: reservations.HypervisorTypeCapability(flavorDetails.ExtraSpecs, false), Flavor: compute.FlavorSpec{ Name: flavorDetails.Name, VCPUs: flavorDetails.VCPUs, diff --git a/internal/scheduling/reservations/extra_specs.go b/internal/scheduling/reservations/extra_specs.go new file mode 100644 index 000000000..a63d9bd23 --- /dev/null +++ b/internal/scheduling/reservations/extra_specs.go @@ -0,0 +1,49 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package reservations + +import ( + "strings" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" +) + +// ExtractTraits splits a flavor's "trait:" extra specs into required and +// forbidden trait names, mirroring the Nova spec adapter so reservation +// scheduling honors the same trait constraints as regular placement. The +// reservation controllers build the typed compute.SchedulerRequest directly +// (not via the Nova wire adapter), so they must populate RequiredTraits/ +// ForbiddenTraits themselves — otherwise filter_has_requested_traits sees empty +// slices and becomes a no-op. +func ExtractTraits(extraSpecs map[string]string) (required, forbidden []string) { + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "trait:") + if !ok { + continue + } + switch value { + case "required": + required = append(required, name) + case "forbidden": + forbidden = append(forbidden, name) + } + } + return required, forbidden +} + +// HypervisorTypeCapability returns the capabilities:hypervisor_type match (QEMU +// vs CH) for a reservation scheduler request, so filter_capabilities keeps a +// workload off an incompatible hypervisor. When the flavor names no hypervisor +// type, it returns "qemu" if defaultQEMU is set (the failover controller seeds +// from running VMs whose flavor may omit it) and nil otherwise. +func HypervisorTypeCapability(extraSpecs map[string]string, defaultQEMU bool) []compute.Capability { + hvType := extraSpecs["capabilities:hypervisor_type"] + if hvType == "" { + if !defaultQEMU { + return nil + } + hvType = "qemu" + } + return []compute.Capability{{Name: "hypervisor_type", Value: hvType}} +} diff --git a/internal/scheduling/reservations/extra_specs_test.go b/internal/scheduling/reservations/extra_specs_test.go new file mode 100644 index 000000000..c1f13356d --- /dev/null +++ b/internal/scheduling/reservations/extra_specs_test.go @@ -0,0 +1,66 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package reservations + +import ( + "slices" + "testing" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" +) + +func TestExtractTraits(t *testing.T) { + required, forbidden := ExtractTraits(map[string]string{ + "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required", + "trait:CUSTOM_NO_GPU": "forbidden", + "trait:IGNORED": "preferred", // neither required nor forbidden + "capabilities:hypervisor_type": "qemu", // not a trait + }) + if !slices.Equal(required, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}) { + t.Errorf("required = %v, want [CUSTOM_HANA_EXCLUSIVE_HOST]", required) + } + if !slices.Equal(forbidden, []string{"CUSTOM_NO_GPU"}) { + t.Errorf("forbidden = %v, want [CUSTOM_NO_GPU]", forbidden) + } +} + +func TestHypervisorTypeCapability(t *testing.T) { + tests := []struct { + name string + extraSpecs map[string]string + defaultQEMU bool + want []compute.Capability + }{ + { + name: "value passed through", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, + }, + { + name: "missing without default -> nil", + extraSpecs: map[string]string{}, + want: nil, + }, + { + name: "missing with default -> qemu", + extraSpecs: map[string]string{}, + defaultQEMU: true, + want: []compute.Capability{{Name: "hypervisor_type", Value: "qemu"}}, + }, + { + name: "explicit value wins over default", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}, + defaultQEMU: true, + want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := HypervisorTypeCapability(tt.extraSpecs, tt.defaultQEMU) + if !slices.Equal(got, tt.want) { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 22c65ab7d..ab6cf5943 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -40,25 +40,6 @@ func InferFailoverPipeline(extraSpecs map[string]string) string { return "kvm-general-purpose-load-balancing" } -// traitsFromExtraSpecs splits the flavor extra specs' "trait:" entries into -// required and forbidden trait names, mirroring the Nova spec adapter so failover -// reservation scheduling honors the same trait constraints as regular placement. -func traitsFromExtraSpecs(extraSpecs map[string]string) (required, forbidden []string) { - for key, value := range extraSpecs { - name, ok := strings.CutPrefix(key, "trait:") - if !ok { - continue - } - switch value { - case "required": - required = append(required, name) - case "forbidden": - forbidden = append(forbidden, name) - } - } - return required, forbidden -} - // imagePropertiesFromFlavor synthesizes Glance-style image properties from a flavor's // capabilities:hypervisor_type extra spec. The failover controller does not have access // to the real Glance image metadata of the VM, but for reservation scheduling the VM @@ -124,23 +105,24 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex hosts = append(hosts, compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) weights[h.ComputeHost] = 0.0 } - requiredTraits, forbiddenTraits := traitsFromExtraSpecs(vm.FlavorExtraSpecs) + requiredTraits, forbiddenTraits := reservations.ExtractTraits(vm.FlavorExtraSpecs) schedReq := compute.SchedulerRequest{ - Pipeline: pipeline, - GatherAllCandidates: true, - HypervisorFamily: compute.HypervisorFamilyKVM, - Hosts: hosts, - Weights: weights, - Options: opts, - Intent: intent, - InstanceUUID: vm.UUID, - NumInstances: 1, // one reservation per VM; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free - ProjectID: vm.ProjectID, - AvailabilityZone: vm.AvailabilityZone, - IgnoreHosts: ignoreHypervisors, - ResourceGroup: resSpec.ResourceGroup(vm.FlavorName), - RequiredTraits: requiredTraits, - ForbiddenTraits: forbiddenTraits, + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: hosts, + Weights: weights, + Options: opts, + Intent: intent, + InstanceUUID: vm.UUID, + NumInstances: 1, // one reservation per VM; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free + ProjectID: vm.ProjectID, + AvailabilityZone: vm.AvailabilityZone, + IgnoreHosts: ignoreHypervisors, + ResourceGroup: resSpec.ResourceGroup(vm.FlavorName), + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + RequestedCapabilities: reservations.HypervisorTypeCapability(vm.FlavorExtraSpecs, true), Flavor: compute.FlavorSpec{ Name: resSpec.FlavorName, VCPUs: resSpec.VCPUs, @@ -278,25 +260,26 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( // match it in the reservation's allocations. KVM gathers all candidates; the // reservation host must come back ranked for the VM to be compatible. pipeline := InferFailoverPipeline(flavorExtraSpecs) - requiredTraits, forbiddenTraits := traitsFromExtraSpecs(flavorExtraSpecs) + requiredTraits, forbiddenTraits := reservations.ExtractTraits(flavorExtraSpecs) evacuateOpts := DefaultFailoverOptions evacuateOpts.ReadOnly = true evacuateOpts.LockReservations = true schedReq := compute.SchedulerRequest{ - Pipeline: pipeline, - GatherAllCandidates: true, - HypervisorFamily: compute.HypervisorFamilyKVM, - Hosts: []compute.Host{{ComputeHost: reservationHost, HypervisorHostname: reservationHost}}, - Weights: map[string]float64{reservationHost: 0.0}, - Options: evacuateOpts, - Intent: api.EvacuateIntent, - InstanceUUID: vm.UUID, - NumInstances: 1, // validating one VM; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free - ProjectID: vm.ProjectID, - AvailabilityZone: vm.AvailabilityZone, - IgnoreHosts: []string{vm.CurrentHypervisor}, - RequiredTraits: requiredTraits, - ForbiddenTraits: forbiddenTraits, + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: []compute.Host{{ComputeHost: reservationHost, HypervisorHostname: reservationHost}}, + Weights: map[string]float64{reservationHost: 0.0}, + Options: evacuateOpts, + Intent: api.EvacuateIntent, + InstanceUUID: vm.UUID, + NumInstances: 1, // validating one VM; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free + ProjectID: vm.ProjectID, + AvailabilityZone: vm.AvailabilityZone, + IgnoreHosts: []string{vm.CurrentHypervisor}, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + RequestedCapabilities: reservations.HypervisorTypeCapability(flavorExtraSpecs, true), Flavor: compute.FlavorSpec{ Name: vm.FlavorName, VCPUs: vcpus, diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go index aef2b584a..2295952c1 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling_test.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -5,6 +5,7 @@ package failover import ( "context" + "slices" "testing" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -591,6 +592,53 @@ func TestFailoverSchedulerOptions(t *testing.T) { } } +// ============================================================================ +// Test: trait and hypervisor_type constraints sent to the scheduler +// ============================================================================ + +// TestFailoverSchedulerFlavorConstraints verifies the scheduler request carries +// the flavor's traits and capabilities:hypervisor_type, so +// filter_has_requested_traits / filter_capabilities keep a workload off an +// incompatible host — including the qemu default when the flavor omits the +// hypervisor type. +func TestFailoverSchedulerFlavorConstraints(t *testing.T) { + tests := []struct { + name string + extraSpecs map[string]string + wantHVType string + wantTraits []string + }{ + {name: "flavor without hypervisor_type defaults to qemu", extraSpecs: nil, wantHVType: "qemu"}, + {name: "explicit CH preserved", extraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}, wantHVType: "CH"}, + { + name: "required trait is forwarded", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu", "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required"}, + wantHVType: "qemu", + wantTraits: []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vm := buildSchedulingTestVM("vm-1", "host-1") + vm.FlavorExtraSpecs = tt.extraSpecs + resolved := resolveVMSpecForScheduling(context.Background(), vm, false, nil) + + var capturedReq sched.SchedulerRequest + scheduler := newCapturingScheduler("host-2", &capturedReq) + c := buildMinimalController(t, scheduler) + _, _ = c.scheduleAndBuildNewFailoverReservation(context.Background(), vm, []string{"host-1", "host-2"}, nil, nil, resolved) //nolint:errcheck + + wantCap := []sched.Capability{{Name: "hypervisor_type", Value: tt.wantHVType}} + if len(capturedReq.RequestedCapabilities) != 1 || capturedReq.RequestedCapabilities[0] != wantCap[0] { + t.Errorf("RequestedCapabilities = %v, want %v", capturedReq.RequestedCapabilities, wantCap) + } + if !slices.Equal(capturedReq.RequiredTraits, tt.wantTraits) { + t.Errorf("RequiredTraits = %v, want %v", capturedReq.RequiredTraits, tt.wantTraits) + } + }) + } +} + // ============================================================================ // Test: imagePropertiesFromFlavor // ============================================================================ diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go index ecd4f5a01..f13b814c9 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding.go @@ -15,6 +15,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -248,6 +249,11 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See hosts = append(hosts, compute.Host{ComputeHost: name}) weights[name] = 0 } + // Honor the flavor's trait and hypervisor-type constraints so the probe only + // considers compatible hosts (filter_has_requested_traits/filter_capabilities + // would otherwise see empty slices and become no-ops). The seeder only emits + // seeds whose flavor names a supported hypervisor type, so no default is needed. + requiredTraits, forbiddenTraits := reservations.ExtractTraits(seed.Flavor.ExtraSpecs) req := compute.SchedulerRequest{ // Same HANA vs general-purpose split as regular failover placement. Pipeline: failover.InferFailoverPipeline(seed.Flavor.ExtraSpecs), @@ -269,6 +275,12 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See // bin-pack math treats the host as free. NumInstances: 1, AvailabilityZone: seed.AvailabilityZone, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + // Match the flavor's hypervisor type (QEMU vs CH). The seeder only emits + // seeds whose flavor names a supported hypervisor type, so this is always + // set (no default needed). + RequestedCapabilities: reservations.HypervisorTypeCapability(seed.Flavor.ExtraSpecs, false), Flavor: compute.FlavorSpec{ Name: seed.Flavor.Name, VCPUs: seed.Flavor.VCPUs, From d9dbdb9b12e597006925feb5d43be7fd2d5e02a8 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 4 Nov 2026 09:11:11 +0100 Subject: [PATCH 14/21] refactor(pipeline): use DeepCopy for machine processing in FilterWeigherPipelineController Signed-off-by: Markus Wieland --- .../scheduling/machines/filter_weigher_pipeline_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/scheduling/machines/filter_weigher_pipeline_controller.go b/internal/scheduling/machines/filter_weigher_pipeline_controller.go index aa708e22a..672e287d6 100644 --- a/internal/scheduling/machines/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/machines/filter_weigher_pipeline_controller.go @@ -61,7 +61,7 @@ func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Reques func (c *FilterWeigherPipelineController) ProcessNewMachine(ctx context.Context, machine *ironcorev1alpha1.Machine) error { c.processMu.Lock() defer c.processMu.Unlock() - return c.process(ctx, machine) + return c.process(ctx, machine.DeepCopy()) } func (c *FilterWeigherPipelineController) process(ctx context.Context, machine *ironcorev1alpha1.Machine) error { From 953899986a2e87ea7e90f0782b6e2cdcb2de9335 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 4 Nov 2026 09:24:41 +0100 Subject: [PATCH 15/21] Remove dead fields Signed-off-by: Markus Wieland --- internal/scheduling/compute/request.go | 70 ++++------- internal/scheduling/compute/request_test.go | 19 --- internal/scheduling/nova/adapter.go | 60 ++------- .../nova/candidate_gatherer_test.go | 6 - .../filter_has_enough_capacity_test.go | 118 +++++++++--------- .../filter_instance_group_affinity_test.go | 2 +- ...ilter_instance_group_anti_affinity_test.go | 2 +- .../nova/plugins/weighers/kvm_binpack_test.go | 2 - .../weighers/kvm_failover_evacuation_test.go | 2 - ...failover_reservation_consolidation_test.go | 2 - .../kvm_instance_group_soft_affinity_test.go | 2 - .../weighers/kvm_prefer_smaller_hosts_test.go | 2 - .../reservations/capacity/controller.go | 1 - .../commitments/reservation_controller.go | 1 - .../scheduling/reservations/extra_specs.go | 19 +-- .../failover/reservation_scheduling.go | 2 - .../reservations/failoverhost/pathfinding.go | 1 - .../failoverhost/pathfinding_test.go | 16 ++- .../reservations/failoverhost/seeding.go | 4 +- 19 files changed, 114 insertions(+), 217 deletions(-) diff --git a/internal/scheduling/compute/request.go b/internal/scheduling/compute/request.go index b5fd99577..c3145683c 100644 --- a/internal/scheduling/compute/request.go +++ b/internal/scheduling/compute/request.go @@ -26,10 +26,8 @@ import ( ) // HypervisorFamily is the coarse hypervisor family a request targets. It is the -// distinction the pipeline and request routing care about; the more specific -// HypervisorType (QEMU vs CH within the KVM family) is only needed by Nova side -// effects. The adapter resolves the family from Nova's -// "capabilities:hypervisor_type" flavor extra spec. +// distinction the pipeline and request routing care about. The adapter resolves +// the family from Nova's "capabilities:hypervisor_type" flavor extra spec. type HypervisorFamily string const ( @@ -39,30 +37,6 @@ const ( HypervisorFamilyVMware HypervisorFamily = "vmware" ) -// HypervisorType is the specific hypervisor type of a request. The adapter -// resolves it from Nova's "capabilities:hypervisor_type" flavor extra spec. -type HypervisorType string - -const ( - // HypervisorTypeQEMU represents QEMU/KVM hypervisors. - HypervisorTypeQEMU HypervisorType = "qemu" - // HypervisorTypeCH represents Cloud-Hypervisor/KVM hypervisors. - HypervisorTypeCH HypervisorType = "ch" - // HypervisorTypeVMware represents VMware hypervisors. - HypervisorTypeVMware HypervisorType = "vmware" -) - -// FlavorType classifies the workload type. The adapter resolves it from Nova's -// "trait:CUSTOM_HANA_EXCLUSIVE_HOST" flavor extra spec. -type FlavorType string - -const ( - // FlavorTypeGeneralPurpose represents general purpose workloads. - FlavorTypeGeneralPurpose FlavorType = "general-purpose" - // FlavorTypeHANA represents HANA workloads. - FlavorTypeHANA FlavorType = "hana" -) - // ImageHypervisorType is the hypervisor type declared by the image. The adapter // resolves it from Nova's image properties ("img_hv_type"/"hypervisor_type"). type ImageHypervisorType string @@ -104,22 +78,12 @@ type Capability struct { // It carries no extra-spec map: the adapter resolves every extra spec the // pipeline needs into the typed fields on this struct and on SchedulerRequest. type FlavorSpec struct { - // Name is the flavor name. Empty only affects CR/flavor-group lookups in - // side effects, not the pipeline math. - Name string - // FlavorID is the flavor ID. Empty only affects the VM resource written by - // side effects. - FlavorID string // MemoryMB is the requested memory per instance. 0 means a zero-memory // instance to capacity/bin-pack steps (fits anywhere), so the adapter sets it. MemoryMB uint64 // VCPUs is the requested vCPUs per instance. 0 means a zero-CPU instance to // capacity/bin-pack steps (fits anywhere), so the adapter sets it. VCPUs uint64 - // Type classifies the workload (general-purpose/hana), resolved by the - // adapter from the flavor's extra specs. Empty is treated as general-purpose - // by pipeline inference. - Type FlavorType } // RequestedDestination mirrors the subset of Nova's requested_destination that @@ -229,10 +193,6 @@ type SchedulerRequest struct { // routing. Empty means unresolved: KVM-only gates (e.g. VM creation) treat it // as not-KVM and skip. HypervisorFamily HypervisorFamily - // HypervisorType is the specific type (qemu/ch/vmware), needed only by Nova - // side effects. Empty means unresolved; the VM-upsert side effect errors - // rather than guessing. - HypervisorType HypervisorType // ImageHypervisorType is the hypervisor type declared by the image. The empty // value (ImageHypervisorTypeUnset) means the image declared none ("no image // constraint"); ImageHypervisorTypeUnparseable means it declared one that @@ -247,9 +207,6 @@ type SchedulerRequest struct { // bin-pack steps see a zero-sized instance (fits anywhere) — the adapter // always populates it for real placements. Flavor FlavorSpec - // ImageID is the Glance image UUID. Empty only affects the VM resource - // written by side effects, not pipeline decisions. - ImageID string // RequiredTraits are trait names the host must have (adapter stripped the // "trait:" prefix and kept the "required" ones). Empty/nil means no required @@ -296,7 +253,7 @@ type SchedulerRequest struct { GlobalRequestID string } -// TODO: REMOVE ME Compile-time check that SchedulerRequest satisfies the pipeline request contract. +// Compile-time check that SchedulerRequest satisfies the pipeline request contract. var _ lib.FilterWeigherPipelineRequest = SchedulerRequest{} // GetIntent returns the resolved scheduling intent. @@ -389,3 +346,24 @@ func (r SchedulerRequest) ForcedHosts() []string { } return matched } + +// SplitTraits splits a flavor's "trait:" extra specs into required and +// forbidden trait names, classified by the "required"/"forbidden" value. It is +// the single place that understands Nova's trait extra-spec convention, so both +// the Nova wire adapter and the reservation controllers (which build a +// SchedulerRequest directly) populate RequiredTraits/ForbiddenTraits the same way. +func SplitTraits(extraSpecs map[string]string) (required, forbidden []string) { + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "trait:") + if !ok { + continue + } + switch value { + case "required": + required = append(required, name) + case "forbidden": + forbidden = append(forbidden, name) + } + } + return required, forbidden +} diff --git a/internal/scheduling/compute/request_test.go b/internal/scheduling/compute/request_test.go index 2d41642f5..8f96f9176 100644 --- a/internal/scheduling/compute/request_test.go +++ b/internal/scheduling/compute/request_test.go @@ -10,25 +10,6 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) -func TestHypervisorFamily(t *testing.T) { - // The family is the distinction the pipeline uses; the specific type carries - // the QEMU/CH detail only Nova side effects need. - cases := []struct { - hvType HypervisorType - family HypervisorFamily - }{ - {HypervisorTypeQEMU, HypervisorFamilyKVM}, - {HypervisorTypeCH, HypervisorFamilyKVM}, - {HypervisorTypeVMware, HypervisorFamilyVMware}, - } - for _, c := range cases { - r := SchedulerRequest{HypervisorType: c.hvType, HypervisorFamily: c.family} - if r.HypervisorFamily != c.family { - t.Errorf("type %q: family = %q, want %q", c.hvType, r.HypervisorFamily, c.family) - } - } -} - func TestIsForcedDestination(t *testing.T) { cases := []struct { name string diff --git a/internal/scheduling/nova/adapter.go b/internal/scheduling/nova/adapter.go index 9554abe85..9d2a19840 100644 --- a/internal/scheduling/nova/adapter.go +++ b/internal/scheduling/nova/adapter.go @@ -64,17 +64,14 @@ func ToSchedulerRequest( GlobalRequestID: derefString(wire.Context.GlobalRequestID), } - // Flavor sizing + classification. + // Flavor sizing. req.Flavor = compute.FlavorSpec{ - Name: spec.Flavor.Data.Name, - FlavorID: spec.Flavor.Data.FlavorID, VCPUs: spec.Flavor.Data.VCPUs, MemoryMB: spec.Flavor.Data.MemoryMB, - Type: toFlavorType(wire), } - // Hypervisor family + specific type. - req.HypervisorFamily, req.HypervisorType = toHypervisorFamilyAndType(wire) + // Hypervisor family. + req.HypervisorFamily = toHypervisorFamily(wire) // Whether to gather all placement candidates (ignore the caller's host // preselection) is determined by the hypervisor family: cortex can enumerate @@ -83,11 +80,10 @@ func ToSchedulerRequest( req.GatherAllCandidates = req.HypervisorFamily == compute.HypervisorFamilyKVM // Image. - req.ImageID = spec.Image.Data.ID req.ImageHypervisorType = toImageHypervisorType(spec.Image.Data) // Traits, capabilities, accelerators from flavor extra specs. - req.RequiredTraits, req.ForbiddenTraits = toTraits(spec.Flavor.Data.ExtraSpecs) + req.RequiredTraits, req.ForbiddenTraits = compute.SplitTraits(spec.Flavor.Data.ExtraSpecs) req.RequestedCapabilities = toCapabilities(spec.Flavor.Data.ExtraSpecs) _, req.AcceleratorsRequested = spec.Flavor.Data.ExtraSpecs["accel:device_profile"] @@ -106,34 +102,20 @@ func toComputeHosts(hosts []api.ExternalSchedulerHost) []compute.Host { return out } -// toFlavorType maps the wire flavor type enum onto the compute enum. -func toFlavorType(wire api.ExternalSchedulerRequest) compute.FlavorType { - ft, err := wire.GetFlavorType() - if err != nil { - return compute.FlavorTypeGeneralPurpose - } - if ft == api.FlavorTypeHANA { - return compute.FlavorTypeHANA - } - return compute.FlavorTypeGeneralPurpose -} - -// toHypervisorFamilyAndType resolves the coarse family and the specific type -// from the wire hypervisor type. Both are empty when unresolved. -func toHypervisorFamilyAndType(wire api.ExternalSchedulerRequest) (compute.HypervisorFamily, compute.HypervisorType) { +// toHypervisorFamily resolves the coarse family from the wire hypervisor type. +// It is empty when unresolved. +func toHypervisorFamily(wire api.ExternalSchedulerRequest) compute.HypervisorFamily { hv, err := wire.GetHypervisorType() if err != nil { - return "", "" + return "" } switch hv { - case api.HypervisorTypeQEMU: - return compute.HypervisorFamilyKVM, compute.HypervisorTypeQEMU - case api.HypervisorTypeCH: - return compute.HypervisorFamilyKVM, compute.HypervisorTypeCH + case api.HypervisorTypeQEMU, api.HypervisorTypeCH: + return compute.HypervisorFamilyKVM case api.HypervisorTypeVMware: - return compute.HypervisorFamilyVMware, compute.HypervisorTypeVMware + return compute.HypervisorFamilyVMware } - return "", "" + return "" } // toImageHypervisorType maps the image-declared hypervisor type. It returns @@ -165,24 +147,6 @@ func toImageHypervisorType(image api.NovaImageMeta) compute.ImageHypervisorType return compute.ImageHypervisorTypeUnset } -// toTraits splits the flavor extra specs' "trait:" entries into required -// and forbidden trait names (classified by the "required"/"forbidden" value). -func toTraits(extraSpecs map[string]string) (required, forbidden []string) { - for key, value := range extraSpecs { - name, ok := strings.CutPrefix(key, "trait:") - if !ok { - continue - } - switch value { - case "required": - required = append(required, name) - case "forbidden": - forbidden = append(forbidden, name) - } - } - return required, forbidden -} - // toCapabilities extracts the "capabilities:" extra specs as typed Capability // equalities. If any value uses an unsupported comparison operator, it returns // nil so the capabilities filter is skipped entirely (matching Nova behavior). diff --git a/internal/scheduling/nova/candidate_gatherer_test.go b/internal/scheduling/nova/candidate_gatherer_test.go index 6f3694339..6c9b15e81 100644 --- a/internal/scheduling/nova/candidate_gatherer_test.go +++ b/internal/scheduling/nova/candidate_gatherer_test.go @@ -80,7 +80,6 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, request: compute.SchedulerRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, }, expectError: false, expectedHosts: []string{"host1", "host2", "host3"}, @@ -97,7 +96,6 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, request: compute.SchedulerRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, }, expectError: false, expectedHosts: []string{"host1"}, @@ -119,7 +117,6 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, request: compute.SchedulerRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeCH, }, expectError: false, expectedHosts: []string{"ch-host1", "ch-host2"}, @@ -136,7 +133,6 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, request: compute.SchedulerRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeCH, }, expectError: false, expectedHosts: []string{"ch-host1"}, @@ -147,7 +143,6 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { hypervisors: []client.Object{}, request: compute.SchedulerRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, }, expectError: false, expectedHosts: []string{}, @@ -164,7 +159,6 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, request: compute.SchedulerRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, Hosts: []compute.Host{ {ComputeHost: "old-host", HypervisorHostname: "old-hv"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go index afe8b969d..15aed3daf 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go @@ -249,13 +249,13 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) compute.SchedulerRequest { - return newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup, vcpus, memory, "", evacuation, hosts) +func newNovaRequest(instanceUUID, projectID, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) compute.SchedulerRequest { + return newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup, vcpus, memory, "", evacuation, hosts) } // newNovaRequestWithIntent creates a nova request with a specific intent. // intentHint can be: "evacuate", "reserve_for_committed_resource", "reserve_for_failover", or "" for create. -func newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory, intentHint string, evacuation bool, hosts []string) compute.SchedulerRequest { +func newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup string, vcpus int, memory, intentHint string, evacuation bool, hosts []string) compute.SchedulerRequest { hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} @@ -285,9 +285,7 @@ func newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup s Intent: intent, HardwareVersion: flavorGroup, HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: flavorName, VCPUs: uint64(vcpus), //nolint:gosec // test code MemoryMB: memoryMB, }, @@ -361,7 +359,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { { name: "No reservations - all hosts with capacity pass", reservations: []*v1alpha1.Reservation{}, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -372,7 +370,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-1", "host1", "project-A", "m1.large", "gp-1", "8", "16Gi", nil, nil), newCommittedReservation("res-2", "host2", "project-A", "m1.large", "gp-1", "4", "8Gi", nil, nil), }, - request: newNovaRequest("instance-123", "project-B", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-B", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -383,7 +381,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-1 half used", "host1", "project-A", "m1.large", "gp-1", "8", "16Gi", crSpecAllocs(crVm("vm-1", "4", "8Gi")), map[string]string{"vm-1": "host1"}), newCommittedReservation("res-2 fully used", "host2", "project-A", "m1.large", "gp-1", "4", "8Gi", crSpecAllocs(crVm("vm-2", "4", "8Gi")), map[string]string{"vm-2": "host2"}), }, - request: newNovaRequest("instance-123", "project-B", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-B", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -400,7 +398,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { map[string]string{"vm-1": "host1", "vm-2": "host1"}, ), }, - request: newNovaRequest("instance-123", "project-B", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-B", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, // host1: 5cpu/10Gi free → 4cpu/8Gi request passes filteredHosts: []string{"host4"}, @@ -412,7 +410,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newMigratingReservation("res-2", "host1", "host2", "project-A", "m1.large", "gp-1", "2", "4Gi", nil, nil), // migration reservation from host1 to host2 newMigratingReservation("res-3", "host2", "host1", "project-A", "m1.large", "gp-1", "2", "4Gi", crSpecAllocs(crVm("vm-1", "2", "4Gi")), map[string]string{"vm-1": "host1"}), // migration reservation from host2 to host1 }, - request: newNovaRequest("instance-123", "project-B", "m1.small", "gp-1", 2, "4Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-B", "gp-1", 2, "4Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -426,7 +424,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-3-used", "host1", "project-A", "some flavor", "gp-1", "2", "4Gi", crSpecAllocs(crVm("vm-2", "1", "1Gi")), map[string]string{"vm-2": "host1"}), // used reservation newCommittedReservation("res-4", "host2", "project-A", "some flavor", "gp-2", "4", "8Gi", nil, nil), // different flavor group, should still block }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host3"}, filteredHosts: []string{"host4", "host2"}, @@ -437,7 +435,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-1", "host1", "project-A", "m1.large", "gp-1", "8", "16Gi", nil, nil), newCommittedReservation("res-2", "host3", "project-A", "m1.large", "gp-1", "16", "32Gi", nil, nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: true}, expectedHosts: []string{"host2"}, filteredHosts: []string{"host1", "host3", "host4"}, @@ -447,7 +445,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newCommittedReservation("legacy-res", "host1", "project-A", "m1.large", "gp-1", "4", "8Gi", nil, nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, @@ -459,7 +457,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-2", "host2", "project-X", "m1.xlarge", "gp-1", "4", "8Gi", nil, nil), newCommittedReservation("res-3", "host3", "project-X", "m1.xlarge", "gp-1", "16", "32Gi", nil, nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{}, filteredHosts: []string{"host1", "host2", "host3", "host4"}, @@ -469,7 +467,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newMigratingReservation("pending-res", "host1", "", "project-X", "m1.large", "gp-1", "8", "16Gi", nil, nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2", "host3"}, // host1 blocked by pending reservation filteredHosts: []string{"host1", "host4"}, @@ -480,7 +478,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newMigratingReservation("pending-res", "host1", "", "project-X", "m1.large", "gp-1", "8", "16Gi", nil, nil), newMigratingReservation("placed-res", "host2", "host3", "project-X", "m1.large", "gp-1", "4", "8Gi", nil, nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, // host1 blocked by pending, host2 blocked by placed, host3 still has capacity filteredHosts: []string{"host1", "host2", "host4"}, @@ -490,7 +488,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newCommittedReservation("no-host-res", "", "project-X", "m1.large", "gp-1", "8", "16Gi", nil, nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -507,7 +505,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { map[string]string{"vm-1": "host1"}, ), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 9, "17Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 9, "17Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, // host1: 8 free CPU < 9, 16Gi < 17Gi → filtered; host2: 4 < 9 → filtered; host4: no capacity → filtered expectedHosts: []string{"host3"}, @@ -526,7 +524,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { map[string]string{"vm-1": "host1"}, // vm-2 is spec-only ), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 3, "5Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 3, "5Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, // host1: 2 free CPU < 3 requested, 4Gi < 5Gi → filtered; host4: no capacity → filtered expectedHosts: []string{"host2", "host3"}, @@ -538,7 +536,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newFailoverReservation("failover-1", "host1", "8", "16Gi", map[string]string{"instance-123": "host5"}), newFailoverReservation("failover-2", "host2", "4", "8Gi", map[string]string{"instance-123": "host6"}), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -549,7 +547,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newFailoverReservation("failover-1", "host1", "4", "8Gi", map[string]string{"instance-123": "host5"}), newFailoverReservation("failover-2", "host2", "4", "8Gi", map[string]string{"instance-123": "host6"}), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -560,7 +558,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newFailoverReservation("failover-1", "host1", "8", "16Gi", map[string]string{"other-instance": "host5"}), newFailoverReservation("failover-2", "host2", "4", "8Gi", map[string]string{"another-instance": "host6"}), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -570,7 +568,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newFailoverReservation("failover-1", "host1", "8", "16Gi", map[string]string{}), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2", "host3"}, filteredHosts: []string{"host1"}, @@ -580,7 +578,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newFailoverReservation("failover-1", "host1", "4", "8Gi", map[string]string{"instance-111": "host5", "instance-222": "host6", "instance-333": "host7"}), }, - request: newNovaRequest("instance-222", "project-A", "m1.large", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-222", "project-A", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -593,7 +591,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newUnconfirmedReservation("unconfirmed-res", "host1", "project-X", "gp-1", "8", "16Gi"), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2", "host3"}, filteredHosts: []string{"host1", "host4"}, @@ -606,7 +604,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newUnconfirmedReservation("unconfirmed-1", "host1", "project-X", "gp-1", "8", "16Gi"), newUnconfirmedReservation("unconfirmed-2", "host2", "project-X", "gp-1", "4", "8Gi"), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -631,7 +629,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { Status: v1alpha1.ReservationStatus{}, }, }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2", "host3"}, filteredHosts: []string{"host1", "host4"}, @@ -721,7 +719,7 @@ func TestFilterHasEnoughCapacity_LockReservations(t *testing.T) { }, } - request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}) + request := newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { step := &FilterHasEnoughCapacity{} @@ -776,7 +774,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Two hosts: No ignore - both hosts blocked by reservations", hypervisors: twoHostHVs, reservations: twoHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), ignoredReservationTypes: nil, expectedHosts: []string{}, filteredHosts: []string{"host1", "host2"}, @@ -785,7 +783,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Two hosts: Ignore CR only - host1 passes, host2 blocked by failover", hypervisors: twoHostHVs, reservations: twoHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, expectedHosts: []string{"host1"}, filteredHosts: []string{"host2"}, @@ -794,7 +792,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Two hosts: Ignore Failover only - host2 passes, host1 blocked by CR", hypervisors: twoHostHVs, reservations: twoHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host2"}, filteredHosts: []string{"host1"}, @@ -803,7 +801,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Two hosts: Ignore both - both hosts pass", hypervisors: twoHostHVs, reservations: twoHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource, v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, @@ -816,7 +814,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Large VM (12 CPU): No ignore - blocked (6 free < 12 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 12, "24Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 12, "24Gi", false, []string{"host1"}), ignoredReservationTypes: nil, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -825,7 +823,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Large VM (12 CPU): Ignore CR - blocked (10 free < 12 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 12, "24Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 12, "24Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -834,7 +832,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Large VM (12 CPU): Ignore Failover - blocked (8 free < 12 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 12, "24Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 12, "24Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -843,7 +841,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Large VM (12 CPU): Ignore both - passes (12 free = 12 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 12, "24Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 12, "24Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource, v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -854,7 +852,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Failover-size VM (10 CPU): No ignore - blocked (6 free < 10 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), ignoredReservationTypes: nil, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -863,7 +861,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Failover-size VM (10 CPU): Ignore CR - passes (10 free = 10 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -872,7 +870,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Failover-size VM (10 CPU): Ignore Failover - blocked (8 free < 10 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -881,7 +879,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Failover-size VM (10 CPU): Ignore both - passes (12 free > 10 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource, v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -892,7 +890,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, CR-size VM (8 CPU): No ignore - blocked (6 free < 8 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1"}), ignoredReservationTypes: nil, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -901,7 +899,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, CR-size VM (8 CPU): Ignore CR - passes (10 free > 8 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -910,7 +908,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, CR-size VM (8 CPU): Ignore Failover - passes (8 free = 8 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -919,7 +917,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, CR-size VM (8 CPU): Ignore both - passes (12 free > 8 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource, v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -960,7 +958,7 @@ func TestFilterHasEnoughCapacity_AssumeEmptyHosts(t *testing.T) { hypervisors := []*hv1.Hypervisor{ newHypervisor("host1", "8", "6", "32Gi", "0"), } - request := newNovaRequest("vm", "proj", "m1.large", "gp", 4, "1Gi", false, []string{"host1"}) + request := newNovaRequest("vm", "proj", "gp", 4, "1Gi", false, []string{"host1"}) objects := make([]client.Object, 0, len(hypervisors)) for _, h := range hypervisors { @@ -1000,7 +998,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) newCommittedReservation("cr-res", "host1", "project-X", "m1.large", "gp-1", "4", "8Gi", nil, nil), newFailoverReservation("failover-res", "host2", "4", "8Gi", map[string]string{"other-vm": "host3"}), } - request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) + request := newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) for _, h := range hypervisors { @@ -1039,7 +1037,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTimeMerge(t *testin newCommittedReservation("cr-res", "host1", "project-X", "m1.large", "gp-1", "4", "8Gi", nil, nil), newFailoverReservation("failover-res", "host2", "4", "8Gi", map[string]string{"other-vm": "host3"}), } - request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) + request := newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) for _, h := range hypervisors { @@ -1095,7 +1093,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Request with reserve_for_committed_resource intent (scheduling a new CR reservation). // The intent causes the filter to keep CR reservations locked (prevents overbooking). - request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "m1.large", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1", "host2"}), + request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1", "host2"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2"}, // host1 blocked because existing-cr stays locked filteredHosts: []string{"host1"}, @@ -1111,7 +1109,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) newCommittedReservation("existing-cr", "host1", "project-A", "m1.large", "gp-1", "8", "16Gi", nil, nil), }, // Normal VM create request (no special intent) - CR reservation should be unlocked - request: newNovaRequest("vm-instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("vm-instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2"}, // host1 passes because existing-cr is unlocked for matching project+flavor filteredHosts: []string{}, @@ -1127,7 +1125,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) newCommittedReservation("other-project-cr", "host1", "project-B", "m1.large", "gp-1", "8", "16Gi", nil, nil), }, // Request with reserve_for_committed_resource intent - request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "m1.large", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1", "host2"}), + request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1", "host2"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2"}, filteredHosts: []string{"host1"}, // host1 blocked by other project's reservation (would be blocked anyway) @@ -1145,7 +1143,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Request with reserve_for_committed_resource intent, needs 10 CPU // After blocking all 3 reservations (24 CPU), only 8 CPU free -> should fail - request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "m1.large", "gp-1", 10, "20Gi", "reserve_for_committed_resource", false, []string{"host1"}), + request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "gp-1", 10, "20Gi", "reserve_for_committed_resource", false, []string{"host1"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{}, filteredHosts: []string{"host1"}, // All reservations stay locked (intent prevents unlock), not enough capacity @@ -1163,7 +1161,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Normal VM create request, needs 10 CPU // All 3 reservations unlocked for matching project+flavor -> 32 CPU free -> should pass - request: newNovaRequest("vm-instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("vm-instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1"}, // All reservations unlocked, enough capacity filteredHosts: []string{}, @@ -1179,7 +1177,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Request with reserve_for_committed_resource intent. // IgnoredReservationTypes is a YAML-level safety override that bypasses the intent check. - request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "m1.large", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1"}), + request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1"}), opts: FilterHasEnoughCapacityOpts{ LockReserved: false, IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, @@ -1198,7 +1196,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Normal VM create request (different project, so unlocking via project match won't work) // But IgnoredReservationTypes should make it work - request: newNovaRequest("vm-instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1"}), + request: newNovaRequest("vm-instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1"}), opts: FilterHasEnoughCapacityOpts{ LockReserved: false, IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, @@ -1271,7 +1269,7 @@ func TestFilterHasEnoughCapacity_PlannedCRDoesNotBlock(t *testing.T) { step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() step.Options = FilterHasEnoughCapacityOpts{LockReserved: false} - request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1"}) + request := newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1"}) result, err := step.Run(logr.Discard(), request) if err != nil { t.Fatalf("expected no error, got %v", err) @@ -1299,7 +1297,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { newHypervisorWithCapacityOnly("host3", "2", "4Gi"), // nil EffectiveCapacity, uses Capacity: 2 CPU free (not enough) newHypervisorWithCapacityOnly("host4", "16", "32Gi"), // nil EffectiveCapacity, uses Capacity: 16 CPU free, 32Gi free }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), expectedHosts: []string{"host1", "host2", "host4"}, filteredHosts: []string{"host3"}, }, @@ -1309,7 +1307,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { newHypervisorWithCapacityOnly("host1", "8", "16Gi"), newHypervisorWithCapacityOnly("host2", "4", "8Gi"), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, }, @@ -1328,7 +1326,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { // Both: 4 free -> filtered (need 5) newHypervisorWithBothCapacities("host3", "4", "4", "8Gi", "8Gi"), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 5, "8Gi", false, []string{"host1", "host2", "host3"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 5, "8Gi", false, []string{"host1", "host2", "host3"}), expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{"host3"}, }, @@ -1460,7 +1458,7 @@ func TestFilterHasEnoughCapacity_CapacityBoundaries(t *testing.T) { step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.hv.DeepCopy()).Build() step.Options = FilterHasEnoughCapacityOpts{LockReserved: false} - req := newNovaRequest("inst-1", "project-A", "m1.small", "gp-1", flavorVCPUs, flavorMem, false, []string{"h"}) + req := newNovaRequest("inst-1", "project-A", "gp-1", flavorVCPUs, flavorMem, false, []string{"h"}) result, err := step.Run(logr.Discard(), req) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -1663,7 +1661,7 @@ func TestFilterHasEnoughCapacity_VMInterReservationMigration(t *testing.T) { step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() step.Options = FilterHasEnoughCapacityOpts{LockReserved: false} - request := newNovaRequest("instance-new", thirdParty, "m1.small", flavorGroup, 3, "6Gi", false, []string{"hv-a", "hv-b"}) + request := newNovaRequest("instance-new", thirdParty, flavorGroup, 3, "6Gi", false, []string{"hv-a", "hv-b"}) result, err := step.Run(logr.Discard(), request) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go index 029fe6d90..b32f514fb 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go @@ -234,7 +234,7 @@ func TestFilterInstanceGroupAffinityStep_SkipsForNonPlacementIntent(t *testing.T // Affinity group on host1 only — host2 would normally be filtered. for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { t.Run(intent, func(t *testing.T) { - request := newNovaRequest("vm", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request := newNovaRequest("vm", "proj", "gp", 1, "1Gi", false, []string{"host1", "host2"}) request.InstanceGroup = &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1"}} request.Intent = v1alpha1.SchedulingIntent(intent) step := &FilterInstanceGroupAffinityStep{} diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go index 27bce62c0..c4bc2c3f4 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go @@ -393,7 +393,7 @@ func TestFilterInstanceGroupAntiAffinityStep_SkipsForNonPlacementIntent(t *testi for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { t.Run(intent, func(t *testing.T) { - request := newNovaRequest("vm-new", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request := newNovaRequest("vm-new", "proj", "gp", 1, "1Gi", false, []string{"host1", "host2"}) request.InstanceGroup = &compute.InstanceGroupSpec{ Policy: "anti-affinity", Members: []string{"vm-existing"}, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go index 5ef1d2af8..3ffa3d2c5 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go @@ -50,9 +50,7 @@ func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) com InstanceUUID: "instance-123", NumInstances: numInstances, HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", VCPUs: vcpus, MemoryMB: memoryMB, }, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go index 5f1548227..4f8a02308 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go @@ -124,9 +124,7 @@ func newNovaRequest(instanceUUID string, evacuation bool, hosts []string) comput InstanceUUID: instanceUUID, NumInstances: 1, HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", VCPUs: 4, MemoryMB: parseMemoryToMB("8Gi"), }, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go index 1c41e796c..047700ebd 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go @@ -69,9 +69,7 @@ func newFailoverReservationRequest(resourceGroup string, hosts []string) compute Intent: api.ReserveForFailoverIntent, ResourceGroup: resourceGroup, HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", VCPUs: 4, MemoryMB: 8192, }, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go index 1ad1f0dc3..e13a1cf58 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go @@ -44,9 +44,7 @@ func newInstanceGroupRequest(policy string, members, hosts []string) compute.Sch InstanceUUID: "new-instance-uuid", NumInstances: 1, HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", VCPUs: 4, MemoryMB: 8192, }, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go index 9b4d642b6..fef4de526 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go @@ -46,9 +46,7 @@ func newPreferSmallerHostsRequest(hosts []string) compute.SchedulerRequest { InstanceUUID: "instance-123", NumInstances: 1, HypervisorFamily: compute.HypervisorFamilyKVM, - HypervisorType: compute.HypervisorTypeQEMU, Flavor: compute.FlavorSpec{ - Name: "m1.large", VCPUs: 4, MemoryMB: 8192, }, diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index a1f01279f..c18ea696f 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -842,7 +842,6 @@ func (c *Reconciler) probeScheduler( ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavor.ExtraSpecs, false), Flavor: sched.FlavorSpec{ - Name: flavor.Name, VCPUs: flavor.VCPUs, MemoryMB: flavor.MemoryMB, }, diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 932fdd859..b0867be5c 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -376,7 +376,6 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavorDetails.ExtraSpecs, false), Flavor: compute.FlavorSpec{ - Name: flavorDetails.Name, VCPUs: flavorDetails.VCPUs, MemoryMB: flavorDetails.MemoryMB, }, diff --git a/internal/scheduling/reservations/extra_specs.go b/internal/scheduling/reservations/extra_specs.go index a63d9bd23..7d878607f 100644 --- a/internal/scheduling/reservations/extra_specs.go +++ b/internal/scheduling/reservations/extra_specs.go @@ -4,8 +4,6 @@ package reservations import ( - "strings" - "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) @@ -15,21 +13,10 @@ import ( // reservation controllers build the typed compute.SchedulerRequest directly // (not via the Nova wire adapter), so they must populate RequiredTraits/ // ForbiddenTraits themselves — otherwise filter_has_requested_traits sees empty -// slices and becomes a no-op. +// slices and becomes a no-op. It delegates to compute.SplitTraits so the trait +// convention lives in one place. func ExtractTraits(extraSpecs map[string]string) (required, forbidden []string) { - for key, value := range extraSpecs { - name, ok := strings.CutPrefix(key, "trait:") - if !ok { - continue - } - switch value { - case "required": - required = append(required, name) - case "forbidden": - forbidden = append(forbidden, name) - } - } - return required, forbidden + return compute.SplitTraits(extraSpecs) } // HypervisorTypeCapability returns the capabilities:hypervisor_type match (QEMU diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index ab6cf5943..4310c1566 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -124,7 +124,6 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(vm.FlavorExtraSpecs, true), Flavor: compute.FlavorSpec{ - Name: resSpec.FlavorName, VCPUs: resSpec.VCPUs, MemoryMB: resSpec.MemoryMB, }, @@ -281,7 +280,6 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavorExtraSpecs, true), Flavor: compute.FlavorSpec{ - Name: vm.FlavorName, VCPUs: vcpus, MemoryMB: memoryMB, }, diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go index f13b814c9..1f696e52d 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding.go @@ -282,7 +282,6 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See // set (no default needed). RequestedCapabilities: reservations.HypervisorTypeCapability(seed.Flavor.ExtraSpecs, false), Flavor: compute.FlavorSpec{ - Name: seed.Flavor.Name, VCPUs: seed.Flavor.VCPUs, MemoryMB: seed.Flavor.MemoryMB, }, diff --git a/internal/scheduling/reservations/failoverhost/pathfinding_test.go b/internal/scheduling/reservations/failoverhost/pathfinding_test.go index 7c91042f8..df9fca88a 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding_test.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding_test.go @@ -53,7 +53,12 @@ type stubPipeline struct { } func (s *stubPipeline) Run(req compute.SchedulerRequest) (compute.SchedulerResult, error) { - flavor := req.Flavor.Name + // Identify which seed this probe is for by the trait the seed constructor + // puts on the flavor (see seed()), which the probe carries in RequiredTraits. + flavor := "" + if len(req.RequiredTraits) > 0 { + flavor = req.RequiredTraits[0] + } if err := req.Options.Validate(); err != nil { s.t.Errorf("invalid scheduling options: %v", err) } @@ -108,9 +113,14 @@ func hostTopo(hostMemGi map[string]int64) topology { return topology{hypervisors: hypervisors} } -// seed is a tiny constructor for a flavor seed named name in az1. +// seed is a tiny constructor for a flavor seed named name in az1. The name is +// also set as a required trait so the stub pipeline can tell which seed a probe +// request belongs to (the probe carries the flavor's traits in RequiredTraits). func seed(name string) Seed { - return Seed{Flavor: Flavor{Name: name}, AvailabilityZone: "az1"} + return Seed{ + Flavor: Flavor{Name: name, ExtraSpecs: map[string]string{"trait:" + name: "required"}}, + AvailabilityZone: "az1", + } } func TestFindPathsErrorsOnLookupFailure(t *testing.T) { diff --git a/internal/scheduling/reservations/failoverhost/seeding.go b/internal/scheduling/reservations/failoverhost/seeding.go index 1af4c98fc..4dd9d7508 100644 --- a/internal/scheduling/reservations/failoverhost/seeding.go +++ b/internal/scheduling/reservations/failoverhost/seeding.go @@ -25,8 +25,8 @@ import ( // fields fed into api.NovaFlavor, so the rest of the flavor-group shape never // leaks into the pathfinder. type Flavor struct { - // Name is the flavor name, forwarded as the probe's flavor name so - // flavor-name-sensitive filters behave as in a real placement. + // Name is the flavor name, used for pipeline inference, trait/capability + // extraction and to label the failover edges a seed supports. Name string // MemoryMB and VCPUs size the probe. RAM is in MiB (despite the "MB" label), // matching what the scheduler expects on NovaFlavor.MemoryMB. From 1ee05585ec063007569b8b4067ed60e2ab3b8978 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 4 Nov 2026 09:47:19 +0100 Subject: [PATCH 16/21] refactor(reservations): consolidate trait and capability handling by utilizing shared functions from the Nova API Signed-off-by: Markus Wieland --- api/external/nova/messages.go | 166 ++++++++++++++++++ internal/scheduling/compute/request.go | 21 --- internal/scheduling/nova/adapter.go | 39 +--- .../reservations/capacity/controller.go | 3 +- .../commitments/reservation_controller.go | 3 +- .../scheduling/reservations/extra_specs.go | 43 +++-- .../reservations/extra_specs_test.go | 37 ++-- .../failover/reservation_scheduling.go | 6 +- .../failover/reservation_scheduling_test.go | 6 +- .../reservations/failoverhost/pathfinding.go | 3 +- 10 files changed, 222 insertions(+), 105 deletions(-) diff --git a/api/external/nova/messages.go b/api/external/nova/messages.go index 33ed4c66f..afe818f32 100644 --- a/api/external/nova/messages.go +++ b/api/external/nova/messages.go @@ -6,10 +6,13 @@ package api import ( "errors" "fmt" + "slices" "strings" + "testing" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -221,6 +224,90 @@ func (req ExternalSchedulerRequest) IsKVMRequest() bool { return hvType == HypervisorTypeQEMU || hvType == HypervisorTypeCH } +// The helpers below resolve Nova's loosely-typed flavor extra-spec string +// conventions into the typed values the scheduling pipeline consumes. They live +// here, with the Nova wire message types, because the "trait:"/"capabilities:"/ +// "accel:" key conventions are Nova's — not the pipeline's. Both the Nova wire +// adapter and the in-process reservation controllers (which hold a flavor's +// extra-spec map directly) call them, so the conventions are understood in exactly +// one place. + +// unsupportedCapabilityOps are the Nova extra-spec comparison operators the +// capabilities filter does not implement. A capability value containing any of +// these means the request uses an advanced match the pipeline can't honor, so +// SplitCapabilities drops capabilities entirely (the filter then becomes a +// no-op), matching Nova's "skip" behavior. +// See: https://github.com/sapcc/nova/blob/3ebf80/nova/scheduler/filters/extra_specs_ops.py#L23 +var unsupportedCapabilityOps = []string{ + "=", "", "", "==", "!=", ">=", "<=", + "s==", "s!=", "s<", "s<=", "s>", "s>=", "", +} + +// SplitTraits splits a flavor's "trait:" extra specs into required and +// forbidden trait names, classified by the "required"/"forbidden" value. +func SplitTraits(extraSpecs map[string]string) (required, forbidden []string) { + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "trait:") + if !ok { + continue + } + switch value { + case "required": + required = append(required, name) + case "forbidden": + forbidden = append(forbidden, name) + } + } + return required, forbidden +} + +// SplitCapabilities extracts a flavor's "capabilities:" extra specs as typed +// compute.Capability equalities (prefix stripped). The "hypervisor_type" value is +// canonicalized (qemu->QEMU, ch->CH) so it matches what hypervisors advertise +// (filter_capabilities compares case-sensitively); all other values pass through +// unchanged. If any value uses an unsupported comparison operator it returns nil so +// the capabilities filter is skipped entirely (matching Nova behavior). +func SplitCapabilities(extraSpecs map[string]string) []compute.Capability { + var caps []compute.Capability + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "capabilities:") + if !ok { + continue + } + for _, op := range unsupportedCapabilityOps { + if strings.Contains(value, op) { + return nil // Unsupported operator: skip the whole filter. + } + } + if name == "hypervisor_type" { + value = CanonicalHypervisorType(value) + } + caps = append(caps, compute.Capability{Name: name, Value: value}) + } + return caps +} + +// CanonicalHypervisorType normalizes a flavor's capabilities:hypervisor_type value +// to the casing hypervisors advertise (QEMU/CH), since filter_capabilities compares +// values case-sensitively. Unrecognized values pass through unchanged. +func CanonicalHypervisorType(value string) string { + switch strings.ToLower(value) { + case "qemu": + return "QEMU" + case "ch": + return "CH" + default: + return value + } +} + +// RequestsAccelerators reports whether a flavor requests a hardware accelerator, +// resolved from the presence of the "accel:device_profile" extra spec. +func RequestsAccelerators(extraSpecs map[string]string) bool { + _, ok := extraSpecs["accel:device_profile"] + return ok +} + const ( // LiveMigrationIntent indicates that the request is intended for live migration. LiveMigrationIntent v1alpha1.SchedulingIntent = "live_migration" @@ -519,3 +606,82 @@ type NovaRequestContext struct { UserName string `json:"user_name"` ProjectName string `json:"project_name"` } + +func TestSplitTraits(t *testing.T) { + required, forbidden := SplitTraits(map[string]string{ + "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required", + "trait:CUSTOM_NO_GPU": "forbidden", + "trait:IGNORED": "preferred", // neither required nor forbidden + "capabilities:hypervisor_type": "qemu", // not a trait + }) + if !slices.Equal(required, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}) { + t.Errorf("required = %v, want [CUSTOM_HANA_EXCLUSIVE_HOST]", required) + } + if !slices.Equal(forbidden, []string{"CUSTOM_NO_GPU"}) { + t.Errorf("forbidden = %v, want [CUSTOM_NO_GPU]", forbidden) + } +} + +func TestSplitCapabilities(t *testing.T) { + sortCaps := func(c []compute.Capability) { + slices.SortFunc(c, func(a, b compute.Capability) int { return strings.Compare(a.Name, b.Name) }) + } + cases := []struct { + name string + extraSpecs map[string]string + want []compute.Capability + }{ + { + name: "canonicalizes lowercase hypervisor_type to match hosts", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, + }, + { + name: "ch canonicalized", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "ch"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, + }, + { + name: "non-hypervisor_type capabilities preserved verbatim", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "QEMU", + "capabilities:cpu_arch": "aarch64", + }, + want: []compute.Capability{ + {Name: "cpu_arch", Value: "aarch64"}, + {Name: "hypervisor_type", Value: "QEMU"}, + }, + }, + { + name: "unsupported operator drops all capabilities", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "QEMU", + "capabilities:cpu_arch": " aarch64 x86_64", + }, + want: nil, + }, + { + name: "non-capability extra specs ignored", + extraSpecs: map[string]string{"trait:CUSTOM_X": "required", "quota:hw_version": "2101"}, + want: nil, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := SplitCapabilities(c.extraSpecs) + sortCaps(got) + if !slices.Equal(got, c.want) { + t.Errorf("got %v, want %v", got, c.want) + } + }) + } +} + +func TestRequestsAccelerators(t *testing.T) { + if !RequestsAccelerators(map[string]string{"accel:device_profile": "gpu-v100"}) { + t.Error("expected accelerators requested when accel:device_profile is present") + } + if RequestsAccelerators(map[string]string{"capabilities:cpu_arch": "x86_64"}) { + t.Error("expected no accelerators requested without accel:device_profile") + } +} diff --git a/internal/scheduling/compute/request.go b/internal/scheduling/compute/request.go index c3145683c..3079cf2e5 100644 --- a/internal/scheduling/compute/request.go +++ b/internal/scheduling/compute/request.go @@ -346,24 +346,3 @@ func (r SchedulerRequest) ForcedHosts() []string { } return matched } - -// SplitTraits splits a flavor's "trait:" extra specs into required and -// forbidden trait names, classified by the "required"/"forbidden" value. It is -// the single place that understands Nova's trait extra-spec convention, so both -// the Nova wire adapter and the reservation controllers (which build a -// SchedulerRequest directly) populate RequiredTraits/ForbiddenTraits the same way. -func SplitTraits(extraSpecs map[string]string) (required, forbidden []string) { - for key, value := range extraSpecs { - name, ok := strings.CutPrefix(key, "trait:") - if !ok { - continue - } - switch value { - case "required": - required = append(required, name) - case "forbidden": - forbidden = append(forbidden, name) - } - } - return required, forbidden -} diff --git a/internal/scheduling/nova/adapter.go b/internal/scheduling/nova/adapter.go index 9d2a19840..08200c718 100644 --- a/internal/scheduling/nova/adapter.go +++ b/internal/scheduling/nova/adapter.go @@ -4,24 +4,11 @@ package nova import ( - "strings" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) -// unsupportedCapabilityOps are the Nova extra-spec comparison operators the -// capabilities filter does not implement. A capability value containing any of -// these means the request uses an advanced match the pipeline can't honor, so -// we drop capabilities entirely (the filter then becomes a no-op), matching the -// old filter's "skip" behavior. -// See: https://github.com/sapcc/nova/blob/3ebf80/nova/scheduler/filters/extra_specs_ops.py#L23 -var unsupportedCapabilityOps = []string{ - "=", "", "", "==", "!=", ">=", "<=", - "s==", "s!=", "s<", "s<=", "s>", "s>=", "", -} - // ToSchedulerRequest translates the Nova over-the-wire request into the typed, // Nova-agnostic compute.SchedulerRequest the pipeline consumes. This is the one // place that reads Nova's loosely-typed scheduler hints and flavor/image extra @@ -83,9 +70,9 @@ func ToSchedulerRequest( req.ImageHypervisorType = toImageHypervisorType(spec.Image.Data) // Traits, capabilities, accelerators from flavor extra specs. - req.RequiredTraits, req.ForbiddenTraits = compute.SplitTraits(spec.Flavor.Data.ExtraSpecs) - req.RequestedCapabilities = toCapabilities(spec.Flavor.Data.ExtraSpecs) - _, req.AcceleratorsRequested = spec.Flavor.Data.ExtraSpecs["accel:device_profile"] + req.RequiredTraits, req.ForbiddenTraits = api.SplitTraits(spec.Flavor.Data.ExtraSpecs) + req.RequestedCapabilities = api.SplitCapabilities(spec.Flavor.Data.ExtraSpecs) + req.AcceleratorsRequested = api.RequestsAccelerators(spec.Flavor.Data.ExtraSpecs) // Nested objects. req.RequestedDestination = toRequestedDestination(spec.RequestedDestination) @@ -147,26 +134,6 @@ func toImageHypervisorType(image api.NovaImageMeta) compute.ImageHypervisorType return compute.ImageHypervisorTypeUnset } -// toCapabilities extracts the "capabilities:" extra specs as typed Capability -// equalities. If any value uses an unsupported comparison operator, it returns -// nil so the capabilities filter is skipped entirely (matching Nova behavior). -func toCapabilities(extraSpecs map[string]string) []compute.Capability { - var caps []compute.Capability - for key, value := range extraSpecs { - name, ok := strings.CutPrefix(key, "capabilities:") - if !ok { - continue - } - for _, op := range unsupportedCapabilityOps { - if strings.Contains(value, op) { - return nil // Unsupported operator: skip the whole filter. - } - } - caps = append(caps, compute.Capability{Name: name, Value: value}) - } - return caps -} - func toRequestedDestination(rd *api.NovaObject[api.NovaRequestedDestination]) *compute.RequestedDestination { if rd == nil { return nil diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index c18ea696f..991d7be1a 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -826,7 +826,7 @@ func (c *Reconciler) probeScheduler( // filter_has_requested_traits/filter_capabilities would otherwise see empty // slices and become no-ops). No hypervisor-type default — only match when the // flavor names one. - requiredTraits, forbiddenTraits := reservations.ExtractTraits(flavor.ExtraSpecs) + requiredTraits, forbiddenTraits := schedulerapi.SplitTraits(flavor.ExtraSpecs) out, err := c.scheduler.RunWithoutLock(ctx, sched.SchedulerRequest{ Pipeline: pipeline, GatherAllCandidates: true, @@ -841,6 +841,7 @@ func (c *Reconciler) probeScheduler( RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavor.ExtraSpecs, false), + AcceleratorsRequested: schedulerapi.RequestsAccelerators(flavor.ExtraSpecs), Flavor: sched.FlavorSpec{ VCPUs: flavor.VCPUs, MemoryMB: flavor.MemoryMB, diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index b0867be5c..9aca58de8 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -359,7 +359,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // request is built directly, so filter_has_requested_traits/filter_capabilities // would otherwise see empty slices and become no-ops). No hypervisor-type // default here — only match when the flavor names one. - requiredTraits, forbiddenTraits := reservations.ExtractTraits(flavorDetails.ExtraSpecs) + requiredTraits, forbiddenTraits := schedulerdelegationapi.SplitTraits(flavorDetails.ExtraSpecs) schedReq := compute.SchedulerRequest{ Pipeline: pipelineName, GatherAllCandidates: true, @@ -375,6 +375,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavorDetails.ExtraSpecs, false), + AcceleratorsRequested: schedulerdelegationapi.RequestsAccelerators(flavorDetails.ExtraSpecs), Flavor: compute.FlavorSpec{ VCPUs: flavorDetails.VCPUs, MemoryMB: flavorDetails.MemoryMB, diff --git a/internal/scheduling/reservations/extra_specs.go b/internal/scheduling/reservations/extra_specs.go index 7d878607f..bd252add8 100644 --- a/internal/scheduling/reservations/extra_specs.go +++ b/internal/scheduling/reservations/extra_specs.go @@ -4,33 +4,30 @@ package reservations import ( + api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) -// ExtractTraits splits a flavor's "trait:" extra specs into required and -// forbidden trait names, mirroring the Nova spec adapter so reservation -// scheduling honors the same trait constraints as regular placement. The -// reservation controllers build the typed compute.SchedulerRequest directly -// (not via the Nova wire adapter), so they must populate RequiredTraits/ -// ForbiddenTraits themselves — otherwise filter_has_requested_traits sees empty -// slices and becomes a no-op. It delegates to compute.SplitTraits so the trait -// convention lives in one place. -func ExtractTraits(extraSpecs map[string]string) (required, forbidden []string) { - return compute.SplitTraits(extraSpecs) -} +// The reservation controllers build the typed compute.SchedulerRequest directly +// (not via the Nova wire adapter), yet their flavors still carry Nova's extra-spec +// string conventions. This thin wrapper delegates to the shared parsers in +// api/external/nova so reservation scheduling honors the same capability +// constraints as regular placement, with the conventions understood in one place. -// HypervisorTypeCapability returns the capabilities:hypervisor_type match (QEMU -// vs CH) for a reservation scheduler request, so filter_capabilities keeps a -// workload off an incompatible hypervisor. When the flavor names no hypervisor -// type, it returns "qemu" if defaultQEMU is set (the failover controller seeds -// from running VMs whose flavor may omit it) and nil otherwise. +// HypervisorTypeCapability returns the required (key=value) hypervisor capabilities +// for a reservation scheduler request, so filter_capabilities keeps a workload off +// incompatible hosts. It extracts ALL of the flavor's "capabilities:" extra +// specs (via api.SplitCapabilities, which canonicalizes the hypervisor_type casing +// and drops everything on an unsupported operator) — not just hypervisor_type. +// +// When the flavor names no hypervisor type and defaultQEMU is set (the failover +// controller seeds from running VMs whose flavor may omit it), the canonical "QEMU" +// hypervisor_type is added. Without defaultQEMU, a missing hypervisor type adds no +// hypervisor_type constraint. func HypervisorTypeCapability(extraSpecs map[string]string, defaultQEMU bool) []compute.Capability { - hvType := extraSpecs["capabilities:hypervisor_type"] - if hvType == "" { - if !defaultQEMU { - return nil - } - hvType = "qemu" + caps := api.SplitCapabilities(extraSpecs) + if extraSpecs["capabilities:hypervisor_type"] == "" && defaultQEMU { + caps = append(caps, compute.Capability{Name: "hypervisor_type", Value: "QEMU"}) } - return []compute.Capability{{Name: "hypervisor_type", Value: hvType}} + return caps } diff --git a/internal/scheduling/reservations/extra_specs_test.go b/internal/scheduling/reservations/extra_specs_test.go index c1f13356d..c268b243e 100644 --- a/internal/scheduling/reservations/extra_specs_test.go +++ b/internal/scheduling/reservations/extra_specs_test.go @@ -5,26 +5,12 @@ package reservations import ( "slices" + "strings" "testing" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) -func TestExtractTraits(t *testing.T) { - required, forbidden := ExtractTraits(map[string]string{ - "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required", - "trait:CUSTOM_NO_GPU": "forbidden", - "trait:IGNORED": "preferred", // neither required nor forbidden - "capabilities:hypervisor_type": "qemu", // not a trait - }) - if !slices.Equal(required, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}) { - t.Errorf("required = %v, want [CUSTOM_HANA_EXCLUSIVE_HOST]", required) - } - if !slices.Equal(forbidden, []string{"CUSTOM_NO_GPU"}) { - t.Errorf("forbidden = %v, want [CUSTOM_NO_GPU]", forbidden) - } -} - func TestHypervisorTypeCapability(t *testing.T) { tests := []struct { name string @@ -37,16 +23,21 @@ func TestHypervisorTypeCapability(t *testing.T) { extraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}, want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, }, + { + name: "lowercase value canonicalized to match hosts", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, + }, { name: "missing without default -> nil", extraSpecs: map[string]string{}, want: nil, }, { - name: "missing with default -> qemu", + name: "missing with default -> canonical QEMU", extraSpecs: map[string]string{}, defaultQEMU: true, - want: []compute.Capability{{Name: "hypervisor_type", Value: "qemu"}}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, }, { name: "explicit value wins over default", @@ -54,10 +45,22 @@ func TestHypervisorTypeCapability(t *testing.T) { defaultQEMU: true, want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, }, + { + name: "non-hypervisor_type capabilities preserved", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "qemu", + "capabilities:cpu_arch": "aarch64", + }, + want: []compute.Capability{ + {Name: "cpu_arch", Value: "aarch64"}, + {Name: "hypervisor_type", Value: "QEMU"}, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := HypervisorTypeCapability(tt.extraSpecs, tt.defaultQEMU) + slices.SortFunc(got, func(a, b compute.Capability) int { return strings.Compare(a.Name, b.Name) }) if !slices.Equal(got, tt.want) { t.Errorf("got %v, want %v", got, tt.want) } diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 4310c1566..2b7f5edc3 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -105,7 +105,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex hosts = append(hosts, compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) weights[h.ComputeHost] = 0.0 } - requiredTraits, forbiddenTraits := reservations.ExtractTraits(vm.FlavorExtraSpecs) + requiredTraits, forbiddenTraits := api.SplitTraits(vm.FlavorExtraSpecs) schedReq := compute.SchedulerRequest{ Pipeline: pipeline, GatherAllCandidates: true, @@ -123,6 +123,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(vm.FlavorExtraSpecs, true), + AcceleratorsRequested: api.RequestsAccelerators(vm.FlavorExtraSpecs), Flavor: compute.FlavorSpec{ VCPUs: resSpec.VCPUs, MemoryMB: resSpec.MemoryMB, @@ -259,7 +260,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( // match it in the reservation's allocations. KVM gathers all candidates; the // reservation host must come back ranked for the VM to be compatible. pipeline := InferFailoverPipeline(flavorExtraSpecs) - requiredTraits, forbiddenTraits := reservations.ExtractTraits(flavorExtraSpecs) + requiredTraits, forbiddenTraits := api.SplitTraits(flavorExtraSpecs) evacuateOpts := DefaultFailoverOptions evacuateOpts.ReadOnly = true evacuateOpts.LockReservations = true @@ -279,6 +280,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavorExtraSpecs, true), + AcceleratorsRequested: api.RequestsAccelerators(flavorExtraSpecs), Flavor: compute.FlavorSpec{ VCPUs: vcpus, MemoryMB: memoryMB, diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go index 2295952c1..10d9399ac 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling_test.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -608,12 +608,12 @@ func TestFailoverSchedulerFlavorConstraints(t *testing.T) { wantHVType string wantTraits []string }{ - {name: "flavor without hypervisor_type defaults to qemu", extraSpecs: nil, wantHVType: "qemu"}, + {name: "flavor without hypervisor_type defaults to canonical QEMU", extraSpecs: nil, wantHVType: "QEMU"}, {name: "explicit CH preserved", extraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}, wantHVType: "CH"}, { - name: "required trait is forwarded", + name: "lowercase qemu canonicalized and trait forwarded", extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu", "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required"}, - wantHVType: "qemu", + wantHVType: "QEMU", wantTraits: []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}, }, } diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go index 1f696e52d..a27ae29ea 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding.go @@ -253,7 +253,7 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See // considers compatible hosts (filter_has_requested_traits/filter_capabilities // would otherwise see empty slices and become no-ops). The seeder only emits // seeds whose flavor names a supported hypervisor type, so no default is needed. - requiredTraits, forbiddenTraits := reservations.ExtractTraits(seed.Flavor.ExtraSpecs) + requiredTraits, forbiddenTraits := api.SplitTraits(seed.Flavor.ExtraSpecs) req := compute.SchedulerRequest{ // Same HANA vs general-purpose split as regular failover placement. Pipeline: failover.InferFailoverPipeline(seed.Flavor.ExtraSpecs), @@ -281,6 +281,7 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See // seeds whose flavor names a supported hypervisor type, so this is always // set (no default needed). RequestedCapabilities: reservations.HypervisorTypeCapability(seed.Flavor.ExtraSpecs, false), + AcceleratorsRequested: api.RequestsAccelerators(seed.Flavor.ExtraSpecs), Flavor: compute.FlavorSpec{ VCPUs: seed.Flavor.VCPUs, MemoryMB: seed.Flavor.MemoryMB, From 6e3b3fea3d0712d48d86e41a078dd6fe0f452fb3 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 4 Nov 2026 09:51:38 +0100 Subject: [PATCH 17/21] refactor(tests): add unit tests for SplitTraits and SplitCapabilities functions Signed-off-by: Markus Wieland --- api/external/nova/messages.go | 81 ----------------------------- api/external/nova/messages_test.go | 82 ++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 81 deletions(-) diff --git a/api/external/nova/messages.go b/api/external/nova/messages.go index afe818f32..7d89c9176 100644 --- a/api/external/nova/messages.go +++ b/api/external/nova/messages.go @@ -6,9 +6,7 @@ package api import ( "errors" "fmt" - "slices" "strings" - "testing" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -606,82 +604,3 @@ type NovaRequestContext struct { UserName string `json:"user_name"` ProjectName string `json:"project_name"` } - -func TestSplitTraits(t *testing.T) { - required, forbidden := SplitTraits(map[string]string{ - "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required", - "trait:CUSTOM_NO_GPU": "forbidden", - "trait:IGNORED": "preferred", // neither required nor forbidden - "capabilities:hypervisor_type": "qemu", // not a trait - }) - if !slices.Equal(required, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}) { - t.Errorf("required = %v, want [CUSTOM_HANA_EXCLUSIVE_HOST]", required) - } - if !slices.Equal(forbidden, []string{"CUSTOM_NO_GPU"}) { - t.Errorf("forbidden = %v, want [CUSTOM_NO_GPU]", forbidden) - } -} - -func TestSplitCapabilities(t *testing.T) { - sortCaps := func(c []compute.Capability) { - slices.SortFunc(c, func(a, b compute.Capability) int { return strings.Compare(a.Name, b.Name) }) - } - cases := []struct { - name string - extraSpecs map[string]string - want []compute.Capability - }{ - { - name: "canonicalizes lowercase hypervisor_type to match hosts", - extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}, - want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, - }, - { - name: "ch canonicalized", - extraSpecs: map[string]string{"capabilities:hypervisor_type": "ch"}, - want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, - }, - { - name: "non-hypervisor_type capabilities preserved verbatim", - extraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - "capabilities:cpu_arch": "aarch64", - }, - want: []compute.Capability{ - {Name: "cpu_arch", Value: "aarch64"}, - {Name: "hypervisor_type", Value: "QEMU"}, - }, - }, - { - name: "unsupported operator drops all capabilities", - extraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - "capabilities:cpu_arch": " aarch64 x86_64", - }, - want: nil, - }, - { - name: "non-capability extra specs ignored", - extraSpecs: map[string]string{"trait:CUSTOM_X": "required", "quota:hw_version": "2101"}, - want: nil, - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got := SplitCapabilities(c.extraSpecs) - sortCaps(got) - if !slices.Equal(got, c.want) { - t.Errorf("got %v, want %v", got, c.want) - } - }) - } -} - -func TestRequestsAccelerators(t *testing.T) { - if !RequestsAccelerators(map[string]string{"accel:device_profile": "gpu-v100"}) { - t.Error("expected accelerators requested when accel:device_profile is present") - } - if RequestsAccelerators(map[string]string{"capabilities:cpu_arch": "x86_64"}) { - t.Error("expected no accelerators requested without accel:device_profile") - } -} diff --git a/api/external/nova/messages_test.go b/api/external/nova/messages_test.go index b76cddf03..f575da45e 100644 --- a/api/external/nova/messages_test.go +++ b/api/external/nova/messages_test.go @@ -5,9 +5,12 @@ package api import ( "encoding/json" + "slices" + "strings" "testing" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) func TestGetIntent(t *testing.T) { @@ -744,3 +747,82 @@ func TestForcedHosts(t *testing.T) { }) } } + +func TestSplitTraits(t *testing.T) { + required, forbidden := SplitTraits(map[string]string{ + "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required", + "trait:CUSTOM_NO_GPU": "forbidden", + "trait:IGNORED": "preferred", // neither required nor forbidden + "capabilities:hypervisor_type": "qemu", // not a trait + }) + if !slices.Equal(required, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}) { + t.Errorf("required = %v, want [CUSTOM_HANA_EXCLUSIVE_HOST]", required) + } + if !slices.Equal(forbidden, []string{"CUSTOM_NO_GPU"}) { + t.Errorf("forbidden = %v, want [CUSTOM_NO_GPU]", forbidden) + } +} + +func TestSplitCapabilities(t *testing.T) { + sortCaps := func(c []compute.Capability) { + slices.SortFunc(c, func(a, b compute.Capability) int { return strings.Compare(a.Name, b.Name) }) + } + cases := []struct { + name string + extraSpecs map[string]string + want []compute.Capability + }{ + { + name: "canonicalizes lowercase hypervisor_type to match hosts", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, + }, + { + name: "ch canonicalized", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "ch"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, + }, + { + name: "non-hypervisor_type capabilities preserved verbatim", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "QEMU", + "capabilities:cpu_arch": "aarch64", + }, + want: []compute.Capability{ + {Name: "cpu_arch", Value: "aarch64"}, + {Name: "hypervisor_type", Value: "QEMU"}, + }, + }, + { + name: "unsupported operator drops all capabilities", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "QEMU", + "capabilities:cpu_arch": " aarch64 x86_64", + }, + want: nil, + }, + { + name: "non-capability extra specs ignored", + extraSpecs: map[string]string{"trait:CUSTOM_X": "required", "quota:hw_version": "2101"}, + want: nil, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := SplitCapabilities(c.extraSpecs) + sortCaps(got) + if !slices.Equal(got, c.want) { + t.Errorf("got %v, want %v", got, c.want) + } + }) + } +} + +func TestRequestsAccelerators(t *testing.T) { + if !RequestsAccelerators(map[string]string{"accel:device_profile": "gpu-v100"}) { + t.Error("expected accelerators requested when accel:device_profile is present") + } + if RequestsAccelerators(map[string]string{"capabilities:cpu_arch": "x86_64"}) { + t.Error("expected no accelerators requested without accel:device_profile") + } +} From 1362643439d52dbd3e2cce0ef8f0493c8f178ed4 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 4 Nov 2026 10:51:15 +0100 Subject: [PATCH 18/21] feat(accounting): break import cycle and collect methods in nova adapter Signed-off-by: Markus Wieland --- api/external/nova/messages.go | 85 ----------------- api/external/nova/messages_test.go | 82 ----------------- internal/scheduling/nova/adapter.go | 91 ++++++++++++++++++- internal/scheduling/nova/adapter_test.go | 91 +++++++++++++++++++ internal/scheduling/nova/crs/recorder.go | 6 +- .../filters/filter_has_enough_capacity.go | 4 +- .../accounting.go} | 7 +- .../accounting_test.go} | 2 +- .../{ => accounting}/flavor_groups.go | 2 +- .../reservations/capacity/controller.go | 12 ++- .../commitments/api/change_commitments.go | 7 +- .../reservations/commitments/api/info.go | 3 +- .../reservations/commitments/api/quota.go | 4 +- .../reservations/commitments/capacity.go | 4 +- .../committed_resource_controller.go | 5 +- .../host_oversubscription_controller.go | 7 +- .../commitments/integration_test.go | 3 +- .../commitments/reservation_controller.go | 12 ++- .../reservations/commitments/syncer.go | 4 +- .../reservations/commitments/usage.go | 3 +- .../commitments/usage_reconciler.go | 3 +- .../scheduling/reservations/extra_specs.go | 13 +-- .../reservations/failover/controller.go | 3 +- .../reservations/failover/helpers.go | 3 +- .../failover/reservation_scheduling.go | 9 +- .../reservations/failoverhost/pathfinding.go | 5 +- .../reservations/quota/controller.go | 9 +- 27 files changed, 255 insertions(+), 224 deletions(-) create mode 100644 internal/scheduling/nova/adapter_test.go rename internal/scheduling/reservations/{capacity_accounting.go => accounting/accounting.go} (94%) rename internal/scheduling/reservations/{capacity_accounting_test.go => accounting/accounting_test.go} (99%) rename internal/scheduling/reservations/{ => accounting}/flavor_groups.go (99%) diff --git a/api/external/nova/messages.go b/api/external/nova/messages.go index 7d89c9176..33ed4c66f 100644 --- a/api/external/nova/messages.go +++ b/api/external/nova/messages.go @@ -10,7 +10,6 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -222,90 +221,6 @@ func (req ExternalSchedulerRequest) IsKVMRequest() bool { return hvType == HypervisorTypeQEMU || hvType == HypervisorTypeCH } -// The helpers below resolve Nova's loosely-typed flavor extra-spec string -// conventions into the typed values the scheduling pipeline consumes. They live -// here, with the Nova wire message types, because the "trait:"/"capabilities:"/ -// "accel:" key conventions are Nova's — not the pipeline's. Both the Nova wire -// adapter and the in-process reservation controllers (which hold a flavor's -// extra-spec map directly) call them, so the conventions are understood in exactly -// one place. - -// unsupportedCapabilityOps are the Nova extra-spec comparison operators the -// capabilities filter does not implement. A capability value containing any of -// these means the request uses an advanced match the pipeline can't honor, so -// SplitCapabilities drops capabilities entirely (the filter then becomes a -// no-op), matching Nova's "skip" behavior. -// See: https://github.com/sapcc/nova/blob/3ebf80/nova/scheduler/filters/extra_specs_ops.py#L23 -var unsupportedCapabilityOps = []string{ - "=", "", "", "==", "!=", ">=", "<=", - "s==", "s!=", "s<", "s<=", "s>", "s>=", "", -} - -// SplitTraits splits a flavor's "trait:" extra specs into required and -// forbidden trait names, classified by the "required"/"forbidden" value. -func SplitTraits(extraSpecs map[string]string) (required, forbidden []string) { - for key, value := range extraSpecs { - name, ok := strings.CutPrefix(key, "trait:") - if !ok { - continue - } - switch value { - case "required": - required = append(required, name) - case "forbidden": - forbidden = append(forbidden, name) - } - } - return required, forbidden -} - -// SplitCapabilities extracts a flavor's "capabilities:" extra specs as typed -// compute.Capability equalities (prefix stripped). The "hypervisor_type" value is -// canonicalized (qemu->QEMU, ch->CH) so it matches what hypervisors advertise -// (filter_capabilities compares case-sensitively); all other values pass through -// unchanged. If any value uses an unsupported comparison operator it returns nil so -// the capabilities filter is skipped entirely (matching Nova behavior). -func SplitCapabilities(extraSpecs map[string]string) []compute.Capability { - var caps []compute.Capability - for key, value := range extraSpecs { - name, ok := strings.CutPrefix(key, "capabilities:") - if !ok { - continue - } - for _, op := range unsupportedCapabilityOps { - if strings.Contains(value, op) { - return nil // Unsupported operator: skip the whole filter. - } - } - if name == "hypervisor_type" { - value = CanonicalHypervisorType(value) - } - caps = append(caps, compute.Capability{Name: name, Value: value}) - } - return caps -} - -// CanonicalHypervisorType normalizes a flavor's capabilities:hypervisor_type value -// to the casing hypervisors advertise (QEMU/CH), since filter_capabilities compares -// values case-sensitively. Unrecognized values pass through unchanged. -func CanonicalHypervisorType(value string) string { - switch strings.ToLower(value) { - case "qemu": - return "QEMU" - case "ch": - return "CH" - default: - return value - } -} - -// RequestsAccelerators reports whether a flavor requests a hardware accelerator, -// resolved from the presence of the "accel:device_profile" extra spec. -func RequestsAccelerators(extraSpecs map[string]string) bool { - _, ok := extraSpecs["accel:device_profile"] - return ok -} - const ( // LiveMigrationIntent indicates that the request is intended for live migration. LiveMigrationIntent v1alpha1.SchedulingIntent = "live_migration" diff --git a/api/external/nova/messages_test.go b/api/external/nova/messages_test.go index f575da45e..b76cddf03 100644 --- a/api/external/nova/messages_test.go +++ b/api/external/nova/messages_test.go @@ -5,12 +5,9 @@ package api import ( "encoding/json" - "slices" - "strings" "testing" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) func TestGetIntent(t *testing.T) { @@ -747,82 +744,3 @@ func TestForcedHosts(t *testing.T) { }) } } - -func TestSplitTraits(t *testing.T) { - required, forbidden := SplitTraits(map[string]string{ - "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required", - "trait:CUSTOM_NO_GPU": "forbidden", - "trait:IGNORED": "preferred", // neither required nor forbidden - "capabilities:hypervisor_type": "qemu", // not a trait - }) - if !slices.Equal(required, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}) { - t.Errorf("required = %v, want [CUSTOM_HANA_EXCLUSIVE_HOST]", required) - } - if !slices.Equal(forbidden, []string{"CUSTOM_NO_GPU"}) { - t.Errorf("forbidden = %v, want [CUSTOM_NO_GPU]", forbidden) - } -} - -func TestSplitCapabilities(t *testing.T) { - sortCaps := func(c []compute.Capability) { - slices.SortFunc(c, func(a, b compute.Capability) int { return strings.Compare(a.Name, b.Name) }) - } - cases := []struct { - name string - extraSpecs map[string]string - want []compute.Capability - }{ - { - name: "canonicalizes lowercase hypervisor_type to match hosts", - extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}, - want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, - }, - { - name: "ch canonicalized", - extraSpecs: map[string]string{"capabilities:hypervisor_type": "ch"}, - want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, - }, - { - name: "non-hypervisor_type capabilities preserved verbatim", - extraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - "capabilities:cpu_arch": "aarch64", - }, - want: []compute.Capability{ - {Name: "cpu_arch", Value: "aarch64"}, - {Name: "hypervisor_type", Value: "QEMU"}, - }, - }, - { - name: "unsupported operator drops all capabilities", - extraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - "capabilities:cpu_arch": " aarch64 x86_64", - }, - want: nil, - }, - { - name: "non-capability extra specs ignored", - extraSpecs: map[string]string{"trait:CUSTOM_X": "required", "quota:hw_version": "2101"}, - want: nil, - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got := SplitCapabilities(c.extraSpecs) - sortCaps(got) - if !slices.Equal(got, c.want) { - t.Errorf("got %v, want %v", got, c.want) - } - }) - } -} - -func TestRequestsAccelerators(t *testing.T) { - if !RequestsAccelerators(map[string]string{"accel:device_profile": "gpu-v100"}) { - t.Error("expected accelerators requested when accel:device_profile is present") - } - if RequestsAccelerators(map[string]string{"capabilities:cpu_arch": "x86_64"}) { - t.Error("expected no accelerators requested without accel:device_profile") - } -} diff --git a/internal/scheduling/nova/adapter.go b/internal/scheduling/nova/adapter.go index 08200c718..0269bbb72 100644 --- a/internal/scheduling/nova/adapter.go +++ b/internal/scheduling/nova/adapter.go @@ -4,6 +4,8 @@ package nova import ( + "strings" + api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" @@ -70,9 +72,9 @@ func ToSchedulerRequest( req.ImageHypervisorType = toImageHypervisorType(spec.Image.Data) // Traits, capabilities, accelerators from flavor extra specs. - req.RequiredTraits, req.ForbiddenTraits = api.SplitTraits(spec.Flavor.Data.ExtraSpecs) - req.RequestedCapabilities = api.SplitCapabilities(spec.Flavor.Data.ExtraSpecs) - req.AcceleratorsRequested = api.RequestsAccelerators(spec.Flavor.Data.ExtraSpecs) + req.RequiredTraits, req.ForbiddenTraits = SplitTraits(spec.Flavor.Data.ExtraSpecs) + req.RequestedCapabilities = SplitCapabilities(spec.Flavor.Data.ExtraSpecs) + req.AcceleratorsRequested = RequestsAccelerators(spec.Flavor.Data.ExtraSpecs) // Nested objects. req.RequestedDestination = toRequestedDestination(spec.RequestedDestination) @@ -192,3 +194,86 @@ func firstNonEmpty(a, b string) string { } return b } + +// The helpers below resolve Nova's loosely-typed flavor extra-spec string +// conventions into the typed values the scheduling pipeline consumes. They live +// here, with the Nova spec adapter, because the "trait:"/"capabilities:"/"accel:" +// key conventions are Nova's — not the pipeline's. Both this adapter and the +// in-process reservation controllers (which hold a flavor's extra-spec map +// directly) call them, so the conventions are understood in exactly one place. + +// unsupportedCapabilityOps are the Nova extra-spec comparison operators the +// capabilities filter does not implement. A capability value containing any of +// these means the request uses an advanced match the pipeline can't honor, so +// SplitCapabilities drops capabilities entirely (the filter then becomes a +// no-op), matching Nova's "skip" behavior. +// See: https://github.com/sapcc/nova/blob/3ebf80/nova/scheduler/filters/extra_specs_ops.py#L23 +var unsupportedCapabilityOps = []string{ + "=", "", "", "==", "!=", ">=", "<=", + "s==", "s!=", "s<", "s<=", "s>", "s>=", "", +} + +// SplitTraits splits a flavor's "trait:" extra specs into required and +// forbidden trait names, classified by the "required"/"forbidden" value. +func SplitTraits(extraSpecs map[string]string) (required, forbidden []string) { + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "trait:") + if !ok { + continue + } + switch value { + case "required": + required = append(required, name) + case "forbidden": + forbidden = append(forbidden, name) + } + } + return required, forbidden +} + +// SplitCapabilities extracts a flavor's "capabilities:" extra specs as typed +// compute.Capability equalities (prefix stripped). The "hypervisor_type" value is +// canonicalized (qemu->QEMU, ch->CH) so it matches what hypervisors advertise +// (filter_capabilities compares case-sensitively); all other values pass through +// unchanged. If any value uses an unsupported comparison operator it returns nil so +// the capabilities filter is skipped entirely (matching Nova behavior). +func SplitCapabilities(extraSpecs map[string]string) []compute.Capability { + var caps []compute.Capability + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "capabilities:") + if !ok { + continue + } + for _, op := range unsupportedCapabilityOps { + if strings.Contains(value, op) { + return nil // Unsupported operator: skip the whole filter. + } + } + if name == "hypervisor_type" { + value = CanonicalHypervisorType(value) + } + caps = append(caps, compute.Capability{Name: name, Value: value}) + } + return caps +} + +// CanonicalHypervisorType normalizes a flavor's capabilities:hypervisor_type value +// to the casing hypervisors advertise (QEMU/CH), since filter_capabilities compares +// values case-sensitively. Unrecognized values pass through unchanged. +func CanonicalHypervisorType(value string) string { + switch strings.ToLower(value) { + case "qemu": + return "QEMU" + case "ch": + return "CH" + default: + return value + } +} + +// RequestsAccelerators reports whether a flavor requests a hardware accelerator, +// resolved from the presence of the "accel:device_profile" extra spec. +func RequestsAccelerators(extraSpecs map[string]string) bool { + _, ok := extraSpecs["accel:device_profile"] + return ok +} diff --git a/internal/scheduling/nova/adapter_test.go b/internal/scheduling/nova/adapter_test.go new file mode 100644 index 000000000..d45a6659e --- /dev/null +++ b/internal/scheduling/nova/adapter_test.go @@ -0,0 +1,91 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package nova + +import ( + "slices" + "strings" + "testing" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" +) + +func TestSplitTraits(t *testing.T) { + required, forbidden := SplitTraits(map[string]string{ + "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required", + "trait:CUSTOM_NO_GPU": "forbidden", + "trait:IGNORED": "preferred", // neither required nor forbidden + "capabilities:hypervisor_type": "qemu", // not a trait + }) + if !slices.Equal(required, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}) { + t.Errorf("required = %v, want [CUSTOM_HANA_EXCLUSIVE_HOST]", required) + } + if !slices.Equal(forbidden, []string{"CUSTOM_NO_GPU"}) { + t.Errorf("forbidden = %v, want [CUSTOM_NO_GPU]", forbidden) + } +} + +func TestSplitCapabilities(t *testing.T) { + sortCaps := func(c []compute.Capability) { + slices.SortFunc(c, func(a, b compute.Capability) int { return strings.Compare(a.Name, b.Name) }) + } + cases := []struct { + name string + extraSpecs map[string]string + want []compute.Capability + }{ + { + name: "canonicalizes lowercase hypervisor_type to match hosts", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, + }, + { + name: "ch canonicalized", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "ch"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, + }, + { + name: "non-hypervisor_type capabilities preserved verbatim", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "QEMU", + "capabilities:cpu_arch": "aarch64", + }, + want: []compute.Capability{ + {Name: "cpu_arch", Value: "aarch64"}, + {Name: "hypervisor_type", Value: "QEMU"}, + }, + }, + { + name: "unsupported operator drops all capabilities", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "QEMU", + "capabilities:cpu_arch": " aarch64 x86_64", + }, + want: nil, + }, + { + name: "non-capability extra specs ignored", + extraSpecs: map[string]string{"trait:CUSTOM_X": "required", "quota:hw_version": "2101"}, + want: nil, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := SplitCapabilities(c.extraSpecs) + sortCaps(got) + if !slices.Equal(got, c.want) { + t.Errorf("got %v, want %v", got, c.want) + } + }) + } +} + +func TestRequestsAccelerators(t *testing.T) { + if !RequestsAccelerators(map[string]string{"accel:device_profile": "gpu-v100"}) { + t.Error("expected accelerators requested when accel:device_profile is present") + } + if RequestsAccelerators(map[string]string{"capabilities:cpu_arch": "x86_64"}) { + t.Error("expected no accelerators requested without accel:device_profile") + } +} diff --git a/internal/scheduling/nova/crs/recorder.go b/internal/scheduling/nova/crs/recorder.go index 71177213e..fd4f477cb 100644 --- a/internal/scheduling/nova/crs/recorder.go +++ b/internal/scheduling/nova/crs/recorder.go @@ -12,7 +12,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/api/resource" @@ -42,12 +42,12 @@ var errFlavorNotInGroup = errors.New("flavor not in any group") // Returns errFlavorNotInGroup (PAYG) if the flavor is not in any group. // Returns a different error for transient failures (Knowledge CRD unavailable, etc). func (r *Recorder) resolveFlavorGroup(ctx context.Context, flavorName string) (string, *compute.FlavorInGroup, error) { - fgClient := reservations.FlavorGroupKnowledgeClient{Client: r.Client} + fgClient := accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := fgClient.GetAllFlavorGroups(ctx, nil) if err != nil { return "", nil, err } - groupName, flavor, err := reservations.FindFlavorInGroups(flavorName, flavorGroups) + groupName, flavor, err := accounting.FindFlavorInGroups(flavorName, flavorGroups) if err != nil { return "", nil, errFlavorNotInGroup } diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index d1d80b7d8..c42d19c0d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -14,7 +14,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" - resv "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -231,7 +231,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.Sche // Oversize spec-only: if a pending VM is larger than the remaining slot, block its full size. // // FailoverReservations: block = Spec.Resources (always fully blocked). - resourcesToBlock := resv.UnusedReservationCapacity(&reservation, ignoreAllocations) + resourcesToBlock := accounting.UnusedReservationCapacity(&reservation, ignoreAllocations) // Block the calculated resources on each host for host := range hostsToBlock { diff --git a/internal/scheduling/reservations/capacity_accounting.go b/internal/scheduling/reservations/accounting/accounting.go similarity index 94% rename from internal/scheduling/reservations/capacity_accounting.go rename to internal/scheduling/reservations/accounting/accounting.go index f73debff0..edba54dbc 100644 --- a/internal/scheduling/reservations/capacity_accounting.go +++ b/internal/scheduling/reservations/accounting/accounting.go @@ -1,7 +1,12 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package reservations +// Package accounting holds the reservation capacity-accounting math and flavor +// group lookup. Both are pure, self-contained helpers over Reservation/Knowledge +// CRDs with no dependency on the scheduling pipeline, so they can live below both +// the reservation controllers and the Nova scheduler tree without forming an +// import cycle. +package accounting import ( hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" diff --git a/internal/scheduling/reservations/capacity_accounting_test.go b/internal/scheduling/reservations/accounting/accounting_test.go similarity index 99% rename from internal/scheduling/reservations/capacity_accounting_test.go rename to internal/scheduling/reservations/accounting/accounting_test.go index 140b58f8f..939ba7b16 100644 --- a/internal/scheduling/reservations/capacity_accounting_test.go +++ b/internal/scheduling/reservations/accounting/accounting_test.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package reservations +package accounting import ( "testing" diff --git a/internal/scheduling/reservations/flavor_groups.go b/internal/scheduling/reservations/accounting/flavor_groups.go similarity index 99% rename from internal/scheduling/reservations/flavor_groups.go rename to internal/scheduling/reservations/accounting/flavor_groups.go index b2ce7d859..9d115592d 100644 --- a/internal/scheduling/reservations/flavor_groups.go +++ b/internal/scheduling/reservations/accounting/flavor_groups.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package reservations +package accounting import ( "context" diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 991d7be1a..68c8281c8 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -30,7 +30,9 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/go-logr/logr" ) @@ -224,7 +226,7 @@ func (c *Reconciler) reconcileAll(ctx context.Context) error { logger := LoggerFromContext(ctx) startTime := time.Now() - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { return fmt.Errorf("failed to get flavor groups: %w", err) @@ -826,7 +828,7 @@ func (c *Reconciler) probeScheduler( // filter_has_requested_traits/filter_capabilities would otherwise see empty // slices and become no-ops). No hypervisor-type default — only match when the // flavor names one. - requiredTraits, forbiddenTraits := schedulerapi.SplitTraits(flavor.ExtraSpecs) + requiredTraits, forbiddenTraits := nova.SplitTraits(flavor.ExtraSpecs) out, err := c.scheduler.RunWithoutLock(ctx, sched.SchedulerRequest{ Pipeline: pipeline, GatherAllCandidates: true, @@ -841,7 +843,7 @@ func (c *Reconciler) probeScheduler( RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavor.ExtraSpecs, false), - AcceleratorsRequested: schedulerapi.RequestsAccelerators(flavor.ExtraSpecs), + AcceleratorsRequested: nova.RequestsAccelerators(flavor.ExtraSpecs), Flavor: sched.FlavorSpec{ VCPUs: flavor.VCPUs, MemoryMB: flavor.MemoryMB, @@ -919,7 +921,7 @@ func (c *Reconciler) blockedResourcesByHost(ctx context.Context) (map[string]map continue } - resourcesToBlock := reservations.UnusedReservationCapacity(res, false) + resourcesToBlock := accounting.UnusedReservationCapacity(res, false) for host := range hostsToBlock { if blocked[host] == nil { blocked[host] = make(map[string]int64) @@ -990,7 +992,7 @@ func (c *Reconciler) reservedResourcesByGroupAZ( "reservation", res.Name, "type", res.Spec.Type, "az", az) continue } - blockedRes := reservations.UnusedReservationCapacity(res, false) + blockedRes := accounting.UnusedReservationCapacity(res, false) // Route each reservation to exactly one bucket by type; the two are disjoint. bucket := committed if res.Spec.Type == v1alpha1.ReservationTypeFailover { diff --git a/internal/scheduling/reservations/commitments/api/change_commitments.go b/internal/scheduling/reservations/commitments/api/change_commitments.go index e4c04d70a..47a78e653 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments.go @@ -15,6 +15,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/go-logr/logr" @@ -180,7 +181,7 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque } { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} if knowledgeCRD, err := knowledge.Get(ctx); err == nil && knowledgeCRD != nil { var currentVersion int64 = -1 if !knowledgeCRD.Status.LastContentChange.IsZero() { @@ -231,7 +232,7 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque } func (api *HTTPAPI) processCommitmentChanges(ctx context.Context, w http.ResponseWriter, logger logr.Logger, req liquid.CommitmentChangeRequest, resp *liquid.CommitmentChangeResponse) error { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Info("failed to get flavor groups from knowledge extractor", "error", err) @@ -640,7 +641,7 @@ func rollbackCR(ctx context.Context, logger logr.Logger, k8sClient client.Client // performDryRun checks whether the net capacity delta in req can be placed by running // temporary probe CommittedResource CRDs through the controller, then always cleaning them up. func (api *HTTPAPI) performDryRun(ctx context.Context, logger logr.Logger, req liquid.CommitmentChangeRequest, resp *liquid.CommitmentChangeResponse) { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { resp.RejectionReason = "dry run: caches not ready, please retry later" diff --git a/internal/scheduling/reservations/commitments/api/info.go b/internal/scheduling/reservations/commitments/api/info.go index 7ab5bcac8..d4b7957a9 100644 --- a/internal/scheduling/reservations/commitments/api/info.go +++ b/internal/scheduling/reservations/commitments/api/info.go @@ -14,6 +14,7 @@ import ( "time" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" "github.com/go-logr/logr" "github.com/google/uuid" @@ -105,7 +106,7 @@ func mibToGiB(v *uint64) *uint64 { // All flavor groups report usage; only those with fixed RAM/core ratio accept commitments. func (api *HTTPAPI) buildServiceInfo(ctx context.Context, logger logr.Logger) (liquid.ServiceInfo, error) { // Get all flavor groups from Knowledge CRDs - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { // Return -1 as version when knowledge is not ready diff --git a/internal/scheduling/reservations/commitments/api/quota.go b/internal/scheduling/reservations/commitments/api/quota.go index 59d759a9a..ff989fd32 100644 --- a/internal/scheduling/reservations/commitments/api/quota.go +++ b/internal/scheduling/reservations/commitments/api/quota.go @@ -13,7 +13,7 @@ import ( "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/google/uuid" @@ -97,7 +97,7 @@ func (api *HTTPAPI) HandleQuota(w http.ResponseWriter, r *http.Request) { // Fetch flavor groups to determine per-resource RAM unit. // The ProjectQuota CRD stores RAM values in GiB; Limes sends in the declared unit // (slots for fixed-ratio groups, GiB for variable-ratio). Convert on receipt. - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(r.Context(), nil) if err != nil { log.Info("flavor groups not available for quota unit conversion", "error", err.Error()) diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index 78381d0f5..b9f01f8c0 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -15,7 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" ) // ErrCapacityNotReady is returned by CalculateCapacity when one or more FlavorGroupCapacity @@ -64,7 +64,7 @@ func NewCapacityCalculator(client client.Client, conf APIConfig) *CapacityCalcul // - Usage: always None. Limes derives project usage from the separate Report-Usage endpoint, so // a usage value here is unused and would only invite misinterpretation of reserved capacity. func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.ServiceCapacityRequest) (liquid.ServiceCapacityReport, error) { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { return liquid.ServiceCapacityReport{}, fmt.Errorf("failed to get flavor groups: %w", err) diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go index e5101f0d2..4ea44f016 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go @@ -24,6 +24,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" ) @@ -442,7 +443,7 @@ func (r *CommittedResourceController) reconcileCoresHeadroom(ctx context.Context } func (r *CommittedResourceController) applyReservationState(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (*ApplyResult, error) { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { return nil, fmt.Errorf("flavor knowledge not ready: %w", err) @@ -646,7 +647,7 @@ func (r *CommittedResourceController) rollbackToAccepted(ctx context.Context, lo if cr.Status.AcceptedSpec == nil { return r.deleteChildReservations(ctx, cr) } - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { // Can't compute the rollback target — fall back to full delete rather than leaving diff --git a/internal/scheduling/reservations/commitments/host_oversubscription_controller.go b/internal/scheduling/reservations/commitments/host_oversubscription_controller.go index 325640e07..ecb353886 100644 --- a/internal/scheduling/reservations/commitments/host_oversubscription_controller.go +++ b/internal/scheduling/reservations/commitments/host_oversubscription_controller.go @@ -22,6 +22,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -267,7 +268,7 @@ func (r *HostOversubscriptionController) SetupWithManager(mgr ctrl.Manager, mcl // computeViolations returns the per-resource excess (positive quantity) for any resource where // the host is over-subscribed. Returns nil if capacity data is unavailable, empty map if no violation. func computeViolations(allReservations []v1alpha1.Reservation, hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity { - free := reservations.HostFreeCapacity(allReservations, hv) + free := accounting.HostFreeCapacity(allReservations, hv) if free == nil { return nil } @@ -333,7 +334,7 @@ func selectEvictionTarget( } entries := make([]allocatedEntry, len(allocated)) for i, res := range allocated { - unusedCap := reservations.UnusedReservationCapacity(res, false) + unusedCap := accounting.UnusedReservationCapacity(res, false) buckets := make(map[hv1.ResourceName]int64) for rn, total := range res.Spec.Resources { if t := total.Value(); t != 0 { @@ -381,7 +382,7 @@ func evictReservation( logger := LoggerFromContext(ctx).WithValues("component", "host-oversubscription", "reservation", res.Name) - freed := reservations.UnusedReservationCapacity(res, true) + freed := accounting.UnusedReservationCapacity(res, true) old := res.DeepCopy() res.Spec.TargetHost = "" diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go index 5e0bab167..0cc6d3165 100644 --- a/internal/scheduling/reservations/commitments/integration_test.go +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -30,6 +30,7 @@ import ( sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" @@ -1384,7 +1385,7 @@ func TestCROversubscriptionRemediation(t *testing.T) { placedSlots = append(placedSlots, r) } } - free := reservations.HostFreeCapacity(placedSlots, *hv) + free := accounting.HostFreeCapacity(placedSlots, *hv) zero := resource.MustParse("0") for rn, f := range free { if f.Cmp(zero) < 0 { diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 9aca58de8..437e9e78b 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -28,7 +28,9 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/cobaltcore-dev/cortex/pkg/sso" @@ -282,7 +284,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr } // Get flavor details from flavor group knowledge CRD - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { logger.V(1).Info("flavor knowledge not ready, requeueing", @@ -292,7 +294,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr } // Search for the flavor across all flavor groups - flavorGroupName, flavorDetails, err := reservations.FindFlavorInGroups(resourceName, flavorGroups) + flavorGroupName, flavorDetails, err := accounting.FindFlavorInGroups(resourceName, flavorGroups) if err != nil { logger.Error(err, "flavor not found in any flavor group", "resourceName", resourceName) @@ -359,7 +361,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // request is built directly, so filter_has_requested_traits/filter_capabilities // would otherwise see empty slices and become no-ops). No hypervisor-type // default here — only match when the flavor names one. - requiredTraits, forbiddenTraits := schedulerdelegationapi.SplitTraits(flavorDetails.ExtraSpecs) + requiredTraits, forbiddenTraits := nova.SplitTraits(flavorDetails.ExtraSpecs) schedReq := compute.SchedulerRequest{ Pipeline: pipelineName, GatherAllCandidates: true, @@ -375,7 +377,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavorDetails.ExtraSpecs, false), - AcceleratorsRequested: schedulerdelegationapi.RequestsAccelerators(flavorDetails.ExtraSpecs), + AcceleratorsRequested: nova.RequestsAccelerators(flavorDetails.ExtraSpecs), Flavor: compute.FlavorSpec{ VCPUs: flavorDetails.VCPUs, MemoryMB: flavorDetails.MemoryMB, @@ -627,7 +629,7 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte // multiple VMs, or the new host is at capacity, remove this VM so the slot // on the original host remains available for re-use. isSingleVM := len(res.Spec.CommittedResourceReservation.Allocations) == 1 - if isSingleVM && reservations.HostHasCapacityForReservation(allReservations.Items, foundHV, res) { + if isSingleVM && accounting.HostHasCapacityForReservation(allReservations.Items, foundHV, res) { logger.Info("VM live-migrated to host with capacity, updating TargetHost", "vm", vmUUID, "reservation", res.Name, diff --git a/internal/scheduling/reservations/commitments/syncer.go b/internal/scheduling/reservations/commitments/syncer.go index ef5bcf97a..8728972e3 100644 --- a/internal/scheduling/reservations/commitments/syncer.go +++ b/internal/scheduling/reservations/commitments/syncer.go @@ -12,7 +12,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/go-logr/logr" "github.com/sapcc/go-api-declarations/liquid" corev1 "k8s.io/api/core/v1" @@ -248,7 +248,7 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { }() // Check if flavor group knowledge is ready - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: s.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: s.Client} knowledgeCRD, err := knowledge.Get(ctx) if err != nil { logger.Error(err, "failed to check flavor group knowledge readiness") diff --git a/internal/scheduling/reservations/commitments/usage.go b/internal/scheduling/reservations/commitments/usage.go index 90ab813d9..5561b51a4 100644 --- a/internal/scheduling/reservations/commitments/usage.go +++ b/internal/scheduling/reservations/commitments/usage.go @@ -14,6 +14,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/go-logr/logr" "github.com/sapcc/go-api-declarations/liquid" . "go.xyrillian.de/gg/option" @@ -98,7 +99,7 @@ func (c *UsageCalculator) CalculateUsage( allAZs []liquid.AvailabilityZone, ) (liquid.ServiceUsageReport, error) { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { return liquid.ServiceUsageReport{}, fmt.Errorf("failed to get flavor groups: %w", err) diff --git a/internal/scheduling/reservations/commitments/usage_reconciler.go b/internal/scheduling/reservations/commitments/usage_reconciler.go index 0e350036a..0933245ed 100644 --- a/internal/scheduling/reservations/commitments/usage_reconciler.go +++ b/internal/scheduling/reservations/commitments/usage_reconciler.go @@ -25,6 +25,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" ) @@ -123,7 +124,7 @@ func (r *UsageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl logger := log logger.V(1).Info("usage reconcile starting", "trigger", trigger, "generation", cr.Generation) - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { r.Monitor.reconcileDuration.WithLabelValues("error").Observe(time.Since(start).Seconds()) diff --git a/internal/scheduling/reservations/extra_specs.go b/internal/scheduling/reservations/extra_specs.go index bd252add8..c55563f94 100644 --- a/internal/scheduling/reservations/extra_specs.go +++ b/internal/scheduling/reservations/extra_specs.go @@ -4,20 +4,21 @@ package reservations import ( - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" ) // The reservation controllers build the typed compute.SchedulerRequest directly // (not via the Nova wire adapter), yet their flavors still carry Nova's extra-spec -// string conventions. This thin wrapper delegates to the shared parsers in -// api/external/nova so reservation scheduling honors the same capability -// constraints as regular placement, with the conventions understood in one place. +// string conventions. HypervisorTypeCapability reuses the shared parser in +// internal/scheduling/nova and layers the reservation-specific defaultQEMU seeding +// on top, so reservation scheduling honors the same capability constraints as +// regular placement with the conventions understood in one place. // HypervisorTypeCapability returns the required (key=value) hypervisor capabilities // for a reservation scheduler request, so filter_capabilities keeps a workload off // incompatible hosts. It extracts ALL of the flavor's "capabilities:" extra -// specs (via api.SplitCapabilities, which canonicalizes the hypervisor_type casing +// specs (via nova.SplitCapabilities, which canonicalizes the hypervisor_type casing // and drops everything on an unsupported operator) — not just hypervisor_type. // // When the flavor names no hypervisor type and defaultQEMU is set (the failover @@ -25,7 +26,7 @@ import ( // hypervisor_type is added. Without defaultQEMU, a missing hypervisor type adds no // hypervisor_type constraint. func HypervisorTypeCapability(extraSpecs map[string]string, defaultQEMU bool) []compute.Capability { - caps := api.SplitCapabilities(extraSpecs) + caps := nova.SplitCapabilities(extraSpecs) if extraSpecs["capabilities:hypervisor_type"] == "" && defaultQEMU { caps = append(caps, compute.Capability{Name: "hypervisor_type", Value: "QEMU"}) } diff --git a/internal/scheduling/reservations/failover/controller.go b/internal/scheduling/reservations/failover/controller.go index 4de28cca8..69032871f 100644 --- a/internal/scheduling/reservations/failover/controller.go +++ b/internal/scheduling/reservations/failover/controller.go @@ -16,6 +16,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/google/uuid" @@ -347,7 +348,7 @@ func (c *FailoverReservationController) ReconcilePeriodic(ctx context.Context) ( // 6. Fetch flavor groups for reservation sizing (if configured) var flavorGroups map[string]compute.FlavorGroupFeature if c.Config.UseFlavorGroupResources { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} fg, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Info("flavor group knowledge not available, will fall back to VM resources for sizing", diff --git a/internal/scheduling/reservations/failover/helpers.go b/internal/scheduling/reservations/failover/helpers.go index 862674f95..7d2671ad9 100644 --- a/internal/scheduling/reservations/failover/helpers.go +++ b/internal/scheduling/reservations/failover/helpers.go @@ -12,6 +12,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -85,7 +86,7 @@ func resolveVMSpecForScheduling( logger := LoggerFromContext(ctx) if useFlavorGroupResources && flavorGroups != nil { - groupName, _, err := reservations.FindFlavorInGroups(vm.FlavorName, flavorGroups) + groupName, _, err := accounting.FindFlavorInGroups(vm.FlavorName, flavorGroups) if err == nil { fg := flavorGroups[groupName] largest := fg.LargestFlavor diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 2b7f5edc3..333af97bb 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -14,6 +14,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) @@ -105,7 +106,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex hosts = append(hosts, compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) weights[h.ComputeHost] = 0.0 } - requiredTraits, forbiddenTraits := api.SplitTraits(vm.FlavorExtraSpecs) + requiredTraits, forbiddenTraits := nova.SplitTraits(vm.FlavorExtraSpecs) schedReq := compute.SchedulerRequest{ Pipeline: pipeline, GatherAllCandidates: true, @@ -123,7 +124,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(vm.FlavorExtraSpecs, true), - AcceleratorsRequested: api.RequestsAccelerators(vm.FlavorExtraSpecs), + AcceleratorsRequested: nova.RequestsAccelerators(vm.FlavorExtraSpecs), Flavor: compute.FlavorSpec{ VCPUs: resSpec.VCPUs, MemoryMB: resSpec.MemoryMB, @@ -260,7 +261,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( // match it in the reservation's allocations. KVM gathers all candidates; the // reservation host must come back ranked for the VM to be compatible. pipeline := InferFailoverPipeline(flavorExtraSpecs) - requiredTraits, forbiddenTraits := api.SplitTraits(flavorExtraSpecs) + requiredTraits, forbiddenTraits := nova.SplitTraits(flavorExtraSpecs) evacuateOpts := DefaultFailoverOptions evacuateOpts.ReadOnly = true evacuateOpts.LockReservations = true @@ -280,7 +281,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( RequiredTraits: requiredTraits, ForbiddenTraits: forbiddenTraits, RequestedCapabilities: reservations.HypervisorTypeCapability(flavorExtraSpecs, true), - AcceleratorsRequested: api.RequestsAccelerators(flavorExtraSpecs), + AcceleratorsRequested: nova.RequestsAccelerators(flavorExtraSpecs), Flavor: compute.FlavorSpec{ VCPUs: vcpus, MemoryMB: memoryMB, diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go index a27ae29ea..ace3b51a3 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding.go @@ -15,6 +15,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -253,7 +254,7 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See // considers compatible hosts (filter_has_requested_traits/filter_capabilities // would otherwise see empty slices and become no-ops). The seeder only emits // seeds whose flavor names a supported hypervisor type, so no default is needed. - requiredTraits, forbiddenTraits := api.SplitTraits(seed.Flavor.ExtraSpecs) + requiredTraits, forbiddenTraits := nova.SplitTraits(seed.Flavor.ExtraSpecs) req := compute.SchedulerRequest{ // Same HANA vs general-purpose split as regular failover placement. Pipeline: failover.InferFailoverPipeline(seed.Flavor.ExtraSpecs), @@ -281,7 +282,7 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See // seeds whose flavor names a supported hypervisor type, so this is always // set (no default needed). RequestedCapabilities: reservations.HypervisorTypeCapability(seed.Flavor.ExtraSpecs, false), - AcceleratorsRequested: api.RequestsAccelerators(seed.Flavor.ExtraSpecs), + AcceleratorsRequested: nova.RequestsAccelerators(seed.Flavor.ExtraSpecs), Flavor: compute.FlavorSpec{ VCPUs: seed.Flavor.VCPUs, MemoryMB: seed.Flavor.MemoryMB, diff --git a/internal/scheduling/reservations/quota/controller.go b/internal/scheduling/reservations/quota/controller.go index a64ae07a5..dc398f62d 100644 --- a/internal/scheduling/reservations/quota/controller.go +++ b/internal/scheduling/reservations/quota/controller.go @@ -13,6 +13,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -74,7 +75,7 @@ func (c *QuotaController) ReconcilePeriodic(ctx context.Context) error { logger.Info("starting full quota reconcile") // Fetch flavor groups from Knowledge CRD - flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + flavorGroupClient := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Error(err, "failed to get flavor groups") @@ -228,7 +229,7 @@ func (c *QuotaController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl } // Fetch flavor groups for CRUsage computation - flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + flavorGroupClient := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Error(err, "failed to get flavor groups") @@ -267,7 +268,7 @@ func (c *QuotaController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // a ProjectQuota is first created and has no persisted TotalUsage yet. func (c *QuotaController) computeTotalUsageForProject(ctx context.Context, projectID string) (map[string]map[string]int64, error) { // Fetch flavor groups from Knowledge CRD - flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + flavorGroupClient := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) if err != nil { return nil, fmt.Errorf("failed to get flavor groups: %w", err) @@ -362,7 +363,7 @@ func (c *QuotaController) ReconcileHVDiff(ctx context.Context, oldHV, newHV *hv1 logger.V(1).Info("HV instance diff detected", "added", len(added), "removed", len(removed)) // Get flavor groups for mapping - flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + flavorGroupClient := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Error(err, "failed to get flavor groups for incremental update") From 42a5ac5469f070ffa779a8056d185a4b741f67cb Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 4 Nov 2026 13:32:02 +0100 Subject: [PATCH 19/21] refactor(values): remove deprecated Decision and DecisionList gvks from multiple values.yaml files Signed-off-by: Markus Wieland --- helm/bundles/cortex-cinder/values.yaml | 2 -- helm/bundles/cortex-ironcore/values.yaml | 2 -- helm/bundles/cortex-manila/values.yaml | 2 -- helm/bundles/cortex-nova/values.yaml | 2 -- helm/bundles/cortex-pods/values.yaml | 2 -- 5 files changed, 10 deletions(-) diff --git a/helm/bundles/cortex-cinder/values.yaml b/helm/bundles/cortex-cinder/values.yaml index b11cbfc38..98d8e5f4b 100644 --- a/helm/bundles/cortex-cinder/values.yaml +++ b/helm/bundles/cortex-cinder/values.yaml @@ -90,8 +90,6 @@ cortex: &cortex apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/bundles/cortex-ironcore/values.yaml b/helm/bundles/cortex-ironcore/values.yaml index 6f577b04b..e54147f15 100644 --- a/helm/bundles/cortex-ironcore/values.yaml +++ b/helm/bundles/cortex-ironcore/values.yaml @@ -33,8 +33,6 @@ cortex: apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/bundles/cortex-manila/values.yaml b/helm/bundles/cortex-manila/values.yaml index 5d48da8f8..39f177f6d 100644 --- a/helm/bundles/cortex-manila/values.yaml +++ b/helm/bundles/cortex-manila/values.yaml @@ -90,8 +90,6 @@ cortex: &cortex apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 12d9af450..822886bc7 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -121,8 +121,6 @@ cortex: &cortex apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/bundles/cortex-pods/values.yaml b/helm/bundles/cortex-pods/values.yaml index efd5ee9b1..3815f67b8 100644 --- a/helm/bundles/cortex-pods/values.yaml +++ b/helm/bundles/cortex-pods/values.yaml @@ -33,8 +33,6 @@ cortex: apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline From 2bb2cb6d188450691544d15ed791b7b3ede57ae8 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 4 Nov 2026 15:01:45 +0100 Subject: [PATCH 20/21] refactor: Rename Scheduler to Placement Signed-off-by: Markus Wieland --- cmd/manager/main.go | 4 +- .../compute/{scheduler.go => placement.go} | 44 ++++----- .../{scheduler_test.go => placement_test.go} | 90 +++++++++---------- internal/scheduling/compute/request.go | 36 ++++---- internal/scheduling/compute/request_test.go | 18 ++-- internal/scheduling/compute/result.go | 4 +- internal/scheduling/lib/pipeline_result.go | 2 +- internal/scheduling/nova/adapter.go | 6 +- .../scheduling/nova/candidate_gatherer.go | 6 +- .../nova/candidate_gatherer_test.go | 20 ++--- internal/scheduling/nova/cr_metrics_test.go | 6 +- internal/scheduling/nova/crs/recorder.go | 2 +- internal/scheduling/nova/crs/recorder_test.go | 6 +- .../scheduling/nova/external_scheduler_api.go | 12 +-- .../nova/external_scheduler_api_test.go | 42 ++++----- .../filter_weigher_pipeline_controller.go | 10 +-- ...filter_weigher_pipeline_controller_test.go | 2 +- internal/scheduling/nova/integration_test.go | 8 +- .../filters/filter_aggregate_metadata.go | 4 +- .../filters/filter_aggregate_metadata_test.go | 40 ++++----- .../filters/filter_allowed_projects.go | 4 +- .../filters/filter_allowed_projects_test.go | 32 +++---- .../plugins/filters/filter_capabilities.go | 4 +- .../filters/filter_capabilities_test.go | 38 ++++---- .../nova/plugins/filters/filter_correct_az.go | 4 +- .../plugins/filters/filter_correct_az_test.go | 14 +-- .../plugins/filters/filter_exclude_hosts.go | 4 +- .../filters/filter_exclude_hosts_test.go | 36 ++++---- .../filters/filter_external_customer.go | 4 +- .../filters/filter_external_customer_test.go | 38 ++++---- .../filters/filter_has_accelerators.go | 4 +- .../filters/filter_has_accelerators_test.go | 26 +++--- .../filters/filter_has_enough_capacity.go | 4 +- .../filter_has_enough_capacity_test.go | 14 +-- .../filters/filter_has_requested_traits.go | 4 +- .../filter_has_requested_traits_test.go | 34 +++---- .../filters/filter_host_instructions.go | 4 +- .../filters/filter_host_instructions_test.go | 36 ++++---- .../filters/filter_image_properties.go | 4 +- .../filters/filter_image_properties_test.go | 8 +- .../filters/filter_instance_group_affinity.go | 4 +- .../filter_instance_group_affinity_test.go | 30 +++---- .../filter_instance_group_anti_affinity.go | 4 +- ...ilter_instance_group_anti_affinity_test.go | 38 ++++---- .../plugins/filters/filter_live_migratable.go | 4 +- .../filters/filter_live_migratable_test.go | 38 ++++---- .../filters/filter_quota_enforcement.go | 4 +- .../filters/filter_quota_enforcement_test.go | 6 +- .../filters/filter_requested_destination.go | 4 +- .../filter_requested_destination_test.go | 44 ++++----- .../filters/filter_status_conditions.go | 4 +- .../filters/filter_status_conditions_test.go | 24 ++--- .../nova/plugins/filters/zz_index.go | 2 +- .../nova/plugins/weighers/kvm_binpack.go | 6 +- .../nova/plugins/weighers/kvm_binpack_test.go | 8 +- .../kvm_committed_resource_reservation.go | 4 +- ...kvm_committed_resource_reservation_test.go | 10 +-- .../plugins/weighers/kvm_cr_migration_slot.go | 4 +- .../weighers/kvm_cr_migration_slot_test.go | 8 +- .../weighers/kvm_failover_evacuation.go | 4 +- .../weighers/kvm_failover_evacuation_test.go | 6 +- .../kvm_failover_reservation_consolidation.go | 4 +- ...failover_reservation_consolidation_test.go | 6 +- .../kvm_instance_group_soft_affinity.go | 4 +- .../kvm_instance_group_soft_affinity_test.go | 6 +- .../weighers/kvm_prefer_smaller_hosts.go | 4 +- .../weighers/kvm_prefer_smaller_hosts_test.go | 6 +- .../vmware_anti_affinity_noisy_projects.go | 6 +- ...mware_anti_affinity_noisy_projects_test.go | 8 +- .../vmware_avoid_long_term_contended_hosts.go | 6 +- ...re_avoid_long_term_contended_hosts_test.go | 6 +- ...vmware_avoid_short_term_contended_hosts.go | 6 +- ...e_avoid_short_term_contended_hosts_test.go | 6 +- .../nova/plugins/weighers/vmware_binpack.go | 6 +- .../plugins/weighers/vmware_binpack_test.go | 12 +-- .../nova/plugins/weighers/zz_index.go | 2 +- internal/scheduling/nova/vm_handler.go | 2 +- internal/scheduling/nova/vm_handler_test.go | 4 +- .../reservations/capacity/controller.go | 6 +- .../reservations/capacity/controller_test.go | 46 +++++----- .../api/change_commitments_e2e_test.go | 20 ++--- .../commitments/integration_test.go | 30 +++---- .../commitments/reservation_controller.go | 4 +- .../reservation_controller_test.go | 28 +++--- .../scheduling/reservations/extra_specs.go | 2 +- .../reservations/failover/controller.go | 4 +- .../reservations/failover/integration_test.go | 18 ++-- .../failover/reservation_scheduling.go | 6 +- .../failover/reservation_scheduling_test.go | 20 ++--- .../reservations/failoverhost/controller.go | 2 +- .../failoverhost/integration_test.go | 10 +-- .../reservations/failoverhost/pathfinding.go | 4 +- .../failoverhost/pathfinding_test.go | 10 +-- 93 files changed, 619 insertions(+), 619 deletions(-) rename internal/scheduling/compute/{scheduler.go => placement.go} (80%) rename internal/scheduling/compute/{scheduler_test.go => placement_test.go} (68%) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 5dc7cd3f3..59a8b18e6 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -420,7 +420,7 @@ func main() { // nova-pipeline-controllers are enabled and consumed in-process by the Nova // HTTP API and the reservation/commitment/capacity/failover controllers (which // must run in the same process — see the cortex-nova-scheduling deployment). - var novaScheduler *compute.Scheduler + var novaScheduler *compute.Placement if slices.Contains(mainConfig.EnabledControllers, "nova-pipeline-controllers") { featureGates := conf.GetConfigOrDie[nova.FeatureGates]() @@ -441,7 +441,7 @@ func main() { // Shared in-process scheduler: owns the scheduling lock and runs the // pipeline. The Nova HTTP API and the reservation controllers call it // directly (no self-directed HTTP). - novaScheduler = compute.NewScheduler(filterWeigherController, nova.NewCandidateGatherer(multiclusterClient)) + novaScheduler = compute.NewPlacement(filterWeigherController, nova.NewCandidateGatherer(multiclusterClient)) vmHandler := &nova.VirtualMachineHandler{Client: multiclusterClient} crRecorder := crs.Recorder{ Client: multiclusterClient, diff --git a/internal/scheduling/compute/scheduler.go b/internal/scheduling/compute/placement.go similarity index 80% rename from internal/scheduling/compute/scheduler.go rename to internal/scheduling/compute/placement.go index 4d9842e51..7244016b2 100644 --- a/internal/scheduling/compute/scheduler.go +++ b/internal/scheduling/compute/placement.go @@ -17,7 +17,7 @@ import ( // lib.BasePipelineController. type PipelineProvider interface { // GetPipeline returns the initialized pipeline for the given name, if ready. - GetPipeline(name string) (lib.FilterWeigherPipeline[SchedulerRequest], bool) + GetPipeline(name string) (lib.FilterWeigherPipeline[PlacementRequest], bool) } // CandidateGatherer gathers all placement candidates for a request when the @@ -25,27 +25,27 @@ type PipelineProvider interface { type CandidateGatherer interface { // MutateWithAllCandidates overrides the request's hosts and weights with all // available placement candidates. - MutateWithAllCandidates(ctx context.Context, request *SchedulerRequest) error + MutateWithAllCandidates(ctx context.Context, request *PlacementRequest) error } // PreHook is a caller-provided hook executed under the scheduling lock before // the pipeline runs. It may mutate the request (e.g. to create a pessimistic // in-flight reservation and narrow the candidate set). Returning an error aborts // the run before the pipeline executes. A nil PreHook is a no-op. -type PreHook func(ctx context.Context, req *SchedulerRequest) error +type PreHook func(ctx context.Context, req *PlacementRequest) error // PostHook is a caller-provided side effect executed after the pipeline run, // under the scheduling lock (e.g. upserting the VM resource, recording the // decision). It receives the typed pipeline result. Returning an error aborts // the run; non-fatal side effects should log and return nil so the placement // decision still stands. A nil PostHook is a no-op. -type PostHook func(ctx context.Context, req SchedulerRequest, result SchedulerResult) error +type PostHook func(ctx context.Context, req PlacementRequest, result PlacementResult) error -// Scheduler is the single in-process owner of Nova placement decisions. It holds +// Placement is the single in-process owner of Nova placement decisions. It holds // the one shared scheduling lock and runs the pipeline plus a caller-provided // pre-hook and post-hook. It owns no side-effect policy itself — the caller // decides, via the hooks it passes, what happens before and after the run. -type Scheduler struct { +type Placement struct { // mu is the single shared scheduling lock, held across the pre-hook, the // pipeline run, and the post-hook in Run. RunWithoutLock bypasses it. mu sync.Mutex @@ -56,22 +56,22 @@ type Scheduler struct { gatherer CandidateGatherer } -// NewScheduler builds a Scheduler over the given pipeline provider and candidate +// NewPlacement builds a Placement over the given pipeline provider and candidate // gatherer. -func NewScheduler(pipelines PipelineProvider, gatherer CandidateGatherer) *Scheduler { - return &Scheduler{pipelines: pipelines, gatherer: gatherer} +func NewPlacement(pipelines PipelineProvider, gatherer CandidateGatherer) *Placement { + return &Placement{pipelines: pipelines, gatherer: gatherer} } // Run executes the pipeline and hooks while holding the shared scheduling lock // across the whole operation, so concurrent runs observe a consistent view of // shared scheduling state. The pre-hook runs before the pipeline; the post-hook // after. Either hook may be nil. Use this for any run that mutates shared state. -func (s *Scheduler) Run( +func (s *Placement) Run( ctx context.Context, - req SchedulerRequest, + req PlacementRequest, pre PreHook, post PostHook, -) (SchedulerResult, error) { +) (PlacementResult, error) { s.mu.Lock() defer s.mu.Unlock() @@ -82,36 +82,36 @@ func (s *Scheduler) Run( // lock. Use this only for read-only runs (e.g. capacity probes) that do not // mutate shared scheduling state and can safely run concurrently with a locked // run. Either hook may be nil. -func (s *Scheduler) RunWithoutLock( +func (s *Placement) RunWithoutLock( ctx context.Context, - req SchedulerRequest, + req PlacementRequest, pre PreHook, post PostHook, -) (SchedulerResult, error) { +) (PlacementResult, error) { return s.run(ctx, req, pre, post) } // run is the shared body of Run and RunWithoutLock. The caller is responsible // for holding (or deliberately not holding) the lock. -func (s *Scheduler) run( +func (s *Placement) run( ctx context.Context, - req SchedulerRequest, + req PlacementRequest, pre PreHook, post PostHook, -) (SchedulerResult, error) { +) (PlacementResult, error) { log := ctrl.LoggerFrom(ctx) pipeline, ok := s.pipelines.GetPipeline(req.Pipeline) if !ok { - return SchedulerResult{}, fmt.Errorf("pipeline not found or not ready: %q", req.Pipeline) + return PlacementResult{}, fmt.Errorf("pipeline not found or not ready: %q", req.Pipeline) } // The pre-hook runs before the pipeline and may mutate the request. if pre != nil { if err := pre(ctx, &req); err != nil { - return SchedulerResult{}, err + return PlacementResult{}, err } } @@ -120,14 +120,14 @@ func (s *Scheduler) run( if req.GatherAllCandidates { log.V(1).Info("gathering all placement candidates before filtering") if err := s.gatherer.MutateWithAllCandidates(ctx, &req); err != nil { - return SchedulerResult{}, fmt.Errorf("failed to gather all placement candidates: %w", err) + return PlacementResult{}, fmt.Errorf("failed to gather all placement candidates: %w", err) } log.V(1).Info("gathered all placement candidates", "numHosts", len(req.Hosts)) } result, runErr := pipeline.Run(req) if runErr != nil { - return SchedulerResult{}, runErr + return PlacementResult{}, runErr } // The post-hook runs after a successful pipeline run, under the lock. diff --git a/internal/scheduling/compute/scheduler_test.go b/internal/scheduling/compute/placement_test.go similarity index 68% rename from internal/scheduling/compute/scheduler_test.go rename to internal/scheduling/compute/placement_test.go index 74d941f84..ab58e687a 100644 --- a/internal/scheduling/compute/scheduler_test.go +++ b/internal/scheduling/compute/placement_test.go @@ -13,17 +13,17 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -// fakePipeline is a stub lib.FilterWeigherPipeline[SchedulerRequest] that records +// fakePipeline is a stub lib.FilterWeigherPipeline[PlacementRequest] that records // the request it was run with and returns a configurable result/error. The run // function, if set, is invoked inside Run (used to observe lock behavior). type fakePipeline struct { - result SchedulerResult + result PlacementResult err error - onRun func(SchedulerRequest) - lastReq SchedulerRequest + onRun func(PlacementRequest) + lastReq PlacementRequest } -func (p *fakePipeline) Run(req SchedulerRequest) (SchedulerResult, error) { +func (p *fakePipeline) Run(req PlacementRequest) (PlacementResult, error) { p.lastReq = req if p.onRun != nil { p.onRun(req) @@ -35,10 +35,10 @@ func (p *fakePipeline) Run(req SchedulerRequest) (SchedulerResult, error) { // configured name. type fakeProvider struct { name string - pipeline lib.FilterWeigherPipeline[SchedulerRequest] + pipeline lib.FilterWeigherPipeline[PlacementRequest] } -func (f *fakeProvider) GetPipeline(name string) (lib.FilterWeigherPipeline[SchedulerRequest], bool) { +func (f *fakeProvider) GetPipeline(name string) (lib.FilterWeigherPipeline[PlacementRequest], bool) { if name != f.name || f.pipeline == nil { return nil, false } @@ -51,7 +51,7 @@ type fakeGatherer struct { err error } -func (g *fakeGatherer) MutateWithAllCandidates(_ context.Context, req *SchedulerRequest) error { +func (g *fakeGatherer) MutateWithAllCandidates(_ context.Context, req *PlacementRequest) error { g.called = true if g.err != nil { return g.err @@ -60,31 +60,31 @@ func (g *fakeGatherer) MutateWithAllCandidates(_ context.Context, req *Scheduler return nil } -// newTestScheduler builds a Scheduler whose only ready pipeline is named "p". -func newTestScheduler(p *fakePipeline, g CandidateGatherer) *Scheduler { - return NewScheduler(&fakeProvider{name: "p", pipeline: p}, g) +// newTestPlacement builds a Placement whose only ready pipeline is named "p". +func newTestPlacement(p *fakePipeline, g CandidateGatherer) *Placement { + return NewPlacement(&fakeProvider{name: "p", pipeline: p}, g) } func TestRunPipelineNotFound(t *testing.T) { - s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) - _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "missing"}, nil, nil) + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) + _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "missing"}, nil, nil) if err == nil { t.Fatal("expected error for missing pipeline, got nil") } } func TestRunPreThenPost(t *testing.T) { - s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) var order []string - pre := func(_ context.Context, _ *SchedulerRequest) error { + pre := func(_ context.Context, _ *PlacementRequest) error { order = append(order, "pre") return nil } - post := func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { + post := func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { order = append(order, "post") return nil } - if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, pre, post); err != nil { + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, pre, post); err != nil { t.Fatalf("unexpected error: %v", err) } if len(order) != 2 || order[0] != "pre" || order[1] != "post" { @@ -93,17 +93,17 @@ func TestRunPreThenPost(t *testing.T) { } func TestRunNilHooks(t *testing.T) { - s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) - if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, nil); err != nil { + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, nil); err != nil { t.Fatalf("nil hooks should be a no-op, got error: %v", err) } } func TestRunPostHookErrorAborts(t *testing.T) { - s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) boom := errors.New("boom") - post := func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { return boom } - _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, post) + post := func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { return boom } + _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, post) if !errors.Is(err, boom) { t.Fatalf("expected boom error, got %v", err) } @@ -111,10 +111,10 @@ func TestRunPostHookErrorAborts(t *testing.T) { func TestRunPipelineErrorSkipsPostHook(t *testing.T) { boom := errors.New("pipeline failed") - s := newTestScheduler(&fakePipeline{err: boom}, &fakeGatherer{}) + s := newTestPlacement(&fakePipeline{err: boom}, &fakeGatherer{}) var ran bool - post := func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { ran = true; return nil } - _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, post) + post := func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { ran = true; return nil } + _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, post) if !errors.Is(err, boom) { t.Fatalf("expected pipeline error, got %v", err) } @@ -125,17 +125,17 @@ func TestRunPipelineErrorSkipsPostHook(t *testing.T) { func TestRunPreHookRunsBeforePipelineAndMutates(t *testing.T) { var sawHostInPipeline string - p := &fakePipeline{onRun: func(r SchedulerRequest) { + p := &fakePipeline{onRun: func(r PlacementRequest) { if len(r.Hosts) > 0 { sawHostInPipeline = r.Hosts[0].ComputeHost } }} - s := newTestScheduler(p, &fakeGatherer{}) - pre := func(_ context.Context, r *SchedulerRequest) error { + s := newTestPlacement(p, &fakeGatherer{}) + pre := func(_ context.Context, r *PlacementRequest) error { r.Hosts = []Host{{ComputeHost: "from-prehook"}} return nil } - if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, pre, nil); err != nil { + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, pre, nil); err != nil { t.Fatalf("unexpected error: %v", err) } if sawHostInPipeline != "from-prehook" { @@ -145,11 +145,11 @@ func TestRunPreHookRunsBeforePipelineAndMutates(t *testing.T) { func TestRunPreHookErrorAbortsBeforePipeline(t *testing.T) { var pipelineRan bool - p := &fakePipeline{onRun: func(SchedulerRequest) { pipelineRan = true }} - s := newTestScheduler(p, &fakeGatherer{}) + p := &fakePipeline{onRun: func(PlacementRequest) { pipelineRan = true }} + s := newTestPlacement(p, &fakeGatherer{}) boom := errors.New("prehook boom") - pre := func(_ context.Context, _ *SchedulerRequest) error { return boom } - _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, pre, nil) + pre := func(_ context.Context, _ *PlacementRequest) error { return boom } + _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, pre, nil) if !errors.Is(err, boom) { t.Fatalf("expected prehook error, got %v", err) } @@ -161,8 +161,8 @@ func TestRunPreHookErrorAbortsBeforePipeline(t *testing.T) { func TestRunGathersCandidatesWhenRequested(t *testing.T) { g := &fakeGatherer{} p := &fakePipeline{} - s := newTestScheduler(p, g) - req := SchedulerRequest{Pipeline: "p", GatherAllCandidates: true} + s := newTestPlacement(p, g) + req := PlacementRequest{Pipeline: "p", GatherAllCandidates: true} if _, err := s.Run(context.Background(), req, nil, nil); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -176,8 +176,8 @@ func TestRunGathersCandidatesWhenRequested(t *testing.T) { func TestRunDoesNotGatherWithoutGatherAllCandidates(t *testing.T) { g := &fakeGatherer{} - s := newTestScheduler(&fakePipeline{}, g) - if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, nil); err != nil { + s := newTestPlacement(&fakePipeline{}, g) + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, nil); err != nil { t.Fatalf("unexpected error: %v", err) } if g.called { @@ -189,13 +189,13 @@ func TestRunDoesNotGatherWithoutGatherAllCandidates(t *testing.T) { // first Run's post-hook completes — i.e. the lock spans the whole run, not just // the pipeline. func TestRunHoldsLockAcrossPostHook(t *testing.T) { - s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) firstInPost := make(chan struct{}) releaseFirst := make(chan struct{}) secondDone := make(chan struct{}) - var slowPost PostHook = func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { + var slowPost PostHook = func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { close(firstInPost) <-releaseFirst return nil @@ -205,7 +205,7 @@ func TestRunHoldsLockAcrossPostHook(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, slowPost); err != nil { + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, slowPost); err != nil { t.Errorf("first Run: %v", err) } }() @@ -213,7 +213,7 @@ func TestRunHoldsLockAcrossPostHook(t *testing.T) { <-firstInPost // first Run is parked inside its post-hook, still holding the lock. go func() { - if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, nil); err != nil { + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, nil); err != nil { t.Errorf("second Run: %v", err) } close(secondDone) @@ -234,18 +234,18 @@ func TestRunHoldsLockAcrossPostHook(t *testing.T) { // TestRunWithoutLockDoesNotBlock verifies that RunWithoutLock proceeds even while // a locked Run holds the lock inside its post-hook. func TestRunWithoutLockDoesNotBlock(t *testing.T) { - s := newTestScheduler(&fakePipeline{}, &fakeGatherer{}) + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) firstInPost := make(chan struct{}) releaseFirst := make(chan struct{}) - var slowPost PostHook = func(_ context.Context, _ SchedulerRequest, _ SchedulerResult) error { + var slowPost PostHook = func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { close(firstInPost) <-releaseFirst return nil } go func() { - if _, err := s.Run(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, slowPost); err != nil { + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, slowPost); err != nil { t.Errorf("locked Run: %v", err) } }() @@ -253,7 +253,7 @@ func TestRunWithoutLockDoesNotBlock(t *testing.T) { done := make(chan struct{}) go func() { - if _, err := s.RunWithoutLock(context.Background(), SchedulerRequest{Pipeline: "p"}, nil, nil); err != nil { + if _, err := s.RunWithoutLock(context.Background(), PlacementRequest{Pipeline: "p"}, nil, nil); err != nil { t.Errorf("RunWithoutLock: %v", err) } close(done) diff --git a/internal/scheduling/compute/request.go b/internal/scheduling/compute/request.go index 3079cf2e5..82ba20889 100644 --- a/internal/scheduling/compute/request.go +++ b/internal/scheduling/compute/request.go @@ -3,11 +3,11 @@ // Package compute is the in-process entrypoint for Nova placement scheduling. // -// It defines a well-typed, Nova-agnostic SchedulerRequest that the -// filter-weigher pipeline consumes, and a Scheduler that owns the single shared +// It defines a well-typed, Nova-agnostic PlacementRequest that the +// filter-weigher pipeline consumes, and a Placement that owns the single shared // scheduling lock and runs the pipeline plus caller-provided side-effect // closures. Callers that run in the same process (the Nova HTTP adapter, the -// reservation/commitment/capacity/failover controllers) call the Scheduler +// reservation/commitment/capacity/failover controllers) call the Placement // directly instead of POSTing to the external HTTP endpoint. // // The spec deliberately contains NO loosely-typed maps or Nova string @@ -76,7 +76,7 @@ type Capability struct { // FlavorSpec is the typed, Nova-agnostic description of the requested flavor. // It carries no extra-spec map: the adapter resolves every extra spec the -// pipeline needs into the typed fields on this struct and on SchedulerRequest. +// pipeline needs into the typed fields on this struct and on PlacementRequest. type FlavorSpec struct { // MemoryMB is the requested memory per instance. 0 means a zero-memory // instance to capacity/bin-pack steps (fits anywhere), so the adapter sets it. @@ -118,20 +118,20 @@ type InstanceGroupSpec struct { MaxServerPerHost int } -// SchedulerRequest is the well-typed, Nova-agnostic input to the scheduling +// PlacementRequest is the well-typed, Nova-agnostic input to the scheduling // pipeline. Every field is a resolved scalar, enum, or typed collection — there // are no maps of Nova-specific string keys and no values the pipeline must parse. // // Each field documents what leaving it at its zero value means, so the Nova spec // adapter (and any other caller) can see exactly which fields are load-bearing. // -// SchedulerRequest implements lib.FilterWeigherPipelineRequest so it can be the +// PlacementRequest implements lib.FilterWeigherPipelineRequest so it can be the // RequestType of the filter-weigher pipeline. -type SchedulerRequest struct { +type PlacementRequest struct { // Pipeline is the name of the pipeline to execute. Required: an empty name // resolves no pipeline and the run fails with "pipeline not found". Pipeline string - // GatherAllCandidates, when true, tells the Scheduler to ignore the Hosts and + // GatherAllCandidates, when true, tells the Placement to ignore the Hosts and // Weights the caller provided and instead gather ALL available placement // candidates (via the CandidateGatherer) before filtering. It is set for // requests where cortex can enumerate the candidate hosts itself — today the @@ -253,14 +253,14 @@ type SchedulerRequest struct { GlobalRequestID string } -// Compile-time check that SchedulerRequest satisfies the pipeline request contract. -var _ lib.FilterWeigherPipelineRequest = SchedulerRequest{} +// Compile-time check that PlacementRequest satisfies the pipeline request contract. +var _ lib.FilterWeigherPipelineRequest = PlacementRequest{} // GetIntent returns the resolved scheduling intent. -func (r SchedulerRequest) GetIntent() v1alpha1.SchedulingIntent { return r.Intent } +func (r PlacementRequest) GetIntent() v1alpha1.SchedulingIntent { return r.Intent } // GetHosts returns the compute host names of all candidates. -func (r SchedulerRequest) GetHosts() []string { +func (r PlacementRequest) GetHosts() []string { hosts := make([]string, len(r.Hosts)) for i, h := range r.Hosts { hosts[i] = h.ComputeHost @@ -269,13 +269,13 @@ func (r SchedulerRequest) GetHosts() []string { } // GetWeights returns the per-host input weights. -func (r SchedulerRequest) GetWeights() map[string]float64 { return r.Weights } +func (r PlacementRequest) GetWeights() map[string]float64 { return r.Weights } // GetOptions returns the call-time options for this run. -func (r SchedulerRequest) GetOptions() scheduling.Options { return r.Options } +func (r PlacementRequest) GetOptions() scheduling.Options { return r.Options } // GetTraceLogArgs returns alternating key/value pairs for the step trace log. -func (r SchedulerRequest) GetTraceLogArgs() []any { +func (r PlacementRequest) GetTraceLogArgs() []any { return []any{ "greq", r.GlobalRequestID, "req", r.RequestID, @@ -287,7 +287,7 @@ func (r SchedulerRequest) GetTraceLogArgs() []any { // Filter returns a copy of the request narrowed to the given hosts, with the // weights updated to the provided values. Hosts not present in includedHosts are // dropped and won't be reconsidered by later steps. -func (r SchedulerRequest) Filter(includedHosts map[string]float64) lib.FilterWeigherPipelineRequest { +func (r PlacementRequest) Filter(includedHosts map[string]float64) lib.FilterWeigherPipelineRequest { filteredHosts := make([]Host, 0, len(includedHosts)) for _, h := range r.Hosts { if _, ok := includedHosts[h.ComputeHost]; ok { @@ -308,7 +308,7 @@ func (r SchedulerRequest) Filter(includedHosts map[string]float64) lib.FilterWei // request that still carries a non-create intent (rebuild/evacuate/resize) must // be validated by the filters, so it is not treated as forced here. Intent // resolution happens in the adapter; this only inspects the typed Intent field. -func (r SchedulerRequest) IsForcedDestination() bool { +func (r PlacementRequest) IsForcedDestination() bool { if len(r.ForceHosts) == 0 && len(r.ForceNodes) == 0 { return false } @@ -319,7 +319,7 @@ func (r SchedulerRequest) IsForcedDestination() bool { // destinations. force_hosts matches on compute host name (case-insensitive); // force_nodes matches on hypervisor hostname (case-sensitive); when both are set // a host must match both. Order is preserved and compute hosts are deduplicated. -func (r SchedulerRequest) ForcedHosts() []string { +func (r PlacementRequest) ForcedHosts() []string { forceHostSet := make(map[string]bool, len(r.ForceHosts)) for _, h := range r.ForceHosts { forceHostSet[strings.ToLower(h)] = true diff --git a/internal/scheduling/compute/request_test.go b/internal/scheduling/compute/request_test.go index 8f96f9176..8244e7dbe 100644 --- a/internal/scheduling/compute/request_test.go +++ b/internal/scheduling/compute/request_test.go @@ -13,14 +13,14 @@ import ( func TestIsForcedDestination(t *testing.T) { cases := []struct { name string - req SchedulerRequest + req PlacementRequest forced bool }{ - {"no force", SchedulerRequest{}, false}, - {"force host, no intent", SchedulerRequest{ForceHosts: []string{"h1"}}, true}, - {"force node, no intent", SchedulerRequest{ForceNodes: []string{"n1"}}, true}, - {"force host, unknown intent", SchedulerRequest{ForceHosts: []string{"h1"}, Intent: v1alpha1.SchedulingIntentUnknown}, true}, - {"force host, concrete intent must still filter", SchedulerRequest{ForceHosts: []string{"h1"}, Intent: "evacuate"}, false}, + {"no force", PlacementRequest{}, false}, + {"force host, no intent", PlacementRequest{ForceHosts: []string{"h1"}}, true}, + {"force node, no intent", PlacementRequest{ForceNodes: []string{"n1"}}, true}, + {"force host, unknown intent", PlacementRequest{ForceHosts: []string{"h1"}, Intent: v1alpha1.SchedulingIntentUnknown}, true}, + {"force host, concrete intent must still filter", PlacementRequest{ForceHosts: []string{"h1"}, Intent: "evacuate"}, false}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -51,7 +51,7 @@ func TestForcedHosts(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - r := SchedulerRequest{Hosts: hosts, ForceHosts: c.force, ForceNodes: c.nodes} + r := PlacementRequest{Hosts: hosts, ForceHosts: c.force, ForceNodes: c.nodes} got := r.ForcedHosts() if !slices.Equal(got, c.want) { t.Errorf("ForcedHosts() = %v, want %v", got, c.want) @@ -61,13 +61,13 @@ func TestForcedHosts(t *testing.T) { } func TestFilterNarrowsHostsAndWeights(t *testing.T) { - r := SchedulerRequest{ + r := PlacementRequest{ Hosts: []Host{ {ComputeHost: "a"}, {ComputeHost: "b"}, {ComputeHost: "c"}, }, Weights: map[string]float64{"a": 1, "b": 2, "c": 3}, } - filtered := r.Filter(map[string]float64{"a": 10, "c": 30}).(SchedulerRequest) + filtered := r.Filter(map[string]float64{"a": 10, "c": 30}).(PlacementRequest) got := filtered.GetHosts() slices.Sort(got) if !slices.Equal(got, []string{"a", "c"}) { diff --git a/internal/scheduling/compute/result.go b/internal/scheduling/compute/result.go index 5b1ddbe7f..56cfde8a0 100644 --- a/internal/scheduling/compute/result.go +++ b/internal/scheduling/compute/result.go @@ -7,11 +7,11 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -// SchedulerResult is the well-defined output of a scheduling pipeline run: +// PlacementResult is the well-defined output of a scheduling pipeline run: // ordered hosts, the target host, and the input/output weights plus per-step // activations used to rank them. It is an alias of lib.PipelineResult, the // domain-neutral type the generic pipeline returns. The human-readable // explanation is NOT part of the result — it is a concern of writing the // decision onto the scheduled resource (e.g. the Nova VM's CortexSpec), so the // relevant side effect generates it from the result via lib.GenerateExplanation. -type SchedulerResult = lib.PipelineResult +type PlacementResult = lib.PipelineResult diff --git a/internal/scheduling/lib/pipeline_result.go b/internal/scheduling/lib/pipeline_result.go index 156558317..938c08d78 100644 --- a/internal/scheduling/lib/pipeline_result.go +++ b/internal/scheduling/lib/pipeline_result.go @@ -14,7 +14,7 @@ type StepResult struct { // PipelineResult is the domain-neutral output of a filter-weigher pipeline run: // the ordered hosts, the target host, and the input/output weights plus per-step // activations used to rank them. Domain-facing aliases (e.g. -// compute.SchedulerResult) point at this type. +// compute.PlacementResult) point at this type. type PipelineResult struct { // RawInWeights are the input weights to the pipeline, before normalization. RawInWeights map[string]float64 diff --git a/internal/scheduling/nova/adapter.go b/internal/scheduling/nova/adapter.go index 0269bbb72..ce236111d 100644 --- a/internal/scheduling/nova/adapter.go +++ b/internal/scheduling/nova/adapter.go @@ -12,7 +12,7 @@ import ( ) // ToSchedulerRequest translates the Nova over-the-wire request into the typed, -// Nova-agnostic compute.SchedulerRequest the pipeline consumes. This is the one +// Nova-agnostic compute.PlacementRequest the pipeline consumes. This is the one // place that reads Nova's loosely-typed scheduler hints and flavor/image extra // specs; every downstream filter/weigher sees only resolved typed fields. // @@ -22,11 +22,11 @@ func ToSchedulerRequest( wire api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent, pipeline string, -) compute.SchedulerRequest { +) compute.PlacementRequest { spec := wire.Spec.Data - req := compute.SchedulerRequest{ + req := compute.PlacementRequest{ Pipeline: pipeline, Hosts: toComputeHosts(wire.Hosts), Weights: wire.Weights, diff --git a/internal/scheduling/nova/candidate_gatherer.go b/internal/scheduling/nova/candidate_gatherer.go index 8d20cc595..1da100cc1 100644 --- a/internal/scheduling/nova/candidate_gatherer.go +++ b/internal/scheduling/nova/candidate_gatherer.go @@ -16,7 +16,7 @@ import ( // for a given Nova scheduling request. type CandidateGatherer interface { // Gather all placement candidates and mutate the request accordingly. - MutateWithAllCandidates(ctx context.Context, request *compute.SchedulerRequest) error + MutateWithAllCandidates(ctx context.Context, request *compute.PlacementRequest) error } // candidateGatherer is the default implementation of CandidateGatherer @@ -24,7 +24,7 @@ type CandidateGatherer interface { type candidateGatherer struct{ client.Client } // NewCandidateGatherer builds the default Nova candidate gatherer backed by the -// given client. It is handed to the compute.Scheduler, which invokes it when a +// given client. It is handed to the compute.Placement, which invokes it when a // request opts into gathering all placement candidates. func NewCandidateGatherer(c client.Client) CandidateGatherer { return &candidateGatherer{Client: c} @@ -32,7 +32,7 @@ func NewCandidateGatherer(c client.Client) CandidateGatherer { // MutateWithAllCandidates gathers all placement candidates and mutates // the request accordingly. -func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.SchedulerRequest) error { +func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.PlacementRequest) error { // Currently we can only get candidates for kvm placements. if request.HypervisorFamily != compute.HypervisorFamilyKVM { return fmt.Errorf( diff --git a/internal/scheduling/nova/candidate_gatherer_test.go b/internal/scheduling/nova/candidate_gatherer_test.go index 6c9b15e81..2d0fd14c7 100644 --- a/internal/scheduling/nova/candidate_gatherer_test.go +++ b/internal/scheduling/nova/candidate_gatherer_test.go @@ -26,7 +26,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { tests := []struct { name string hypervisors []client.Object - request compute.SchedulerRequest + request compute.PlacementRequest expectError bool errorContains string expectedHosts []string @@ -35,7 +35,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { { name: "unresolved hypervisor family", hypervisors: []client.Object{}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: "", }, expectError: true, @@ -44,7 +44,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { { name: "unsupported hypervisor family vmware", hypervisors: []client.Object{}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: compute.HypervisorFamilyVMware, }, expectError: true, @@ -53,7 +53,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { { name: "unsupported hypervisor family unknown", hypervisors: []client.Object{}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: "unknown-family", }, expectError: true, @@ -78,7 +78,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, @@ -94,7 +94,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, @@ -115,7 +115,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, @@ -131,7 +131,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, @@ -141,7 +141,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { { name: "empty hypervisor list", hypervisors: []client.Object{}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, @@ -157,7 +157,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ HypervisorFamily: compute.HypervisorFamilyKVM, Hosts: []compute.Host{ {ComputeHost: "old-host", HypervisorHostname: "old-hv"}, diff --git a/internal/scheduling/nova/cr_metrics_test.go b/internal/scheduling/nova/cr_metrics_test.go index a7417e382..c158b244f 100644 --- a/internal/scheduling/nova/cr_metrics_test.go +++ b/internal/scheduling/nova/cr_metrics_test.go @@ -362,9 +362,9 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { reg.MustRegister(counter) controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[sched.SchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[sched.PlacementRequest]]{ Client: fakeClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.SchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, } @@ -377,7 +377,7 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { // Drive the committed-resource tracking path through the real HTTP API // handler, where the feature gate now lives (as a post-hook). - scheduler := sched.NewScheduler(controller, passthroughGatherer{}) + scheduler := sched.NewPlacement(controller, passthroughGatherer{}) crRecorder := crs.Recorder{Client: fakeClient, NoHostFoundCounter: counter} api := newTestAPIWithScheduler( HTTPAPIConfig{}, diff --git a/internal/scheduling/nova/crs/recorder.go b/internal/scheduling/nova/crs/recorder.go index fd4f477cb..043003099 100644 --- a/internal/scheduling/nova/crs/recorder.go +++ b/internal/scheduling/nova/crs/recorder.go @@ -56,7 +56,7 @@ func (r *Recorder) resolveFlavorGroup(ctx context.Context, flavorName string) (s // RecordPlacement writes the placed VM UUID into the matching Reservation slot // and emits placement metrics. Called after a successful Nova placement. -func (r *Recorder) RecordPlacement(ctx context.Context, result sched.SchedulerResult, intent v1alpha1.SchedulingIntent, request api.ExternalSchedulerRequest) { +func (r *Recorder) RecordPlacement(ctx context.Context, result sched.PlacementResult, intent v1alpha1.SchedulingIntent, request api.ExternalSchedulerRequest) { log := ctrl.LoggerFrom(ctx) instanceUUID := request.Spec.Data.InstanceUUID diff --git a/internal/scheduling/nova/crs/recorder_test.go b/internal/scheduling/nova/crs/recorder_test.go index 29fee664d..5eee058ff 100644 --- a/internal/scheduling/nova/crs/recorder_test.go +++ b/internal/scheduling/nova/crs/recorder_test.go @@ -105,9 +105,9 @@ func TestRecordCRAllocation(t *testing.T) { } } - makeResult := func(host string, candidates ...string) sched.SchedulerResult { + makeResult := func(host string, candidates ...string) sched.PlacementResult { h := host - return sched.SchedulerResult{ + return sched.PlacementResult{ TargetHost: &h, OrderedHosts: candidates, } @@ -151,7 +151,7 @@ func TestRecordCRAllocation(t *testing.T) { name string objects []client.Object request api.ExternalSchedulerRequest - result sched.SchedulerResult + result sched.PlacementResult intent v1alpha1.SchedulingIntent checkSlot string expectAllocation bool diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index e7836370e..1e14a4ac5 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -49,11 +49,11 @@ type HTTPAPI interface { Init(*http.ServeMux) } -// runner is the subset of compute.Scheduler the HTTP API depends on. It is an -// interface so the handler can be unit-tested with a mock. *compute.Scheduler +// runner is the subset of compute.Placement the HTTP API depends on. It is an +// interface so the handler can be unit-tested with a mock. *compute.Placement // satisfies it. type runner interface { - Run(ctx context.Context, req compute.SchedulerRequest, pre compute.PreHook, post compute.PostHook) (compute.SchedulerResult, error) + Run(ctx context.Context, req compute.PlacementRequest, pre compute.PreHook, post compute.PostHook) (compute.PlacementResult, error) } type httpAPI struct { @@ -66,11 +66,11 @@ type httpAPI struct { } // NewAPI builds the Nova external-scheduler HTTP API. It calls the shared -// compute.Scheduler directly; VM-resource creation and committed-resource +// compute.Placement directly; VM-resource creation and committed-resource // tracking run as post-hooks after the pipeline, under the scheduler's lock. func NewAPI( config HTTPAPIConfig, - scheduler *compute.Scheduler, + scheduler *compute.Placement, vmHandler *VirtualMachineHandler, crRecorder crs.Recorder, featureGates FeatureGates, @@ -280,7 +280,7 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req // Side effects run as a single post-hook after the pipeline, under the // scheduler lock. It operates on the Nova wire request (which carries the // fields the VM resource and CR tracking need) plus the typed result. - postHook := func(ctx context.Context, _ compute.SchedulerRequest, result compute.SchedulerResult) error { + postHook := func(ctx context.Context, _ compute.PlacementRequest, result compute.PlacementResult) error { // VM-resource creation. switch { case requestData.Options.ReadOnly: diff --git a/internal/scheduling/nova/external_scheduler_api_test.go b/internal/scheduling/nova/external_scheduler_api_test.go index 0e21f0538..5a2014b27 100644 --- a/internal/scheduling/nova/external_scheduler_api_test.go +++ b/internal/scheduling/nova/external_scheduler_api_test.go @@ -23,14 +23,14 @@ import ( // receives the mapped request and may invoke the post-hook to exercise the // handler's side-effect path; it returns the result/error the handler sees. type mockRunner struct { - runFunc func(ctx context.Context, req compute.SchedulerRequest, post compute.PostHook) (compute.SchedulerResult, error) + runFunc func(ctx context.Context, req compute.PlacementRequest, post compute.PostHook) (compute.PlacementResult, error) } -func (m *mockRunner) Run(ctx context.Context, req compute.SchedulerRequest, _ compute.PreHook, post compute.PostHook) (compute.SchedulerResult, error) { +func (m *mockRunner) Run(ctx context.Context, req compute.PlacementRequest, _ compute.PreHook, post compute.PostHook) (compute.PlacementResult, error) { if m.runFunc != nil { return m.runFunc(ctx, req, post) } - return compute.SchedulerResult{}, nil + return compute.PlacementResult{}, nil } // newTestAPI builds an *httpAPI with the given config and mock runner, leaving @@ -44,7 +44,7 @@ func newTestAPI(config HTTPAPIConfig, r runner) *httpAPI { } } -// newTestAPIWithScheduler builds an *httpAPI over a real compute.Scheduler plus +// newTestAPIWithScheduler builds an *httpAPI over a real compute.Placement plus // the CR recorder and feature gates, for tests that exercise the committed- // resource post-hook end-to-end. func newTestAPIWithScheduler(config HTTPAPIConfig, scheduler runner, crRecorder crs.Recorder, featureGates FeatureGates) *httpAPI { @@ -61,7 +61,7 @@ func newTestAPIWithScheduler(config HTTPAPIConfig, scheduler runner, crRecorder // unchanged (tests seed their own host sets). type passthroughGatherer struct{} -func (passthroughGatherer) MutateWithAllCandidates(context.Context, *compute.SchedulerRequest) error { +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *compute.PlacementRequest) error { return nil } @@ -169,7 +169,7 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { method string body string processDecisionErr error - result *compute.SchedulerResult + result *compute.PlacementResult expectedStatus int expectedHosts []string }{ @@ -208,7 +208,7 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { } return string(data) }(), - result: &compute.SchedulerResult{ + result: &compute.PlacementResult{ OrderedHosts: []string{"host1"}, }, expectedStatus: http.StatusOK, @@ -273,14 +273,14 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r := &mockRunner{ - runFunc: func(_ context.Context, _ compute.SchedulerRequest, _ compute.PostHook) (compute.SchedulerResult, error) { + runFunc: func(_ context.Context, _ compute.PlacementRequest, _ compute.PostHook) (compute.PlacementResult, error) { if tt.processDecisionErr != nil { - return compute.SchedulerResult{}, tt.processDecisionErr + return compute.PlacementResult{}, tt.processDecisionErr } if tt.result != nil { return *tt.result, nil } - return compute.SchedulerResult{}, nil + return compute.PlacementResult{}, nil }, } @@ -327,8 +327,8 @@ func TestHTTPAPI_NovaExternalScheduler_EmptyHostsSerializedAsArray(t *testing.T) // response must serialize hosts as an empty array, not null, since Nova's // schema requires an array. r := &mockRunner{ - runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.SchedulerResult, error) { - return compute.SchedulerResult{OrderedHosts: nil}, nil + runFunc: func(context.Context, compute.PlacementRequest, compute.PostHook) (compute.PlacementResult, error) { + return compute.PlacementResult{OrderedHosts: nil}, nil }, } api := newTestAPI(HTTPAPIConfig{}, r) @@ -363,11 +363,11 @@ func TestHTTPAPI_NovaExternalScheduler_EmptyHostsSerializedAsArray(t *testing.T) } func TestHTTPAPI_NovaExternalScheduler_RequestMapping(t *testing.T) { - var captured compute.SchedulerRequest + var captured compute.PlacementRequest r := &mockRunner{ - runFunc: func(_ context.Context, req compute.SchedulerRequest, _ compute.PostHook) (compute.SchedulerResult, error) { + runFunc: func(_ context.Context, req compute.PlacementRequest, _ compute.PostHook) (compute.PlacementResult, error) { captured = req - return compute.SchedulerResult{OrderedHosts: []string{"host1"}}, nil + return compute.PlacementResult{OrderedHosts: []string{"host1"}}, nil }, } @@ -435,9 +435,9 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestination(t *testing.T) { schedulerCalled := false r := &mockRunner{ - runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.SchedulerResult, error) { + runFunc: func(context.Context, compute.PlacementRequest, compute.PostHook) (compute.PlacementResult, error) { schedulerCalled = true - return compute.SchedulerResult{}, nil + return compute.PlacementResult{}, nil }, } api := newTestAPI(HTTPAPIConfig{}, r) @@ -485,9 +485,9 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationNoWeights(t *testing.T) schedulerCalled := false r := &mockRunner{ - runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.SchedulerResult, error) { + runFunc: func(context.Context, compute.PlacementRequest, compute.PostHook) (compute.PlacementResult, error) { schedulerCalled = true - return compute.SchedulerResult{}, nil + return compute.PlacementResult{}, nil }, } api := newTestAPI(HTTPAPIConfig{}, r) @@ -538,9 +538,9 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationDisabled(t *testing.T) { schedulerCalled := false r := &mockRunner{ - runFunc: func(context.Context, compute.SchedulerRequest, compute.PostHook) (compute.SchedulerResult, error) { + runFunc: func(context.Context, compute.PlacementRequest, compute.PostHook) (compute.PlacementResult, error) { schedulerCalled = true - return compute.SchedulerResult{}, nil + return compute.PlacementResult{}, nil }, } disabled := false diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index 25bab98a6..7ee25cf55 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -23,13 +23,13 @@ import ( // FilterWeigherPipelineController owns the Nova filter-weigher pipelines: it // watches Pipeline and Knowledge resources to (re)configure the pipelines, and -// exposes them via GetPipeline for the compute.Scheduler to run. Placement -// requests no longer flow through this controller — the compute.Scheduler, +// exposes them via GetPipeline for the compute.Placement to run. Placement +// requests no longer flow through this controller — the compute.Placement, // driven by the external-scheduler HTTP API and the reservation controllers, is // the entrypoint. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. - lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]] + lib.BasePipelineController[lib.FilterWeigherPipeline[compute.PlacementRequest]] // Monitor to pass down to all pipelines. Monitor lib.FilterWeigherPipelineMonitor @@ -43,7 +43,7 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { func (c *FilterWeigherPipelineController) InitPipeline( ctx context.Context, p v1alpha1.Pipeline, -) lib.PipelineInitResult[lib.FilterWeigherPipeline[compute.SchedulerRequest]] { +) lib.PipelineInitResult[lib.FilterWeigherPipeline[compute.PlacementRequest]] { return lib.InitNewFilterWeigherPipeline( ctx, c.Client, p.Name, @@ -118,7 +118,7 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, // Reconcile is a no-op: this controller reconfigures pipelines through its // Pipeline/Knowledge watch handlers, and placement runs through the -// compute.Scheduler, not through object reconciliation. A Reconcile method is +// compute.Placement, not through object reconciliation. A Reconcile method is // required to satisfy the controller builder. func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go index 32079e9e5..26576100c 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go @@ -21,7 +21,7 @@ type mockCandidateGatherer struct { gatheredHosts []compute.Host } -func (m *mockCandidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.SchedulerRequest) error { +func (m *mockCandidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.PlacementRequest) error { m.called = true if m.err != nil { return m.err diff --git a/internal/scheduling/nova/integration_test.go b/internal/scheduling/nova/integration_test.go index 882ccaf4f..d87241bb3 100644 --- a/internal/scheduling/nova/integration_test.go +++ b/internal/scheduling/nova/integration_test.go @@ -250,9 +250,9 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec // Create the pipeline controller with shared monitor to avoid duplicate registration controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.PlacementRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: getSharedMonitor(), @@ -284,11 +284,11 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec controller.Pipelines[testPipeline.Name] = result.Pipeline controller.PipelineConfigs[testPipeline.Name] = testPipeline - // Create the HTTP API backed by a real compute.Scheduler over the controller. + // Create the HTTP API backed by a real compute.Placement over the controller. // Set EvacuationShuffleK=0 to disable shuffle for deterministic test results. // KVM requests trigger candidate gathering; use a passthrough gatherer // (nil gatheredHosts) so the test's preselected hosts are preserved. - scheduler := compute.NewScheduler(controller, &mockCandidateGatherer{}) + scheduler := compute.NewPlacement(controller, &mockCandidateGatherer{}) api := &httpAPI{ config: HTTPAPIConfig{EvacuationShuffleK: 0}, monitor: lib.NewSchedulerMonitor(), // Create new monitor but don't register diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go index a39bf83b5..7a2aa17bc 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go @@ -16,12 +16,12 @@ import ( ) type FilterAggregateMetadata struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Restrict hosts to specific projects if they are in an aggregate that has // the "filter_tenant_id" metadata key set. -func (s *FilterAggregateMetadata) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterAggregateMetadata) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Capacity probe calls are synthetic and not placed on behalf of a tenant // project, so they must ignore tenant pinning. Failover reservations DO respect diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go index fcc4e2774..c23479858 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go @@ -22,14 +22,14 @@ import ( func TestFilterAggregateMetadata_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string }{ { name: "No aggregates with filter_tenant_id - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -62,7 +62,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host with filter_tenant_id matching project - host passes", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -88,7 +88,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host with filter_tenant_id not matching project - host filtered", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-b", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -114,7 +114,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host in multiple aggregates with different filter_tenant_id - project matches one", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-b", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -136,7 +136,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host in multiple aggregates - project matches none", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-c", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -158,7 +158,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Mixed hosts - some restricted, some open", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -198,7 +198,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{}, }, @@ -215,7 +215,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "All hosts filtered out", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-nonexistent", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -241,7 +241,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host not in hypervisor list - passes (no restrictions found)", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -261,7 +261,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Aggregate with nil metadata - treated as no restriction", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -280,7 +280,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id - project matches one of many", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-b", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -299,7 +299,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id - project matches none", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-d", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -318,7 +318,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id with surrounding whitespace - project matches", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-b", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -337,7 +337,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys - project matches one of the numbered keys", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-c", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -360,7 +360,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys with comma-separated values - project matches", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-e", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -383,7 +383,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys - project matches none", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-z", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -448,7 +448,7 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { } func TestFilterAggregateMetadata_Run_ClientError(t *testing.T) { - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -509,7 +509,7 @@ func TestFilterAggregateMetadata_SkipsForCapacityProbeIntent(t *testing.T) { step := &FilterAggregateMetadata{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ ProjectID: "project-y", Intent: api.CapacityProbeIntent, Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, @@ -546,7 +546,7 @@ func TestFilterAggregateMetadata_AppliesForFailoverIntents(t *testing.T) { t.Run(string(intent), func(t *testing.T) { step := &FilterAggregateMetadata{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ ProjectID: "project-y", Intent: intent, Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go index 101acf51a..ee0cac37f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go @@ -15,12 +15,12 @@ import ( ) type FilterAllowedProjectsStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Lock certain hosts for certain projects, based on the hypervisor spec. // Note that hosts without specified projects are still accessible. -func (s *FilterAllowedProjectsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterAllowedProjectsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Capacity probe calls are synthetic and not placed on behalf of a tenant // project, so they must ignore tenant pinning. Failover reservations DO respect diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go index 6bd68cf0d..3b4e9d60a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go @@ -69,13 +69,13 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No project ID in request - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -88,7 +88,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches allowed projects on host2", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -101,7 +101,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches allowed projects on host3 only", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host2"}, @@ -114,7 +114,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project does not match any allowed projects", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-d", Hosts: []compute.Host{ {ComputeHost: "host2"}, @@ -127,7 +127,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Hosts without allowed projects are accessible", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-x", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -139,7 +139,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Mixed hosts - some with empty allowed projects, some with matching project", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-b", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -153,7 +153,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "All hosts filtered out", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-nonexistent", Hosts: []compute.Host{ {ComputeHost: "host2"}, @@ -166,7 +166,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{}, }, @@ -175,7 +175,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches on multiple hosts with allowed projects", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-a", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -190,7 +190,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches second item in allowed projects list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-b", Hosts: []compute.Host{ {ComputeHost: "host2"}, @@ -201,7 +201,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "All hosts have empty allowed projects", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "any-project", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -213,7 +213,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Mixed allowed projects configuration", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-c", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -228,7 +228,7 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Case sensitive project matching", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "Project-A", Hosts: []compute.Host{ {ComputeHost: "host2"}, @@ -290,7 +290,7 @@ func TestFilterAllowedProjectsStep_SkipsForCapacityProbeIntent(t *testing.T) { step := &FilterAllowedProjectsStep{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ ProjectID: "project-y", Intent: api.CapacityProbeIntent, Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, @@ -322,7 +322,7 @@ func TestFilterAllowedProjectsStep_AppliesForFailoverIntents(t *testing.T) { t.Run(string(intent), func(t *testing.T) { step := &FilterAllowedProjectsStep{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ ProjectID: "project-y", Intent: intent, Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, diff --git a/internal/scheduling/nova/plugins/filters/filter_capabilities.go b/internal/scheduling/nova/plugins/filters/filter_capabilities.go index 2a204d963..392ef4d06 100644 --- a/internal/scheduling/nova/plugins/filters/filter_capabilities.go +++ b/internal/scheduling/nova/plugins/filters/filter_capabilities.go @@ -14,7 +14,7 @@ import ( ) type FilterCapabilitiesStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Get the provided capabilities of a hypervisor resource in the format Nova expects. @@ -44,7 +44,7 @@ func hvToNovaCapabilities(hv hv1.Hypervisor) (map[string]string, error) { // Check the capabilities of each host and if they match the extra spec provided // in the request spec flavor. -func (s *FilterCapabilitiesStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterCapabilitiesStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) requestedCapabilities := request.RequestedCapabilities if len(requestedCapabilities) == 0 { diff --git a/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go b/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go index 36fe46525..ce3000d95 100644 --- a/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go @@ -225,14 +225,14 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string expectedEvents []lib.FilterWeigherPipelineStepEvent }{ { name: "No extra specs in request - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -244,7 +244,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Non-capability extra specs - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -255,7 +255,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match CH hypervisor type", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "CH"}, }, @@ -270,7 +270,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match QEMU hypervisor type", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "QEMU"}, }, @@ -286,7 +286,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match x86_64 CPU architecture", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "cpu_arch", Value: "x86_64"}, }, @@ -302,7 +302,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match aarch64 CPU architecture", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "cpu_arch", Value: "aarch64"}, }, @@ -318,7 +318,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match both hypervisor type and CPU architecture", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "CH"}, {Name: "cpu_arch", Value: "x86_64"}, @@ -335,7 +335,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match QEMU with aarch64", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "QEMU"}, {Name: "cpu_arch", Value: "aarch64"}, @@ -352,7 +352,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Ignore hypervisors not in request even with unknown type", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "CH"}, }, @@ -366,7 +366,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Candidate hypervisor with empty hypervisor type emits event and is filtered out", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "CH"}, }, @@ -382,7 +382,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Candidate hypervisor with unknown hypervisor type emits event and is filtered out", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "CH"}, }, @@ -398,7 +398,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "No matching hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "KVM"}, }, @@ -412,7 +412,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Mixed capability and non-capability extra specs", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "CH"}, {Name: "cpu_arch", Value: "x86_64"}, @@ -428,7 +428,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "CH"}, }, @@ -439,7 +439,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Case sensitive capability matching", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedCapabilities: []compute.Capability{ {Name: "hypervisor_type", Value: "ch"}, // lowercase should not match }, @@ -453,7 +453,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Unsupported operator in extra specs - skip filter", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ // An unsupported comparison operator is dropped by the adapter, // so the filter sees no requested capabilities and is a no-op. RequestedCapabilities: nil, @@ -468,7 +468,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Unsupported operator - skip filter", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ // An unsupported comparison operator is dropped by the adapter, // so the filter sees no requested capabilities and is a no-op. RequestedCapabilities: nil, @@ -482,7 +482,7 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "All hosts match when no capabilities requested", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, diff --git a/internal/scheduling/nova/plugins/filters/filter_correct_az.go b/internal/scheduling/nova/plugins/filters/filter_correct_az.go index 2b1a12d8d..730ca968d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_correct_az.go +++ b/internal/scheduling/nova/plugins/filters/filter_correct_az.go @@ -14,11 +14,11 @@ import ( ) type FilterCorrectAZStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Only get hosts in the requested az. -func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) if request.AvailabilityZone == "" { traceLog.V(1).Info("no availability zone requested, skipping filter_correct_az step") diff --git a/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go b/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go index 2fe0c0829..8901fa81d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go @@ -57,13 +57,13 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "Filter hosts in az-1", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AvailabilityZone: "az-1", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -78,7 +78,7 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Filter hosts in az-2", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AvailabilityZone: "az-2", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -92,7 +92,7 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Filter hosts in az-3", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AvailabilityZone: "az-3", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -106,7 +106,7 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "No hosts in requested AZ", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AvailabilityZone: "az-nonexistent", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -119,7 +119,7 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AvailabilityZone: "az-1", Hosts: []compute.Host{}, }, @@ -128,7 +128,7 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Host not in database", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AvailabilityZone: "az-1", Hosts: []compute.Host{ {ComputeHost: "host1"}, diff --git a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go index 6bd31579a..de1241fa8 100644 --- a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go +++ b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go @@ -14,7 +14,7 @@ import ( // This step is useful to circumvent scheduling issues on specific hosts by // excluding them from all scheduling decisions. type FilterExcludeHostsStep struct { - lib.BaseFilter[compute.SchedulerRequest, FilterExcludeHostsStepOpts] + lib.BaseFilter[compute.PlacementRequest, FilterExcludeHostsStepOpts] } type FilterExcludeHostsStepOpts struct { @@ -29,7 +29,7 @@ func (opts FilterExcludeHostsStepOpts) Validate() error { return nil } func (s *FilterExcludeHostsStep) Run( traceLog logr.Logger, - request compute.SchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) diff --git a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go index a0c232b26..6a4256f4b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go @@ -15,14 +15,14 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { tests := []struct { name string excludedHosts []string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No excluded hosts - all hosts pass", excludedHosts: []string{}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -35,7 +35,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single excluded host", excludedHosts: []string{"host2"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -48,7 +48,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Multiple excluded hosts", excludedHosts: []string{"host1", "host3"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -62,7 +62,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "All hosts excluded", excludedHosts: []string{"host1", "host2", "host3"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -75,7 +75,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Excluded host not in request - no effect", excludedHosts: []string{"host-nonexistent"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -87,7 +87,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Some excluded hosts in request, some not", excludedHosts: []string{"host1", "host-nonexistent", "host3"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -100,7 +100,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Empty host list in request", excludedHosts: []string{"host1", "host2"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{}, }, expectedHosts: []string{}, @@ -109,7 +109,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Nil excluded hosts list", excludedHosts: nil, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -121,7 +121,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Duplicate excluded hosts", excludedHosts: []string{"host1", "host1", "host2"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -134,7 +134,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single host request - excluded", excludedHosts: []string{"host1"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, }, @@ -145,7 +145,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single host request - not excluded", excludedHosts: []string{"host2"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, }, @@ -156,7 +156,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Case sensitive matching", excludedHosts: []string{"Host1", "HOST2"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -169,7 +169,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Hosts with special characters", excludedHosts: []string{"nova-compute-bb123.region.example.com"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "nova-compute-bb123.region.example.com"}, {ComputeHost: "nova-compute-bb124.region.example.com"}, @@ -181,7 +181,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Empty string in excluded hosts", excludedHosts: []string{"", "host1"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -193,7 +193,7 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Large number of excluded hosts", excludedHosts: []string{"host1", "host2", "host3", "host4", "host5", "host6", "host7", "host8", "host9", "host10"}, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, @@ -210,8 +210,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { step := &FilterExcludeHostsStep{ - BaseFilter: lib.BaseFilter[compute.SchedulerRequest, FilterExcludeHostsStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, FilterExcludeHostsStepOpts]{}, + BaseFilter: lib.BaseFilter[compute.PlacementRequest, FilterExcludeHostsStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, FilterExcludeHostsStepOpts]{}, }, } step.Options = FilterExcludeHostsStepOpts{ diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer.go b/internal/scheduling/nova/plugins/filters/filter_external_customer.go index 2697fe63c..e724e6c4d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer.go @@ -30,12 +30,12 @@ func (opts FilterExternalCustomerStepOpts) Validate() error { } type FilterExternalCustomerStep struct { - lib.BaseFilter[compute.SchedulerRequest, FilterExternalCustomerStepOpts] + lib.BaseFilter[compute.PlacementRequest, FilterExternalCustomerStepOpts] } // Prefix-match the domain name for external customer domains and filter out hosts // that are not intended for external customers. -func (s *FilterExternalCustomerStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterExternalCustomerStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Failover and capacity probe calls are not associated with a user domain; diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go index c99e19a61..fe765695e 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go @@ -79,7 +79,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { tests := []struct { name string opts FilterExternalCustomerStepOpts - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string expectError bool @@ -89,7 +89,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext-customer1", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -106,7 +106,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "internal-customer", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -123,7 +123,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"external-", "ext-", "customer-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "customer-abc", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -140,7 +140,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { CustomerDomainNamePrefixes: []string{"ext-"}, CustomerIgnoredDomainNames: []string{"ext-special-domain"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext-special-domain", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -156,7 +156,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext-customer2", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -172,7 +172,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext-customer3", Hosts: []compute.Host{ {ComputeHost: "host3"}, @@ -187,7 +187,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext-customer", Hosts: []compute.Host{}, }, @@ -199,7 +199,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext-customer", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -214,7 +214,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, @@ -228,7 +228,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -242,7 +242,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "EXT-customer", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -258,7 +258,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -274,7 +274,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { CustomerDomainNamePrefixes: []string{"ext-"}, CustomerIgnoredDomainNames: []string{"ext-test", "ext-dev", "ext-staging"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext-dev", Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -289,7 +289,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.ReserveForFailoverIntent, DomainName: "ext-customer1", Hosts: []compute.Host{ @@ -305,7 +305,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.ReserveForCommittedResourceIntent, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -320,7 +320,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.ReserveForCommittedResourceIntent, DomainName: "ext-customer1", Hosts: []compute.Host{ @@ -337,7 +337,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ DomainName: "ext-customer1", Hosts: []compute.Host{ {ComputeHost: "host-custom-trait"}, @@ -467,7 +467,7 @@ func TestFilterExternalCustomerStep_SkipsForNonPlacementIntent(t *testing.T) { for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { t.Run(intent, func(t *testing.T) { - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ Intent: v1alpha1.SchedulingIntent(intent), DomainName: "iaas-customer", Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, diff --git a/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go b/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go index ed03766a1..8b27f50f1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go @@ -14,11 +14,11 @@ import ( ) type FilterHasAcceleratorsStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // If requested, only get hosts with accelerators. -func (s *FilterHasAcceleratorsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHasAcceleratorsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) if !request.AcceleratorsRequested { traceLog.V(1).Info("no accelerators requested, skipping filter") diff --git a/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go b/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go index 971845e87..57bef26fc 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go @@ -67,13 +67,13 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No accelerators requested - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -86,7 +86,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - filter to hosts with COMPUTE_ACCELERATORS trait", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -100,7 +100,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested with different device profile value", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -113,7 +113,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - all hosts have the trait", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -126,7 +126,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - no hosts have the trait", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{ {ComputeHost: "host3"}, @@ -138,7 +138,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Other extra specs present but no accelerator request - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, @@ -150,7 +150,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested with empty device profile value", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -163,7 +163,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Empty host list with accelerators requested", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{}, }, @@ -172,7 +172,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Empty host list without accelerators requested", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{}, }, expectedHosts: []string{}, @@ -180,7 +180,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Mixed hosts with and without accelerators trait", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -195,7 +195,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators with additional extra specs", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -207,7 +207,7 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Host not in database with accelerators requested", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ AcceleratorsRequested: true, Hosts: []compute.Host{ {ComputeHost: "host1"}, diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index c42d19c0d..920650273 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -37,7 +37,7 @@ type FilterHasEnoughCapacityOpts struct { func (FilterHasEnoughCapacityOpts) Validate() error { return nil } type FilterHasEnoughCapacity struct { - lib.BaseFilter[compute.SchedulerRequest, FilterHasEnoughCapacityOpts] + lib.BaseFilter[compute.PlacementRequest, FilterHasEnoughCapacityOpts] } // Filter hosts that don't have enough capacity to run the requested flavor. @@ -64,7 +64,7 @@ type FilterHasEnoughCapacity struct { // known at this point. // // Please also note that disk space is currently not considered by this filter. -func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { opts := request.GetOptions() result := s.IncludeAllHostsFromRequest(request) diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go index 15aed3daf..8f979f37f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go @@ -249,13 +249,13 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID, projectID, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) compute.SchedulerRequest { +func newNovaRequest(instanceUUID, projectID, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) compute.PlacementRequest { return newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup, vcpus, memory, "", evacuation, hosts) } // newNovaRequestWithIntent creates a nova request with a specific intent. // intentHint can be: "evacuate", "reserve_for_committed_resource", "reserve_for_failover", or "" for create. -func newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup string, vcpus int, memory, intentHint string, evacuation bool, hosts []string) compute.SchedulerRequest { +func newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup string, vcpus int, memory, intentHint string, evacuation bool, hosts []string) compute.PlacementRequest { hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} @@ -278,7 +278,7 @@ func newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup string, vcpus weights[h] = 1.0 } - return compute.SchedulerRequest{ + return compute.PlacementRequest{ ProjectID: projectID, InstanceUUID: instanceUUID, NumInstances: 1, @@ -351,7 +351,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request compute.SchedulerRequest + request compute.PlacementRequest opts FilterHasEnoughCapacityOpts expectedHosts []string filteredHosts []string @@ -763,7 +763,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name string hypervisors []*hv1.Hypervisor reservations []*v1alpha1.Reservation - request compute.SchedulerRequest + request compute.PlacementRequest ignoredReservationTypes []v1alpha1.ReservationType expectedHosts []string filteredHosts []string @@ -1076,7 +1076,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) name string hypervisors []*hv1.Hypervisor reservations []*v1alpha1.Reservation - request compute.SchedulerRequest + request compute.PlacementRequest opts FilterHasEnoughCapacityOpts expectedHosts []string filteredHosts []string @@ -1285,7 +1285,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ diff --git a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go index 55d087193..a8c00db7c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go @@ -14,13 +14,13 @@ import ( ) type FilterHasRequestedTraits struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Filter hosts based on the traits requested in the flavor: // - a forbidden trait means the host must not have the specified trait. // - a required trait means the host must have the specified trait. -func (s *FilterHasRequestedTraits) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHasRequestedTraits) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) requiredTraits := request.RequiredTraits forbiddenTraits := request.ForbiddenTraits diff --git a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go index 7ba8f7b4c..e5276054a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go @@ -75,13 +75,13 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No traits requested - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -93,7 +93,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Single required trait - filter hosts with trait", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -107,7 +107,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Multiple required traits - filter hosts with all traits", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -121,7 +121,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Single forbidden trait - filter hosts without trait", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForbiddenTraits: []string{"CUSTOM_TRAIT_A"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -136,7 +136,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Multiple forbidden traits - filter hosts without any of them", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForbiddenTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -151,7 +151,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Mixed required and forbidden traits", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A"}, ForbiddenTraits: []string{"CUSTOM_TRAIT_D"}, Hosts: []compute.Host{ @@ -166,7 +166,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Required trait that no host has", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_NONEXISTENT"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -179,7 +179,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Forbidden trait that no host has - all pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForbiddenTraits: []string{"CUSTOM_TRAIT_NONEXISTENT"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -192,7 +192,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Host with no traits - required trait filters it out", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -204,7 +204,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Host with no traits - forbidden trait lets it pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForbiddenTraits: []string{"CUSTOM_TRAIT_A"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -216,7 +216,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Non-trait extra specs are ignored", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -228,7 +228,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Invalid trait value (not required or forbidden) - ignored", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -240,7 +240,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A"}, Hosts: []compute.Host{}, }, @@ -249,7 +249,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Host not in database", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -261,7 +261,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Complex scenario with many traits", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, ForbiddenTraits: []string{"CUSTOM_TRAIT_D"}, Hosts: []compute.Host{ @@ -278,7 +278,7 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "All hosts match required and forbidden traits", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequiredTraits: []string{"CUSTOM_TRAIT_A"}, ForbiddenTraits: []string{"CUSTOM_TRAIT_E"}, Hosts: []compute.Host{ diff --git a/internal/scheduling/nova/plugins/filters/filter_host_instructions.go b/internal/scheduling/nova/plugins/filters/filter_host_instructions.go index 6b8c8a38e..88b1b48b6 100644 --- a/internal/scheduling/nova/plugins/filters/filter_host_instructions.go +++ b/internal/scheduling/nova/plugins/filters/filter_host_instructions.go @@ -12,13 +12,13 @@ import ( ) type FilterHostInstructionsStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Filter hosts based on instructions given in the request spec. Supported are: // - spec.ignore_hosts: Filter out all hosts in this list. // - spec.force_hosts: Include only hosts in this list. -func (s *FilterHostInstructionsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHostInstructionsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) if len(request.IgnoreHosts) > 0 { for _, host := range request.IgnoreHosts { diff --git a/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go b/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go index da07468a5..973b40dc1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go @@ -13,13 +13,13 @@ import ( func TestFilterHostInstructionsStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No host instructions - no filtering", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -32,7 +32,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore single host", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host2"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -46,7 +46,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore multiple hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host1", "host3", "host4"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -60,7 +60,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore all hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host1", "host2", "host3", "host4"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -74,7 +74,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore non-existent host", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host-nonexistent"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -87,7 +87,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force single host", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForceHosts: []string{"host2"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -101,7 +101,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force multiple hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForceHosts: []string{"host1", "host3"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -115,7 +115,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force non-existent host", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForceHosts: []string{"host-nonexistent"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -128,7 +128,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force and ignore same host - ignore takes precedence", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host2"}, ForceHosts: []string{"host2"}, Hosts: []compute.Host{ @@ -142,7 +142,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force and ignore different hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host1"}, ForceHosts: []string{"host2", "host3"}, Hosts: []compute.Host{ @@ -157,7 +157,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Complex scenario - multiple ignore and force", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host2", "host5"}, ForceHosts: []string{"host1", "host3", "host4"}, Hosts: []compute.Host{ @@ -174,7 +174,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host1"}, ForceHosts: []string{"host2"}, Hosts: []compute.Host{}, @@ -184,7 +184,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Empty ignore list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -196,7 +196,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Empty force list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForceHosts: []string{}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -210,7 +210,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Both lists empty", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{}, ForceHosts: []string{}, Hosts: []compute.Host{ @@ -223,7 +223,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Duplicate hosts in ignore list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ IgnoreHosts: []string{"host1", "host1", "host2"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -236,7 +236,7 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Duplicate hosts in force list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ForceHosts: []string{"host1", "host1", "host2"}, Hosts: []compute.Host{ {ComputeHost: "host1"}, diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties.go b/internal/scheduling/nova/plugins/filters/filter_image_properties.go index 944797d81..96d595d64 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties.go @@ -16,11 +16,11 @@ import ( ) type FilterImagePropertiesStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Filter hosts based on image properties given in the request spec. -func (s *FilterImagePropertiesStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterImagePropertiesStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Apply this filter to all requests, unless we know from the request's diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go index 2e58e3b8d..be98bb96c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go @@ -20,12 +20,12 @@ import ( // requestWith builds a SchedulerRequest with the given hosts and image // hypervisor type. -func requestWith(hosts []string, hvType compute.ImageHypervisorType) compute.SchedulerRequest { +func requestWith(hosts []string, hvType compute.ImageHypervisorType) compute.PlacementRequest { schedHosts := make([]compute.Host, 0, len(hosts)) for _, h := range hosts { schedHosts = append(schedHosts, compute.Host{ComputeHost: h}) } - return compute.SchedulerRequest{ + return compute.PlacementRequest{ Hosts: schedHosts, ImageHypervisorType: hvType, } @@ -34,7 +34,7 @@ func requestWith(hosts []string, hvType compute.ImageHypervisorType) compute.Sch // requestWithIntent builds a SchedulerRequest with the given intent. The image // hypervisor type is irrelevant to the intent-skip paths these callers exercise, // so it is fixed to VMware. -func requestWithIntent(hosts []string, intent v1alpha1.SchedulingIntent) compute.SchedulerRequest { +func requestWithIntent(hosts []string, intent v1alpha1.SchedulingIntent) compute.PlacementRequest { request := requestWith(hosts, compute.ImageHypervisorTypeVMware) request.Intent = intent return request @@ -54,7 +54,7 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string expectedEvents []lib.FilterWeigherPipelineStepEvent diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go index e2821f5b0..4430139d9 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go @@ -14,13 +14,13 @@ import ( ) type FilterInstanceGroupAffinityStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Select hosts in spec.instance_group. func (s *FilterInstanceGroupAffinityStep) Run( traceLog logr.Logger, - request compute.SchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go index b32f514fb..66c773bf2 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go @@ -14,13 +14,13 @@ import ( func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No instance group - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: nil, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -33,7 +33,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Instance group with anti-affinity policy - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Hosts: []string{"host1", "host2"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -46,7 +46,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Instance group with soft-affinity policy - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "soft-affinity", Hosts: []string{"host1"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -58,7 +58,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy with empty hosts list - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -71,7 +71,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - only hosts in instance group pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host3"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -84,7 +84,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - single host in group", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host2"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -98,7 +98,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - all hosts filtered out", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host5", "host6"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -111,7 +111,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - all hosts in group pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2", "host3"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -124,7 +124,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - partial overlap", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host2", "host3", "host5"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -138,7 +138,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - empty request hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2"}}, Hosts: []compute.Host{}, }, @@ -147,7 +147,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - case sensitive host matching", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"Host1", "HOST2"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -160,7 +160,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - with instance UUID and project ID", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-123", ProjectID: "project-abc", InstanceGroup: &compute.InstanceGroupSpec{UUID: "ig-uuid-456", Policy: "affinity", Hosts: []string{"host1", "host2"}, Members: []string{"vm-uuid-789"}}, @@ -176,7 +176,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - duplicate hosts in group list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2", "host1", "host2"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -189,7 +189,7 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - single host scenario", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1"}}, Hosts: []compute.Host{ {ComputeHost: "host1"}, diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go index 84036fb42..2840d3f13 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go @@ -16,14 +16,14 @@ import ( ) type FilterInstanceGroupAntiAffinityStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Select hosts not in spec_obj.instance_group but only until // max_server_per_host is reached (by default = 1). func (s *FilterInstanceGroupAntiAffinityStep) Run( traceLog logr.Logger, - request compute.SchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go index c4bc2c3f4..87ef4ef6c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go @@ -80,13 +80,13 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No instance group - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: nil, Hosts: []compute.Host{ @@ -100,7 +100,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Instance group with affinity policy - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, Hosts: []compute.Host{ @@ -114,7 +114,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Instance group with soft-anti-affinity policy - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "soft-anti-affinity", Members: []string{"vm-uuid-1"}}, Hosts: []compute.Host{ @@ -127,7 +127,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy with empty members list - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{}}, Hosts: []compute.Host{ @@ -141,7 +141,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - default max_server_per_host=1", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, Hosts: []compute.Host{ @@ -156,7 +156,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=2", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 2}, Hosts: []compute.Host{ @@ -171,7 +171,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=3", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-4", "vm-uuid-5", "vm-uuid-6"}, MaxServerPerHost: 3}, Hosts: []compute.Host{ @@ -184,7 +184,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - host running same VM (resize scenario)", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-1", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, Hosts: []compute.Host{ @@ -198,7 +198,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - all hosts filtered out", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}}, Hosts: []compute.Host{ @@ -212,7 +212,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - mixed hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3", "vm-uuid-4", "vm-uuid-5"}}, Hosts: []compute.Host{ @@ -228,7 +228,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - empty request hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, Hosts: []compute.Host{}, @@ -238,7 +238,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - host with non-member instances", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-100", "vm-uuid-101"}}, Hosts: []compute.Host{ @@ -254,7 +254,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=2 with mixed instances", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}, MaxServerPerHost: 2}, Hosts: []compute.Host{ @@ -270,7 +270,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - with instance UUID and project ID", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", ProjectID: "project-abc", InstanceGroup: &compute.InstanceGroupSpec{UUID: "ig-uuid-456", Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, @@ -285,7 +285,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - multiple members on same host with max=2", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 2}, Hosts: []compute.Host{ @@ -298,7 +298,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - resize with VM on multiple hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-2", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}}, Hosts: []compute.Host{ @@ -313,7 +313,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - single host scenario", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, Hosts: []compute.Host{ @@ -325,7 +325,7 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - high max_server_per_host value", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: "vm-uuid-new", InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 10}, Hosts: []compute.Host{ diff --git a/internal/scheduling/nova/plugins/filters/filter_live_migratable.go b/internal/scheduling/nova/plugins/filters/filter_live_migratable.go index 85abc3d97..a7cbc4f13 100644 --- a/internal/scheduling/nova/plugins/filters/filter_live_migratable.go +++ b/internal/scheduling/nova/plugins/filters/filter_live_migratable.go @@ -16,7 +16,7 @@ import ( ) type FilterLiveMigratableStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Check if a vm can be live migrated from a source to a given target host. @@ -51,7 +51,7 @@ func (s *FilterLiveMigratableStep) checkHasSufficientFeatures( // Ensure the target host of a live migration can accept the migrating VM. func (s *FilterLiveMigratableStep) Run( traceLog logr.Logger, - request compute.SchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) diff --git a/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go b/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go index e5b743a4f..5bae46da5 100644 --- a/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go @@ -201,7 +201,7 @@ func TestFilterLiveMigratableStep_checkHasSufficientFeatures(t *testing.T) { func TestFilterLiveMigratableStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string @@ -209,7 +209,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }{ { name: "Not a live migration - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.EvacuateIntent, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -236,7 +236,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration without source_host - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -263,7 +263,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with compatible hosts", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{ @@ -313,7 +313,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with incompatible architecture", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{ @@ -357,7 +357,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing CPU modes", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{ @@ -407,7 +407,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing features", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{ @@ -441,7 +441,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing devices", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{ @@ -475,7 +475,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with non-existent target host", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{ @@ -504,7 +504,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with all hosts filtered", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{ @@ -539,7 +539,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{}, @@ -575,8 +575,8 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -618,7 +618,7 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { } func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "nonexistent-host", Hosts: []compute.Host{ @@ -656,8 +656,8 @@ func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -673,7 +673,7 @@ func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { } func TestFilterLiveMigratableStep_Run_ClientError(t *testing.T) { - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ Intent: api.LiveMigrationIntent, SourceHost: "host1", Hosts: []compute.Host{ @@ -696,8 +696,8 @@ func TestFilterLiveMigratableStep_Run_ClientError(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index ae73b26ba..5ebcc141e 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -73,7 +73,7 @@ func (FilterQuotaEnforcementOpts) Validate() error { return nil } // // Disabled by default; activated by adding "filter_quota_enforcement" to the pipeline config. type FilterQuotaEnforcement struct { - lib.BaseFilter[compute.SchedulerRequest, FilterQuotaEnforcementOpts] + lib.BaseFilter[compute.PlacementRequest, FilterQuotaEnforcementOpts] } // recordDecision appends an accept/reject/skip outcome as a named pipeline @@ -94,7 +94,7 @@ func recordDecision(result *lib.FilterWeigherPipelineStepResult, mode, decision, }) } -func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) mode := "shadow" diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index eddf38c88..984fbc8a5 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -45,8 +45,8 @@ func intentFromHint(hints map[string]any) v1alpha1.SchedulingIntent { } } -func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) compute.SchedulerRequest { - return compute.SchedulerRequest{ +func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) compute.PlacementRequest { + return compute.PlacementRequest{ ProjectID: projectID, AvailabilityZone: az, NumInstances: numInstances, @@ -73,7 +73,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { type tc struct { name string objects []client.Object - request compute.SchedulerRequest + request compute.PlacementRequest enforce bool expectAccept bool diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go index 540faf6b6..397cfdd88 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go @@ -17,7 +17,7 @@ import ( ) type FilterRequestedDestinationStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // processRequestedAggregates filters hosts based on the requested aggregates. @@ -102,7 +102,7 @@ func (s *FilterRequestedDestinationStep) processRequestedHost( // The requested destination can include a specific host, aggregates, or both. // When both are specified, aggregate filtering is applied first, followed by // host filtering. -func (s *FilterRequestedDestinationStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterRequestedDestinationStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // The requested_destination hint is only set by Nova for user-directed placement; if slices.Contains([]v1alpha1.SchedulingIntent{ diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go index fe1805e62..f1442e648 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go @@ -23,7 +23,7 @@ import ( func TestFilterRequestedDestinationStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string @@ -31,7 +31,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }{ { name: "No requested_destination - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: nil, Hosts: []compute.Host{ {ComputeHost: "host1"}, @@ -59,7 +59,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Empty requested_destination - all hosts pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{}, Host: "", @@ -85,7 +85,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by specific host", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Host: "host2", }, @@ -115,7 +115,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by single aggregate - hosts in spec aggregates", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{"aggregate1"}, }, @@ -145,7 +145,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by single aggregate - hosts in status aggregates", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{"aggregate1"}, }, @@ -175,7 +175,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by multiple aggregates with OR syntax (comma-separated)", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ // "aggregate1,aggregate3" means host must be in aggregate1 OR aggregate3 Aggregates: []string{"aggregate1,aggregate3"}, @@ -211,7 +211,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by multiple aggregates with AND logic", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ // ["aggregate1", "aggregate2"] means host must be in aggregate1 AND aggregate2 Aggregates: []string{"aggregate1", "aggregate2"}, @@ -242,7 +242,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by mixed AND/OR aggregates", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ // Host must be in (aggregate1 OR aggregate2) AND aggregate3 Aggregates: []string{"aggregate1,aggregate2", "aggregate3"}, @@ -278,7 +278,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by aggregates with OR syntax - hosts in both spec and status", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ // "aggregate1,aggregate2" means host must be in aggregate1 OR aggregate2 Aggregates: []string{"aggregate1,aggregate2"}, @@ -309,7 +309,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by both host and aggregates - host takes precedence", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{"aggregate1"}, Host: "host2", @@ -340,7 +340,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Host not found in hypervisors - filtered out", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{"aggregate1"}, }, @@ -366,7 +366,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Host with no aggregates - filtered out when aggregates requested", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{"aggregate1"}, }, @@ -396,7 +396,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "All hosts filtered by aggregate", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{"nonexistent-aggregate"}, }, @@ -421,7 +421,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Requested host not in candidates", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Host: "host-not-in-list", }, @@ -450,7 +450,7 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{"aggregate1"}, }, @@ -485,8 +485,8 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { Build() step := &FilterRequestedDestinationStep{ - BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -654,7 +654,7 @@ func TestFilterRequestedDestinationStep_processRequestedHost(t *testing.T) { } func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{ Aggregates: []string{"aggregate1"}, }, @@ -678,8 +678,8 @@ func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { Build() step := &FilterRequestedDestinationStep{ - BaseFilter: lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -711,7 +711,7 @@ func TestFilterRequestedDestinationStep_SkipsForNonPlacementIntent(t *testing.T) api.CapacityProbeIntent, } { t.Run(string(intent), func(t *testing.T) { - request := compute.SchedulerRequest{ + request := compute.PlacementRequest{ RequestedDestination: &compute.RequestedDestination{Host: "host1"}, Intent: intent, Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, diff --git a/internal/scheduling/nova/plugins/filters/filter_status_conditions.go b/internal/scheduling/nova/plugins/filters/filter_status_conditions.go index 5097141b2..5bcbfd75f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_status_conditions.go +++ b/internal/scheduling/nova/plugins/filters/filter_status_conditions.go @@ -15,12 +15,12 @@ import ( ) type FilterStatusConditionsStep struct { - lib.BaseFilter[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Check that all status conditions meet the expected values, for example, // that the hypervisor is ready and not disabled. -func (s *FilterStatusConditionsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterStatusConditionsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) hvs := &hv1.HypervisorList{} diff --git a/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go b/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go index 0b936ca77..b38bebf12 100644 --- a/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go @@ -214,13 +214,13 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "Filter hosts with all conditions met", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -233,7 +233,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host not ready should be filtered", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host2"}, }, @@ -243,7 +243,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Terminating host should be filtered", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host3"}, }, @@ -253,7 +253,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Tainted host should not be filtered", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host4"}, }, @@ -263,7 +263,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Disabled hypervisor should be filtered", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host8"}, }, @@ -273,7 +273,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host with optional conditions in any state should pass", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host5"}, }, @@ -283,7 +283,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host missing Ready condition should be kept", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host6"}, }, @@ -293,7 +293,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host with no conditions should be kept", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host7"}, }, @@ -303,7 +303,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{}, }, expectedHosts: []string{}, @@ -311,7 +311,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host not in database", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, @@ -322,7 +322,7 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Mixed condition states", - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, diff --git a/internal/scheduling/nova/plugins/filters/zz_index.go b/internal/scheduling/nova/plugins/filters/zz_index.go index afe9b1280..b72812ffd 100644 --- a/internal/scheduling/nova/plugins/filters/zz_index.go +++ b/internal/scheduling/nova/plugins/filters/zz_index.go @@ -8,7 +8,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type NovaFilter = lib.Filter[compute.SchedulerRequest] +type NovaFilter = lib.Filter[compute.PlacementRequest] // Configuration of filters supported by the nova scheduler. var Index = map[string]func() NovaFilter{} diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go index a1b9932c7..9f19b47d1 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go @@ -65,11 +65,11 @@ func (o KVMBinpackStepOpts) Validate() error { // Explanation of the algorithm: https://volcano.sh/en/docs/plugins/#binpack type KVMBinpackStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[compute.SchedulerRequest, KVMBinpackStepOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMBinpackStepOpts] } // Run this weigher in the pipeline after filters have been executed. -func (s *KVMBinpackStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMBinpackStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["binpack score"] = s.PrepareStats(request, "float") @@ -141,7 +141,7 @@ func (s *KVMBinpackStep) Run(traceLog logr.Logger, request compute.SchedulerRequ } // calcVMResources calculates the total resource requests for the VM to be scheduled. -func (s *KVMBinpackStep) calcVMResources(req compute.SchedulerRequest) map[hv1.ResourceName]resource.Quantity { +func (s *KVMBinpackStep) calcVMResources(req compute.PlacementRequest) map[hv1.ResourceName]resource.Quantity { resources := make(map[hv1.ResourceName]resource.Quantity) resourcesMemBytes := int64(req.Flavor.MemoryMB) * (1024 * 1024) //nolint:gosec // memory values are bounded by Nova resourcesMemBytes *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go index 3ffa3d2c5..d7df0dc22 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go @@ -34,7 +34,7 @@ func newHypervisor(name, capacityCPU, capacityMem, allocationCPU, allocationMem } } -func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) compute.SchedulerRequest { +func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) compute.PlacementRequest { hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} @@ -45,7 +45,7 @@ func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) com weights[h] = 1.0 } - return compute.SchedulerRequest{ + return compute.PlacementRequest{ ProjectID: "project-A", InstanceUUID: "instance-123", NumInstances: numInstances, @@ -172,7 +172,7 @@ func TestKVMBinpackStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request compute.SchedulerRequest + request compute.PlacementRequest opts KVMBinpackStepOpts expectedWeights map[string]float64 wantErr bool @@ -474,7 +474,7 @@ func TestKVMBinpackStep_Run(t *testing.T) { func TestKVMBinpackStep_calcVMResources(t *testing.T) { tests := []struct { name string - request compute.SchedulerRequest + request compute.PlacementRequest expectedMemBytes int64 expectedCPU int64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go index cd1b739b9..cd83113a6 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go @@ -48,12 +48,12 @@ func (o KVMCommittedResourceReservationOpts) GetDefaultHostWeight() float64 { // Hosts that have a ready Reservation matching the request's project and flavor group, // with enough free memory capacity for the requested VM, receive a higher weight. type KVMCommittedResourceReservationStep struct { - lib.BaseWeigher[compute.SchedulerRequest, KVMCommittedResourceReservationOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMCommittedResourceReservationOpts] } // Run the weigher step. // Hosts with a matching CommittedResourceReservation (project + flavor group + free capacity) get a higher weight. -func (s *KVMCommittedResourceReservationStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMCommittedResourceReservationStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Skip for failover reservation scheduling — CR reservations should not influence diff --git a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go index b4a3a8bf7..03a1a543a 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go @@ -54,7 +54,7 @@ func newCRReservation(name, host, projectID, resourceGroup, az, totalMemory stri } } -func newCRRequest(hosts []string) compute.SchedulerRequest { +func newCRRequest(hosts []string) compute.PlacementRequest { hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} @@ -63,7 +63,7 @@ func newCRRequest(hosts []string) compute.SchedulerRequest { for _, h := range hosts { weights[h] = 1.0 } - return compute.SchedulerRequest{ + return compute.PlacementRequest{ ProjectID: "project-A", AvailabilityZone: "qa-de-1a", ResourceGroup: "group-2101", @@ -83,7 +83,7 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request compute.SchedulerRequest + request compute.PlacementRequest opts KVMCommittedResourceReservationOpts expectedWeights map[string]float64 }{ @@ -166,7 +166,7 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { { name: "no resource group hint skips weigher", reservations: []*v1alpha1.Reservation{newCRReservation("res-1", "host1", "project-A", "group-2101", "qa-de-1a", "16Gi")}, - request: func() compute.SchedulerRequest { + request: func() compute.PlacementRequest { r := newCRRequest([]string{"host1", "host2"}) r.ResourceGroup = "" return r @@ -205,7 +205,7 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { reservations: []*v1alpha1.Reservation{ newCRReservation("res-1", "host1", "project-A", "group-2101", "qa-de-1a", "16Gi"), }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ ProjectID: "project-A", AvailabilityZone: "qa-de-1a", Intent: api.ReserveForFailoverIntent, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go index e1b80b0c9..0e0290431 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go @@ -59,12 +59,12 @@ func (o KVMCRMigrationSlotOpts) GetDefaultHostWeight() float64 { // // Only activates for LiveMigrationIntent. type KVMCRMigrationSlotStep struct { - lib.BaseWeigher[compute.SchedulerRequest, KVMCRMigrationSlotOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMCRMigrationSlotOpts] } func (s *KVMCRMigrationSlotStep) Run( traceLog logr.Logger, - request compute.SchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) diff --git a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go index f1a77cb0a..8cb63b52d 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go @@ -35,12 +35,12 @@ func newCRMigrationSlotWeigher(t *testing.T, opts KVMCRMigrationSlotOpts, objs . } // migrationRequest builds a live-migration request from the given candidate hosts. -func migrationRequest(hosts ...string) compute.SchedulerRequest { +func migrationRequest(hosts ...string) compute.PlacementRequest { hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} } - return compute.SchedulerRequest{ + return compute.PlacementRequest{ InstanceUUID: testInstanceUUID, ProjectID: testProjectID, Intent: api.LiveMigrationIntent, @@ -130,7 +130,7 @@ func TestKVMCRMigrationSlotStep_Run(t *testing.T) { tests := []struct { name string objects []client.Object - request compute.SchedulerRequest + request compute.PlacementRequest opts KVMCRMigrationSlotOpts expectedWeights map[string]float64 }{ @@ -139,7 +139,7 @@ func TestKVMCRMigrationSlotStep_Run(t *testing.T) { objects: []client.Object{ confirmedSourceSlot("16Gi"), }, - request: compute.SchedulerRequest{ + request: compute.PlacementRequest{ InstanceUUID: testInstanceUUID, ProjectID: testProjectID, // no Intent → CreateIntent diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go index 38b21fdd6..f213ae40b 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go @@ -46,13 +46,13 @@ func (o KVMFailoverEvacuationOpts) GetDefaultHostWeight() float64 { // Hosts that match a failover reservation for the VM being scheduled get a higher weight, // encouraging placement on pre-reserved failover capacity. type KVMFailoverEvacuationStep struct { - lib.BaseWeigher[compute.SchedulerRequest, KVMFailoverEvacuationOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMFailoverEvacuationOpts] } // Run the weigher step. // For evacuation requests, hosts matching a failover reservation where the VM is in Allocations get a higher weight. // For non-evacuation requests (e.g., live migration, rebuild), this weigher has no effect. -func (s *KVMFailoverEvacuationStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMFailoverEvacuationStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) if request.Intent != api.EvacuateIntent { diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go index 4f8a02308..d4de183d6 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go @@ -113,13 +113,13 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID string, evacuation bool, hosts []string) compute.SchedulerRequest { //nolint:unparam // instanceUUID varies in real usage +func newNovaRequest(instanceUUID string, evacuation bool, hosts []string) compute.PlacementRequest { //nolint:unparam // instanceUUID varies in real usage hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} } - req := compute.SchedulerRequest{ + req := compute.PlacementRequest{ ProjectID: "project-A", InstanceUUID: instanceUUID, NumInstances: 1, @@ -153,7 +153,7 @@ func TestKVMFailoverEvacuationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request compute.SchedulerRequest + request compute.PlacementRequest opts KVMFailoverEvacuationOpts expectedWeights map[string]float64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go index 8aa42c766..f6b10ee04 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go @@ -74,13 +74,13 @@ func (o KVMFailoverReservationConsolidationOpts) GetSameSpecPenalty() float64 { // // This produces bounded output (~0 to 1) that plays nicely with other weighers. type KVMFailoverReservationConsolidationStep struct { - lib.BaseWeigher[compute.SchedulerRequest, KVMFailoverReservationConsolidationOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMFailoverReservationConsolidationOpts] } // Run the weigher step. // For reserve_for_failover requests, hosts are scored based on existing failover reservation density // and same-spec diversity. For all other request types, this weigher has no effect. -func (s *KVMFailoverReservationConsolidationStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMFailoverReservationConsolidationStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) if request.Intent != api.ReserveForFailoverIntent { diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go index 047700ebd..c2bdf5996 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go @@ -51,7 +51,7 @@ func newFailoverReservationWithGroup(name, targetHost, resourceGroup string) *v1 } } -func newFailoverReservationRequest(resourceGroup string, hosts []string) compute.SchedulerRequest { +func newFailoverReservationRequest(resourceGroup string, hosts []string) compute.PlacementRequest { hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} @@ -62,7 +62,7 @@ func newFailoverReservationRequest(resourceGroup string, hosts []string) compute weights[h] = 1.0 } - return compute.SchedulerRequest{ + return compute.PlacementRequest{ ProjectID: "project-A", InstanceUUID: "test-instance", NumInstances: 1, @@ -88,7 +88,7 @@ func TestKVMFailoverReservationConsolidationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request compute.SchedulerRequest + request compute.PlacementRequest opts KVMFailoverReservationConsolidationOpts expectedWeights map[string]float64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go index 60799ff59..d26947c7c 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go @@ -23,10 +23,10 @@ import ( // respectively, for the scheduler to choose that host for new instances of // the same group. type KVMInstanceGroupSoftAffinityStep struct { - lib.BaseWeigher[compute.SchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseWeigher[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } -func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["affinity"] = s.PrepareStats(request, "float") diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go index e13a1cf58..ec360c361 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go @@ -33,13 +33,13 @@ func newHypervisorWithInstances(name string, instanceIDs ...string) *hv1.Hypervi } } -func newInstanceGroupRequest(policy string, members, hosts []string) compute.SchedulerRequest { +func newInstanceGroupRequest(policy string, members, hosts []string) compute.PlacementRequest { hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} } - req := compute.SchedulerRequest{ + req := compute.PlacementRequest{ ProjectID: "project-A", InstanceUUID: "new-instance-uuid", NumInstances: 1, @@ -75,7 +75,7 @@ func TestKVMInstanceGroupSoftAffinityStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request compute.SchedulerRequest + request compute.PlacementRequest expectedWeights map[string]float64 wantErr bool }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go index 18c837741..87bf495e4 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go @@ -53,11 +53,11 @@ func (o KVMPreferSmallerHostsStepOpts) Validate() error { // accommodate larger VMs when they need to be scheduled. type KVMPreferSmallerHostsStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[compute.SchedulerRequest, KVMPreferSmallerHostsStepOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMPreferSmallerHostsStepOpts] } // Run this weigher in the pipeline after filters have been executed. -func (s *KVMPreferSmallerHostsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMPreferSmallerHostsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["small host score"] = s.PrepareStats(request, "float") diff --git a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go index fef4de526..c6902a101 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go @@ -30,7 +30,7 @@ func newHypervisorWithCapacity(name, capacityCPU, capacityMem string) *hv1.Hyper } } -func newPreferSmallerHostsRequest(hosts []string) compute.SchedulerRequest { +func newPreferSmallerHostsRequest(hosts []string) compute.PlacementRequest { hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { hostList[i] = compute.Host{ComputeHost: h} @@ -41,7 +41,7 @@ func newPreferSmallerHostsRequest(hosts []string) compute.SchedulerRequest { weights[h] = 1.0 } - return compute.SchedulerRequest{ + return compute.PlacementRequest{ ProjectID: "project-A", InstanceUUID: "instance-123", NumInstances: 1, @@ -170,7 +170,7 @@ func TestKVMPreferSmallerHostsStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request compute.SchedulerRequest + request compute.PlacementRequest opts KVMPreferSmallerHostsStepOpts expectedWeights map[string]float64 wantErr bool diff --git a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go index 47bcfbb7a..3b775977b 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go @@ -38,7 +38,7 @@ func (o VMwareAntiAffinityNoisyProjectsStepOpts) Validate() error { // Step to avoid noisy projects by downvoting the hosts they are running on. type VMwareAntiAffinityNoisyProjectsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[sched.SchedulerRequest, VMwareAntiAffinityNoisyProjectsStepOpts] + lib.BaseWeigher[sched.PlacementRequest, VMwareAntiAffinityNoisyProjectsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -53,7 +53,7 @@ func (s *VMwareAntiAffinityNoisyProjectsStep) Init(ctx context.Context, client c } // Downvote the hosts a project is currently running on if it's noisy. -func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request sched.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request sched.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu usage of this project"] = s.PrepareStats(request, "%") @@ -92,7 +92,7 @@ func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request } func init() { - Index["vmware_anti_affinity_noisy_projects"] = func() lib.Weigher[sched.SchedulerRequest] { + Index["vmware_anti_affinity_noisy_projects"] = func() lib.Weigher[sched.PlacementRequest] { return &VMwareAntiAffinityNoisyProjectsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go index 58770d271..38a28de94 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go @@ -219,12 +219,12 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { tests := []struct { name string - request sched.SchedulerRequest + request sched.PlacementRequest downvotedHosts map[string]struct{} }{ { name: "Noisy project", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ ProjectID: "project1", Hosts: []sched.Host{ {ComputeHost: "host1"}, @@ -239,7 +239,7 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { }, { name: "Non-noisy project", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ ProjectID: "project2", Hosts: []sched.Host{ {ComputeHost: "host1"}, @@ -251,7 +251,7 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { }, { name: "No noisy project data", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go index 5d005a496..c52cd921a 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go @@ -47,7 +47,7 @@ func (o VMwareAvoidLongTermContendedHostsStepOpts) Validate() error { // Step to avoid long term contended hosts by downvoting them. type VMwareAvoidLongTermContendedHostsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[sched.SchedulerRequest, VMwareAvoidLongTermContendedHostsStepOpts] + lib.BaseWeigher[sched.PlacementRequest, VMwareAvoidLongTermContendedHostsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -62,7 +62,7 @@ func (s *VMwareAvoidLongTermContendedHostsStep) Init(ctx context.Context, client } // Downvote hosts that are highly contended. -func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, request sched.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, request sched.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu contention"] = s.PrepareStats(request, "%") @@ -110,7 +110,7 @@ func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, reques } func init() { - Index["vmware_avoid_long_term_contended_hosts"] = func() lib.Weigher[sched.SchedulerRequest] { + Index["vmware_avoid_long_term_contended_hosts"] = func() lib.Weigher[sched.PlacementRequest] { return &VMwareAvoidLongTermContendedHostsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go index 138743750..25f697a4d 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go @@ -218,12 +218,12 @@ func TestVMwareAvoidLongTermContendedHostsStep_Run(t *testing.T) { tests := []struct { name string - request sched.SchedulerRequest + request sched.PlacementRequest expected map[string]float64 }{ { name: "Avoid contended hosts", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -240,7 +240,7 @@ func TestVMwareAvoidLongTermContendedHostsStep_Run(t *testing.T) { }, { name: "Missing data", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ Hosts: []sched.Host{ {ComputeHost: "host4"}, {ComputeHost: "host5"}, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go index 779304247..6d2f0fc37 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go @@ -47,7 +47,7 @@ func (o VMwareAvoidShortTermContendedHostsStepOpts) Validate() error { // Step to avoid recently contended hosts by downvoting them. type VMwareAvoidShortTermContendedHostsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[compute.SchedulerRequest, VMwareAvoidShortTermContendedHostsStepOpts] + lib.BaseWeigher[compute.PlacementRequest, VMwareAvoidShortTermContendedHostsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -62,7 +62,7 @@ func (s *VMwareAvoidShortTermContendedHostsStep) Init(ctx context.Context, clien } // Downvote hosts that are highly contended. -func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, request compute.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu contention"] = s.PrepareStats(request, "%") @@ -110,7 +110,7 @@ func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, reque } func init() { - Index["vmware_avoid_short_term_contended_hosts"] = func() lib.Weigher[compute.SchedulerRequest] { + Index["vmware_avoid_short_term_contended_hosts"] = func() lib.Weigher[compute.PlacementRequest] { return &VMwareAvoidShortTermContendedHostsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go index 517989707..768973f1c 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go @@ -218,12 +218,12 @@ func TestVMwareAvoidShortTermContendedHostsStep_Run(t *testing.T) { tests := []struct { name string - request sched.SchedulerRequest + request sched.PlacementRequest expected map[string]float64 }{ { name: "Avoid contended hosts", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, @@ -240,7 +240,7 @@ func TestVMwareAvoidShortTermContendedHostsStep_Run(t *testing.T) { }, { name: "Missing data", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ Hosts: []sched.Host{ {ComputeHost: "host4"}, {ComputeHost: "host5"}, // No data for host5 diff --git a/internal/scheduling/nova/plugins/weighers/vmware_binpack.go b/internal/scheduling/nova/plugins/weighers/vmware_binpack.go index 686a3eb59..27bc33f39 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_binpack.go @@ -69,7 +69,7 @@ func (o VMwareBinpackStepOpts) Validate() error { // Explanation of the algorithm: https://volcano.sh/en/docs/plugins/#binpack type VMwareBinpackStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[sched.SchedulerRequest, VMwareBinpackStepOpts] + lib.BaseWeigher[sched.PlacementRequest, VMwareBinpackStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -86,7 +86,7 @@ func (s *VMwareBinpackStep) Init(ctx context.Context, client client.Client, weig } // Run this weigher in the pipeline after filters have been executed. -func (s *VMwareBinpackStep) Run(traceLog logr.Logger, request sched.SchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareBinpackStep) Run(traceLog logr.Logger, request sched.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["binpack score"] = s.PrepareStats(request, "float") @@ -185,7 +185,7 @@ func (s *VMwareBinpackStep) calcHostAllocation(hostUtilization compute.HostUtili } // calcVMResources calculates the total resource requests for the VM to be scheduled. -func (s *VMwareBinpackStep) calcVMResources(req sched.SchedulerRequest) map[corev1.ResourceName]resource.Quantity { +func (s *VMwareBinpackStep) calcVMResources(req sched.PlacementRequest) map[corev1.ResourceName]resource.Quantity { resources := make(map[corev1.ResourceName]resource.Quantity) resourcesMemBytes := int64(req.Flavor.MemoryMB) * (1 << 20) //nolint:gosec // memory values are bounded by Nova resourcesMemBytes *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova diff --git a/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go b/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go index 682d354cb..7d3226256 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go @@ -156,13 +156,13 @@ func TestVMwareBinpackStep_Run(t *testing.T) { tests := []struct { name string - request sched.SchedulerRequest + request sched.PlacementRequest expectedHigherScore string expectedLowerScore string }{ { name: "binpack prefers hosts with higher utilization", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ NumInstances: 1, Flavor: sched.FlavorSpec{ MemoryMB: 1000, // 1GB @@ -181,7 +181,7 @@ func TestVMwareBinpackStep_Run(t *testing.T) { }, { name: "handles missing host data gracefully", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ NumInstances: 1, Flavor: sched.FlavorSpec{ MemoryMB: 1000, @@ -280,13 +280,13 @@ func TestVMwareBinpackStep_CalcVMResources(t *testing.T) { tests := []struct { name string - request sched.SchedulerRequest + request sched.PlacementRequest expectedMemory int64 expectedCPU int64 }{ { name: "single instance", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ NumInstances: 1, Flavor: sched.FlavorSpec{ MemoryMB: 2048, @@ -298,7 +298,7 @@ func TestVMwareBinpackStep_CalcVMResources(t *testing.T) { }, { name: "multiple instances", - request: sched.SchedulerRequest{ + request: sched.PlacementRequest{ NumInstances: 3, Flavor: sched.FlavorSpec{ MemoryMB: 1024, diff --git a/internal/scheduling/nova/plugins/weighers/zz_index.go b/internal/scheduling/nova/plugins/weighers/zz_index.go index 6271a9b69..45bef3727 100644 --- a/internal/scheduling/nova/plugins/weighers/zz_index.go +++ b/internal/scheduling/nova/plugins/weighers/zz_index.go @@ -8,7 +8,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type NovaWeigher = lib.Weigher[compute.SchedulerRequest] +type NovaWeigher = lib.Weigher[compute.PlacementRequest] // Configuration of weighers supported by the nova scheduler. var Index = map[string]func() NovaWeigher{} diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index 11a2c05ad..bbc22d635 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -39,7 +39,7 @@ type VirtualMachineHandler struct { // written on create; mutable fields (target host, cortex spec, labels) are // patched on updates. A human-readable explanation of the result and the intent // are recorded on the VM's cortex spec. -func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.SchedulerResult, request api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent) error { +func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.PlacementResult, request api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent) error { log := ctrl.LoggerFrom(ctx) if result.TargetHost == nil { diff --git a/internal/scheduling/nova/vm_handler_test.go b/internal/scheduling/nova/vm_handler_test.go index 3c8ff2cac..6c81f6f60 100644 --- a/internal/scheduling/nova/vm_handler_test.go +++ b/internal/scheduling/nova/vm_handler_test.go @@ -56,8 +56,8 @@ func vmTestRequest() api.ExternalSchedulerRequest { return req } -func vmTestResult(targetHost *string) compute.SchedulerResult { - return compute.SchedulerResult{TargetHost: targetHost} +func vmTestResult(targetHost *string) compute.PlacementResult { + return compute.PlacementResult{TargetHost: targetHost} } func vmTestHypervisor(region, az string) *hv1.Hypervisor { diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 68c8281c8..90c9c2b01 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -107,13 +107,13 @@ func capacityMapsEqual(a, b map[hv1.ResourceName]resource.Quantity) bool { type Reconciler struct { client client.Client vmSource reservations.VMSource - scheduler *sched.Scheduler + scheduler *sched.Placement config Config lastReconcileAt time.Time } -func NewController(c client.Client, config Config, vmSource reservations.VMSource, scheduler *sched.Scheduler) *Reconciler { +func NewController(c client.Client, config Config, vmSource reservations.VMSource, scheduler *sched.Placement) *Reconciler { return &Reconciler{ client: c, vmSource: vmSource, @@ -829,7 +829,7 @@ func (c *Reconciler) probeScheduler( // slices and become no-ops). No hypervisor-type default — only match when the // flavor names one. requiredTraits, forbiddenTraits := nova.SplitTraits(flavor.ExtraSpecs) - out, err := c.scheduler.RunWithoutLock(ctx, sched.SchedulerRequest{ + out, err := c.scheduler.RunWithoutLock(ctx, sched.PlacementRequest{ Pipeline: pipeline, GatherAllCandidates: true, HypervisorFamily: sched.HypervisorFamilyKVM, diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 6ccafdbab..5501166ec 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -102,8 +102,8 @@ func newHypervisor(name, az string, memoryBytes int64) *hv1.Hypervisor { // fakePipeline returns a fixed ordered-host list (modelling a scheduler run). type fakePipeline struct{ hosts []string } -func (p fakePipeline) Run(sched.SchedulerRequest) (sched.SchedulerResult, error) { - return sched.SchedulerResult{OrderedHosts: p.hosts}, nil +func (p fakePipeline) Run(sched.PlacementRequest) (sched.PlacementResult, error) { + return sched.PlacementResult{OrderedHosts: p.hosts}, nil } // fakePipelineProvider serves a fakePipeline per pipeline name, returning the @@ -115,7 +115,7 @@ type fakePipelineProvider struct { useAll bool } -func (f fakePipelineProvider) GetPipeline(name string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { +func (f fakePipelineProvider) GetPipeline(name string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { if f.useAll { return fakePipeline{hosts: f.allPipelines}, true } @@ -125,70 +125,70 @@ func (f fakePipelineProvider) GetPipeline(name string) (lib.FilterWeigherPipelin // fakeGatherer is a no-op gatherer (tests preselect their own hosts). type fakeGatherer struct{} -func (fakeGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { +func (fakeGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { return nil } -// newMockScheduler builds an in-process compute.Scheduler whose pipelines return +// newMockScheduler builds an in-process compute.Placement whose pipelines return // the given per-pipeline host lists. Replaces the former httptest mock server. -func newMockScheduler(hostsByPipeline map[string][]string) *sched.Scheduler { - return sched.NewScheduler(fakePipelineProvider{hostsByPipeline: hostsByPipeline}, fakeGatherer{}) +func newMockScheduler(hostsByPipeline map[string][]string) *sched.Placement { + return sched.NewPlacement(fakePipelineProvider{hostsByPipeline: hostsByPipeline}, fakeGatherer{}) } // newMockSchedulerHosts is a convenience for a scheduler that returns the same // host list regardless of pipeline. -func newMockSchedulerHosts(hosts []string) *sched.Scheduler { - return sched.NewScheduler(fakePipelineProvider{allPipelines: hosts, useAll: true}, fakeGatherer{}) +func newMockSchedulerHosts(hosts []string) *sched.Placement { + return sched.NewPlacement(fakePipelineProvider{allPipelines: hosts, useAll: true}, fakeGatherer{}) } // fakeErrPipeline models a scheduler run that fails (replaces the former // HTTP 500 mock server). type fakeErrPipeline struct{} -func (fakeErrPipeline) Run(sched.SchedulerRequest) (sched.SchedulerResult, error) { - return sched.SchedulerResult{}, errors.New("boom") +func (fakeErrPipeline) Run(sched.PlacementRequest) (sched.PlacementResult, error) { + return sched.PlacementResult{}, errors.New("boom") } type fakeErrPipelineProvider struct{} -func (fakeErrPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { +func (fakeErrPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { return fakeErrPipeline{}, true } // newErrScheduler builds a scheduler whose pipeline always returns an error. -func newErrScheduler() *sched.Scheduler { - return sched.NewScheduler(fakeErrPipelineProvider{}, fakeGatherer{}) +func newErrScheduler() *sched.Placement { + return sched.NewPlacement(fakeErrPipelineProvider{}, fakeGatherer{}) } // capturePipeline records the request it is run with and returns a fixed host list. type capturePipeline struct { - captured *sched.SchedulerRequest + captured *sched.PlacementRequest hosts []string } -func (p capturePipeline) Run(req sched.SchedulerRequest) (sched.SchedulerResult, error) { +func (p capturePipeline) Run(req sched.PlacementRequest) (sched.PlacementResult, error) { *p.captured = req - return sched.SchedulerResult{OrderedHosts: p.hosts}, nil + return sched.PlacementResult{OrderedHosts: p.hosts}, nil } type capturePipelineProvider struct { - captured *sched.SchedulerRequest + captured *sched.PlacementRequest hosts []string } -func (f capturePipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { +func (f capturePipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { return capturePipeline(f), true } // captureScheduler builds a scheduler that records the request it is run with // into captured and returns the given hosts. -func captureScheduler(captured *sched.SchedulerRequest, hosts []string) *sched.Scheduler { - return sched.NewScheduler(capturePipelineProvider{captured: captured, hosts: hosts}, fakeGatherer{}) +func captureScheduler(captured *sched.PlacementRequest, hosts []string) *sched.Placement { + return sched.NewPlacement(capturePipelineProvider{captured: captured, hosts: hosts}, fakeGatherer{}) } // newController is a test helper that creates a Reconciler with a nil VMSource // and the given scheduler. -func newController(t *testing.T, c client.Client, cfg Config, scheduler *sched.Scheduler) *Reconciler { +func newController(t *testing.T, c client.Client, cfg Config, scheduler *sched.Placement) *Reconciler { t.Helper() return NewController(c, cfg, nil, scheduler) } @@ -1344,7 +1344,7 @@ func TestProbeScheduler_SetsCapacityProbeIntent(t *testing.T) { // Capture the request the scheduler receives so we can assert the probe // intent and project are set. - var capturedReq sched.SchedulerRequest + var capturedReq sched.PlacementRequest captured := captureScheduler(&capturedReq, []string{"host-1"}) c := NewController(fake.NewClientBuilder().WithScheme(scheme).Build(), Config{}, nil, captured) diff --git a/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go b/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go index fb1f31a52..a10b7de32 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go @@ -57,7 +57,7 @@ type e2eEnv struct { // newE2EEnv creates an e2eEnv with the given flavors and scheduler handler. // The scheduler handler controls what the mock Nova scheduler returns. -func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler *sched.Scheduler) *e2eEnv { +func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler *sched.Placement) *e2eEnv { t.Helper() log.SetLogger(zap.New(zap.WriteTo(os.Stderr), zap.UseDevMode(true))) @@ -272,31 +272,31 @@ func e2eIsTerminalReservation(res v1alpha1.Reservation) bool { // e2eFakePipeline models a scheduler run returning a fixed host list. type e2eFakePipeline struct{ hosts []string } -func (p e2eFakePipeline) Run(sched.SchedulerRequest) (sched.SchedulerResult, error) { - return sched.SchedulerResult{OrderedHosts: p.hosts}, nil +func (p e2eFakePipeline) Run(sched.PlacementRequest) (sched.PlacementResult, error) { + return sched.PlacementResult{OrderedHosts: p.hosts}, nil } type e2eFakeProvider struct{ hosts []string } -func (f e2eFakeProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { +func (f e2eFakeProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { return e2eFakePipeline(f), true } type e2eFakeGatherer struct{} -func (e2eFakeGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { +func (e2eFakeGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { return nil } -func e2eScheduler(hosts []string) *sched.Scheduler { - return sched.NewScheduler(e2eFakeProvider{hosts: hosts}, e2eFakeGatherer{}) +func e2eScheduler(hosts []string) *sched.Placement { + return sched.NewPlacement(e2eFakeProvider{hosts: hosts}, e2eFakeGatherer{}) } // e2eAcceptScheduler returns a scheduler that always places on host-1. -func e2eAcceptScheduler() *sched.Scheduler { return e2eScheduler([]string{"host-1"}) } +func e2eAcceptScheduler() *sched.Placement { return e2eScheduler([]string{"host-1"}) } // e2eRejectScheduler returns a scheduler that finds no host (NoHostsFound). -func e2eRejectScheduler() *sched.Scheduler { return e2eScheduler(nil) } +func e2eRejectScheduler() *sched.Placement { return e2eScheduler(nil) } // assertNoDryRunProbes verifies that all probe CommittedResource CRDs and their child // Reservations have been cleaned up. It only checks objects associated with dry-run probes @@ -337,7 +337,7 @@ var e2eFlavor = &TestFlavor{Name: "m1.small", Group: "hana_1", MemoryMB: 1024, V func TestE2EChangeCommitments(t *testing.T) { testCases := []struct { Name string - Scheduler func() *sched.Scheduler + Scheduler func() *sched.Placement ReqJSON string WantResp APIResponseExpectation WantAbsent []string diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go index 0cc6d3165..0923af1f6 100644 --- a/internal/scheduling/reservations/commitments/integration_test.go +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -410,22 +410,22 @@ type intgEnv struct { // intgSchedulerFn models a scheduler run: given the request it returns the ordered // hosts (an empty list models NoHostsFound). It replaces the former httptest mock // Nova scheduler handler. -type intgSchedulerFn func(sched.SchedulerRequest) sched.SchedulerResult +type intgSchedulerFn func(sched.PlacementRequest) sched.PlacementResult type intgPipeline struct{ fn intgSchedulerFn } -func (p intgPipeline) Run(req sched.SchedulerRequest) (sched.SchedulerResult, error) { +func (p intgPipeline) Run(req sched.PlacementRequest) (sched.PlacementResult, error) { return p.fn(req), nil } type intgPipelineProvider struct{ fn intgSchedulerFn } -func (p intgPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { +func (p intgPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { return intgPipeline(p), true } -func newIntgScheduler(fn intgSchedulerFn) *sched.Scheduler { - return sched.NewScheduler(intgPipelineProvider{fn: fn}, fakeGatherer{}) +func newIntgScheduler(fn intgSchedulerFn) *sched.Placement { + return sched.NewPlacement(intgPipelineProvider{fn: fn}, fakeGatherer{}) } func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn intgSchedulerFn, vmSource reservations.VMSource) *intgEnv { @@ -719,19 +719,19 @@ func intgAssertCRCondition(t *testing.T, k8sClient client.Client, crNames []stri // Scheduler functions // ============================================================================ -func intgAcceptScheduler(sched.SchedulerRequest) sched.SchedulerResult { - return sched.SchedulerResult{OrderedHosts: []string{"host-1"}} +func intgAcceptScheduler(sched.PlacementRequest) sched.PlacementResult { + return sched.PlacementResult{OrderedHosts: []string{"host-1"}} } -func intgRejectScheduler(sched.SchedulerRequest) sched.SchedulerResult { - return sched.SchedulerResult{OrderedHosts: []string{}} +func intgRejectScheduler(sched.PlacementRequest) sched.PlacementResult { + return sched.PlacementResult{OrderedHosts: []string{}} } // intgAcceptFirstScheduler returns a scheduler fn that accepts the first count placement calls // and rejects all subsequent ones. Uses an atomic counter so concurrent calls are safe. func intgAcceptFirstScheduler(count int) intgSchedulerFn { var calls atomic.Int32 - return func(req sched.SchedulerRequest) sched.SchedulerResult { + return func(req sched.PlacementRequest) sched.PlacementResult { if int(calls.Add(1)) <= count { return intgAcceptScheduler(req) } @@ -743,7 +743,7 @@ func intgAcceptFirstScheduler(count int) intgSchedulerFn { // and accepts all subsequent ones. Used to test AllowRejection=false retry-until-success paths. func intgRejectFirstScheduler(count int) intgSchedulerFn { var calls atomic.Int32 - return func(req sched.SchedulerRequest) sched.SchedulerResult { + return func(req sched.PlacementRequest) sched.PlacementResult { if int(calls.Add(1)) <= count { return intgRejectScheduler(req) } @@ -1251,12 +1251,12 @@ func TestCRScheduling_SetsReserveForCommittedResourceIntent(t *testing.T) { // CR slot scheduling has a real project ID and must run tenant-context filters // (filter_allowed_projects, filter_aggregate_metadata, etc.). Verify the intent is // reserve_for_committed_resource so filters can identify the call type correctly. - var capturedReq sched.SchedulerRequest + var capturedReq sched.PlacementRequest var schedulerCalled bool - captureScheduler := func(req sched.SchedulerRequest) sched.SchedulerResult { + captureScheduler := func(req sched.PlacementRequest) sched.PlacementResult { schedulerCalled = true capturedReq = req - return sched.SchedulerResult{OrderedHosts: []string{"host-1"}} + return sched.PlacementResult{OrderedHosts: []string{"host-1"}} } env := newIntgEnv(t, []client.Object{newTestFlavorKnowledge(), intgHypervisor("host-1")}, captureScheduler, nil) @@ -1345,7 +1345,7 @@ func TestCROversubscriptionRemediation(t *testing.T) { } var schedulerCalls atomic.Int32 - schedulerFn := func(req sched.SchedulerRequest) sched.SchedulerResult { + schedulerFn := func(req sched.PlacementRequest) sched.PlacementResult { schedulerCalls.Add(1) return intgRejectScheduler(req) // reject so evicted slots don't land back on host-1 } diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 437e9e78b..a9c188af6 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -48,7 +48,7 @@ type CommitmentReservationController struct { // Configuration for the controller. Conf ReservationControllerConfig // Scheduler is the in-process scheduler used to place reservations. - Scheduler *compute.Scheduler + Scheduler *compute.Placement // DomainResolver resolves OpenStack domain IDs to domain names so the // domain_name scheduler hint can be populated for filter_external_customer. // Nil when KeystoneSecretRef is not configured; hint is omitted in that case. @@ -362,7 +362,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // would otherwise see empty slices and become no-ops). No hypervisor-type // default here — only match when the flavor names one. requiredTraits, forbiddenTraits := nova.SplitTraits(flavorDetails.ExtraSpecs) - schedReq := compute.SchedulerRequest{ + schedReq := compute.PlacementRequest{ Pipeline: pipelineName, GatherAllCandidates: true, HypervisorFamily: compute.HypervisorFamilyKVM, diff --git a/internal/scheduling/reservations/commitments/reservation_controller_test.go b/internal/scheduling/reservations/commitments/reservation_controller_test.go index f8d9c5eb5..fb659c085 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller_test.go +++ b/internal/scheduling/reservations/commitments/reservation_controller_test.go @@ -29,29 +29,29 @@ import ( // fakePipeline returns a fixed ordered-host list (modelling a scheduler run). type fakePipeline struct{ hosts []string } -func (p fakePipeline) Run(sched.SchedulerRequest) (sched.SchedulerResult, error) { - return sched.SchedulerResult{OrderedHosts: p.hosts}, nil +func (p fakePipeline) Run(sched.PlacementRequest) (sched.PlacementResult, error) { + return sched.PlacementResult{OrderedHosts: p.hosts}, nil } // fakePipelineProvider serves a fakePipeline for every pipeline name, returning // the configured host list. type fakePipelineProvider struct{ hosts []string } -func (f fakePipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { +func (f fakePipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { return fakePipeline(f), true } // fakeGatherer is a no-op gatherer (tests supply their own hosts). type fakeGatherer struct{} -func (fakeGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { +func (fakeGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { return nil } -// newMockScheduler builds an in-process compute.Scheduler whose pipeline returns +// newMockScheduler builds an in-process compute.Placement whose pipeline returns // the given host list. Replaces the former httptest mock scheduler server. -func newMockScheduler(hosts []string) *sched.Scheduler { - return sched.NewScheduler(fakePipelineProvider{hosts: hosts}, fakeGatherer{}) +func newMockScheduler(hosts []string) *sched.Placement { + return sched.NewPlacement(fakePipelineProvider{hosts: hosts}, fakeGatherer{}) } // mockDomainResolver is a test double for DomainResolver. @@ -871,16 +871,16 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t // capturingPipeline records the SchedulerRequest it was run with and returns a // single host, so tests can assert on the request the reconciler built. -type capturingPipeline struct{ captured *sched.SchedulerRequest } +type capturingPipeline struct{ captured *sched.PlacementRequest } -func (p capturingPipeline) Run(req sched.SchedulerRequest) (sched.SchedulerResult, error) { +func (p capturingPipeline) Run(req sched.PlacementRequest) (sched.PlacementResult, error) { *p.captured = req - return sched.SchedulerResult{OrderedHosts: []string{"test-host-1"}}, nil + return sched.PlacementResult{OrderedHosts: []string{"test-host-1"}}, nil } -type capturingPipelineProvider struct{ captured *sched.SchedulerRequest } +type capturingPipelineProvider struct{ captured *sched.PlacementRequest } -func (f capturingPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { +func (f capturingPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { return capturingPipeline(f), true } @@ -930,12 +930,12 @@ func TestCommitmentReservationController_DomainNameHint(t *testing.T) { k8sClient := newCRTestClient(scheme, reservation, newTestFlavorKnowledge(), &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "test-host-1"}}) - var captured sched.SchedulerRequest + var captured sched.PlacementRequest reconciler := &CommitmentReservationController{ Client: k8sClient, Scheme: scheme, Conf: ReservationControllerConfig{}, - Scheduler: sched.NewScheduler(capturingPipelineProvider{captured: &captured}, fakeGatherer{}), + Scheduler: sched.NewPlacement(capturingPipelineProvider{captured: &captured}, fakeGatherer{}), DomainResolver: tt.resolver, } diff --git a/internal/scheduling/reservations/extra_specs.go b/internal/scheduling/reservations/extra_specs.go index c55563f94..66e568585 100644 --- a/internal/scheduling/reservations/extra_specs.go +++ b/internal/scheduling/reservations/extra_specs.go @@ -8,7 +8,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" ) -// The reservation controllers build the typed compute.SchedulerRequest directly +// The reservation controllers build the typed compute.PlacementRequest directly // (not via the Nova wire adapter), yet their flavors still carry Nova's extra-spec // string conventions. HypervisorTypeCapability reuses the shared parser in // internal/scheduling/nova and layers the reservation-specific defaultQEMU seeding diff --git a/internal/scheduling/reservations/failover/controller.go b/internal/scheduling/reservations/failover/controller.go index 69032871f..b1bcf0527 100644 --- a/internal/scheduling/reservations/failover/controller.go +++ b/internal/scheduling/reservations/failover/controller.go @@ -44,13 +44,13 @@ type FailoverReservationController struct { client.Client VMSource reservations.VMSource Config FailoverConfig - Scheduler *sched.Scheduler + Scheduler *sched.Placement Recorder events.EventRecorder // Event recorder for emitting Kubernetes events Monitor *FailoverMonitor reconcileCount int64 // Track reconciliation count for rotating VM selection } -func NewFailoverReservationController(c client.Client, vmSource reservations.VMSource, config FailoverConfig, scheduler *sched.Scheduler, monitor *FailoverMonitor) *FailoverReservationController { +func NewFailoverReservationController(c client.Client, vmSource reservations.VMSource, config FailoverConfig, scheduler *sched.Placement, monitor *FailoverMonitor) *FailoverReservationController { return &FailoverReservationController{ Client: c, VMSource: vmSource, diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index 26da3f930..08c219de0 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -37,7 +37,7 @@ import ( // preselected hosts unchanged (the tests seed their own eligible hosts). type passthroughGatherer struct{} -func (passthroughGatherer) MutateWithAllCandidates(context.Context, *compute.SchedulerRequest) error { +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *compute.PlacementRequest) error { return nil } @@ -706,7 +706,7 @@ func (env *IntegrationTestEnv) TriggerFailoverReconcile(flavorRequirements map[s // Drive the same in-process pipeline the testHTTPAPI uses, so the failover // reservation flow exercises the real scheduler over the env's nova controller. - scheduler := compute.NewScheduler(env.NovaController, passthroughGatherer{}) + scheduler := compute.NewPlacement(env.NovaController, passthroughGatherer{}) config := FailoverConfig{ ReconcileInterval: metav1.Duration{Duration: time.Minute}, @@ -1153,9 +1153,9 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h Build() novaController := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.PlacementRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: getSharedMonitor(), @@ -1234,7 +1234,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h novaController.PipelineConfigs[pipeline.Name] = pipeline } - api := &testHTTPAPI{scheduler: compute.NewScheduler(novaController, passthroughGatherer{})} + api := &testHTTPAPI{scheduler: compute.NewPlacement(novaController, passthroughGatherer{})} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) server := httptest.NewServer(mux) @@ -1253,7 +1253,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h // testHTTPAPI is a simplified HTTP API for testing that delegates to the controller. // It does NOT shuffle evacuation hosts, ensuring deterministic test results. type testHTTPAPI struct { - scheduler *compute.Scheduler + scheduler *compute.Placement } // NovaExternalScheduler handles the POST request from the Nova scheduler. @@ -1330,9 +1330,9 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, Build() novaController := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.SchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.PlacementRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.SchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: getSharedMonitor(), @@ -1412,7 +1412,7 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, novaController.PipelineConfigs[pipeline.Name] = pipeline } - api := &testHTTPAPI{scheduler: compute.NewScheduler(novaController, passthroughGatherer{})} + api := &testHTTPAPI{scheduler: compute.NewPlacement(novaController, passthroughGatherer{})} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) server := httptest.NewServer(mux) diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 333af97bb..1a9b22741 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -107,7 +107,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex weights[h.ComputeHost] = 0.0 } requiredTraits, forbiddenTraits := nova.SplitTraits(vm.FlavorExtraSpecs) - schedReq := compute.SchedulerRequest{ + schedReq := compute.PlacementRequest{ Pipeline: pipeline, GatherAllCandidates: true, HypervisorFamily: compute.HypervisorFamilyKVM, @@ -156,7 +156,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex // runScheduler runs the shared scheduler, choosing the lock-free path for // read-only runs (e.g. reuse/evacuate compatibility checks) and the locked path // for runs that mutate shared reservation state. -func (c *FailoverReservationController) runScheduler(ctx context.Context, req compute.SchedulerRequest) (compute.SchedulerResult, error) { +func (c *FailoverReservationController) runScheduler(ctx context.Context, req compute.PlacementRequest) (compute.PlacementResult, error) { if req.Options.ReadOnly { return c.Scheduler.RunWithoutLock(ctx, req, nil, nil) } @@ -265,7 +265,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( evacuateOpts := DefaultFailoverOptions evacuateOpts.ReadOnly = true evacuateOpts.LockReservations = true - schedReq := compute.SchedulerRequest{ + schedReq := compute.PlacementRequest{ Pipeline: pipeline, GatherAllCandidates: true, HypervisorFamily: compute.HypervisorFamilyKVM, diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go index 10d9399ac..24d78a687 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling_test.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -480,36 +480,36 @@ func buildSchedulingTestVMWithResources(uuid, hypervisor string, memoryMB, vcpus // single fixed host, so tests can assert the options the controller built. type capturingPipeline struct { host string - captured *sched.SchedulerRequest + captured *sched.PlacementRequest } -func (p *capturingPipeline) Run(req sched.SchedulerRequest) (sched.SchedulerResult, error) { +func (p *capturingPipeline) Run(req sched.PlacementRequest) (sched.PlacementResult, error) { *p.captured = req - return sched.SchedulerResult{OrderedHosts: []string{p.host}}, nil + return sched.PlacementResult{OrderedHosts: []string{p.host}}, nil } // capturingProvider serves the same capturingPipeline for every pipeline name. type capturingProvider struct{ pipeline *capturingPipeline } -func (f capturingProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.SchedulerRequest], bool) { +func (f capturingProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { return f.pipeline, true } // capturingGatherer is a no-op gatherer (the tests preselect their own hosts). type capturingGatherer struct{} -func (capturingGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { +func (capturingGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { return nil } // newCapturingScheduler builds an in-process scheduler whose pipeline returns // the given host and records every request it ran into *out. -func newCapturingScheduler(host string, out *sched.SchedulerRequest) *sched.Scheduler { - return sched.NewScheduler(capturingProvider{pipeline: &capturingPipeline{host: host, captured: out}}, capturingGatherer{}) +func newCapturingScheduler(host string, out *sched.PlacementRequest) *sched.Placement { + return sched.NewPlacement(capturingProvider{pipeline: &capturingPipeline{host: host, captured: out}}, capturingGatherer{}) } // buildMinimalController returns a FailoverReservationController wired to the given scheduler. -func buildMinimalController(t *testing.T, scheduler *sched.Scheduler) *FailoverReservationController { +func buildMinimalController(t *testing.T, scheduler *sched.Placement) *FailoverReservationController { t.Helper() scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { @@ -577,7 +577,7 @@ func TestFailoverSchedulerOptions(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var capturedReq sched.SchedulerRequest + var capturedReq sched.PlacementRequest scheduler := newCapturingScheduler("host-2", &capturedReq) tt.call(buildMinimalController(t, scheduler), context.Background()) @@ -623,7 +623,7 @@ func TestFailoverSchedulerFlavorConstraints(t *testing.T) { vm.FlavorExtraSpecs = tt.extraSpecs resolved := resolveVMSpecForScheduling(context.Background(), vm, false, nil) - var capturedReq sched.SchedulerRequest + var capturedReq sched.PlacementRequest scheduler := newCapturingScheduler("host-2", &capturedReq) c := buildMinimalController(t, scheduler) _, _ = c.scheduleAndBuildNewFailoverReservation(context.Background(), vm, []string{"host-1", "host-2"}, nil, nil, resolved) //nolint:errcheck diff --git a/internal/scheduling/reservations/failoverhost/controller.go b/internal/scheduling/reservations/failoverhost/controller.go index 0693c60ca..def981f08 100644 --- a/internal/scheduling/reservations/failoverhost/controller.go +++ b/internal/scheduling/reservations/failoverhost/controller.go @@ -128,7 +128,7 @@ type Controller struct { // Scheduler is the shared in-process scheduler the path finder probes for // placement candidates. Required (the controller cannot compute failover // partitions without it). - Scheduler *compute.Scheduler + Scheduler *compute.Placement // Monitor records Prometheus metrics for the calculation loop and the // reconciler. It may be nil (e.g. in tests); every call site nil-guards it. diff --git a/internal/scheduling/reservations/failoverhost/integration_test.go b/internal/scheduling/reservations/failoverhost/integration_test.go index 31d9b5dc9..c34dae82d 100644 --- a/internal/scheduling/reservations/failoverhost/integration_test.go +++ b/internal/scheduling/reservations/failoverhost/integration_test.go @@ -27,7 +27,7 @@ import ( // hosts, so gathering is a no-op here. type passthroughGatherer struct{} -func (passthroughGatherer) MutateWithAllCandidates(context.Context, *sched.SchedulerRequest) error { +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { return nil } @@ -244,7 +244,7 @@ func TestIntegration(t *testing.T) { // end. type integrationTestEnv struct { k8sClient client.Client - scheduler *sched.Scheduler + scheduler *sched.Placement flavors []compute.FlavorInGroup hypervisors map[string]hv1.Hypervisor } @@ -264,9 +264,9 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors k8sClient := newTestClient(t, objects...) controller := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[sched.SchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[sched.PlacementRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.SchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: sharedPipelineMonitor(), @@ -298,7 +298,7 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors // KVM requests trigger candidate gathering; the test seeds its own eligible // hosts, so use a passthrough gatherer that preserves them. - scheduler := sched.NewScheduler(controller, passthroughGatherer{}) + scheduler := sched.NewPlacement(controller, passthroughGatherer{}) return &integrationTestEnv{ k8sClient: k8sClient, diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go index ace3b51a3..2e592d18e 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding.go @@ -83,7 +83,7 @@ type evacuationPathFinder struct { // scheduler is the shared in-process scheduler used to probe placement. The // pathfinder issues read-only runs via RunWithoutLock. Separately injected so // tests can point it at a fake pipeline. - scheduler *compute.Scheduler + scheduler *compute.Placement // monitor records scheduler request metrics. May be nil (e.g. in tests). monitor *Monitor } @@ -255,7 +255,7 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See // would otherwise see empty slices and become no-ops). The seeder only emits // seeds whose flavor names a supported hypervisor type, so no default is needed. requiredTraits, forbiddenTraits := nova.SplitTraits(seed.Flavor.ExtraSpecs) - req := compute.SchedulerRequest{ + req := compute.PlacementRequest{ // Same HANA vs general-purpose split as regular failover placement. Pipeline: failover.InferFailoverPipeline(seed.Flavor.ExtraSpecs), // We offer the host set ourselves; the scheduler must not gather the diff --git a/internal/scheduling/reservations/failoverhost/pathfinding_test.go b/internal/scheduling/reservations/failoverhost/pathfinding_test.go index df9fca88a..72c451f38 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding_test.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding_test.go @@ -29,7 +29,7 @@ func stubScheduler(t *testing.T, candidatesByFlavor map[string][]string, errFlav candidatesByFlavor: candidatesByFlavor, errFlavors: errFlavors, }} - scheduler := compute.NewScheduler(provider, passthroughGatherer{}) + scheduler := compute.NewPlacement(provider, passthroughGatherer{}) return &evacuationPathFinder{scheduler: scheduler} } @@ -39,7 +39,7 @@ type stubPipelineProvider struct { pipeline *stubPipeline } -func (p *stubPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[compute.SchedulerRequest], bool) { +func (p *stubPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[compute.PlacementRequest], bool) { return p.pipeline, true } @@ -52,7 +52,7 @@ type stubPipeline struct { errFlavors map[string]bool } -func (s *stubPipeline) Run(req compute.SchedulerRequest) (compute.SchedulerResult, error) { +func (s *stubPipeline) Run(req compute.PlacementRequest) (compute.PlacementResult, error) { // Identify which seed this probe is for by the trait the seed constructor // puts on the flavor (see seed()), which the probe carries in RequiredTraits. flavor := "" @@ -86,9 +86,9 @@ func (s *stubPipeline) Run(req compute.SchedulerRequest) (compute.SchedulerResul s.t.Errorf("request for flavor %q must not set a source host", flavor) } if s.errFlavors[flavor] { - return compute.SchedulerResult{}, fmt.Errorf("probe failed for flavor %q", flavor) + return compute.PlacementResult{}, fmt.Errorf("probe failed for flavor %q", flavor) } - return compute.SchedulerResult{OrderedHosts: s.candidatesByFlavor[flavor]}, nil + return compute.PlacementResult{OrderedHosts: s.candidatesByFlavor[flavor]}, nil } // pathNames collapses failover edges into sorted "source->target" strings, for From 0997ae5ed4e765c8f77b2d82308d141e0ccbf33d Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 4 Nov 2026 15:30:16 +0100 Subject: [PATCH 21/21] resolve merge conflicts Signed-off-by: Markus Wieland --- internal/scheduling/nova/cr_metrics_test.go | 1 + internal/scheduling/nova/crs/recorder.go | 16 ++--- internal/scheduling/nova/crs/recorder_test.go | 66 +++++++++++------- .../filters/filter_has_enough_capacity.go | 67 +++++++++---------- .../filters/filter_quota_enforcement_test.go | 4 +- .../reservation_controller_test.go | 1 + .../reservations/commitments/syncer.go | 1 + 7 files changed, 85 insertions(+), 71 deletions(-) diff --git a/internal/scheduling/nova/cr_metrics_test.go b/internal/scheduling/nova/cr_metrics_test.go index 47bc3ba09..a05013b7c 100644 --- a/internal/scheduling/nova/cr_metrics_test.go +++ b/internal/scheduling/nova/cr_metrics_test.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/internal/scheduling/nova/crs/recorder.go b/internal/scheduling/nova/crs/recorder.go index 6e6dabf75..31b1e0265 100644 --- a/internal/scheduling/nova/crs/recorder.go +++ b/internal/scheduling/nova/crs/recorder.go @@ -10,6 +10,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/prometheus/client_golang/prometheus" ctrl "sigs.k8s.io/controller-runtime" @@ -50,14 +51,14 @@ func (r *Recorder) resolveFlavorGroup(ctx context.Context, flavorName string) (s // RecordPlacement writes the placed VM UUID into the matching Reservation slot // and emits placement metrics. Called after a successful Nova placement. -func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decision, request api.ExternalSchedulerRequest) { +func (r *Recorder) RecordPlacement(ctx context.Context, result sched.PlacementResult, intent v1alpha1.SchedulingIntent, request api.ExternalSchedulerRequest) { log := ctrl.LoggerFrom(ctx) instanceUUID := request.Spec.Data.InstanceUUID projectID := request.Context.ProjectID flavorName := request.Spec.Data.Flavor.Data.Name - selectedHost := *decision.Status.Result.TargetHost - intent := string(decision.Spec.Intent) + selectedHost := *result.TargetHost + intentStr := string(intent) flavorGroupName, flavorInGroup, err := r.resolveFlavorGroup(ctx, flavorName) if err != nil { @@ -67,7 +68,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis log.Error(err, "CR allocation: failed to resolve flavor group", "flavor", flavorName, "instanceUUID", instanceUUID) if r.PlacementCounter != nil { - r.PlacementCounter.WithLabelValues("unknown", intent, "error").Inc() + r.PlacementCounter.WithLabelValues("unknown", intentStr, "error").Inc() } } return @@ -92,7 +93,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis activeCRs = append(activeCRs, cr) } - candidateHosts := decision.Status.Result.OrderedHosts + candidateHosts := result.OrderedHosts crOutcome := ClassifyPlacement(evaluator, activeCRs, candidateHosts, projectID, flavorGroupName) log.V(1).Info("CR allocation: placement classified", @@ -103,7 +104,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis "flavorGroup", flavorGroupName, ) if r.PlacementCounter != nil { - r.PlacementCounter.WithLabelValues(flavorGroupName, intent, crOutcome).Inc() + r.PlacementCounter.WithLabelValues(flavorGroupName, intentStr, crOutcome).Inc() } if crOutcome != "slot_used" { @@ -159,13 +160,12 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis } // RecordNoHostFound classifies a no-host-found result and emits a log line and metric. -func (r *Recorder) RecordNoHostFound(ctx context.Context, decision *v1alpha1.Decision, request api.ExternalSchedulerRequest) { +func (r *Recorder) RecordNoHostFound(ctx context.Context, intent v1alpha1.SchedulingIntent, request api.ExternalSchedulerRequest) { log := ctrl.LoggerFrom(ctx) projectID := request.Context.ProjectID flavorName := request.Spec.Data.Flavor.Data.Name instanceUUID := request.Spec.Data.InstanceUUID - intent := decision.Spec.Intent flavorGroupName, flavorInGroup, err := r.resolveFlavorGroup(ctx, flavorName) if err != nil { diff --git a/internal/scheduling/nova/crs/recorder_test.go b/internal/scheduling/nova/crs/recorder_test.go index 6e9aba55c..90ee32104 100644 --- a/internal/scheduling/nova/crs/recorder_test.go +++ b/internal/scheduling/nova/crs/recorder_test.go @@ -19,6 +19,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -110,15 +111,11 @@ func TestRecordCRAllocation(t *testing.T) { } } - makeDecision := func(host string, candidates ...string) *v1alpha1.Decision { + makeResult := func(host string, candidates ...string) sched.PlacementResult { h := host - return &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &h, - OrderedHosts: candidates, - }, - }, + return sched.PlacementResult{ + TargetHost: &h, + OrderedHosts: candidates, } } @@ -159,33 +156,39 @@ func TestRecordCRAllocation(t *testing.T) { name string objects []client.Object request api.ExternalSchedulerRequest - decision *v1alpha1.Decision + result sched.PlacementResult + intent v1alpha1.SchedulingIntent expectedCRSlot string // if non-empty, asserts PlacementCounter cr_slot label expectedFG string // flavor_group label; defaults to "unknown" if empty + // annotateSlot, when non-empty, asserts that the named reservation slot + // was annotated with the candidate VM UUID (the #1438 recorder delegates + // the confirmed write to the reservation controller via this annotation). + annotateSlot string }{ { - name: "classifies slot_used and logs placement (recorder delegates write to reservation controller)", + name: "classifies slot_used and annotates reservation (recorder delegates write to reservation controller)", objects: []client.Object{ flavorKnowledge(), makeCRObject(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), expectedCRSlot: "slot_used", expectedFG: flavorGroup, + annotateSlot: "slot-1", }, { // With status.allocs set, ClassifyPlacement sees remaining=0 → "slot_missed". - // The recorder returns early at that point, so no write occurs (idempotency maintained via no-op). - name: "idempotent: UUID already confirmed in status.allocations — skips early", + // The recorder returns early at that point, so no annotation occurs (idempotency maintained via no-op). + name: "idempotent: UUID already confirmed in status.allocation — skips early", objects: []client.Object{ flavorKnowledge(), makeCRObject(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, vmAlloc())), }, - request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, flavorName), + result: makeResult(selectedHost, selectedHost), // No expectedCRSlot: with status.allocs set, slot appears exhausted (remaining=0), // crOutcome="slot_missed" — counter not checked for this idempotent path. }, @@ -195,8 +198,8 @@ func TestRecordCRAllocation(t *testing.T) { flavorKnowledge(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, - request: makeRequest(instanceUUID, projectID, "unknown-flavor"), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, "unknown-flavor"), + result: makeResult(selectedHost, selectedHost), }, { name: "no matching reservation: host mismatch", @@ -205,8 +208,8 @@ func TestRecordCRAllocation(t *testing.T) { makeCRObject(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, "other-host", nil)), }, - request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, flavorName), + result: makeResult(selectedHost, selectedHost), }, { name: "no slot fits: all capacity confirmed (status.allocs non-empty → 0 remaining memory)", @@ -223,8 +226,8 @@ func TestRecordCRAllocation(t *testing.T) { }, })), }, - request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, flavorName), + result: makeResult(selectedHost, selectedHost), }, { name: "inactive CR (pending state) is filtered by IsActive(): no slot_used", @@ -237,8 +240,8 @@ func TestRecordCRAllocation(t *testing.T) { }(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, - request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, flavorName), + result: makeResult(selectedHost, selectedHost), }, { name: "no knowledge CRD: logs error, no allocation", @@ -246,7 +249,7 @@ func TestRecordCRAllocation(t *testing.T) { makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), expectedCRSlot: "error", }, } @@ -267,10 +270,10 @@ func TestRecordCRAllocation(t *testing.T) { PlacementCounter: counter, } - recorder.RecordPlacement(context.Background(), tt.decision, tt.request) + recorder.RecordPlacement(context.Background(), tt.result, tt.intent, tt.request) if tt.expectedCRSlot != "" { - intent := string(tt.decision.Spec.Intent) + intent := string(tt.intent) fg := tt.expectedFG if fg == "" { fg = "unknown" @@ -281,6 +284,17 @@ func TestRecordCRAllocation(t *testing.T) { fg, intent, tt.expectedCRSlot, got) } } + + if tt.annotateSlot != "" { + var res v1alpha1.Reservation + if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: tt.annotateSlot}, &res); err != nil { + t.Fatalf("Get reservation %q: %v", tt.annotateSlot, err) + } + if got := res.Annotations[v1alpha1.AnnotationCandidateAllocation]; got != instanceUUID { + t.Errorf("annotation %q on %q = %q, want %q", + v1alpha1.AnnotationCandidateAllocation, tt.annotateSlot, got, instanceUUID) + } + } }) } } diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index c68b8e82c..345c59b90 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -12,6 +12,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" resv "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -36,7 +37,7 @@ type FilterHasEnoughCapacityOpts struct { func (FilterHasEnoughCapacityOpts) Validate() error { return nil } type FilterHasEnoughCapacity struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterHasEnoughCapacityOpts] + lib.BaseFilter[compute.PlacementRequest, FilterHasEnoughCapacityOpts] } // Filter hosts that don't have enough capacity to run the requested flavor. @@ -63,7 +64,7 @@ type FilterHasEnoughCapacity struct { // known at this point. // // Please also note that disk space is currently not considered by this filter. -func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { opts := request.GetOptions() result := s.IncludeAllHostsFromRequest(request) @@ -157,22 +158,21 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // Check if this is a CR reservation scheduling request. // If so, we should NOT unlock any CR reservations to prevent overbooking. // CR capacity should only be unlocked for actual VM scheduling. - intent, err := request.GetIntent() switch { - case err == nil && intent == api.ReserveForCommittedResourceIntent: + case request.Intent == api.ReserveForCommittedResourceIntent: traceLog.V(1).Info("keeping CR reservation locked for CR reservation scheduling", "reservation", reservation.Name, - "intent", intent) + "intent", request.Intent) // Don't continue - fall through to block the resources case !s.Options.LockReserved && !opts.LockReservations && // For committed resource reservations: unlock resources only if: // 1. Project ID matches // 2. ResourceGroup matches the flavor's hw_version - reservation.Spec.CommittedResourceReservation.MatchesGroup(request.Spec.Data.ProjectID, request.Spec.Data.Flavor.Data.ExtraSpecs["quota:hw_version"]): + reservation.Spec.CommittedResourceReservation.MatchesGroup(request.ProjectID, request.HardwareVersion): traceLog.V(1).Info("unlocking resources reserved by matching committed resource reservation with allocation", "reservation", reservation.Name, - "instanceUUID", request.Spec.Data.InstanceUUID, - "projectID", request.Spec.Data.ProjectID, + "instanceUUID", request.InstanceUUID, + "projectID", request.ProjectID, "resourceGroup", reservation.Spec.CommittedResourceReservation.ResourceGroup) continue } @@ -184,25 +184,22 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // 1. Failover reservations are specifically for HA/evacuation scenarios. // 2. During live migrations or other operations, we don't want to use failover capacity. // Note: we cannot use failover reservations from other VMs, as that can invalidate our HA guarantees. - intent, err := request.GetIntent() - if err == nil { - switch intent { - case api.EvacuateIntent: - if reservation.Status.FailoverReservation != nil { - if _, contained := reservation.Status.FailoverReservation.Allocations[request.Spec.Data.InstanceUUID]; contained { - traceLog.V(1).Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", - "reservation", reservation.Name, - "instanceUUID", request.Spec.Data.InstanceUUID) - continue - } + switch request.Intent { + case api.EvacuateIntent: + if reservation.Status.FailoverReservation != nil { + if _, contained := reservation.Status.FailoverReservation.Allocations[request.InstanceUUID]; contained { + traceLog.V(1).Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", + "reservation", reservation.Name, + "instanceUUID", request.InstanceUUID) + continue } - case api.ReuseFailoverReservationIntent: - // Reuse check: the reservation already pre-blocks the right capacity for this VM. - // Don't subtract it from free capacity to avoid double-counting. - traceLog.V(1).Info("skipping failover reservation block for reuse compatibility check", - "reservation", reservation.Name) - continue } + case api.ReuseFailoverReservationIntent: + // Reuse check: the reservation already pre-blocks the right capacity for this VM. + // Don't subtract it from free capacity to avoid double-counting. + traceLog.V(1).Info("skipping failover reservation block for reuse compatibility check", + "reservation", reservation.Name) + continue } traceLog.V(1).Info("processing failover reservation", "reservation", reservation.Name) } @@ -276,7 +273,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External hostsEncountered[host] = struct{}{} // Check cpu capacity. - if request.Spec.Data.Flavor.Data.VCPUs == 0 { + if request.Flavor.VCPUs == 0 { return nil, errors.New("flavor has 0 vcpus") } freeCPU, ok := free["cpu"] @@ -291,11 +288,11 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // Calculate how many instances can fit on this host, based on cpu. //nolint:gosec // We're checking for underflows above (< 0). vcpuSlots := uint64(freeCPU.Value()) / - request.Spec.Data.Flavor.Data.VCPUs - if vcpuSlots < request.Spec.Data.NumInstances { + request.Flavor.VCPUs + if vcpuSlots < request.NumInstances { traceLog.V(1).Info( "filtering host due to insufficient CPU capacity", - "host", host, "requested", request.Spec.Data.Flavor.Data.VCPUs, + "host", host, "requested", request.Flavor.VCPUs, "available", freeCPU.String(), ) delete(result.Activations, host) @@ -303,7 +300,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External } // Check memory capacity. - if request.Spec.Data.Flavor.Data.MemoryMB == 0 { + if request.Flavor.MemoryMB == 0 { return nil, errors.New("flavor has 0 memory") } freeMemory, ok := free["memory"] @@ -320,11 +317,11 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // are in mebibytes (1 MiB = 2^20 bytes). Multiply by 1<<20 to convert to bytes. //nolint:gosec // We're checking for underflows above (< 0). memorySlots := uint64(freeMemory.Value()) / - (request.Spec.Data.Flavor.Data.MemoryMB * (1 << 20)) - if memorySlots < request.Spec.Data.NumInstances { + (request.Flavor.MemoryMB * (1 << 20)) + if memorySlots < request.NumInstances { traceLog.V(1).Info( "filtering host due to insufficient RAM capacity", - "host", host, "requested_mb", request.Spec.Data.Flavor.Data.MemoryMB, + "host", host, "requested_mb", request.Flavor.MemoryMB, "available_mb", freeMemory.String(), ) delete(result.Activations, host) @@ -332,9 +329,9 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External } traceLog.V(1).Info( "host has enough capacity", "host", host, - "requested_cpus", request.Spec.Data.Flavor.Data.VCPUs, + "requested_cpus", request.Flavor.VCPUs, "available_cpus", freeCPU.String(), - "requested_memory_mb", request.Spec.Data.Flavor.Data.MemoryMB, + "requested_memory_mb", request.Flavor.MemoryMB, "available_memory", freeMemory.String(), ) } diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 256275555..bb9b3d856 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -479,7 +479,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { // Fully used, so it provides no CR headroom itself — the request // falls through to the PAYG check where its amount is deducted. Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}, + Billing: &v1alpha1.BillingStatus{UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}}, }, }, &v1alpha1.ProjectQuota{ @@ -518,7 +518,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("100Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}, + Billing: &v1alpha1.BillingStatus{UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}}, }, }, &v1alpha1.ProjectQuota{ diff --git a/internal/scheduling/reservations/commitments/reservation_controller_test.go b/internal/scheduling/reservations/commitments/reservation_controller_test.go index 39b01d3c7..d5439852f 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller_test.go +++ b/internal/scheduling/reservations/commitments/reservation_controller_test.go @@ -23,6 +23,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) // fakePipeline returns a fixed ordered-host list (modelling a scheduler run). diff --git a/internal/scheduling/reservations/commitments/syncer.go b/internal/scheduling/reservations/commitments/syncer.go index 0185e90b0..9e0175ff7 100644 --- a/internal/scheduling/reservations/commitments/syncer.go +++ b/internal/scheduling/reservations/commitments/syncer.go @@ -12,6 +12,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/go-logr/logr" "github.com/sapcc/go-api-declarations/liquid"