diff --git a/api/v1alpha1/decision_types.go b/api/v1alpha1/decision_types.go deleted file mode 100644 index fb1e07ca9..000000000 --- a/api/v1alpha1/decision_types.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// SchedulingIntent defines the intent of a scheduling decision. -type SchedulingIntent string - -// Other intents can be defined by the operators. -const ( - // Used as default intent if the operator does not specify one. - SchedulingIntentUnknown SchedulingIntent = "Unknown" -) - -type DecisionSpec struct { - // SchedulingDomain defines in which scheduling domain this decision - // was or is processed (e.g., nova, cinder, manila). - SchedulingDomain SchedulingDomain `json:"schedulingDomain"` - - // A reference to the pipeline that should be used for this decision. - // This reference can be used to look up the pipeline definition and its - // scheduler step configuration for additional context. - PipelineRef corev1.ObjectReference `json:"pipelineRef"` - - // An identifier for the underlying resource to be scheduled. - // For example, this can be the UUID of a nova instance or cinder volume. - // This can be used to correlate multiple decisions for the same resource. - ResourceID string `json:"resourceID"` - - // If the type is "nova", this field contains the raw nova decision request. - // +kubebuilder:validation:Optional - NovaRaw *runtime.RawExtension `json:"novaRaw,omitempty"` - // If the type is "cinder", this field contains the raw cinder decision request. - // +kubebuilder:validation:Optional - CinderRaw *runtime.RawExtension `json:"cinderRaw,omitempty"` - // If the type is "manila", this field contains the raw manila decision request. - // +kubebuilder:validation:Optional - ManilaRaw *runtime.RawExtension `json:"manilaRaw,omitempty"` - // If the type is "machine", this field contains the machine reference. - // +kubebuilder:validation:Optional - MachineRef *corev1.ObjectReference `json:"machineRef,omitempty"` - // If the type is "pod", this field contains the pod reference. - // +kubebuilder:validation:Optional - PodRef *corev1.ObjectReference `json:"podRef,omitempty"` - - // The intent of the scheduling decision (e.g., initial scheduling, rescheduling, etc.). - // +kubebuilder:validation:Optional - Intent SchedulingIntent `json:"intent,omitempty"` -} - -type StepResult struct { - // object reference to the scheduler step. - StepName string `json:"stepName"` - // Activations of the step for each host. - Activations map[string]float64 `json:"activations"` -} - -type DecisionResult struct { - // Raw input weights to the pipeline. - // +kubebuilder:validation:Optional - RawInWeights map[string]float64 `json:"rawInWeights"` - // Normalized input weights to the pipeline. - // +kubebuilder:validation:Optional - NormalizedInWeights map[string]float64 `json:"normalizedInWeights"` - // Outputs of the decision pipeline including the activations used - // to make the final ordering of compute hosts. - // +kubebuilder:validation:Optional - StepResults []StepResult `json:"stepResults,omitempty"` - // Aggregated output weights from the pipeline. - // +kubebuilder:validation:Optional - AggregatedOutWeights map[string]float64 `json:"aggregatedOutWeights"` - // Final ordered list of hosts from most preferred to least preferred. - // +kubebuilder:validation:Optional - OrderedHosts []string `json:"orderedHosts,omitempty"` - // The first element of the ordered hosts is considered the target host. - // +kubebuilder:validation:Optional - TargetHost *string `json:"targetHost,omitempty"` -} - -const ( - // The decision was successfully processed. - DecisionConditionReady = "Ready" -) - -type DecisionStatus struct { - // The result of this decision. - // +kubebuilder:validation:Optional - Result *DecisionResult `json:"result,omitempty"` - - // If there were previous decisions for the underlying resource, they can - // be resolved here to provide historical context for the decision. - // +kubebuilder:validation:Optional - History *[]corev1.ObjectReference `json:"history,omitempty"` - - // The number of decisions that preceded this one for the same resource. - // +kubebuilder:validation:Optional - Precedence *int `json:"precedence,omitempty"` - - // A human-readable explanation of the decision result. - // +kubebuilder:validation:Optional - Explanation string `json:"explanation,omitempty"` - - // The current status conditions of the decision. - // +kubebuilder:validation:Optional - Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster -// +kubebuilder:printcolumn:name="Domain",type="string",JSONPath=".spec.schedulingDomain" -// +kubebuilder:printcolumn:name="Resource ID",type="string",JSONPath=".spec.resourceID" -// +kubebuilder:printcolumn:name="#",type="string",JSONPath=".status.precedence" -// +kubebuilder:printcolumn:name="Created",type="date",JSONPath=".metadata.creationTimestamp" -// +kubebuilder:printcolumn:name="Pipeline",type="string",JSONPath=".spec.pipelineRef.name" -// +kubebuilder:printcolumn:name="TargetHost",type="string",JSONPath=".status.result.targetHost" -// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" -// +kubebuilder:selectablefield:JSONPath=".spec.resourceID" - -// Currently the Decision CRD is an in-memory scheduling object used by the external scheduler API -// and filter-weigher pipelines to compute a placement result. It is currently NOT persisted -// to etcd — the scheduling outcome (including a human-readable explanation) is written onto the -// scheduled resource instead, e.g. the nova VM's spec.cortex fields. -// The long-term shape of this CRD is still under discussion; until that is -// settled the Decision serves only as a transient carrier within a single -// scheduling run. -type Decision struct { - metav1.TypeMeta `json:",inline"` - - // metadata is a standard object metadata - // +optional - metav1.ObjectMeta `json:"metadata,omitempty,omitzero"` - - // spec defines the desired state of Decision - // +required - Spec DecisionSpec `json:"spec"` - - // status defines the observed state of Decision - // +optional - Status DecisionStatus `json:"status,omitempty,omitzero"` -} - -// +kubebuilder:object:root=true - -// DecisionList contains a list of Decision -type DecisionList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Decision `json:"items"` -} - -func init() { - SchemeBuilder.Register(&Decision{}, &DecisionList{}) -} diff --git a/api/v1alpha1/pipeline_types.go b/api/v1alpha1/pipeline_types.go index 1e5cfdca3..252718a8d 100644 --- a/api/v1alpha1/pipeline_types.go +++ b/api/v1alpha1/pipeline_types.go @@ -78,12 +78,6 @@ type PipelineSpec struct { // +kubebuilder:validation:Optional Description string `json:"description,omitempty"` - // If this pipeline should ignore host preselection and gather all - // available placement candidates before applying filters, instead of - // relying on a pre-filtered set and weights. - // +kubebuilder:default=false - IgnorePreselection bool `json:"ignorePreselection,omitempty"` - // The type of the pipeline, used to differentiate between // filter-weigher and detector pipelines within the same // scheduling domain. diff --git a/api/v1alpha1/scheduling_intent.go b/api/v1alpha1/scheduling_intent.go new file mode 100644 index 000000000..03eff7f86 --- /dev/null +++ b/api/v1alpha1/scheduling_intent.go @@ -0,0 +1,14 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +// SchedulingIntent defines the intent of a scheduling decision (e.g. initial +// scheduling, rescheduling, failover). It is the base type for the Nova intent +// constants in api/external/nova and is carried on Reservation.Spec.Intent. +type SchedulingIntent string + +const ( + // SchedulingIntentUnknown is the default intent when none is specified. + SchedulingIntentUnknown SchedulingIntent = "Unknown" +) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 5ba04a7e4..557d76c18 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -526,200 +526,6 @@ func (in *DatasourceStatus) DeepCopy() *DatasourceStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Decision) DeepCopyInto(out *Decision) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Decision. -func (in *Decision) DeepCopy() *Decision { - if in == nil { - return nil - } - out := new(Decision) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Decision) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DecisionList) DeepCopyInto(out *DecisionList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Decision, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecisionList. -func (in *DecisionList) DeepCopy() *DecisionList { - if in == nil { - return nil - } - out := new(DecisionList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *DecisionList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DecisionResult) DeepCopyInto(out *DecisionResult) { - *out = *in - if in.RawInWeights != nil { - in, out := &in.RawInWeights, &out.RawInWeights - *out = make(map[string]float64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.NormalizedInWeights != nil { - in, out := &in.NormalizedInWeights, &out.NormalizedInWeights - *out = make(map[string]float64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.StepResults != nil { - in, out := &in.StepResults, &out.StepResults - *out = make([]StepResult, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.AggregatedOutWeights != nil { - in, out := &in.AggregatedOutWeights, &out.AggregatedOutWeights - *out = make(map[string]float64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.OrderedHosts != nil { - in, out := &in.OrderedHosts, &out.OrderedHosts - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.TargetHost != nil { - in, out := &in.TargetHost, &out.TargetHost - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecisionResult. -func (in *DecisionResult) DeepCopy() *DecisionResult { - if in == nil { - return nil - } - out := new(DecisionResult) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DecisionSpec) DeepCopyInto(out *DecisionSpec) { - *out = *in - out.PipelineRef = in.PipelineRef - if in.NovaRaw != nil { - in, out := &in.NovaRaw, &out.NovaRaw - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } - if in.CinderRaw != nil { - in, out := &in.CinderRaw, &out.CinderRaw - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } - if in.ManilaRaw != nil { - in, out := &in.ManilaRaw, &out.ManilaRaw - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } - if in.MachineRef != nil { - in, out := &in.MachineRef, &out.MachineRef - *out = new(v1.ObjectReference) - **out = **in - } - if in.PodRef != nil { - in, out := &in.PodRef, &out.PodRef - *out = new(v1.ObjectReference) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecisionSpec. -func (in *DecisionSpec) DeepCopy() *DecisionSpec { - if in == nil { - return nil - } - out := new(DecisionSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DecisionStatus) DeepCopyInto(out *DecisionStatus) { - *out = *in - if in.Result != nil { - in, out := &in.Result, &out.Result - *out = new(DecisionResult) - (*in).DeepCopyInto(*out) - } - if in.History != nil { - in, out := &in.History, &out.History - *out = new([]v1.ObjectReference) - if **in != nil { - in, out := *in, *out - *out = make([]v1.ObjectReference, len(*in)) - copy(*out, *in) - } - } - if in.Precedence != nil { - in, out := &in.Precedence, &out.Precedence - *out = new(int) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]metav1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecisionStatus. -func (in *DecisionStatus) DeepCopy() *DecisionStatus { - if in == nil { - return nil - } - out := new(DecisionStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Descheduling) DeepCopyInto(out *Descheduling) { *out = *in @@ -1993,28 +1799,6 @@ func (in *ReservationStatus) DeepCopy() *ReservationStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StepResult) DeepCopyInto(out *StepResult) { - *out = *in - if in.Activations != nil { - in, out := &in.Activations, &out.Activations - *out = make(map[string]float64, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StepResult. -func (in *StepResult) DeepCopy() *StepResult { - if in == nil { - return nil - } - out := new(StepResult) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WeigherSpec) DeepCopyInto(out *WeigherSpec) { *out = *in diff --git a/cmd/manager/main.go b/cmd/manager/main.go index d29a1ebeb..a567fd7f4 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -54,6 +54,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/external" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" schedulinglib "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/machines" "github.com/cobaltcore-dev/cortex/internal/scheduling/manila" @@ -415,33 +416,43 @@ func main() { commitmentsAPI.Init(mux, metrics.Registry, ctrl.Log.WithName("commitments-api")) } + // novaScheduler is the shared in-process scheduler. Built when the + // nova-pipeline-controllers are enabled and consumed in-process by the Nova + // HTTP API and the reservation/commitment/capacity/failover controllers (which + // must run in the same process — see the cortex-nova-scheduling deployment). + var novaScheduler *compute.Placement + if slices.Contains(mainConfig.EnabledControllers, "nova-pipeline-controllers") { featureGates := conf.GetConfigOrDie[nova.FeatureGates]() noHostFoundCounter := crs.NewNoHostFoundCounter() placementCounter := crs.NewPlacementCounter() // Filter-weigher pipeline controller setup. filterWeigherController := &nova.FilterWeigherPipelineController{ - Monitor: filterWeigherPipelineMonitor, - FeatureGates: featureGates, - CRRecorder: crs.Recorder{ - NoHostFoundCounter: noHostFoundCounter, - PlacementCounter: placementCounter, - }, + Monitor: filterWeigherPipelineMonitor, } metrics.Registry.MustRegister(noHostFoundCounter) metrics.Registry.MustRegister(placementCounter) // Inferred through the base controller. filterWeigherController.Client = multiclusterClient - filterWeigherController.CRRecorder.Client = multiclusterClient if err := filterWeigherController.SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "nova FilterWeigherPipelineController") os.Exit(1) } + // Shared in-process scheduler: owns the scheduling lock and runs the + // pipeline. The Nova HTTP API and the reservation controllers call it + // directly (no self-directed HTTP). + novaScheduler = compute.NewPlacement(filterWeigherController, nova.NewCandidateGatherer(multiclusterClient)) + vmHandler := &nova.VirtualMachineHandler{Client: multiclusterClient} + crRecorder := crs.Recorder{ + Client: multiclusterClient, + NoHostFoundCounter: noHostFoundCounter, + PlacementCounter: placementCounter, + } novaAPIConfig := conf.GetConfigOrDie[nova.HTTPAPIConfig]() setupLog.Info("loaded nova API config", "evacuationShuffleK", novaAPIConfig.EvacuationShuffleK, "novaLimitHostsToRequest", novaAPIConfig.NovaLimitHostsToRequest) - nova.NewAPI(novaAPIConfig, filterWeigherController).Init(mux) + nova.NewAPI(novaAPIConfig, novaScheduler, vmHandler, crRecorder, featureGates).Init(mux) // Detector pipeline controller setup. novaClient := nova.NewNovaClient() @@ -604,11 +615,16 @@ func main() { if slices.Contains(mainConfig.EnabledControllers, "committed-resource-reservations-controller") { setupLog.Info("enabling controller", "controller", "committed-resource-reservations-controller") + if novaScheduler == nil { + setupLog.Error(nil, "committed-resource-reservations-controller requires nova-pipeline-controllers (in-process scheduler) to be enabled") + os.Exit(1) + } if err := (&commitments.CommitmentReservationController{ - Client: multiclusterClient, - Scheme: mgr.GetScheme(), - Conf: commitmentsConfig.ReservationController, - VMSource: commitmentsVMSource, + Client: multiclusterClient, + Scheme: mgr.GetScheme(), + Conf: commitmentsConfig.ReservationController, + Scheduler: novaScheduler, + VMSource: commitmentsVMSource, }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CommitmentReservation") os.Exit(1) @@ -726,9 +742,10 @@ func main() { setupLog.Error(nil, "failover-reservations-controller requires datasourceName to be configured") os.Exit(1) } - - // The scheduler client calls the nova external scheduler API to get placement decisions - schedulerClient := reservations.NewSchedulerClient(failoverConfig.SchedulerURL) + if novaScheduler == nil { + setupLog.Error(nil, "failover-reservations-controller requires nova-pipeline-controllers (in-process scheduler) to be enabled") + os.Exit(1) + } failoverMonitor := failover.NewFailoverMonitor() if err := metrics.Registry.Register(failoverMonitor); err != nil { @@ -760,7 +777,7 @@ func main() { multiclusterClient, vmSource, failoverConfig, - schedulerClient, + novaScheduler, failoverMonitor, ) @@ -772,7 +789,6 @@ func main() { setupLog.Info("failover-reservations-controller starting", "datasourceName", failoverConfig.DatasourceName, - "schedulerURL", failoverConfig.SchedulerURL, "reconcileInterval", failoverConfig.ReconcileInterval, "revalidationInterval", failoverConfig.RevalidationInterval) @@ -784,7 +800,6 @@ func main() { } setupLog.Info("failover-reservations-controller registered", "datasourceName", failoverConfig.DatasourceName, - "schedulerURL", failoverConfig.SchedulerURL, "reconcileInterval", failoverConfig.ReconcileInterval, "revalidationInterval", failoverConfig.RevalidationInterval, "trustHypervisorLocation", failoverConfig.TrustHypervisorLocation, @@ -799,10 +814,15 @@ func main() { setupLog.Error(err, "failed to register failover host monitor metrics, continuing without metrics") failoverHostMonitor = nil } + if novaScheduler == nil { + setupLog.Error(nil, "failover-host-reservations-controller requires nova-pipeline-controllers (in-process scheduler) to be enabled") + os.Exit(1) + } controller := &failoverhost.Controller{ - Client: multiclusterClient, - Config: failoverHostConfig, - Monitor: failoverHostMonitor, + Client: multiclusterClient, + Config: failoverHostConfig, + Scheduler: novaScheduler, + Monitor: failoverHostMonitor, } if err := controller.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", @@ -812,8 +832,7 @@ func main() { setupLog.Info("failover-host-reservations-controller registered", "ratio", failoverHostConfig.Ratio, "minimum", failoverHostConfig.Minimum, - "generationInterval", failoverHostConfig.GenerationInterval, - "externalSchedulerURL", failoverHostConfig.ExternalSchedulerURL) + "generationInterval", failoverHostConfig.GenerationInterval) } if slices.Contains(mainConfig.EnabledControllers, "capacity-controller") { setupLog.Info("enabling controller", "controller", "capacity-controller") @@ -824,18 +843,21 @@ func main() { os.Exit(1) } + if novaScheduler == nil { + setupLog.Error(nil, "capacity-controller requires nova-pipeline-controllers (in-process scheduler) to be enabled") + os.Exit(1) + } capacityMonitor := capacity.NewMonitor(multiclusterClient) if err := metrics.Registry.Register(&capacityMonitor); err != nil { setupLog.Error(err, "failed to register capacity monitor metrics, continuing without metrics") } - if err := capacity.NewController(multiclusterClient, capacityConfig, commitmentsVMSource). + if err := capacity.NewController(multiclusterClient, capacityConfig, commitmentsVMSource, novaScheduler). SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "capacity") os.Exit(1) } setupLog.Info("capacity-controller registered", - "schedulerURL", capacityConfig.SchedulerURL, "reconcileInterval", capacityConfig.ReconcileInterval, "minReconcileInterval", capacityConfig.MinReconcileInterval, "totalPipeline", capacityConfig.TotalPipeline, diff --git a/helm/bundles/cortex-cinder/templates/alerts.yaml b/helm/bundles/cortex-cinder/templates/alerts.yaml index f15cfb755..b0c6c6557 100644 --- a/helm/bundles/cortex-cinder/templates/alerts.yaml +++ b/helm/bundles/cortex-cinder/templates/alerts.yaml @@ -252,46 +252,6 @@ spec: configuration. It is recommended to investigate the knowledge status and logs for more details. - - alert: CortexCinderDecisionsWithErrors - expr: cortex_decision_state{domain="cinder",state="error"} > 0 - for: 5m - labels: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status" -{{- end }} - context: decisions - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Some decisions are in error state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" - description: > - The cortex scheduling pipeline generated decisions that are in error state. - This may indicate issues with the decision logic or the underlying infrastructure. - It is recommended to investigate the decision logs and the state of the - VMs being processed. - - - alert: CortexCinderTooManyDecisionsWaiting - expr: cortex_decision_state{domain="cinder",state="waiting"} > 10 - for: 5m - labels: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status" -{{- end }} - context: decisions - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Too many decisions are in waiting state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" - description: > - The cortex scheduling pipeline has a high number of decisions for which - no target host has been assigned yet. - - This may indicate a backlog in processing or issues with the decision logic. - It is recommended to investigate the decision logs and the state of the - VMs being processed. - - alert: CortexCinderKPIUnready expr: | cortex_kpi_state{domain="cinder",state!="ready"} != 0 diff --git a/helm/bundles/cortex-cinder/templates/kpis.yaml b/helm/bundles/cortex-cinder/templates/kpis.yaml index fb3958c08..78fd25dd9 100644 --- a/helm/bundles/cortex-cinder/templates/kpis.yaml +++ b/helm/bundles/cortex-cinder/templates/kpis.yaml @@ -25,18 +25,6 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI -metadata: - name: cortex-cinder-decision-state -spec: - schedulingDomain: cinder - impl: decision_state_kpi - opts: - decisionSchedulingDomain: cinder - description: | - This aggregated KPI tracks the state of decision resources managed by cortex. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: KPI metadata: name: cortex-cinder-kpi-state spec: diff --git a/helm/bundles/cortex-cinder/values.yaml b/helm/bundles/cortex-cinder/values.yaml index b11cbfc38..98d8e5f4b 100644 --- a/helm/bundles/cortex-cinder/values.yaml +++ b/helm/bundles/cortex-cinder/values.yaml @@ -90,8 +90,6 @@ cortex: &cortex apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/bundles/cortex-ironcore/values.yaml b/helm/bundles/cortex-ironcore/values.yaml index 6f577b04b..e54147f15 100644 --- a/helm/bundles/cortex-ironcore/values.yaml +++ b/helm/bundles/cortex-ironcore/values.yaml @@ -33,8 +33,6 @@ cortex: apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/bundles/cortex-manila/templates/kpis.yaml b/helm/bundles/cortex-manila/templates/kpis.yaml index d05910f3b..6289d0262 100644 --- a/helm/bundles/cortex-manila/templates/kpis.yaml +++ b/helm/bundles/cortex-manila/templates/kpis.yaml @@ -38,18 +38,6 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI -metadata: - name: cortex-manila-decision-state -spec: - schedulingDomain: manila - impl: decision_state_kpi - opts: - decisionSchedulingDomain: manila - description: | - This aggregated KPI tracks the state of decision resources managed by cortex. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: KPI metadata: name: cortex-manila-kpi-state spec: diff --git a/helm/bundles/cortex-manila/values.yaml b/helm/bundles/cortex-manila/values.yaml index 5d48da8f8..39f177f6d 100644 --- a/helm/bundles/cortex-manila/values.yaml +++ b/helm/bundles/cortex-manila/values.yaml @@ -90,8 +90,6 @@ cortex: &cortex apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index bb82e889a..18322417b 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -278,46 +278,6 @@ spec: configuration. It is recommended to investigate the knowledge status and logs for more details. - - alert: CortexNovaDecisionsWithErrors - expr: cortex_decision_state{domain="nova",state="error"} > 0 - for: 5m - labels: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics" -{{- end }} - context: decisions - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Some decisions are in error state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" - description: > - The cortex scheduling pipeline generated decisions that are in error state. - This may indicate issues with the decision logic or the underlying infrastructure. - It is recommended to investigate the decision logs and the state of the - VMs being processed. - - - alert: CortexNovaTooManyDecisionsWaiting - expr: cortex_decision_state{domain="nova",state="waiting"} > 10 - for: 5m - labels: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics" -{{- end }} - context: decisions - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Too many decisions are in waiting state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" - description: > - The cortex scheduling pipeline has a high number of decisions for which - no target host has been assigned yet. - - This may indicate a backlog in processing or issues with the decision logic. - It is recommended to investigate the decision logs and the state of the - VMs being processed. - - alert: CortexNovaKPIUnready expr: | cortex_kpi_state{domain="nova",state!="ready"} != 0 diff --git a/helm/bundles/cortex-nova/templates/kpis.yaml b/helm/bundles/cortex-nova/templates/kpis.yaml index 4c8dee11a..d52b6099e 100644 --- a/helm/bundles/cortex-nova/templates/kpis.yaml +++ b/helm/bundles/cortex-nova/templates/kpis.yaml @@ -110,18 +110,6 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI -metadata: - name: cortex-nova-decision-state -spec: - schedulingDomain: nova - impl: decision_state_kpi - opts: - decisionSchedulingDomain: nova - description: | - This aggregated KPI tracks the state of decision resources managed by cortex. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: KPI metadata: name: cortex-nova-kpi-state spec: diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index 817e75ab6..dab714b8c 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -15,7 +15,6 @@ spec: Specifically, this pipeline is used for general purpose workloads. type: filter-weigher # Fetch all placement candidates, ignoring nova's preselection. - ignorePreselection: true filters: - name: filter_status_conditions description: | @@ -189,7 +188,6 @@ spec: Specifically, this pipeline is used for hana virtual machines. type: filter-weigher # Fetch all placement candidates, ignoring nova's preselection. - ignorePreselection: true filters: - name: filter_status_conditions description: | @@ -355,7 +353,6 @@ spec: Deprecated: no longer used by the failover reservation controller. Kept for backwards compatibility with deployments that reference this pipeline by name in Helm values. type: filter-weigher - ignorePreselection: true filters: - name: filter_status_conditions description: | @@ -637,7 +634,6 @@ spec: considered. type: filter-weigher # Fetch all placement candidates, ignoring nova's preselection. - ignorePreselection: true filters: - name: filter_status_conditions description: | diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index c0771fdd7..4d2287ab4 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -121,8 +121,6 @@ cortex: &cortex apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline @@ -387,10 +385,6 @@ cortex-scheduling-controllers: # How often the failover partitions are recalculated and the reservations # updated to match. Must be at least 1m. generationInterval: 5m - # URL of the nova external scheduler API, used to discover which hosts a - # given host's workload could be evacuated onto. Typically the same - # scheduler endpoint the flavor-based failover controller uses. - externalSchedulerURL: "http://localhost:8080/scheduler/nova/external" cortex-knowledge-controllers: <<: *cortex diff --git a/helm/bundles/cortex-pods/values.yaml b/helm/bundles/cortex-pods/values.yaml index efd5ee9b1..3815f67b8 100644 --- a/helm/bundles/cortex-pods/values.yaml +++ b/helm/bundles/cortex-pods/values.yaml @@ -33,8 +33,6 @@ cortex: apiservers: home: gvks: - - cortex.cloud/v1alpha1/Decision - - cortex.cloud/v1alpha1/DecisionList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/library/cortex-shim/templates/clusterrole.yaml b/helm/library/cortex-shim/templates/clusterrole.yaml index 74f8e7ad4..867b45ede 100644 --- a/helm/library/cortex-shim/templates/clusterrole.yaml +++ b/helm/library/cortex-shim/templates/clusterrole.yaml @@ -14,7 +14,6 @@ rules: - knowledges - datasources - reservations - - decisions - deschedulings - pipelines - kpis @@ -33,7 +32,6 @@ rules: - knowledges/finalizers - datasources/finalizers - reservations/finalizers - - decisions/finalizers - deschedulings/finalizers - pipelines/finalizers - kpis/finalizers @@ -46,7 +44,6 @@ rules: - knowledges/status - datasources/status - reservations/status - - decisions/status - deschedulings/status - pipelines/status - kpis/status diff --git a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml b/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml deleted file mode 100644 index ec4c99649..000000000 --- a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml +++ /dev/null @@ -1,412 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.23.0 - name: decisions.cortex.cloud -spec: - group: cortex.cloud - names: - kind: Decision - listKind: DecisionList - plural: decisions - singular: decision - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.schedulingDomain - name: Domain - type: string - - jsonPath: .spec.resourceID - name: Resource ID - type: string - - jsonPath: .status.precedence - name: '#' - type: string - - jsonPath: .metadata.creationTimestamp - name: Created - type: date - - jsonPath: .spec.pipelineRef.name - name: Pipeline - type: string - - jsonPath: .status.result.targetHost - name: TargetHost - type: string - - jsonPath: .status.conditions[?(@.type=='Ready')].status - name: Ready - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - Currently the Decision CRD is an in-memory scheduling object used by the external scheduler API - and filter-weigher pipelines to compute a placement result. It is currently NOT persisted - to etcd — the scheduling outcome (including a human-readable explanation) is written onto the - scheduled resource instead, e.g. the nova VM's spec.cortex fields. - The long-term shape of this CRD is still under discussion; until that is - settled the Decision serves only as a transient carrier within a single - scheduling run. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: spec defines the desired state of Decision - properties: - cinderRaw: - description: If the type is "cinder", this field contains the raw - cinder decision request. - type: object - x-kubernetes-preserve-unknown-fields: true - intent: - description: The intent of the scheduling decision (e.g., initial - scheduling, rescheduling, etc.). - type: string - machineRef: - description: If the type is "machine", this field contains the machine - reference. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - manilaRaw: - description: If the type is "manila", this field contains the raw - manila decision request. - type: object - x-kubernetes-preserve-unknown-fields: true - novaRaw: - description: If the type is "nova", this field contains the raw nova - decision request. - type: object - x-kubernetes-preserve-unknown-fields: true - pipelineRef: - description: |- - A reference to the pipeline that should be used for this decision. - This reference can be used to look up the pipeline definition and its - scheduler step configuration for additional context. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - podRef: - description: If the type is "pod", this field contains the pod reference. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - resourceID: - description: |- - An identifier for the underlying resource to be scheduled. - For example, this can be the UUID of a nova instance or cinder volume. - This can be used to correlate multiple decisions for the same resource. - type: string - schedulingDomain: - description: |- - SchedulingDomain defines in which scheduling domain this decision - was or is processed (e.g., nova, cinder, manila). - type: string - required: - - pipelineRef - - resourceID - - schedulingDomain - type: object - status: - description: status defines the observed state of Decision - properties: - conditions: - description: The current status conditions of the decision. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - explanation: - description: A human-readable explanation of the decision result. - type: string - history: - description: |- - If there were previous decisions for the underlying resource, they can - be resolved here to provide historical context for the decision. - items: - description: ObjectReference contains enough information to let - you inspect or modify the referred object. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - type: array - precedence: - description: The number of decisions that preceded this one for the - same resource. - type: integer - result: - description: The result of this decision. - properties: - aggregatedOutWeights: - additionalProperties: - type: number - description: Aggregated output weights from the pipeline. - type: object - normalizedInWeights: - additionalProperties: - type: number - description: Normalized input weights to the pipeline. - type: object - orderedHosts: - description: Final ordered list of hosts from most preferred to - least preferred. - items: - type: string - type: array - rawInWeights: - additionalProperties: - type: number - description: Raw input weights to the pipeline. - type: object - stepResults: - description: |- - Outputs of the decision pipeline including the activations used - to make the final ordering of compute hosts. - items: - properties: - activations: - additionalProperties: - type: number - description: Activations of the step for each host. - type: object - stepName: - description: object reference to the scheduler step. - type: string - required: - - activations - - stepName - type: object - type: array - targetHost: - description: The first element of the ordered hosts is considered - the target host. - type: string - type: object - type: object - required: - - spec - type: object - selectableFields: - - jsonPath: .spec.resourceID - served: true - storage: true - subresources: - status: {} diff --git a/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml b/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml index 21a69b62f..f77625aaf 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml @@ -180,13 +180,6 @@ spec: - name type: object type: array - ignorePreselection: - default: false - description: |- - If this pipeline should ignore host preselection and gather all - available placement candidates before applying filters, instead of - relying on a pre-filtered set and weights. - type: boolean schedulingDomain: description: |- SchedulingDomain defines in which scheduling domain this pipeline diff --git a/helm/library/cortex/templates/rbac/role.yaml b/helm/library/cortex/templates/rbac/role.yaml index 54991d0b4..16e6b9012 100644 --- a/helm/library/cortex/templates/rbac/role.yaml +++ b/helm/library/cortex/templates/rbac/role.yaml @@ -17,7 +17,6 @@ rules: - committedresourcechanges - projectquotas - flavorgroupcapacities - - decisions - deschedulings - pipelines - kpis @@ -40,7 +39,6 @@ rules: - committedresourcechanges/finalizers - projectquotas/finalizers - flavorgroupcapacities/finalizers - - decisions/finalizers - deschedulings/finalizers - pipelines/finalizers - kpis/finalizers @@ -57,7 +55,6 @@ rules: - committedresourcechanges/status - projectquotas/status - flavorgroupcapacities/status - - decisions/status - deschedulings/status - pipelines/status - kpis/status diff --git a/internal/knowledge/kpis/plugins/deployment/decision_state.go b/internal/knowledge/kpis/plugins/deployment/decision_state.go deleted file mode 100644 index 1d07ac7f9..000000000 --- a/internal/knowledge/kpis/plugins/deployment/decision_state.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package deployment - -import ( - "context" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/db" - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" - "github.com/cobaltcore-dev/cortex/pkg/conf" - "github.com/prometheus/client_golang/prometheus" - "k8s.io/apimachinery/pkg/api/meta" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type DecisionStateKPIOpts struct { - // The scheduling domain to filter decisions by. - DecisionSchedulingDomain v1alpha1.SchedulingDomain `json:"decisionSchedulingDomain"` -} - -// KPI observing the state of decision resources managed by cortex. -type DecisionStateKPI struct { - // Common base for all KPIs that provides standard functionality. - plugins.BaseKPI[DecisionStateKPIOpts] - - // Prometheus descriptor for the decision state metric. - counter *prometheus.Desc -} - -func (DecisionStateKPI) GetName() string { return "decision_state_kpi" } - -// Initialize the KPI. -func (k *DecisionStateKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error { - if err := k.BaseKPI.Init(db, client, opts); err != nil { - return err - } - k.counter = prometheus.NewDesc( - "cortex_decision_state", - "State of cortex managed decisions (aggregated)", - []string{"domain", "state"}, - nil, - ) - return nil -} - -// Conform to the prometheus collector interface by providing the descriptor. -func (k *DecisionStateKPI) Describe(ch chan<- *prometheus.Desc) { ch <- k.counter } - -// Collect the decision state metrics. -func (k *DecisionStateKPI) Collect(ch chan<- prometheus.Metric) { - // Get all decisions with the specified decision operator. - decisionList := &v1alpha1.DecisionList{} - if err := k.Client.List(context.Background(), decisionList); err != nil { - return - } - var decisions []v1alpha1.Decision - for _, d := range decisionList.Items { - if d.Spec.SchedulingDomain != k.Options.DecisionSchedulingDomain { - continue - } - decisions = append(decisions, d) - } - // For each decision, categorize by state: error, waiting, or success - var errorCount, waitingCount, successCount float64 - for _, d := range decisions { - switch { - // Error state: decision has a false Ready condition - case meta.IsStatusConditionFalse(d.Status.Conditions, v1alpha1.DecisionConditionReady): - errorCount++ - // Waiting state: decision has a target host set (waiting for migration/placement) - case d.Status.Result != nil && d.Status.Result.TargetHost != nil: - waitingCount++ - // Success state: decision is complete (has result with ordered hosts or no result needed) - default: - successCount++ - } - } - // Emit metrics for all three states - ch <- prometheus.MustNewConstMetric( - k.counter, prometheus.GaugeValue, errorCount, - string(k.Options.DecisionSchedulingDomain), "error", - ) - ch <- prometheus.MustNewConstMetric( - k.counter, prometheus.GaugeValue, waitingCount, - string(k.Options.DecisionSchedulingDomain), "waiting", - ) - ch <- prometheus.MustNewConstMetric( - k.counter, prometheus.GaugeValue, successCount, - string(k.Options.DecisionSchedulingDomain), "success", - ) -} - -func init() { - plugins.Index["decision_state_kpi"] = func() plugins.KPI { return &DecisionStateKPI{} } -} diff --git a/internal/knowledge/kpis/plugins/deployment/decision_state_test.go b/internal/knowledge/kpis/plugins/deployment/decision_state_test.go deleted file mode 100644 index 0d6bd4568..000000000 --- a/internal/knowledge/kpis/plugins/deployment/decision_state_test.go +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package deployment - -import ( - "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/pkg/conf" - "github.com/prometheus/client_golang/prometheus" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestDecisionStateKPI_Init(t *testing.T) { - kpi := &DecisionStateKPI{} - if err := kpi.Init(nil, nil, conf.NewRawOpts(`{"decisionSchedulingDomain": "test-operator"}`)); err != nil { - t.Fatalf("expected no error, got %v", err) - } -} - -func TestDecisionStateKPI_Collect(t *testing.T) { - scheme, err := v1alpha1.SchemeBuilder.Build() - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - targetHost := "host1" - - tests := []struct { - name string - decisions []v1alpha1.Decision - operator string - expectedCount int - description string - expectedError int - expectedWaiting int - expectedSuccess int - }{ - { - name: "no decisions", - decisions: []v1alpha1.Decision{}, - operator: "test-operator", - expectedCount: 3, // always emits 3 metrics: error, waiting, success - description: "should collect metrics with zero counts when no decisions exist", - expectedError: 0, - expectedWaiting: 0, - expectedSuccess: 0, - }, - { - name: "single decision in error state", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec1"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count decision in error state", - expectedError: 1, - expectedWaiting: 0, - expectedSuccess: 0, - }, - { - name: "single decision in waiting state", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec2"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &targetHost, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count decision with target host as waiting", - expectedError: 0, - expectedWaiting: 1, - expectedSuccess: 0, - }, - { - name: "single decision in success state", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec3"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1", "host2"}, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count decision without target host and no error as success", - expectedError: 0, - expectedWaiting: 0, - expectedSuccess: 1, - }, - { - name: "multiple decisions in different states", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-error"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - { - ObjectMeta: v1.ObjectMeta{Name: "dec-waiting"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &targetHost, - }, - }, - }, - { - ObjectMeta: v1.ObjectMeta{Name: "dec-success"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should correctly count decisions across all states", - expectedError: 1, - expectedWaiting: 1, - expectedSuccess: 1, - }, - { - name: "filter by operator", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-correct-operator"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - { - ObjectMeta: v1.ObjectMeta{Name: "dec-wrong-operator"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "other-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should only count decisions with matching operator", - expectedError: 1, - expectedWaiting: 0, - expectedSuccess: 0, - }, - { - name: "multiple decisions same state", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-error-1"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - { - ObjectMeta: v1.ObjectMeta{Name: "dec-error-2"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should correctly aggregate multiple decisions in same state", - expectedError: 2, - expectedWaiting: 0, - expectedSuccess: 0, - }, - { - name: "decision with no result", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-no-result"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{}, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count decision with no result as success", - expectedError: 0, - expectedWaiting: 0, - expectedSuccess: 1, - }, - { - name: "error condition takes precedence", - decisions: []v1alpha1.Decision{ - { - ObjectMeta: v1.ObjectMeta{Name: "dec-error-with-target"}, - Spec: v1alpha1.DecisionSpec{SchedulingDomain: "test-operator"}, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &targetHost, - }, - Conditions: []v1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: v1.ConditionFalse, - }, - }, - }, - }, - }, - operator: "test-operator", - expectedCount: 3, - description: "should count as error even if target host is present", - expectedError: 1, - expectedWaiting: 0, - expectedSuccess: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := make([]v1alpha1.Decision, len(tt.decisions)) - copy(objects, tt.decisions) - - clientBuilder := fake.NewClientBuilder().WithScheme(scheme) - for i := range objects { - clientBuilder = clientBuilder.WithObjects(&objects[i]) - } - client := clientBuilder.Build() - - kpi := &DecisionStateKPI{} - if err := kpi.Init(nil, client, conf.NewRawOpts(`{"decisionSchedulingDomain": "`+tt.operator+`"}`)); err != nil { - t.Fatalf("expected no error, got %v", err) - } - - ch := make(chan prometheus.Metric, 10) - kpi.Collect(ch) - close(ch) - - metricsCount := 0 - for range ch { - metricsCount++ - } - - if metricsCount != tt.expectedCount { - t.Errorf("%s: expected %d metrics, got %d", tt.description, tt.expectedCount, metricsCount) - } - }) - } -} - -func TestDecisionStateKPI_GetName(t *testing.T) { - kpi := &DecisionStateKPI{} - expectedName := "decision_state_kpi" - if name := kpi.GetName(); name != expectedName { - t.Errorf("expected name %q, got %q", expectedName, name) - } -} - -func TestDecisionStateKPI_Describe(t *testing.T) { - kpi := &DecisionStateKPI{} - if err := kpi.Init(nil, nil, conf.NewRawOpts(`{"decisionSchedulingDomain": "test-operator"}`)); err != nil { - t.Fatalf("expected no error, got %v", err) - } - - ch := make(chan *prometheus.Desc, 1) - kpi.Describe(ch) - close(ch) - - descCount := 0 - for range ch { - descCount++ - } - - if descCount != 1 { - t.Errorf("expected 1 descriptor, got %d", descCount) - } -} diff --git a/internal/scheduling/cinder/external_scheduler_api.go b/internal/scheduling/cinder/external_scheduler_api.go index 96bea5590..8ae12e3de 100644 --- a/internal/scheduling/cinder/external_scheduler_api.go +++ b/internal/scheduling/cinder/external_scheduler_api.go @@ -7,29 +7,24 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "net/http" api "github.com/cobaltcore-dev/cortex/api/external/cinder" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/metrics" ) var log = logf.Log.WithName("cinder-scheduler-api") -type HTTPAPIDelegate interface { - // Process the decision from the API. Should create and return the updated decision. - ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error +// Scheduler runs a cinder pipeline for a request and returns the result. +// *FilterWeigherPipelineController satisfies it. +type Scheduler interface { + Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (scheduling.PipelineResult, error) } type HTTPAPI interface { @@ -38,14 +33,14 @@ type HTTPAPI interface { } type httpAPI struct { - monitor scheduling.APIMonitor - delegate HTTPAPIDelegate + monitor scheduling.APIMonitor + scheduler Scheduler } -func NewAPI(delegate HTTPAPIDelegate) HTTPAPI { +func NewAPI(scheduler Scheduler) HTTPAPI { return &httpAPI{ - monitor: scheduling.NewSchedulerMonitor(), - delegate: delegate, + monitor: scheduling.NewSchedulerMonitor(), + scheduler: scheduler, } } @@ -109,7 +104,6 @@ func (httpAPI *httpAPI) CinderExternalScheduler(w http.ResponseWriter, r *http.R c.Respond(logr.Logger{}, http.StatusInternalServerError, err, "failed to read request body") return } - raw := runtime.RawExtension{Raw: body} var requestData api.ExternalSchedulerRequest // Copy the raw body to a io.Reader for json deserialization. cp := body @@ -138,36 +132,12 @@ func (httpAPI *httpAPI) CinderExternalScheduler(w http.ResponseWriter, r *http.R logger.V(1).Info("inferred pipeline name", "pipeline", requestData.Pipeline) } - // Create the decision object in kubernetes. - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "cinder-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: requestData.Pipeline, - }, - ResourceID: "", // TODO - CinderRaw: &raw, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - } - ctx := r.Context() - if err := httpAPI.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { + result, err := httpAPI.scheduler.Schedule(r.Context(), requestData.Pipeline, requestData) + if err != nil { c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return } - // Check if the decision contains status conditions indicating an error. - if meta.IsStatusConditionFalse(decision.Status.Conditions, v1alpha1.DecisionConditionReady) { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision contains error condition"), "decision failed") - return - } - if decision.Status.Result == nil { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision didn't produce a result"), "decision failed") - return - } - hosts := decision.Status.Result.OrderedHosts + hosts := result.OrderedHosts // Ensure we always return an array, never null. The schema requires hosts // to be an array, and a nil slice would serialize to null. if hosts == nil { diff --git a/internal/scheduling/cinder/external_scheduler_api_test.go b/internal/scheduling/cinder/external_scheduler_api_test.go index d5f7f394f..ec8c34feb 100644 --- a/internal/scheduling/cinder/external_scheduler_api_test.go +++ b/internal/scheduling/cinder/external_scheduler_api_test.go @@ -14,26 +14,24 @@ import ( "testing" cinderapi "github.com/cobaltcore-dev/cortex/api/external/cinder" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type mockHTTPAPIDelegate struct { - processDecisionFunc func(ctx context.Context, decision *v1alpha1.Decision) error +type mockScheduler struct { + scheduleFunc func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, error) } -func (m *mockHTTPAPIDelegate) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { - if m.processDecisionFunc != nil { - return m.processDecisionFunc(ctx, decision) +func (m *mockScheduler) Schedule(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { + if m.scheduleFunc != nil { + return m.scheduleFunc(ctx, pipelineName, request) } - return nil + return lib.PipelineResult{}, nil } func TestNewAPI(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} + scheduler := &mockScheduler{} - api := NewAPI(delegate) + api := NewAPI(scheduler) if api == nil { t.Fatal("NewAPI returned nil") @@ -44,8 +42,8 @@ func TestNewAPI(t *testing.T) { t.Fatal("NewAPI did not return httpAPI type") } - if httpAPI.delegate != delegate { - t.Error("Delegate not set correctly") + if httpAPI.scheduler != scheduler { + t.Error("Scheduler not set correctly") } if httpAPI.monitor.ApiRequestsTimer == nil { @@ -54,8 +52,7 @@ func TestNewAPI(t *testing.T) { } func TestHTTPAPI_Init(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate) + api := NewAPI(&mockScheduler{}) mux := http.NewServeMux() api.Init(mux) @@ -72,8 +69,7 @@ func TestHTTPAPI_Init(t *testing.T) { } func TestHTTPAPI_canRunScheduler(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(&mockScheduler{}).(*httpAPI) tests := []struct { name string @@ -141,14 +137,27 @@ func TestHTTPAPI_canRunScheduler(t *testing.T) { } func TestHTTPAPI_CinderExternalScheduler(t *testing.T) { + validBody := func(t *testing.T) string { + req := cinderapi.ExternalSchedulerRequest{ + Hosts: []cinderapi.ExternalSchedulerHost{{VolumeHost: "host1"}}, + Weights: map[string]float64{"host1": 1.0}, + Pipeline: "test-pipeline", + } + data, err := json.Marshal(req) + if err != nil { + t.Fatalf("Failed to marshal request data: %v", err) + } + return string(data) + } + tests := []struct { - name string - method string - body string - processDecisionErr error - decisionResult *v1alpha1.Decision - expectedStatus int - expectedHosts []string + name string + method string + body string + scheduleResult lib.PipelineResult + scheduleErr error + expectedStatus int + expectedHosts []string }{ { name: "invalid method", @@ -162,106 +171,31 @@ func TestHTTPAPI_CinderExternalScheduler(t *testing.T) { expectedStatus: http.StatusBadRequest, }, { - name: "valid request with successful processing", - method: http.MethodPost, - body: func() string { - req := cinderapi.ExternalSchedulerRequest{ - Hosts: []cinderapi.ExternalSchedulerHost{ - {VolumeHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - }, - }, - }, + name: "valid request with successful processing", + method: http.MethodPost, + body: validBody(t), + scheduleResult: lib.PipelineResult{OrderedHosts: []string{"host1"}}, expectedStatus: http.StatusOK, expectedHosts: []string{"host1"}, }, { - name: "processing error", - method: http.MethodPost, - body: func() string { - req := cinderapi.ExternalSchedulerRequest{ - Hosts: []cinderapi.ExternalSchedulerHost{ - {VolumeHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - processDecisionErr: errors.New("processing failed"), - expectedStatus: http.StatusInternalServerError, - }, - { - name: "decision failed", - method: http.MethodPost, - body: func() string { - req := cinderapi.ExternalSchedulerRequest{ - Hosts: []cinderapi.ExternalSchedulerHost{ - {VolumeHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "SchedulingError", - }, - }, - }, - }, + name: "processing error", + method: http.MethodPost, + body: validBody(t), + scheduleErr: errors.New("processing failed"), expectedStatus: http.StatusInternalServerError, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - if tt.processDecisionErr != nil { - return tt.processDecisionErr - } - if tt.decisionResult != nil { - decision.Status = tt.decisionResult.Status - return nil - } - return nil + scheduler := &mockScheduler{ + scheduleFunc: func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { + return tt.scheduleResult, tt.scheduleErr }, } - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(scheduler).(*httpAPI) var body *strings.Reader if tt.body != "" { @@ -300,8 +234,7 @@ func TestHTTPAPI_CinderExternalScheduler(t *testing.T) { } func TestHTTPAPI_inferPipelineName(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(&mockScheduler{}).(*httpAPI) tests := []struct { name string @@ -347,31 +280,25 @@ func TestHTTPAPI_inferPipelineName(t *testing.T) { } } -func TestHTTPAPI_CinderExternalScheduler_DecisionCreation(t *testing.T) { - var capturedDecision *v1alpha1.Decision - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - capturedDecision = decision - // Set a successful result to avoid "decision didn't produce a result" error - decision.Status.Result = &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - } - return nil +// When the request carries no pipeline, the handler infers the default name and +// passes it to the scheduler. +func TestHTTPAPI_CinderExternalScheduler_InfersPipeline(t *testing.T) { + var capturedPipeline string + var capturedRequest cinderapi.ExternalSchedulerRequest + scheduler := &mockScheduler{ + scheduleFunc: func(ctx context.Context, pipelineName string, request cinderapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { + capturedPipeline = pipelineName + capturedRequest = request + return lib.PipelineResult{OrderedHosts: []string{"host1"}}, nil }, } - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(scheduler).(*httpAPI) requestData := cinderapi.ExternalSchedulerRequest{ - Hosts: []cinderapi.ExternalSchedulerHost{ - {VolumeHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", + Hosts: []cinderapi.ExternalSchedulerHost{{VolumeHost: "host1"}}, + Weights: map[string]float64{"host1": 1.0}, } - body, err := json.Marshal(requestData) if err != nil { t.Fatalf("Failed to marshal request data: %v", err) @@ -384,25 +311,10 @@ func TestHTTPAPI_CinderExternalScheduler_DecisionCreation(t *testing.T) { if w.Code != http.StatusOK { t.Errorf("Expected status %d, got %d", http.StatusOK, w.Code) } - - if capturedDecision == nil { - t.Fatal("Decision was not captured") + if capturedPipeline != "cinder-external-scheduler" { + t.Errorf("Expected inferred pipeline 'cinder-external-scheduler', got %q", capturedPipeline) } - - // Verify decision fields - if capturedDecision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainCinder { - t.Errorf("Expected scheduling domain %s, got %s", v1alpha1.SchedulingDomainCinder, capturedDecision.Spec.SchedulingDomain) - } - - if capturedDecision.Spec.PipelineRef.Name != "test-pipeline" { - t.Errorf("Expected pipeline 'test-pipeline', got %s", capturedDecision.Spec.PipelineRef.Name) - } - - if capturedDecision.GenerateName != "cinder-" { - t.Errorf("Expected generate name 'cinder-', got %s", capturedDecision.GenerateName) - } - - if capturedDecision.Spec.CinderRaw == nil { - t.Error("CinderRaw should not be nil") + if len(capturedRequest.Hosts) != 1 || capturedRequest.Hosts[0].VolumeHost != "host1" { + t.Errorf("Request not passed through to scheduler: %+v", capturedRequest) } } diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go index 6d2593147..815571e4c 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go @@ -5,16 +5,12 @@ package cinder import ( "context" - "encoding/json" - "errors" "fmt" "sync" "time" api "github.com/cobaltcore-dev/cortex/api/external/cinder" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cobaltcore-dev/cortex/internal/scheduling/cinder/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/cinder/plugins/weighers" @@ -27,17 +23,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) -// The decision pipeline controller takes decision resources containing a -// placement request spec and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. -// -// Additionally, the controller watches for pipeline and step changes to -// reconfigure the pipelines as needed. +// FilterWeigherPipelineController owns the cinder filter-weigher pipelines: it +// watches Pipeline and Knowledge resources to (re)configure the pipelines and +// runs a pipeline on demand via Schedule, called directly by the external +// scheduler HTTP API. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]] - // Mutex to only allow one process at a time + // Mutex to only allow one scheduling run at a time. processMu sync.Mutex // Monitor to pass down to all pipelines. @@ -49,83 +43,34 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -// Callback executed when kubernetes asks to reconcile a decision resource. -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - c.processMu.Lock() - defer c.processMu.Unlock() - - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - old := decision.DeepCopy() - if err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } +// Reconcile is a no-op: this controller reconfigures pipelines through its +// Pipeline/Knowledge watch handlers and runs placements via Schedule. A +// Reconcile method is required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } -// Process the decision from the API. Should create and return the updated decision. -func (c *FilterWeigherPipelineController) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { +// Schedule runs the named pipeline for the given request and returns the +// result. It serializes runs with the process lock. +func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (lib.PipelineResult, error) { c.processMu.Lock() defer c.processMu.Unlock() - - err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - return err -} - -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) error { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) + pipeline, ok := c.GetPipeline(pipelineName) if !ok { - log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) - } - if decision.Spec.CinderRaw == nil { - log.Error(nil, "skipping decision, no cinderRaw spec defined") - return errors.New("no cinderRaw spec defined") - } - var request api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.CinderRaw.Raw, &request); err != nil { - log.Error(err, "failed to unmarshal cinderRaw spec") - return err + log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) + return lib.PipelineResult{}, fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } result, err := pipeline.Run(request) - if err == nil { - decision.Status.Result = &result - } - // Record a human-readable explanation of the decision for downstream - // consumers. On error the result is nil and the explanation carries the - // failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.Error(err, "failed to run pipeline") - return err + return lib.PipelineResult{}, err } log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return nil + return result, nil } // The base controller will delegate the pipeline creation down to this method. @@ -186,25 +131,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainCinder { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } - return bldr.Named("cortex-cinder-decisions"). + return bldr.Named("cortex-cinder-pipelines"). Complete(c) } diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go index 8ec56c40f..4daf8fee0 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go @@ -4,15 +4,8 @@ package cinder import ( - "context" - "encoding/json" "testing" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" api "github.com/cobaltcore-dev/cortex/api/external/cinder" @@ -21,25 +14,16 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { +func TestFilterWeigherPipelineController_Schedule(t *testing.T) { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { t.Fatalf("Failed to add v1alpha1 scheme: %v", err) } - cinderRequest := api.ExternalSchedulerRequest{ - Spec: map[string]any{ - "volume_id": "test-volume-id", - "size": 10, - }, - Context: api.CinderRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: "global-req-123", - }, + request := api.ExternalSchedulerRequest{ Hosts: []api.ExternalSchedulerHost{ {VolumeHost: "cinder-volume-1"}, {VolumeHost: "cinder-volume-2"}, @@ -49,127 +33,41 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { Options: scheduling.Options{}, } - cinderRaw, err := json.Marshal(cinderRequest) - if err != nil { - t.Fatalf("Failed to marshal cinder request: %v", err) - } - tests := []struct { name string - decision *v1alpha1.Decision - pipeline *v1alpha1.Pipeline + pipelineName string + configured bool expectError bool - expectResult bool }{ { - name: "successful cinder decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, + name: "successful scheduling run", + pipelineName: "test-pipeline", + configured: true, expectError: false, - expectResult: true, }, { - name: "decision without cinderRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: nil, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - { - name: "pipeline not found", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipeline: nil, + name: "pipeline not found", + pipelineName: "nonexistent-pipeline", + configured: false, expectError: true, - expectResult: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{tt.decision} - if tt.pipeline != nil { - objects = append(objects, tt.pipeline) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() + cl := fake.NewClientBuilder().WithScheme(scheme).Build() controller := &FilterWeigherPipelineController{ BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, + Client: cl, Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), }, Monitor: lib.FilterWeigherPipelineMonitor{}, } - if tt.pipeline != nil { + if tt.configured { initResult := controller.InitPipeline(t.Context(), v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, + ObjectMeta: metav1.ObjectMeta{Name: "test-pipeline"}, Spec: v1alpha1.PipelineSpec{ Type: v1alpha1.PipelineTypeFilterWeigher, SchedulingDomain: v1alpha1.SchedulingDomainCinder, @@ -180,238 +78,10 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { t.Fatalf("Failed to init pipeline: %v", initResult) } - controller.Pipelines[tt.pipeline.Name] = initResult.Pipeline - } - - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - Namespace: tt.decision.Namespace, - }, - } - - result, err := controller.Reconcile(context.Background(), req) - - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - - if result.RequeueAfter > 0 { - t.Error("Expected no requeue") - } - - var updatedDecision v1alpha1.Decision - if err := client.Get(context.Background(), req.NamespacedName, &updatedDecision); err != nil { - t.Fatalf("Failed to get updated decision: %v", err) - } - - if tt.expectResult && updatedDecision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - if !tt.expectResult && updatedDecision.Status.Result != nil { - t.Error("Expected result to be nil but was set") - } - }) - } -} - -func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - cinderRequest := api.ExternalSchedulerRequest{ - Spec: map[string]any{ - "volume_id": "test-volume-id", - "size": 10, - }, - Context: api.CinderRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: "global-req-123", - }, - Hosts: []api.ExternalSchedulerHost{ - {VolumeHost: "cinder-volume-1"}, - {VolumeHost: "cinder-volume-2"}, - }, - Weights: map[string]float64{"cinder-volume-1": 1.0, "cinder-volume-2": 0.5}, - Pipeline: "test-pipeline", - } - - cinderRaw, err := json.Marshal(cinderRequest) - if err != nil { - t.Fatalf("Failed to marshal cinder request: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - pipelineConfig *v1alpha1.Pipeline - expectError bool - expectResult bool - }{ - { - name: "successful decision processing with creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-decision-", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "test-uuid-1", - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "successful decision processing without creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-create", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "pipeline not configured", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - CinderRaw: &runtime.RawExtension{ - Raw: cinderRaw, - }, - }, - }, - pipelineConfig: nil, - expectError: true, - expectResult: false, - }, - { - name: "decision without cinderRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - CinderRaw: nil, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{} - if tt.pipelineConfig != nil { - objects = append(objects, tt.pipelineConfig) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - - if tt.pipelineConfig != nil { - controller.PipelineConfigs[tt.pipelineConfig.Name] = *tt.pipelineConfig - initResult := controller.InitPipeline(t.Context(), *tt.pipelineConfig) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to init pipeline: %v", initResult) - } - controller.Pipelines[tt.pipelineConfig.Name] = initResult.Pipeline + controller.Pipelines["test-pipeline"] = initResult.Pipeline } - if tt.decision.Spec.CinderRaw != nil { - req := cinderRequest - req.Options = scheduling.Options{} - raw, marshalErr := json.Marshal(req) - if marshalErr != nil { - t.Fatalf("Failed to marshal request with options: %v", marshalErr) - } - tt.decision.Spec.CinderRaw = &runtime.RawExtension{Raw: raw} - } - - err := controller.ProcessNewDecisionFromAPI(context.Background(), tt.decision) + result, err := controller.Schedule(t.Context(), tt.pipelineName, request) if tt.expectError && err == nil { t.Error("Expected error but got none") @@ -419,10 +89,9 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) if !tt.expectError && err != nil { t.Errorf("Expected no error but got: %v", err) } - if !tt.expectError { - if tt.expectResult && tt.decision.Status.Result == nil { - t.Error("expected decision result to be set in original decision object") + if len(result.OrderedHosts) == 0 { + t.Error("Expected ordered hosts but got none") } } }) diff --git a/internal/scheduling/compute/placement.go b/internal/scheduling/compute/placement.go new file mode 100644 index 000000000..7244016b2 --- /dev/null +++ b/internal/scheduling/compute/placement.go @@ -0,0 +1,140 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "context" + "fmt" + "sync" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + ctrl "sigs.k8s.io/controller-runtime" +) + +// PipelineProvider supplies initialized pipelines by name. The Nova +// filter-weigher pipeline controller satisfies this interface via its embedded +// lib.BasePipelineController. +type PipelineProvider interface { + // GetPipeline returns the initialized pipeline for the given name, if ready. + GetPipeline(name string) (lib.FilterWeigherPipeline[PlacementRequest], bool) +} + +// CandidateGatherer gathers all placement candidates for a request when the +// pipeline is configured to ignore the caller's preselection. +type CandidateGatherer interface { + // MutateWithAllCandidates overrides the request's hosts and weights with all + // available placement candidates. + MutateWithAllCandidates(ctx context.Context, request *PlacementRequest) error +} + +// PreHook is a caller-provided hook executed under the scheduling lock before +// the pipeline runs. It may mutate the request (e.g. to create a pessimistic +// in-flight reservation and narrow the candidate set). Returning an error aborts +// the run before the pipeline executes. A nil PreHook is a no-op. +type PreHook func(ctx context.Context, req *PlacementRequest) error + +// PostHook is a caller-provided side effect executed after the pipeline run, +// under the scheduling lock (e.g. upserting the VM resource, recording the +// decision). It receives the typed pipeline result. Returning an error aborts +// the run; non-fatal side effects should log and return nil so the placement +// decision still stands. A nil PostHook is a no-op. +type PostHook func(ctx context.Context, req PlacementRequest, result PlacementResult) error + +// Placement is the single in-process owner of Nova placement decisions. It holds +// the one shared scheduling lock and runs the pipeline plus a caller-provided +// pre-hook and post-hook. It owns no side-effect policy itself — the caller +// decides, via the hooks it passes, what happens before and after the run. +type Placement struct { + // mu is the single shared scheduling lock, held across the pre-hook, the + // pipeline run, and the post-hook in Run. RunWithoutLock bypasses it. + mu sync.Mutex + + // pipelines resolves pipelines + their config by name. + pipelines PipelineProvider + // gatherer gathers all candidates when a pipeline ignores preselection. + gatherer CandidateGatherer +} + +// NewPlacement builds a Placement over the given pipeline provider and candidate +// gatherer. +func NewPlacement(pipelines PipelineProvider, gatherer CandidateGatherer) *Placement { + return &Placement{pipelines: pipelines, gatherer: gatherer} +} + +// Run executes the pipeline and hooks while holding the shared scheduling lock +// across the whole operation, so concurrent runs observe a consistent view of +// shared scheduling state. The pre-hook runs before the pipeline; the post-hook +// after. Either hook may be nil. Use this for any run that mutates shared state. +func (s *Placement) Run( + ctx context.Context, + req PlacementRequest, + pre PreHook, + post PostHook, +) (PlacementResult, error) { + + s.mu.Lock() + defer s.mu.Unlock() + return s.run(ctx, req, pre, post) +} + +// RunWithoutLock executes the pipeline and hooks WITHOUT acquiring the shared +// lock. Use this only for read-only runs (e.g. capacity probes) that do not +// mutate shared scheduling state and can safely run concurrently with a locked +// run. Either hook may be nil. +func (s *Placement) RunWithoutLock( + ctx context.Context, + req PlacementRequest, + pre PreHook, + post PostHook, +) (PlacementResult, error) { + + return s.run(ctx, req, pre, post) +} + +// run is the shared body of Run and RunWithoutLock. The caller is responsible +// for holding (or deliberately not holding) the lock. +func (s *Placement) run( + ctx context.Context, + req PlacementRequest, + pre PreHook, + post PostHook, +) (PlacementResult, error) { + + log := ctrl.LoggerFrom(ctx) + + pipeline, ok := s.pipelines.GetPipeline(req.Pipeline) + if !ok { + return PlacementResult{}, fmt.Errorf("pipeline not found or not ready: %q", req.Pipeline) + } + + // The pre-hook runs before the pipeline and may mutate the request. + if pre != nil { + if err := pre(ctx, &req); err != nil { + return PlacementResult{}, err + } + } + + // When the request opts into gathering all candidates, override the hosts and + // weights with every available placement candidate before filtering. + if req.GatherAllCandidates { + log.V(1).Info("gathering all placement candidates before filtering") + if err := s.gatherer.MutateWithAllCandidates(ctx, &req); err != nil { + return PlacementResult{}, fmt.Errorf("failed to gather all placement candidates: %w", err) + } + log.V(1).Info("gathered all placement candidates", "numHosts", len(req.Hosts)) + } + + result, runErr := pipeline.Run(req) + if runErr != nil { + return PlacementResult{}, runErr + } + + // The post-hook runs after a successful pipeline run, under the lock. + if post != nil { + if err := post(ctx, req, result); err != nil { + return result, err + } + } + return result, nil +} diff --git a/internal/scheduling/compute/placement_test.go b/internal/scheduling/compute/placement_test.go new file mode 100644 index 000000000..ab58e687a --- /dev/null +++ b/internal/scheduling/compute/placement_test.go @@ -0,0 +1,269 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" +) + +// fakePipeline is a stub lib.FilterWeigherPipeline[PlacementRequest] that records +// the request it was run with and returns a configurable result/error. The run +// function, if set, is invoked inside Run (used to observe lock behavior). +type fakePipeline struct { + result PlacementResult + err error + onRun func(PlacementRequest) + lastReq PlacementRequest +} + +func (p *fakePipeline) Run(req PlacementRequest) (PlacementResult, error) { + p.lastReq = req + if p.onRun != nil { + p.onRun(req) + } + return p.result, p.err +} + +// fakeProvider implements PipelineProvider. A pipeline is returned only for the +// configured name. +type fakeProvider struct { + name string + pipeline lib.FilterWeigherPipeline[PlacementRequest] +} + +func (f *fakeProvider) GetPipeline(name string) (lib.FilterWeigherPipeline[PlacementRequest], bool) { + if name != f.name || f.pipeline == nil { + return nil, false + } + return f.pipeline, true +} + +// fakeGatherer implements CandidateGatherer, recording whether it was called. +type fakeGatherer struct { + called bool + err error +} + +func (g *fakeGatherer) MutateWithAllCandidates(_ context.Context, req *PlacementRequest) error { + g.called = true + if g.err != nil { + return g.err + } + req.Hosts = []Host{{ComputeHost: "gathered", HypervisorHostname: "gathered"}} + return nil +} + +// newTestPlacement builds a Placement whose only ready pipeline is named "p". +func newTestPlacement(p *fakePipeline, g CandidateGatherer) *Placement { + return NewPlacement(&fakeProvider{name: "p", pipeline: p}, g) +} + +func TestRunPipelineNotFound(t *testing.T) { + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) + _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "missing"}, nil, nil) + if err == nil { + t.Fatal("expected error for missing pipeline, got nil") + } +} + +func TestRunPreThenPost(t *testing.T) { + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) + var order []string + pre := func(_ context.Context, _ *PlacementRequest) error { + order = append(order, "pre") + return nil + } + post := func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { + order = append(order, "post") + return nil + } + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, pre, post); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(order) != 2 || order[0] != "pre" || order[1] != "post" { + t.Fatalf("hooks ran out of order: %v", order) + } +} + +func TestRunNilHooks(t *testing.T) { + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, nil); err != nil { + t.Fatalf("nil hooks should be a no-op, got error: %v", err) + } +} + +func TestRunPostHookErrorAborts(t *testing.T) { + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) + boom := errors.New("boom") + post := func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { return boom } + _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, post) + if !errors.Is(err, boom) { + t.Fatalf("expected boom error, got %v", err) + } +} + +func TestRunPipelineErrorSkipsPostHook(t *testing.T) { + boom := errors.New("pipeline failed") + s := newTestPlacement(&fakePipeline{err: boom}, &fakeGatherer{}) + var ran bool + post := func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { ran = true; return nil } + _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, post) + if !errors.Is(err, boom) { + t.Fatalf("expected pipeline error, got %v", err) + } + if ran { + t.Fatal("post-hook must not run when the pipeline errors") + } +} + +func TestRunPreHookRunsBeforePipelineAndMutates(t *testing.T) { + var sawHostInPipeline string + p := &fakePipeline{onRun: func(r PlacementRequest) { + if len(r.Hosts) > 0 { + sawHostInPipeline = r.Hosts[0].ComputeHost + } + }} + s := newTestPlacement(p, &fakeGatherer{}) + pre := func(_ context.Context, r *PlacementRequest) error { + r.Hosts = []Host{{ComputeHost: "from-prehook"}} + return nil + } + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, pre, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if sawHostInPipeline != "from-prehook" { + t.Fatalf("pre-hook mutation not visible to pipeline: got %q", sawHostInPipeline) + } +} + +func TestRunPreHookErrorAbortsBeforePipeline(t *testing.T) { + var pipelineRan bool + p := &fakePipeline{onRun: func(PlacementRequest) { pipelineRan = true }} + s := newTestPlacement(p, &fakeGatherer{}) + boom := errors.New("prehook boom") + pre := func(_ context.Context, _ *PlacementRequest) error { return boom } + _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, pre, nil) + if !errors.Is(err, boom) { + t.Fatalf("expected prehook error, got %v", err) + } + if pipelineRan { + t.Fatal("pipeline must not run after a pre-hook error") + } +} + +func TestRunGathersCandidatesWhenRequested(t *testing.T) { + g := &fakeGatherer{} + p := &fakePipeline{} + s := newTestPlacement(p, g) + req := PlacementRequest{Pipeline: "p", GatherAllCandidates: true} + if _, err := s.Run(context.Background(), req, nil, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !g.called { + t.Fatal("expected gatherer to be called when GatherAllCandidates is set") + } + if len(p.lastReq.Hosts) != 1 || p.lastReq.Hosts[0].ComputeHost != "gathered" { + t.Fatalf("pipeline did not see gathered candidates: %+v", p.lastReq.Hosts) + } +} + +func TestRunDoesNotGatherWithoutGatherAllCandidates(t *testing.T) { + g := &fakeGatherer{} + s := newTestPlacement(&fakePipeline{}, g) + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if g.called { + t.Fatal("gatherer should not be called without GatherAllCandidates") + } +} + +// TestRunHoldsLockAcrossPostHook verifies that a second Run blocks until the +// first Run's post-hook completes — i.e. the lock spans the whole run, not just +// the pipeline. +func TestRunHoldsLockAcrossPostHook(t *testing.T) { + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) + + firstInPost := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondDone := make(chan struct{}) + + var slowPost PostHook = func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { + close(firstInPost) + <-releaseFirst + return nil + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, slowPost); err != nil { + t.Errorf("first Run: %v", err) + } + }() + + <-firstInPost // first Run is parked inside its post-hook, still holding the lock. + + go func() { + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, nil); err != nil { + t.Errorf("second Run: %v", err) + } + close(secondDone) + }() + + // The second Run must be blocked on the lock held by the first. + select { + case <-secondDone: + t.Fatal("second Run completed while the first still held the lock") + case <-time.After(50 * time.Millisecond): + } + + close(releaseFirst) // let the first Run finish and release the lock. + wg.Wait() + <-secondDone // the second Run now proceeds. +} + +// TestRunWithoutLockDoesNotBlock verifies that RunWithoutLock proceeds even while +// a locked Run holds the lock inside its post-hook. +func TestRunWithoutLockDoesNotBlock(t *testing.T) { + s := newTestPlacement(&fakePipeline{}, &fakeGatherer{}) + + firstInPost := make(chan struct{}) + releaseFirst := make(chan struct{}) + var slowPost PostHook = func(_ context.Context, _ PlacementRequest, _ PlacementResult) error { + close(firstInPost) + <-releaseFirst + return nil + } + + go func() { + if _, err := s.Run(context.Background(), PlacementRequest{Pipeline: "p"}, nil, slowPost); err != nil { + t.Errorf("locked Run: %v", err) + } + }() + <-firstInPost // locked Run holds the lock. + + done := make(chan struct{}) + go func() { + if _, err := s.RunWithoutLock(context.Background(), PlacementRequest{Pipeline: "p"}, nil, nil); err != nil { + t.Errorf("RunWithoutLock: %v", err) + } + close(done) + }() + + select { + case <-done: + // Expected: lock-free run completes without waiting for the lock. + case <-time.After(time.Second): + t.Fatal("RunWithoutLock blocked on the scheduling lock") + } + close(releaseFirst) +} diff --git a/internal/scheduling/compute/request.go b/internal/scheduling/compute/request.go new file mode 100644 index 000000000..82ba20889 --- /dev/null +++ b/internal/scheduling/compute/request.go @@ -0,0 +1,348 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +// Package compute is the in-process entrypoint for Nova placement scheduling. +// +// It defines a well-typed, Nova-agnostic PlacementRequest that the +// filter-weigher pipeline consumes, and a Placement that owns the single shared +// scheduling lock and runs the pipeline plus caller-provided side-effect +// closures. Callers that run in the same process (the Nova HTTP adapter, the +// reservation/commitment/capacity/failover controllers) call the Placement +// directly instead of POSTing to the external HTTP endpoint. +// +// The spec deliberately contains NO loosely-typed maps or Nova string +// conventions (flavor extra-spec keys, scheduler-hint keys, image property +// keys). A Nova spec adapter is responsible for translating Nova's over-the-wire +// shape into these typed fields; the pipeline only ever sees well-specced +// values. +package compute + +import ( + "strings" + + "github.com/cobaltcore-dev/cortex/api/scheduling" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" +) + +// HypervisorFamily is the coarse hypervisor family a request targets. It is the +// distinction the pipeline and request routing care about. The adapter resolves +// the family from Nova's "capabilities:hypervisor_type" flavor extra spec. +type HypervisorFamily string + +const ( + // HypervisorFamilyKVM covers KVM-based hypervisors (QEMU and CH). + HypervisorFamilyKVM HypervisorFamily = "kvm" + // HypervisorFamilyVMware covers VMware hypervisors. + HypervisorFamilyVMware HypervisorFamily = "vmware" +) + +// ImageHypervisorType is the hypervisor type declared by the image. The adapter +// resolves it from Nova's image properties ("img_hv_type"/"hypervisor_type"). +type ImageHypervisorType string + +const ( + // ImageHypervisorTypeVMware indicates a VMware image. + ImageHypervisorTypeVMware ImageHypervisorType = "vmware" + // ImageHypervisorTypeBaremetal indicates a baremetal image. + ImageHypervisorTypeBaremetal ImageHypervisorType = "baremetal" + // ImageHypervisorTypeKVM indicates a KVM image. + ImageHypervisorTypeKVM ImageHypervisorType = "kvm" + // ImageHypervisorTypeUnset indicates no image hypervisor type was declared. + ImageHypervisorTypeUnset ImageHypervisorType = "" + // ImageHypervisorTypeUnparseable indicates an image hypervisor type was declared but could not be parsed. + ImageHypervisorTypeUnparseable ImageHypervisorType = "unparseable" +) + +// Host is a placement candidate. For KVM hosts the compute host name and the +// hypervisor hostname are identical; for others they may differ. +type Host struct { + // ComputeHost is the Nova compute host name, e.g. nova-compute-bb123. + ComputeHost string + // HypervisorHostname is the hypervisor hostname, e.g. domain-c123.. + HypervisorHostname string +} + +// Capability is a single required hypervisor capability as a typed key/value +// equality, e.g. {Name: "cpu_arch", Value: "x86_64"}. The adapter resolves these +// from Nova's "capabilities:" flavor extra specs and rejects any entry that +// uses an unsupported comparison operator before it reaches the pipeline. +type Capability struct { + // Name is the capability key without the "capabilities:" prefix. + Name string + // Value is the required exact value. + Value string +} + +// FlavorSpec is the typed, Nova-agnostic description of the requested flavor. +// It carries no extra-spec map: the adapter resolves every extra spec the +// pipeline needs into the typed fields on this struct and on PlacementRequest. +type FlavorSpec struct { + // MemoryMB is the requested memory per instance. 0 means a zero-memory + // instance to capacity/bin-pack steps (fits anywhere), so the adapter sets it. + MemoryMB uint64 + // VCPUs is the requested vCPUs per instance. 0 means a zero-CPU instance to + // capacity/bin-pack steps (fits anywhere), so the adapter sets it. + VCPUs uint64 +} + +// RequestedDestination mirrors the subset of Nova's requested_destination that +// the pipeline reads. A nil *RequestedDestination on the request means no +// destination constraint was requested. +type RequestedDestination struct { + // Host is the requested target host. Empty means no host was pinned (only an + // aggregate constraint applies). + Host string + // Aggregates are the host-aggregate names the destination must belong to. + // Empty/nil means no aggregate constraint. + Aggregates []string +} + +// InstanceGroupSpec is the typed subset of a Nova server group the pipeline +// reads. A nil *InstanceGroupSpec on the request means the instance is not part +// of a server group, so group affinity/anti-affinity filters are skipped. +type InstanceGroupSpec struct { + // UUID identifies the server group. Empty if unknown. + UUID string + // Policy is the group policy (e.g. "anti-affinity", "soft-affinity"). Empty + // means no policy-specific filter/weigher applies. + Policy string + // Hosts are the hosts already occupied by members of this group. Empty/nil + // means the group currently occupies no hosts. + Hosts []string + // Members are the instance UUIDs already in the group. Empty/nil means the + // group has no members yet, so anti-affinity has nothing to exclude. + Members []string + // MaxServerPerHost caps how many group members may share a host. 0 means + // unset, which the anti-affinity filter treats as the Nova default of 1. + MaxServerPerHost int +} + +// PlacementRequest is the well-typed, Nova-agnostic input to the scheduling +// pipeline. Every field is a resolved scalar, enum, or typed collection — there +// are no maps of Nova-specific string keys and no values the pipeline must parse. +// +// Each field documents what leaving it at its zero value means, so the Nova spec +// adapter (and any other caller) can see exactly which fields are load-bearing. +// +// PlacementRequest implements lib.FilterWeigherPipelineRequest so it can be the +// RequestType of the filter-weigher pipeline. +type PlacementRequest struct { + // Pipeline is the name of the pipeline to execute. Required: an empty name + // resolves no pipeline and the run fails with "pipeline not found". + Pipeline string + // GatherAllCandidates, when true, tells the Placement to ignore the Hosts and + // Weights the caller provided and instead gather ALL available placement + // candidates (via the CandidateGatherer) before filtering. It is set for + // requests where cortex can enumerate the candidate hosts itself — today the + // KVM family, whose hypervisors cortex knows from the hypervisor operator; + // VMware placement leaves it false and relies on the caller's preselection. + // When true, Hosts/Weights below are overwritten before filtering. + GatherAllCandidates bool + // Hosts are the placement candidates. An empty list yields an empty result + // (no host found), unless GatherAllCandidates is set, in which case Hosts is + // overwritten before filtering. + Hosts []Host + // Weights are the per-host input weights, keyed by compute host name. A host + // absent from this map is weighed as 0. Empty means every candidate starts + // equally weighted. + Weights map[string]float64 + // Options configure the pipeline behavior for this run. The zero value is a + // normal (non-read-only) run with no candidate limit; see scheduling.Options + // for per-field defaults. + Options scheduling.Options + + // Intent is the resolved scheduling intent (replaces the "_nova_check_type" + // scheduler hint). The zero value is the empty intent; intent-gated + // filters/weighers treat it like an initial placement (create) and a forced + // destination is honored (see IsForcedDestination). + Intent v1alpha1.SchedulingIntent + + // InstanceUUID identifies the instance being placed. Empty is allowed for + // synthetic runs (e.g. capacity probes); it only appears in logs and side + // effects, so an empty value disables nothing in the pipeline itself. + InstanceUUID string + // ProjectID is the owning OpenStack project. Empty disables project-scoped + // filtering (e.g. allowed-projects, quota) — those steps treat it as "no + // project context". + ProjectID string + // UserID is the requesting user. Empty only affects trace logging and the VM + // ownership written by side effects. + UserID string + // AvailabilityZone is the target AZ. Empty means no AZ constraint, so the + // correct-AZ filter keeps all hosts. + AvailabilityZone string + // NumInstances is the number of instances this request represents. It is a + // MULTIPLIER in capacity and bin-pack math, so 0 makes a host look infinitely + // free and must not be left unset — the adapter always sets it (Nova sends + // >= 1). + NumInstances uint64 + + // IgnoreHosts are hosts to exclude from placement. Empty/nil means no hosts + // are excluded. + IgnoreHosts []string + // ForceHosts pins placement to these compute hosts. Empty/nil means no host + // is forced. When set (and the intent does not require filter validation), + // the pipeline is skipped and only forced hosts are returned. + ForceHosts []string + // ForceNodes pins placement to these hypervisor hostnames. Empty/nil means no + // node is forced. Semantics mirror ForceHosts. + ForceNodes []string + + // HypervisorFamily is the coarse family (kvm/vmware) used by the pipeline and + // routing. Empty means unresolved: KVM-only gates (e.g. VM creation) treat it + // as not-KVM and skip. + HypervisorFamily HypervisorFamily + // ImageHypervisorType is the hypervisor type declared by the image. The empty + // value (ImageHypervisorTypeUnset) means the image declared none ("no image + // constraint"); ImageHypervisorTypeUnparseable means it declared one that + // could not be parsed, which the image-properties filter surfaces as a + // diagnostic. + ImageHypervisorType ImageHypervisorType + // HardwareVersion is resolved from the "quota:hw_version" extra spec. Empty + // means no hardware-version constraint. + HardwareVersion string + + // Flavor is the requested flavor sizing. A zero FlavorSpec means capacity and + // bin-pack steps see a zero-sized instance (fits anywhere) — the adapter + // always populates it for real placements. + Flavor FlavorSpec + + // RequiredTraits are trait names the host must have (adapter stripped the + // "trait:" prefix and kept the "required" ones). Empty/nil means no required + // traits, so the traits filter imposes no inclusion constraint. + RequiredTraits []string + // ForbiddenTraits are trait names the host must NOT have. Empty/nil means no + // forbidden traits. If both trait lists are empty the traits filter is a no-op. + ForbiddenTraits []string + // RequestedCapabilities are required (key=value) hypervisor capabilities + // (adapter stripped the "capabilities:" prefix and rejected unsupported + // operators). Empty/nil means the capabilities filter is a no-op. + RequestedCapabilities []Capability + // AcceleratorsRequested reports whether the flavor requests a hardware + // accelerator (adapter resolves it from the presence of the + // "accel:device_profile" extra spec). When true, the accelerators filter keeps + // only hosts carrying the COMPUTE_ACCELERATORS trait; false is a no-op. + AcceleratorsRequested bool + + // SourceHost is the migration source (was hint "source_host"). Empty means + // not a host-to-host move, so the live-migratable filter imposes no source + // constraint. + SourceHost string + // DomainName is the customer domain (was hint "domain_name"). Empty means no + // domain context, so the external-customer filter is skipped. + DomainName string + // ResourceGroup is the failover resource group (was hint + // "_cortex_resource_group"). Empty means no group, so group-consolidation + // weighing is skipped. + ResourceGroup string + + // RequestedDestination is the requested target host/aggregate. Nil means no + // destination constraint (see the type doc). + RequestedDestination *RequestedDestination + // InstanceGroup is the server group the instance belongs to. Nil means the + // instance is not grouped, so affinity/anti-affinity steps are skipped (see + // the type doc). + InstanceGroup *InstanceGroupSpec + + // RequestID identifies this scheduling request for tracing. Empty only + // affects log correlation. + RequestID string + // GlobalRequestID is the cross-service request ID for tracing. Empty only + // affects log correlation. + GlobalRequestID string +} + +// Compile-time check that PlacementRequest satisfies the pipeline request contract. +var _ lib.FilterWeigherPipelineRequest = PlacementRequest{} + +// GetIntent returns the resolved scheduling intent. +func (r PlacementRequest) GetIntent() v1alpha1.SchedulingIntent { return r.Intent } + +// GetHosts returns the compute host names of all candidates. +func (r PlacementRequest) GetHosts() []string { + hosts := make([]string, len(r.Hosts)) + for i, h := range r.Hosts { + hosts[i] = h.ComputeHost + } + return hosts +} + +// GetWeights returns the per-host input weights. +func (r PlacementRequest) GetWeights() map[string]float64 { return r.Weights } + +// GetOptions returns the call-time options for this run. +func (r PlacementRequest) GetOptions() scheduling.Options { return r.Options } + +// GetTraceLogArgs returns alternating key/value pairs for the step trace log. +func (r PlacementRequest) GetTraceLogArgs() []any { + return []any{ + "greq", r.GlobalRequestID, + "req", r.RequestID, + "user", r.UserID, + "project", r.ProjectID, + } +} + +// Filter returns a copy of the request narrowed to the given hosts, with the +// weights updated to the provided values. Hosts not present in includedHosts are +// dropped and won't be reconsidered by later steps. +func (r PlacementRequest) Filter(includedHosts map[string]float64) lib.FilterWeigherPipelineRequest { + filteredHosts := make([]Host, 0, len(includedHosts)) + for _, h := range r.Hosts { + if _, ok := includedHosts[h.ComputeHost]; ok { + filteredHosts = append(filteredHosts, h) + } + } + r.Hosts = filteredHosts + filteredWeights := make(map[string]float64, len(includedHosts)) + for host, weight := range includedHosts { + filteredWeights[host] = weight + } + r.Weights = filteredWeights + return r +} + +// IsForcedDestination reports whether the request is forced onto specific +// hosts/nodes and the scheduler filters should be skipped entirely. A forced +// request that still carries a non-create intent (rebuild/evacuate/resize) must +// be validated by the filters, so it is not treated as forced here. Intent +// resolution happens in the adapter; this only inspects the typed Intent field. +func (r PlacementRequest) IsForcedDestination() bool { + if len(r.ForceHosts) == 0 && len(r.ForceNodes) == 0 { + return false + } + return r.Intent == "" || r.Intent == v1alpha1.SchedulingIntentUnknown +} + +// ForcedHosts returns the subset of request hosts matching the forced +// destinations. force_hosts matches on compute host name (case-insensitive); +// force_nodes matches on hypervisor hostname (case-sensitive); when both are set +// a host must match both. Order is preserved and compute hosts are deduplicated. +func (r PlacementRequest) ForcedHosts() []string { + forceHostSet := make(map[string]bool, len(r.ForceHosts)) + for _, h := range r.ForceHosts { + forceHostSet[strings.ToLower(h)] = true + } + forceNodeSet := make(map[string]bool, len(r.ForceNodes)) + for _, n := range r.ForceNodes { + forceNodeSet[n] = true + } + matched := make([]string, 0, len(r.Hosts)) + seen := make(map[string]bool, len(r.Hosts)) + for _, h := range r.Hosts { + if len(forceHostSet) > 0 && !forceHostSet[strings.ToLower(h.ComputeHost)] { + continue + } + if len(forceNodeSet) > 0 && !forceNodeSet[h.HypervisorHostname] { + continue + } + key := strings.ToLower(h.ComputeHost) + if seen[key] { + continue + } + seen[key] = true + matched = append(matched, h.ComputeHost) + } + return matched +} diff --git a/internal/scheduling/compute/request_test.go b/internal/scheduling/compute/request_test.go new file mode 100644 index 000000000..8244e7dbe --- /dev/null +++ b/internal/scheduling/compute/request_test.go @@ -0,0 +1,82 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "slices" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +func TestIsForcedDestination(t *testing.T) { + cases := []struct { + name string + req PlacementRequest + forced bool + }{ + {"no force", PlacementRequest{}, false}, + {"force host, no intent", PlacementRequest{ForceHosts: []string{"h1"}}, true}, + {"force node, no intent", PlacementRequest{ForceNodes: []string{"n1"}}, true}, + {"force host, unknown intent", PlacementRequest{ForceHosts: []string{"h1"}, Intent: v1alpha1.SchedulingIntentUnknown}, true}, + {"force host, concrete intent must still filter", PlacementRequest{ForceHosts: []string{"h1"}, Intent: "evacuate"}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.req.IsForcedDestination(); got != c.forced { + t.Errorf("IsForcedDestination() = %v, want %v", got, c.forced) + } + }) + } +} + +func TestForcedHosts(t *testing.T) { + hosts := []Host{ + {ComputeHost: "Host-A", HypervisorHostname: "node-a"}, + {ComputeHost: "host-b", HypervisorHostname: "node-b"}, + {ComputeHost: "host-b", HypervisorHostname: "node-b2"}, // duplicate compute host + } + cases := []struct { + name string + force []string + nodes []string + want []string + }{ + {"force_hosts case-insensitive", []string{"host-a"}, nil, []string{"Host-A"}}, + {"force_nodes exact", nil, []string{"node-b"}, []string{"host-b"}}, + {"both intersect", []string{"host-b"}, []string{"node-b2"}, []string{"host-b"}}, + {"no match", []string{"nope"}, nil, []string{}}, + {"dedup compute host", []string{"host-b"}, nil, []string{"host-b"}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + r := PlacementRequest{Hosts: hosts, ForceHosts: c.force, ForceNodes: c.nodes} + got := r.ForcedHosts() + if !slices.Equal(got, c.want) { + t.Errorf("ForcedHosts() = %v, want %v", got, c.want) + } + }) + } +} + +func TestFilterNarrowsHostsAndWeights(t *testing.T) { + r := PlacementRequest{ + Hosts: []Host{ + {ComputeHost: "a"}, {ComputeHost: "b"}, {ComputeHost: "c"}, + }, + Weights: map[string]float64{"a": 1, "b": 2, "c": 3}, + } + filtered := r.Filter(map[string]float64{"a": 10, "c": 30}).(PlacementRequest) + got := filtered.GetHosts() + slices.Sort(got) + if !slices.Equal(got, []string{"a", "c"}) { + t.Fatalf("Filter kept hosts %v, want [a c]", got) + } + if filtered.Weights["a"] != 10 || filtered.Weights["c"] != 30 { + t.Fatalf("Filter weights = %v, want a:10 c:30", filtered.Weights) + } + if _, ok := filtered.Weights["b"]; ok { + t.Fatalf("Filter should have dropped host b from weights: %v", filtered.Weights) + } +} diff --git a/internal/scheduling/compute/result.go b/internal/scheduling/compute/result.go new file mode 100644 index 000000000..56cfde8a0 --- /dev/null +++ b/internal/scheduling/compute/result.go @@ -0,0 +1,17 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" +) + +// PlacementResult is the well-defined output of a scheduling pipeline run: +// ordered hosts, the target host, and the input/output weights plus per-step +// activations used to rank them. It is an alias of lib.PipelineResult, the +// domain-neutral type the generic pipeline returns. The human-readable +// explanation is NOT part of the result — it is a concern of writing the +// decision onto the scheduled resource (e.g. the Nova VM's CortexSpec), so the +// relevant side effect generates it from the result via lib.GenerateExplanation. +type PlacementResult = lib.PipelineResult diff --git a/internal/scheduling/lib/explanation.go b/internal/scheduling/lib/explanation.go index c1deeff0f..4eff7017a 100644 --- a/internal/scheduling/lib/explanation.go +++ b/internal/scheduling/lib/explanation.go @@ -7,8 +7,6 @@ import ( "fmt" "sort" "strings" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) // maxHostsInExplanation caps how many host names are listed per line in a @@ -24,13 +22,12 @@ func joinHostsCapped(hosts []string, maxHosts int) string { return fmt.Sprintf("%s (and %d more)", strings.Join(hosts[:maxHosts], ", "), len(hosts)-maxHosts) } -// GenerateExplanation produces a human-readable explanation from a decision +// GenerateExplanation produces a human-readable explanation from a pipeline // result. On failure it includes the error. On success it describes which // pipeline steps filtered out which hosts and, for the surviving hosts, how the -// weighers ranked them (via ExplainWeighing). The result is written to -// Decision.Status.Explanation so it propagates to the scheduled resource (e.g. -// the VM CortexSpec.Explanation). -func GenerateExplanation(result *v1alpha1.DecisionResult, pipelineErr error) string { +// weighers ranked them (via ExplainWeighing). Consumers propagate the result to +// the scheduled resource (e.g. the Nova VM's CortexSpec.Explanation). +func GenerateExplanation(result *PipelineResult, pipelineErr error) string { if pipelineErr != nil { return fmt.Sprintf("Pipeline run failed: %s.", pipelineErr.Error()) } diff --git a/internal/scheduling/lib/explanation_test.go b/internal/scheduling/lib/explanation_test.go index 2c020f396..8ddcd2af4 100644 --- a/internal/scheduling/lib/explanation_test.go +++ b/internal/scheduling/lib/explanation_test.go @@ -7,8 +7,6 @@ import ( "errors" "fmt" "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) func ptr[T any](v T) *T { return &v } @@ -16,7 +14,7 @@ func ptr[T any](v T) *T { return &v } func TestGenerateExplanation(t *testing.T) { tests := []struct { name string - result *v1alpha1.DecisionResult + result *PipelineResult err error expected string }{ @@ -34,20 +32,20 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "result with target host only no steps", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ TargetHost: ptr("host-1"), }, expected: "Selected host: host-1.", }, { name: "result with RawInWeights and filtering steps", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ RawInWeights: map[string]float64{ "host-a": 1.0, "host-b": 0.5, "host-c": 0.3, }, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ { StepName: "filter_capacity", Activations: map[string]float64{ @@ -72,12 +70,12 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "uses NormalizedInWeights when RawInWeights empty", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ NormalizedInWeights: map[string]float64{ "host-x": 0.6, "host-y": 0.4, }, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ { StepName: "weigher_cpu", Activations: map[string]float64{ @@ -92,8 +90,8 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "no weights no target", - result: &v1alpha1.DecisionResult{ - StepResults: []v1alpha1.StepResult{ + result: &PipelineResult{ + StepResults: []StepResult{ {StepName: "some-step", Activations: map[string]float64{}}, }, }, @@ -101,9 +99,9 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "no weights with target", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ TargetHost: ptr("host-1"), - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "some-step", Activations: map[string]float64{}}, }, }, @@ -111,12 +109,12 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "all hosts survive all steps", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ RawInWeights: map[string]float64{ "host-a": 1.0, "host-b": 0.5, }, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ { StepName: "noop", Activations: map[string]float64{ @@ -131,7 +129,7 @@ func TestGenerateExplanation(t *testing.T) { }, { name: "large cluster caps host lists", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { weights := make(map[string]float64, 50) for i := range 50 { weights[fmt.Sprintf("host-%03d", i)] = 1.0 @@ -141,9 +139,9 @@ func TestGenerateExplanation(t *testing.T) { for i := range 30 { surviving[fmt.Sprintf("host-%03d", i)] = 1.0 } - return &v1alpha1.DecisionResult{ + return &PipelineResult{ RawInWeights: weights, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ { StepName: "filter_big", Activations: surviving, diff --git a/internal/scheduling/lib/filter_weigher_pipeline.go b/internal/scheduling/lib/filter_weigher_pipeline.go index 57ab4e321..d29ff2449 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline.go +++ b/internal/scheduling/lib/filter_weigher_pipeline.go @@ -23,7 +23,7 @@ var pipelineLog = logf.Log.WithName("scheduler-pipeline") type FilterWeigherPipeline[RequestType FilterWeigherPipelineRequest] interface { // Run the scheduling pipeline with the given request. // Call-time options are read from request.GetOptions(). - Run(request RequestType) (v1alpha1.DecisionResult, error) + Run(request RequestType) (PipelineResult, error) } // Pipeline of scheduler steps. @@ -142,7 +142,7 @@ func InitNewFilterWeigherPipeline[RequestType FilterWeigherPipelineRequest]( func (p *filterWeigherPipeline[RequestType]) runFilters( traceLog logr.Logger, request RequestType, -) (filteredRequest RequestType, stepResults []v1alpha1.StepResult) { +) (filteredRequest RequestType, stepResults []StepResult) { filteredRequest = request for _, filterName := range p.filtersOrder { @@ -159,7 +159,7 @@ func (p *filterWeigherPipeline[RequestType]) runFilters( continue } stepLog.V(1).Info("scheduler: finished filter") - stepResults = append(stepResults, v1alpha1.StepResult{ + stepResults = append(stepResults, StepResult{ StepName: filterName, Activations: result.Activations, }) @@ -269,10 +269,10 @@ func (s *filterWeigherPipeline[RequestType]) sortHostsByWeights(weights map[stri } // Evaluate the pipeline and return a list of hosts in order of preference. -func (p *filterWeigherPipeline[RequestType]) Run(request RequestType) (v1alpha1.DecisionResult, error) { +func (p *filterWeigherPipeline[RequestType]) Run(request RequestType) (PipelineResult, error) { opts := request.GetOptions() if err := opts.Validate(); err != nil { - return v1alpha1.DecisionResult{}, err + return PipelineResult{}, err } traceLog := pipelineLog.WithValues(request.GetTraceLogArgs()...) @@ -341,13 +341,13 @@ func (p *filterWeigherPipeline[RequestType]) Run(request RequestType) (v1alpha1. if !ok { continue } - stepResults = append(stepResults, v1alpha1.StepResult{ + stepResults = append(stepResults, StepResult{ StepName: weigherName, Activations: activations, }) } - result := v1alpha1.DecisionResult{ + result := PipelineResult{ RawInWeights: request.GetWeights(), NormalizedInWeights: inWeights, StepResults: stepResults, diff --git a/internal/scheduling/lib/pipeline_result.go b/internal/scheduling/lib/pipeline_result.go new file mode 100644 index 000000000..938c08d78 --- /dev/null +++ b/internal/scheduling/lib/pipeline_result.go @@ -0,0 +1,31 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package lib + +// StepResult holds the per-host activations a single pipeline step produced. +type StepResult struct { + // StepName is the name of the filter or weigher that produced this result. + StepName string + // Activations maps each host to the activation the step assigned it. + Activations map[string]float64 +} + +// PipelineResult is the domain-neutral output of a filter-weigher pipeline run: +// the ordered hosts, the target host, and the input/output weights plus per-step +// activations used to rank them. Domain-facing aliases (e.g. +// compute.PlacementResult) point at this type. +type PipelineResult struct { + // RawInWeights are the input weights to the pipeline, before normalization. + RawInWeights map[string]float64 + // NormalizedInWeights are the input weights after normalization. + NormalizedInWeights map[string]float64 + // StepResults are the activations each pipeline step produced, in order. + StepResults []StepResult + // AggregatedOutWeights are the final weights after applying all weighers. + AggregatedOutWeights map[string]float64 + // OrderedHosts is the final ordering from most to least preferred. + OrderedHosts []string + // TargetHost is the first element of OrderedHosts, if any. + TargetHost *string +} diff --git a/internal/scheduling/lib/weighing_explainer.go b/internal/scheduling/lib/weighing_explainer.go index 2e66bce41..9584afb6f 100644 --- a/internal/scheduling/lib/weighing_explainer.go +++ b/internal/scheduling/lib/weighing_explainer.go @@ -8,8 +8,6 @@ import ( "math" "sort" "strings" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) const ( @@ -55,7 +53,7 @@ type weigherContribution struct { // // Returns an empty string when the result is nil, has fewer than 2 ordered // hosts, or contains no weigher steps. -func ExplainWeighing(result *v1alpha1.DecisionResult) string { +func ExplainWeighing(result *PipelineResult) string { if result == nil || len(result.OrderedHosts) < 2 { return "" } @@ -199,7 +197,7 @@ func ExplainWeighing(result *v1alpha1.DecisionResult) string { // recovery is not possible (e.g., all activations are zero, or the system is // under-determined). It reports only the initial weight bias and raw activation // differentials without exact contribution magnitudes. -func explainWithoutMultipliers(result *v1alpha1.DecisionResult, topHosts []string, _ []v1alpha1.StepResult) string { +func explainWithoutMultipliers(result *PipelineResult, topHosts []string, _ []StepResult) string { topN := len(topHosts) var sb strings.Builder @@ -239,13 +237,13 @@ func explainWithoutMultipliers(result *v1alpha1.DecisionResult, topHosts []strin // weigher (scoring) steps rather than filter steps. A weigher step is one // whose activation map contains entries for ALL hosts in OrderedHosts — // filters reduce the host set while weighers score all remaining hosts. -func identifyWeigherSteps(result *v1alpha1.DecisionResult) []v1alpha1.StepResult { +func identifyWeigherSteps(result *PipelineResult) []StepResult { orderedHostSet := make(map[string]struct{}, len(result.OrderedHosts)) for _, h := range result.OrderedHosts { orderedHostSet[h] = struct{}{} } - var weigherSteps []v1alpha1.StepResult + var weigherSteps []StepResult for _, step := range result.StepResults { if len(step.Activations) == 0 { continue @@ -281,7 +279,7 @@ func identifyWeigherSteps(result *v1alpha1.DecisionResult) []v1alpha1.StepResult // Returns the recovered multipliers and true on success, or nil and false if // the system is under-determined or ill-conditioned. func recoverMultipliers( - weigherSteps []v1alpha1.StepResult, + weigherSteps []StepResult, orderedHosts []string, normalizedIn map[string]float64, aggregatedOut map[string]float64, diff --git a/internal/scheduling/lib/weighing_explainer_test.go b/internal/scheduling/lib/weighing_explainer_test.go index 3e63508d7..f333ecb8b 100644 --- a/internal/scheduling/lib/weighing_explainer_test.go +++ b/internal/scheduling/lib/weighing_explainer_test.go @@ -8,14 +8,12 @@ import ( "sort" "strings" "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) func TestExplainWeighing(t *testing.T) { tests := []struct { name string - result *v1alpha1.DecisionResult + result *PipelineResult contains []string // substrings that must appear in output excludes []string // substrings that must NOT appear in output empty bool // expect empty string @@ -27,7 +25,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "single host returns empty", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ OrderedHosts: []string{"host-a"}, NormalizedInWeights: map[string]float64{"host-a": 0.5}, AggregatedOutWeights: map[string]float64{"host-a": 0.8}, @@ -36,11 +34,11 @@ func TestExplainWeighing(t *testing.T) { }, { name: "only filter steps returns empty", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ NormalizedInWeights: map[string]float64{"host-a": 0.5, "host-b": 0.3, "host-c": 0.1}, AggregatedOutWeights: map[string]float64{"host-a": 0.5, "host-b": 0.3}, OrderedHosts: []string{"host-a", "host-b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ // A true filter: activations do NOT include host-b (it was filtered after this step). {StepName: "filter_az", Activations: map[string]float64{"host-a": 1.0}}, }, @@ -49,7 +47,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "two hosts single weigher positive multiplier", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // Simulate: multiplier = 2.0, act[a] = 1.0, act[b] = -0.5 // contrib[a] = 2.0 * tanh(1.0) = 1.5231 // contrib[b] = 2.0 * tanh(-0.5) = -0.9242 @@ -58,11 +56,11 @@ func TestExplainWeighing(t *testing.T) { inA, inB := 0.3, 0.3 outA := inA + mult*math.Tanh(actA) outB := inB + mult*math.Tanh(actB) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: map[string]float64{"host-a": inA, "host-b": inB}, AggregatedOutWeights: map[string]float64{"host-a": outA, "host-b": outB}, OrderedHosts: []string{"host-a", "host-b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_cpu", Activations: map[string]float64{"host-a": actA, "host-b": actB}}, }, } @@ -71,7 +69,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "two hosts single weigher negative multiplier", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // Simulate: multiplier = -1.0 (inverts behavior) // A host with HIGH raw activation is penalized. // act[a] = -2.0 (high penalty activates less due to neg mult -> boosts) @@ -83,11 +81,11 @@ func TestExplainWeighing(t *testing.T) { outB := inB + mult*math.Tanh(actB) // outA = 0 + (-1)*tanh(-2) = 0 + (-1)*(-0.964) = +0.964 // outB = 0 + (-1)*tanh(3) = 0 + (-1)*(0.995) = -0.995 - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: map[string]float64{"host-a": inA, "host-b": inB}, AggregatedOutWeights: map[string]float64{"host-a": outA, "host-b": outB}, OrderedHosts: []string{"host-a", "host-b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "kvm_binpack", Activations: map[string]float64{"host-a": actA, "host-b": actB}}, }, } @@ -96,7 +94,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "three hosts two weighers with counterfactual", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // weigher_cpu (mult=3.0) strongly favors host-a // weigher_mem (mult=1.0) slightly favors host-b // Without weigher_cpu, host-b should be #1. @@ -111,11 +109,11 @@ func TestExplainWeighing(t *testing.T) { // Sort to determine OrderedHosts. hosts := []string{"h1", "h2", "h3"} sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_cpu", Activations: actCPU}, {StepName: "weigher_mem", Activations: actMem}, }, @@ -125,7 +123,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "opposing weigher reported", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // weigher_boost (mult=2.0) pushes host-a up // weigher_penalty (mult=-0.5) opposes host-a // Net: host-a still wins due to weigher_boost dominance. @@ -139,11 +137,11 @@ func TestExplainWeighing(t *testing.T) { } hosts := []string{"x", "y"} sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_boost", Activations: actBoost}, {StepName: "weigher_penalty", Activations: actPenalty}, }, @@ -153,7 +151,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "negligible impact weigher reported", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // weigher_big (mult=5.0) dominates // weigher_tiny (mult=0.001) is negligible multBig, multTiny := 5.0, 0.001 @@ -166,11 +164,11 @@ func TestExplainWeighing(t *testing.T) { } hosts := []string{"a", "b"} sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_big", Activations: actBig}, {StepName: "weigher_tiny", Activations: actTiny}, }, @@ -180,11 +178,11 @@ func TestExplainWeighing(t *testing.T) { }, { name: "initial weight bias dominates when activations are zero", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ NormalizedInWeights: map[string]float64{"fast": 0.8, "slow": 0.2}, AggregatedOutWeights: map[string]float64{"fast": 0.8, "slow": 0.2}, OrderedHosts: []string{"fast", "slow"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "weigher_noop", Activations: map[string]float64{"fast": 0.0, "slow": 0.0}}, }, }, @@ -193,7 +191,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "mixed filter and weigher steps ignores filters", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { mult := 1.5 actW := map[string]float64{"a": 1.0, "b": -0.5} in := map[string]float64{"a": 0.0, "b": 0.0, "c": 0.0} @@ -203,11 +201,11 @@ func TestExplainWeighing(t *testing.T) { } hosts := []string{"a", "b"} sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: map[string]float64{"a": 0.0, "b": 0.0}, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ // Filter step: removed host c (activations don't include all OrderedHosts... wait, they do include a and b). // Actually a filter that removed c would have activations for a and b only. // Since OrderedHosts = [a, b], this LOOKS like a weigher. @@ -224,11 +222,11 @@ func TestExplainWeighing(t *testing.T) { }, { name: "under-determined system falls back to gap-only explanation", - result: &v1alpha1.DecisionResult{ + result: &PipelineResult{ NormalizedInWeights: map[string]float64{"a": 0.0, "b": 0.0}, AggregatedOutWeights: map[string]float64{"a": 1.0, "b": 0.5}, OrderedHosts: []string{"a", "b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "w1", Activations: map[string]float64{"a": 1.0, "b": 0.5}}, {StepName: "w2", Activations: map[string]float64{"a": 0.8, "b": 0.3}}, {StepName: "w3", Activations: map[string]float64{"a": 0.6, "b": 0.1}}, @@ -239,7 +237,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "multiplier recovery accuracy", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // Known multipliers: [2.5, -1.0, 0.7] mults := []float64{2.5, -1.0, 0.7} acts := []map[string]float64{ @@ -257,11 +255,11 @@ func TestExplainWeighing(t *testing.T) { } } sortByWeight(hosts, out) - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: hosts, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "w_big", Activations: acts[0]}, {StepName: "w_neg", Activations: acts[1]}, {StepName: "w_small", Activations: acts[2]}, @@ -272,7 +270,7 @@ func TestExplainWeighing(t *testing.T) { }, { name: "counterfactual not reported when removal does not change #1", - result: func() *v1alpha1.DecisionResult { + result: func() *PipelineResult { // Two weighers both favor host-a. Removing either still leaves a as #1. mult1, mult2 := 2.0, 2.0 act1 := map[string]float64{"a": 1.0, "b": -1.0} @@ -282,11 +280,11 @@ func TestExplainWeighing(t *testing.T) { for _, h := range []string{"a", "b"} { out[h] = in[h] + mult1*math.Tanh(act1[h]) + mult2*math.Tanh(act2[h]) } - return &v1alpha1.DecisionResult{ + return &PipelineResult{ NormalizedInWeights: in, AggregatedOutWeights: out, OrderedHosts: []string{"a", "b"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "w1", Activations: act1}, {StepName: "w2", Activations: act2}, }, @@ -340,7 +338,7 @@ func TestRecoverMultipliers(t *testing.T) { } } - steps := []v1alpha1.StepResult{ + steps := []StepResult{ {StepName: "w1", Activations: acts[0]}, {StepName: "w2", Activations: acts[1]}, {StepName: "w3", Activations: acts[2]}, @@ -382,9 +380,9 @@ func TestSolveLinearSystem(t *testing.T) { } func TestIdentifyWeigherSteps(t *testing.T) { - result := &v1alpha1.DecisionResult{ + result := &PipelineResult{ OrderedHosts: []string{"a", "b", "c"}, - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ // Filter: missing host c {StepName: "filter_x", Activations: map[string]float64{"a": 1.0, "b": 1.0}}, // Weigher: has all ordered hosts @@ -482,12 +480,12 @@ func TestExplainWeighingDemo(t *testing.T) { // Sort hosts by aggregated weight to get OrderedHosts. sortByWeight(hosts, aggregatedOut) - result := &v1alpha1.DecisionResult{ + result := &PipelineResult{ NormalizedInWeights: normalizedIn, AggregatedOutWeights: aggregatedOut, OrderedHosts: hosts, TargetHost: &hosts[0], - StepResults: []v1alpha1.StepResult{ + StepResults: []StepResult{ {StepName: "kvm_binpack", Activations: actBinpack}, {StepName: "kvm_failover_evacuation", Activations: actFailover}, {StepName: "kvm_prefer_smaller_hosts", Activations: actSmaller}, diff --git a/internal/scheduling/machines/filter_weigher_pipeline_controller.go b/internal/scheduling/machines/filter_weigher_pipeline_controller.go index 3c542487b..672e287d6 100644 --- a/internal/scheduling/machines/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/machines/filter_weigher_pipeline_controller.go @@ -19,8 +19,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/machines/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/machines/plugins/weighers" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,9 +29,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -// The decision pipeline controller takes decision resources containing a -// machine ref and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. +// The machine pipeline controller watches for new machines and runs the +// scheduling pipeline to assign each a machine pool. The result is written back +// directly to the machine's MachinePoolRef. // // Additionally, the controller watches for pipeline and step changes to // reconfigure the pipelines as needed. @@ -53,75 +51,28 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - c.processMu.Lock() - defer c.processMu.Unlock() - - // Determine if this is a decision or machine reconciliation. - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - old := decision.DeepCopy() - if err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } +// Reconcile is a no-op: this controller schedules machines through its Machine +// watch handler and reconfigures pipelines through its Pipeline watch handler. +// A Reconcile method is required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } func (c *FilterWeigherPipelineController) ProcessNewMachine(ctx context.Context, machine *ironcorev1alpha1.Machine) error { c.processMu.Lock() defer c.processMu.Unlock() - - // Create a decision resource to schedule the machine. - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "machine-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainMachines, - ResourceID: machine.Name, - PipelineRef: corev1.ObjectReference{ - Name: "machines-scheduler", - }, - MachineRef: &corev1.ObjectReference{ - Name: machine.Name, - Namespace: machine.Namespace, - }, - }, - } - - err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - return err + return c.process(ctx, machine.DeepCopy()) } -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) error { +func (c *FilterWeigherPipelineController) process(ctx context.Context, machine *ironcorev1alpha1.Machine) error { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) + const pipelineName = "machines-scheduler" + pipeline, ok := c.GetPipeline(pipelineName) if !ok { - log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) + log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) + return fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } // Find all available machine pools. @@ -136,28 +87,12 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision // Execute the scheduling pipeline. request := ironcore.MachinePipelineRequest{Pools: pools.Items} result, err := pipeline.Run(request) - if err == nil { - decision.Status.Result = &result - } - // Record a human-readable explanation of the decision for downstream - // consumers. On error the result is nil and the explanation carries the - // failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.V(1).Error(err, "failed to run scheduler pipeline") return errors.New("failed to run scheduler pipeline") } log.Info("decision processed successfully", "duration", time.Since(startedAt)) - // Set the machine pool ref on the machine. - machine := &ironcorev1alpha1.Machine{} - if err := c.Get(ctx, client.ObjectKey{ - Name: decision.Spec.MachineRef.Name, - Namespace: decision.Spec.MachineRef.Namespace, - }, machine); err != nil { - log.Error(err, "failed to fetch machine for decision") - return err - } // Assign the first machine pool returned by the pipeline. old := machine.DeepCopy() machine.Spec.MachinePoolRef = &corev1.LocalObjectReference{Name: *result.TargetHost} @@ -254,25 +189,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainMachines { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } return bldr.Named("cortex-machine-scheduler"). Complete(c) } diff --git a/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go b/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go index af85f6382..e31032bca 100644 --- a/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go @@ -12,194 +12,12 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheduling scheme: %v", err) - } - if err := ironcorev1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add ironcore scheme: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - machinePools []ironcorev1alpha1.MachinePool - machine *ironcorev1alpha1.Machine - expectError bool - expectDecision bool - expectTargetHost string - expectMachinePool string - }{ - { - name: "successful machine decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainMachines, - ResourceID: "test-machine", - PipelineRef: corev1.ObjectReference{ - Name: "machines-scheduler", - }, - MachineRef: &corev1.ObjectReference{ - Name: "test-machine", - Namespace: "default", - }, - }, - }, - machinePools: []ironcorev1alpha1.MachinePool{ - { - ObjectMeta: metav1.ObjectMeta{Name: "pool1"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "pool2"}, - }, - }, - machine: &ironcorev1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-machine", - Namespace: "default", - }, - Spec: ironcorev1alpha1.MachineSpec{ - Scheduler: "", - }, - }, - expectError: false, - expectDecision: true, - expectTargetHost: "pool1", // NoopFilter returns first pool - expectMachinePool: "pool1", - }, - { - name: "no machine pools available", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pools", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainMachines, - ResourceID: "test-machine", - PipelineRef: corev1.ObjectReference{ - Name: "machines-scheduler", - }, - MachineRef: &corev1.ObjectReference{ - Name: "test-machine", - Namespace: "default", - }, - }, - }, - machinePools: []ironcorev1alpha1.MachinePool{}, - expectError: true, - expectDecision: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []runtime.Object{tt.decision} - for i := range tt.machinePools { - objects = append(objects, &tt.machinePools[i]) - } - if tt.machine != nil { - objects = append(objects, tt.machine) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[ironcore.MachinePipelineRequest]]{ - Pipelines: map[string]lib.FilterWeigherPipeline[ironcore.MachinePipelineRequest]{ - "machines-scheduler": createMockPipeline(), - }, - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - controller.Client = client - - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - }, - } - - result, err := controller.Reconcile(context.Background(), req) - - if tt.expectError && err == nil { - t.Error("expected error but got none") - return - } - - if !tt.expectError && err != nil { - t.Errorf("expected no error, got: %v", err) - return - } - - if result.RequeueAfter > 0 { - t.Errorf("unexpected requeue: %v", result.RequeueAfter) - } - - // Verify decision status if expected - if tt.expectDecision { - var updatedDecision v1alpha1.Decision - err := client.Get(context.Background(), req.NamespacedName, &updatedDecision) - if err != nil { - t.Errorf("Failed to get updated decision: %v", err) - return - } - - if updatedDecision.Status.Result == nil { - t.Error("expected decision result to be set") - return - } - - if updatedDecision.Status.Result.TargetHost == nil { - t.Error("expected target host to be set") - return - } - - if *updatedDecision.Status.Result.TargetHost != tt.expectTargetHost { - t.Errorf("expected target host %q, got %q", tt.expectTargetHost, *updatedDecision.Status.Result.TargetHost) - } - - // Verify machine was updated with machine pool ref - if tt.machine != nil { - var updatedMachine ironcorev1alpha1.Machine - err := client.Get(context.Background(), types.NamespacedName{ - Name: tt.machine.Name, - Namespace: tt.machine.Namespace, - }, &updatedMachine) - if err != nil { - t.Errorf("Failed to get updated machine: %v", err) - return - } - - if updatedMachine.Spec.MachinePoolRef == nil { - t.Error("expected machine pool ref to be set") - return - } - - if updatedMachine.Spec.MachinePoolRef.Name != tt.expectMachinePool { - t.Errorf("expected machine pool %q, got %q", tt.expectMachinePool, updatedMachine.Spec.MachinePoolRef.Name) - } - } - } - }) - } -} - func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { controller := &FilterWeigherPipelineController{ Monitor: lib.FilterWeigherPipelineMonitor{}, @@ -293,6 +111,36 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { expectMachinePoolAssigned bool expectTargetHost string }{ + { + name: "successful machine scheduling", + machine: &ironcorev1alpha1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-machine", + Namespace: "default", + }, + Spec: ironcorev1alpha1.MachineSpec{ + Scheduler: "", + }, + }, + machinePools: []ironcorev1alpha1.MachinePool{ + {ObjectMeta: metav1.ObjectMeta{Name: "pool1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "pool2"}}, + }, + pipelineConfig: &v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machines-scheduler", + }, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + SchedulingDomain: v1alpha1.SchedulingDomainMachines, + Filters: []v1alpha1.FilterSpec{}, + Weighers: []v1alpha1.WeigherSpec{}, + }, + }, + expectError: false, + expectMachinePoolAssigned: true, + expectTargetHost: "pool1", // mock pipeline returns first pool + }, { name: "pipeline not configured", machine: &ironcorev1alpha1.Machine{ @@ -350,7 +198,6 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { client := fake.NewClientBuilder(). WithScheme(scheme). WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ @@ -411,14 +258,14 @@ func createMockPipeline() lib.FilterWeigherPipeline[ironcore.MachinePipelineRequ type mockMachinePipeline struct{} -func (m *mockMachinePipeline) Run(request ironcore.MachinePipelineRequest) (v1alpha1.DecisionResult, error) { +func (m *mockMachinePipeline) Run(request ironcore.MachinePipelineRequest) (lib.PipelineResult, error) { if len(request.Pools) == 0 { - return v1alpha1.DecisionResult{}, nil + return lib.PipelineResult{}, nil } // Return the first pool as the target host targetHost := request.Pools[0].Name - return v1alpha1.DecisionResult{ + return lib.PipelineResult{ TargetHost: &targetHost, }, nil } diff --git a/internal/scheduling/manila/external_scheduler_api.go b/internal/scheduling/manila/external_scheduler_api.go index 2dc18d13a..273c282dd 100644 --- a/internal/scheduling/manila/external_scheduler_api.go +++ b/internal/scheduling/manila/external_scheduler_api.go @@ -7,29 +7,24 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "net/http" api "github.com/cobaltcore-dev/cortex/api/external/manila" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/metrics" ) var log = logf.Log.WithName("manila-scheduler-api") -type HTTPAPIDelegate interface { - // Process the decision from the API. Should create and return the updated decision. - ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error +// Scheduler runs a manila pipeline for a request and returns the result. +// *FilterWeigherPipelineController satisfies it. +type Scheduler interface { + Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (scheduling.PipelineResult, error) } type HTTPAPI interface { @@ -38,14 +33,14 @@ type HTTPAPI interface { } type httpAPI struct { - monitor scheduling.APIMonitor - delegate HTTPAPIDelegate + monitor scheduling.APIMonitor + scheduler Scheduler } -func NewAPI(delegate HTTPAPIDelegate) HTTPAPI { +func NewAPI(scheduler Scheduler) HTTPAPI { return &httpAPI{ - monitor: scheduling.NewSchedulerMonitor(), - delegate: delegate, + monitor: scheduling.NewSchedulerMonitor(), + scheduler: scheduler, } } @@ -109,7 +104,6 @@ func (httpAPI *httpAPI) ManilaExternalScheduler(w http.ResponseWriter, r *http.R c.Respond(logr.Logger{}, http.StatusInternalServerError, err, "failed to read request body") return } - raw := runtime.RawExtension{Raw: body} var requestData api.ExternalSchedulerRequest // Copy the raw body to a io.Reader for json deserialization. cp := body @@ -138,36 +132,12 @@ func (httpAPI *httpAPI) ManilaExternalScheduler(w http.ResponseWriter, r *http.R logger.V(1).Info("inferred pipeline name", "pipeline", requestData.Pipeline) } - // Create the decision object in kubernetes. - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "manila-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: requestData.Pipeline, - }, - ResourceID: "", // TODO model out the spec. - ManilaRaw: &raw, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - } - ctx := r.Context() - if err := httpAPI.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { + result, err := httpAPI.scheduler.Schedule(r.Context(), requestData.Pipeline, requestData) + if err != nil { c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return } - // Check if the decision contains status conditions indicating an error. - if meta.IsStatusConditionFalse(decision.Status.Conditions, v1alpha1.DecisionConditionReady) { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision contains error condition"), "decision failed") - return - } - if decision.Status.Result == nil { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision didn't produce a result"), "decision failed") - return - } - hosts := decision.Status.Result.OrderedHosts + hosts := result.OrderedHosts // Ensure we always return an array, never null. The schema requires hosts // to be an array, and a nil slice would serialize to null. if hosts == nil { diff --git a/internal/scheduling/manila/external_scheduler_api_test.go b/internal/scheduling/manila/external_scheduler_api_test.go index dfdb3f534..7012984ea 100644 --- a/internal/scheduling/manila/external_scheduler_api_test.go +++ b/internal/scheduling/manila/external_scheduler_api_test.go @@ -14,26 +14,24 @@ import ( "testing" manilaapi "github.com/cobaltcore-dev/cortex/api/external/manila" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type mockHTTPAPIDelegate struct { - processDecisionFunc func(ctx context.Context, decision *v1alpha1.Decision) error +type mockScheduler struct { + scheduleFunc func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, error) } -func (m *mockHTTPAPIDelegate) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { - if m.processDecisionFunc != nil { - return m.processDecisionFunc(ctx, decision) +func (m *mockScheduler) Schedule(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { + if m.scheduleFunc != nil { + return m.scheduleFunc(ctx, pipelineName, request) } - return nil + return lib.PipelineResult{}, nil } func TestNewAPI(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} + scheduler := &mockScheduler{} - api := NewAPI(delegate) + api := NewAPI(scheduler) if api == nil { t.Fatal("NewAPI returned nil") @@ -44,8 +42,8 @@ func TestNewAPI(t *testing.T) { t.Fatal("NewAPI did not return httpAPI type") } - if httpAPI.delegate != delegate { - t.Error("Delegate not set correctly") + if httpAPI.scheduler != scheduler { + t.Error("Scheduler not set correctly") } if httpAPI.monitor.ApiRequestsTimer == nil { @@ -54,8 +52,7 @@ func TestNewAPI(t *testing.T) { } func TestHTTPAPI_Init(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate) + api := NewAPI(&mockScheduler{}) mux := http.NewServeMux() api.Init(mux) @@ -72,8 +69,7 @@ func TestHTTPAPI_Init(t *testing.T) { } func TestHTTPAPI_canRunScheduler(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(&mockScheduler{}).(*httpAPI) tests := []struct { name string @@ -141,14 +137,27 @@ func TestHTTPAPI_canRunScheduler(t *testing.T) { } func TestHTTPAPI_ManilaExternalScheduler(t *testing.T) { + validBody := func(t *testing.T) string { + req := manilaapi.ExternalSchedulerRequest{ + Hosts: []manilaapi.ExternalSchedulerHost{{ShareHost: "host1"}}, + Weights: map[string]float64{"host1": 1.0}, + Pipeline: "test-pipeline", + } + data, err := json.Marshal(req) + if err != nil { + t.Fatalf("Failed to marshal request data: %v", err) + } + return string(data) + } + tests := []struct { - name string - method string - body string - processDecisionErr error - decisionResult *v1alpha1.Decision - expectedStatus int - expectedHosts []string + name string + method string + body string + scheduleResult lib.PipelineResult + scheduleErr error + expectedStatus int + expectedHosts []string }{ { name: "invalid method", @@ -162,106 +171,31 @@ func TestHTTPAPI_ManilaExternalScheduler(t *testing.T) { expectedStatus: http.StatusBadRequest, }, { - name: "valid request with successful processing", - method: http.MethodPost, - body: func() string { - req := manilaapi.ExternalSchedulerRequest{ - Hosts: []manilaapi.ExternalSchedulerHost{ - {ShareHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - }, - }, - }, + name: "valid request with successful processing", + method: http.MethodPost, + body: validBody(t), + scheduleResult: lib.PipelineResult{OrderedHosts: []string{"host1"}}, expectedStatus: http.StatusOK, expectedHosts: []string{"host1"}, }, { - name: "processing error", - method: http.MethodPost, - body: func() string { - req := manilaapi.ExternalSchedulerRequest{ - Hosts: []manilaapi.ExternalSchedulerHost{ - {ShareHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - processDecisionErr: errors.New("processing failed"), - expectedStatus: http.StatusInternalServerError, - }, - { - name: "decision failed", - method: http.MethodPost, - body: func() string { - req := manilaapi.ExternalSchedulerRequest{ - Hosts: []manilaapi.ExternalSchedulerHost{ - {ShareHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", - } - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Failed to marshal request data: %v", err) - } - return string(data) - }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "SchedulingError", - }, - }, - }, - }, + name: "processing error", + method: http.MethodPost, + body: validBody(t), + scheduleErr: errors.New("processing failed"), expectedStatus: http.StatusInternalServerError, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - if tt.processDecisionErr != nil { - return tt.processDecisionErr - } - if tt.decisionResult != nil { - decision.Status = tt.decisionResult.Status - return nil - } - return nil + scheduler := &mockScheduler{ + scheduleFunc: func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { + return tt.scheduleResult, tt.scheduleErr }, } - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(scheduler).(*httpAPI) var body *strings.Reader if tt.body != "" { @@ -300,8 +234,7 @@ func TestHTTPAPI_ManilaExternalScheduler(t *testing.T) { } func TestHTTPAPI_inferPipelineName(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(&mockScheduler{}).(*httpAPI) tests := []struct { name string @@ -347,31 +280,25 @@ func TestHTTPAPI_inferPipelineName(t *testing.T) { } } -func TestHTTPAPI_ManilaExternalScheduler_DecisionCreation(t *testing.T) { - var capturedDecision *v1alpha1.Decision - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - capturedDecision = decision - // Set a successful result to avoid "decision didn't produce a result" error - decision.Status.Result = &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - } - return nil +// When the request carries no pipeline, the handler infers the default name and +// passes it to the scheduler. +func TestHTTPAPI_ManilaExternalScheduler_InfersPipeline(t *testing.T) { + var capturedPipeline string + var capturedRequest manilaapi.ExternalSchedulerRequest + scheduler := &mockScheduler{ + scheduleFunc: func(ctx context.Context, pipelineName string, request manilaapi.ExternalSchedulerRequest) (lib.PipelineResult, error) { + capturedPipeline = pipelineName + capturedRequest = request + return lib.PipelineResult{OrderedHosts: []string{"host1"}}, nil }, } - api := NewAPI(delegate).(*httpAPI) + api := NewAPI(scheduler).(*httpAPI) requestData := manilaapi.ExternalSchedulerRequest{ - Hosts: []manilaapi.ExternalSchedulerHost{ - {ShareHost: "host1"}, - }, - Weights: map[string]float64{ - "host1": 1.0, - }, - Pipeline: "test-pipeline", + Hosts: []manilaapi.ExternalSchedulerHost{{ShareHost: "host1"}}, + Weights: map[string]float64{"host1": 1.0}, } - body, err := json.Marshal(requestData) if err != nil { t.Fatalf("Failed to marshal request data: %v", err) @@ -384,25 +311,10 @@ func TestHTTPAPI_ManilaExternalScheduler_DecisionCreation(t *testing.T) { if w.Code != http.StatusOK { t.Errorf("Expected status %d, got %d", http.StatusOK, w.Code) } - - if capturedDecision == nil { - t.Fatal("Decision was not captured") + if capturedPipeline != "manila-external-scheduler" { + t.Errorf("Expected inferred pipeline 'manila-external-scheduler', got %q", capturedPipeline) } - - // Verify decision fields - if capturedDecision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainManila { - t.Errorf("Expected scheduling domain %s, got %s", v1alpha1.SchedulingDomainManila, capturedDecision.Spec.SchedulingDomain) - } - - if capturedDecision.Spec.PipelineRef.Name != "test-pipeline" { - t.Errorf("Expected pipeline 'test-pipeline', got %s", capturedDecision.Spec.PipelineRef.Name) - } - - if capturedDecision.GenerateName != "manila-" { - t.Errorf("Expected generate name 'manila-', got %s", capturedDecision.GenerateName) - } - - if capturedDecision.Spec.ManilaRaw == nil { - t.Error("ManilaRaw should not be nil") + if len(capturedRequest.Hosts) != 1 || capturedRequest.Hosts[0].ShareHost != "host1" { + t.Errorf("Request not passed through to scheduler: %+v", capturedRequest) } } diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller.go b/internal/scheduling/manila/filter_weigher_pipeline_controller.go index f6442f697..cc7ced78e 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller.go @@ -5,16 +5,12 @@ package manila import ( "context" - "encoding/json" - "errors" "fmt" "sync" "time" api "github.com/cobaltcore-dev/cortex/api/external/manila" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/manila/plugins/filters" @@ -27,17 +23,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) -// The decision pipeline controller takes decision resources containing a -// placement request spec and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. -// -// Additionally, the controller watches for pipeline and step changes to -// reconfigure the pipelines as needed. +// FilterWeigherPipelineController owns the manila filter-weigher pipelines: it +// watches Pipeline and Knowledge resources to (re)configure the pipelines and +// runs a pipeline on demand via Schedule, called directly by the external +// scheduler HTTP API. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]] - // Mutex to only allow one process at a time + // Mutex to only allow one scheduling run at a time. processMu sync.Mutex // Monitor to pass down to all pipelines. @@ -49,83 +43,34 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -// Callback executed when kubernetes asks to reconcile a decision resource. -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - c.processMu.Lock() - defer c.processMu.Unlock() - - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - old := decision.DeepCopy() - if err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } +// Reconcile is a no-op: this controller reconfigures pipelines through its +// Pipeline/Knowledge watch handlers and runs placements via Schedule. A +// Reconcile method is required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } -// Process the decision from the API. Should create and return the updated decision. -func (c *FilterWeigherPipelineController) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { +// Schedule runs the named pipeline for the given request and returns the +// result. It serializes runs with the process lock. +func (c *FilterWeigherPipelineController) Schedule(ctx context.Context, pipelineName string, request api.ExternalSchedulerRequest) (lib.PipelineResult, error) { c.processMu.Lock() defer c.processMu.Unlock() - - err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - return err -} - -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) error { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) + pipeline, ok := c.GetPipeline(pipelineName) if !ok { - log.Error(nil, "skipping decision, pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) - } - if decision.Spec.ManilaRaw == nil { - log.Error(nil, "skipping decision, no manilaRaw spec defined") - return errors.New("no manilaRaw spec defined") - } - var request api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.ManilaRaw.Raw, &request); err != nil { - log.Error(err, "failed to unmarshal manilaRaw spec") - return err + log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) + return lib.PipelineResult{}, fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } result, err := pipeline.Run(request) - if err == nil { - decision.Status.Result = &result - } - // Record a human-readable explanation of the decision for downstream - // consumers. On error the result is nil and the explanation carries the - // failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.Error(err, "failed to run pipeline") - return err + return lib.PipelineResult{}, err } log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return nil + return result, nil } // The base controller will delegate the pipeline creation down to this method. @@ -186,25 +131,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainManila { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } - return bldr.Named("cortex-manila-decisions"). + return bldr.Named("cortex-manila-pipelines"). Complete(c) } diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go b/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go index 76dbd6f93..418c9297d 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go @@ -4,44 +4,26 @@ package manila import ( - "context" - "encoding/json" "testing" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" api "github.com/cobaltcore-dev/cortex/api/external/manila" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/sapcc/go-bits/must" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/storage" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { +func TestFilterWeigherPipelineController_Schedule(t *testing.T) { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { t.Fatalf("Failed to add v1alpha1 scheme: %v", err) } - manilaRequest := api.ExternalSchedulerRequest{ - Spec: map[string]any{ - "share_id": "test-share-id", - "size": 10, - }, - Context: api.ManilaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: "global-req-123", - }, + request := api.ExternalSchedulerRequest{ Hosts: []api.ExternalSchedulerHost{ {ShareHost: "manila-share-1@backend1"}, {ShareHost: "manila-share-2@backend2"}, @@ -51,143 +33,55 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { Options: scheduling.Options{}, } - manilaRaw, err := json.Marshal(manilaRequest) - if err != nil { - t.Fatalf("Failed to marshal manila request: %v", err) - } - tests := []struct { name string - decision *v1alpha1.Decision - pipeline *v1alpha1.Pipeline + pipelineName string + configured bool expectError bool - expectResult bool }{ { - name: "successful manila decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, + name: "successful scheduling run", + pipelineName: "test-pipeline", + configured: true, expectError: false, - expectResult: true, }, { - name: "decision without manilaRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: nil, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, + name: "pipeline not found", + pipelineName: "nonexistent-pipeline", + configured: false, expectError: true, - expectResult: false, - }, - { - name: "pipeline not found", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipeline: nil, - expectError: true, - expectResult: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{tt.decision} - if tt.pipeline != nil { - objects = append(objects, tt.pipeline) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() + cl := fake.NewClientBuilder().WithScheme(scheme).Build() controller := &FilterWeigherPipelineController{ BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, + Client: cl, Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), }, Monitor: lib.FilterWeigherPipelineMonitor{}, } - if tt.pipeline != nil { + if tt.configured { initResult := controller.InitPipeline(t.Context(), v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.pipeline.Name, + ObjectMeta: metav1.ObjectMeta{Name: "test-pipeline"}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + SchedulingDomain: v1alpha1.SchedulingDomainManila, + Filters: []v1alpha1.FilterSpec{}, + Weighers: []v1alpha1.WeigherSpec{}, }, - Spec: tt.pipeline.Spec, }) - if err != nil { - t.Fatalf("Failed to init pipeline: %v", err) + if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { + t.Fatalf("Failed to init pipeline: %v", initResult) } - controller.Pipelines[tt.pipeline.Name] = initResult.Pipeline + controller.Pipelines["test-pipeline"] = initResult.Pipeline } - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - Namespace: tt.decision.Namespace, - }, - } - - result, err := controller.Reconcile(context.Background(), req) + result, err := controller.Schedule(t.Context(), tt.pipelineName, request) if tt.expectError && err == nil { t.Error("Expected error but got none") @@ -195,227 +89,9 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { if !tt.expectError && err != nil { t.Errorf("Expected no error but got: %v", err) } - - if result.RequeueAfter > 0 { - t.Error("Expected no requeue") - } - - var updatedDecision v1alpha1.Decision - if err := client.Get(context.Background(), req.NamespacedName, &updatedDecision); err != nil { - t.Fatalf("Failed to get updated decision: %v", err) - } - - if tt.expectResult && updatedDecision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - if !tt.expectResult && updatedDecision.Status.Result != nil { - t.Error("Expected result to be nil but was set") - } - }) - } -} - -func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - manilaRequest := api.ExternalSchedulerRequest{ - Spec: map[string]any{ - "share_id": "test-share-id", - "size": 10, - }, - Context: api.ManilaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: "global-req-123", - }, - Hosts: []api.ExternalSchedulerHost{ - {ShareHost: "manila-share-1@backend1"}, - {ShareHost: "manila-share-2@backend2"}, - }, - Weights: map[string]float64{"manila-share-1@backend1": 1.0, "manila-share-2@backend2": 0.5}, - Pipeline: "test-pipeline", - } - - manilaRaw, err := json.Marshal(manilaRequest) - if err != nil { - t.Fatalf("Failed to marshal manila request: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - pipelineConfig *v1alpha1.Pipeline - expectError bool - expectResult bool - }{ - { - name: "successful decision processing with creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-decision-", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - ResourceID: "test-uuid-1", - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "successful decision processing without creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-create", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "pipeline not configured", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - ManilaRaw: &runtime.RawExtension{ - Raw: manilaRaw, - }, - }, - }, - pipelineConfig: nil, - expectError: true, - expectResult: false, - }, - { - name: "decision without manilaRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - ManilaRaw: nil, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainManila, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{} - if tt.pipelineConfig != nil { - objects = append(objects, tt.pipelineConfig) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - - if tt.pipelineConfig != nil { - controller.PipelineConfigs[tt.pipelineConfig.Name] = *tt.pipelineConfig - initResult := controller.InitPipeline(t.Context(), *tt.pipelineConfig) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to init pipeline: %v", initResult) - } - controller.Pipelines[tt.pipelineConfig.Name] = initResult.Pipeline - } - - if tt.decision.Spec.ManilaRaw != nil { - req := manilaRequest - req.Options = scheduling.Options{} - raw, marshalErr := json.Marshal(req) - if marshalErr != nil { - t.Fatalf("Failed to marshal request with options: %v", marshalErr) - } - tt.decision.Spec.ManilaRaw = &runtime.RawExtension{Raw: raw} - } - - err := controller.ProcessNewDecisionFromAPI(context.Background(), tt.decision) - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - if !tt.expectError { - if tt.expectResult && tt.decision.Status.Result == nil { - t.Error("expected decision result to be set in original decision object") + if len(result.OrderedHosts) == 0 { + t.Error("Expected ordered hosts but got none") } } }) @@ -433,16 +109,14 @@ func TestFilterWeigherPipelineController_PipelineType(t *testing.T) { } func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) + controller := &FilterWeigherPipelineController{ + Monitor: lib.FilterWeigherPipelineMonitor{}, } tests := []struct { name string filters []v1alpha1.FilterSpec weighers []v1alpha1.WeigherSpec - knowledges []client.Object expectNonCriticalError bool expectCriticalError bool expectUnknownFilter bool @@ -452,53 +126,6 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { name: "empty steps", filters: []v1alpha1.FilterSpec{}, weighers: []v1alpha1.WeigherSpec{}, - knowledges: []client.Object{}, - expectNonCriticalError: false, - expectCriticalError: false, - expectUnknownFilter: false, - expectUnknownWeigher: false, - }, - { - name: "supported netapp step", - weighers: []v1alpha1.WeigherSpec{ - { - Name: "netapp_cpu_usage_balancing", - Params: []v1alpha1.Parameter{ - {Key: "AvgCPUUsageLowerBound", FloatValue: new(0.0)}, - {Key: "AvgCPUUsageUpperBound", FloatValue: new(90.0)}, - {Key: "MaxCPUUsageLowerBound", FloatValue: new(0.0)}, - {Key: "MaxCPUUsageUpperBound", FloatValue: new(100.0)}, - }, - }, - }, - knowledges: []client.Object{ - &v1alpha1.Knowledge{ - ObjectMeta: metav1.ObjectMeta{ - Name: "netapp-storage-pool-cpu-usage-manila", - }, - Status: v1alpha1.KnowledgeStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.KnowledgeConditionReady, - Status: metav1.ConditionTrue, - }, - }, - Raw: must.Return(v1alpha1.BoxFeatureList([]storage.StoragePoolCPUUsage{ - { - StoragePoolName: "manila-share-1@backend1", - AvgCPUUsagePct: 50, - MaxCPUUsagePct: 80, - }, - { - StoragePoolName: "manila-share-2@backend2", - AvgCPUUsagePct: 20, - MaxCPUUsagePct: 40, - }, - })), - RawLength: 2, - }, - }, - }, expectNonCriticalError: false, expectCriticalError: false, expectUnknownFilter: false, @@ -508,7 +135,7 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { name: "unsupported step", filters: []v1alpha1.FilterSpec{ { - Name: "unsupported-plugin", + Name: "test-plugin", }, }, expectNonCriticalError: false, @@ -520,16 +147,6 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(tt.knowledges...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - controller := &FilterWeigherPipelineController{ - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - controller.Client = client // Through basepipelinecontroller - initResult := controller.InitPipeline(t.Context(), v1alpha1.Pipeline{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pipeline", @@ -542,19 +159,19 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { }, }) - if !tt.expectCriticalError && len(initResult.FilterErrors) > 0 { - t.Errorf("Expected no critical error but got: %v", initResult.FilterErrors) - } if tt.expectCriticalError && len(initResult.FilterErrors) == 0 { - t.Error("Expected critical error but got none") + t.Error("Expected error but got none") } - - if !tt.expectNonCriticalError && len(initResult.WeigherErrors) > 0 { - t.Errorf("Expected no non-critical error but got: %v", initResult.WeigherErrors) + if !tt.expectCriticalError && len(initResult.FilterErrors) > 0 { + t.Errorf("Expected no error but got: %v", initResult.FilterErrors) } + if tt.expectNonCriticalError && len(initResult.WeigherErrors) == 0 { t.Error("Expected non-critical error but got none") } + if !tt.expectNonCriticalError && len(initResult.WeigherErrors) > 0 { + t.Errorf("Expected no non-critical error but got: %v", initResult.WeigherErrors) + } }) } } diff --git a/internal/scheduling/nova/adapter.go b/internal/scheduling/nova/adapter.go new file mode 100644 index 000000000..ce236111d --- /dev/null +++ b/internal/scheduling/nova/adapter.go @@ -0,0 +1,279 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package nova + +import ( + "strings" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" +) + +// ToSchedulerRequest translates the Nova over-the-wire request into the typed, +// Nova-agnostic compute.PlacementRequest the pipeline consumes. This is the one +// place that reads Nova's loosely-typed scheduler hints and flavor/image extra +// specs; every downstream filter/weigher sees only resolved typed fields. +// +// intent and pipeline are resolved by the caller (the HTTP API layer) and passed +// in. +func ToSchedulerRequest( + wire api.ExternalSchedulerRequest, + intent v1alpha1.SchedulingIntent, + pipeline string, +) compute.PlacementRequest { + + spec := wire.Spec.Data + + req := compute.PlacementRequest{ + Pipeline: pipeline, + Hosts: toComputeHosts(wire.Hosts), + Weights: wire.Weights, + Options: wire.Options, + Intent: intent, + + InstanceUUID: spec.InstanceUUID, + ProjectID: firstNonEmpty(spec.ProjectID, wire.Context.ProjectID), + UserID: spec.UserID, + AvailabilityZone: spec.AvailabilityZone, + NumInstances: spec.NumInstances, + + IgnoreHosts: derefStrings(spec.IgnoreHosts), + ForceHosts: derefStrings(spec.ForceHosts), + ForceNodes: derefStrings(spec.ForceNodes), + + HardwareVersion: spec.Flavor.Data.ExtraSpecs["quota:hw_version"], + + SourceHost: hintStr(spec, "source_host"), + DomainName: hintStr(spec, "domain_name"), + ResourceGroup: hintStr(spec, api.HintKeyResourceGroup), + + RequestID: wire.Context.RequestID, + GlobalRequestID: derefString(wire.Context.GlobalRequestID), + } + + // Flavor sizing. + req.Flavor = compute.FlavorSpec{ + VCPUs: spec.Flavor.Data.VCPUs, + MemoryMB: spec.Flavor.Data.MemoryMB, + } + + // Hypervisor family. + req.HypervisorFamily = toHypervisorFamily(wire) + + // Whether to gather all placement candidates (ignore the caller's host + // preselection) is determined by the hypervisor family: cortex can enumerate + // KVM hypervisors itself, whereas VMware placement relies on the host list + // Nova sent in the request. + req.GatherAllCandidates = req.HypervisorFamily == compute.HypervisorFamilyKVM + + // Image. + req.ImageHypervisorType = toImageHypervisorType(spec.Image.Data) + + // Traits, capabilities, accelerators from flavor extra specs. + req.RequiredTraits, req.ForbiddenTraits = SplitTraits(spec.Flavor.Data.ExtraSpecs) + req.RequestedCapabilities = SplitCapabilities(spec.Flavor.Data.ExtraSpecs) + req.AcceleratorsRequested = RequestsAccelerators(spec.Flavor.Data.ExtraSpecs) + + // Nested objects. + req.RequestedDestination = toRequestedDestination(spec.RequestedDestination) + req.InstanceGroup = toInstanceGroup(spec.InstanceGroup) + + return req +} + +func toComputeHosts(hosts []api.ExternalSchedulerHost) []compute.Host { + out := make([]compute.Host, len(hosts)) + for i, h := range hosts { + out[i] = compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname} + } + return out +} + +// toHypervisorFamily resolves the coarse family from the wire hypervisor type. +// It is empty when unresolved. +func toHypervisorFamily(wire api.ExternalSchedulerRequest) compute.HypervisorFamily { + hv, err := wire.GetHypervisorType() + if err != nil { + return "" + } + switch hv { + case api.HypervisorTypeQEMU, api.HypervisorTypeCH: + return compute.HypervisorFamilyKVM + case api.HypervisorTypeVMware: + return compute.HypervisorFamilyVMware + } + return "" +} + +// toImageHypervisorType maps the image-declared hypervisor type. It returns +// ImageHypervisorTypeUnparseable when the image declared a type that could not +// be parsed, distinct from ImageHypervisorTypeUnset when none was declared. +func toImageHypervisorType(image api.NovaImageMeta) compute.ImageHypervisorType { + hv, err := image.GetHypervisorType() + if err != nil { + // GetHypervisorType errors both when nothing is declared and when a + // declared value is unrecognized. Distinguish them: if a known key is + // present, the value was unparseable. + if image.Properties.Data != nil { + for _, key := range []string{"img_hv_type", "hypervisor_type"} { + if _, ok := image.Properties.Data[key]; ok { + return compute.ImageHypervisorTypeUnparseable + } + } + } + return compute.ImageHypervisorTypeUnset + } + switch hv { + case api.NovaImageMetaHVTypeVMware: + return compute.ImageHypervisorTypeVMware + case api.NovaImageMetaHVTypeBaremetal: + return compute.ImageHypervisorTypeBaremetal + case api.NovaImageMetaHVTypeKVM: + return compute.ImageHypervisorTypeKVM + } + return compute.ImageHypervisorTypeUnset +} + +func toRequestedDestination(rd *api.NovaObject[api.NovaRequestedDestination]) *compute.RequestedDestination { + if rd == nil { + return nil + } + return &compute.RequestedDestination{ + Host: rd.Data.Host, + Aggregates: rd.Data.Aggregates, + } +} + +func toInstanceGroup(ig *api.NovaObject[api.NovaInstanceGroup]) *compute.InstanceGroupSpec { + if ig == nil { + return nil + } + out := &compute.InstanceGroupSpec{ + UUID: ig.Data.UUID, + Policy: ig.Data.Policy, + Hosts: ig.Data.Hosts, + Members: ig.Data.Members, + } + if ig.Data.Rules != nil { + if v, ok := ig.Data.Rules["max_server_per_host"]; ok { + if n, ok := v.(int); ok { + out.MaxServerPerHost = n + } + } + } + return out +} + +func hintStr(spec api.NovaSpec, key string) string { + s, err := spec.GetSchedulerHintStr(key) + if err != nil { + return "" + } + return s +} + +func derefStrings(p *[]string) []string { + if p == nil { + return nil + } + return *p +} + +func derefString(p *string) string { + if p == nil { + return "" + } + return *p +} + +func firstNonEmpty(a, b string) string { + if a != "" { + return a + } + return b +} + +// The helpers below resolve Nova's loosely-typed flavor extra-spec string +// conventions into the typed values the scheduling pipeline consumes. They live +// here, with the Nova spec adapter, because the "trait:"/"capabilities:"/"accel:" +// key conventions are Nova's — not the pipeline's. Both this adapter and the +// in-process reservation controllers (which hold a flavor's extra-spec map +// directly) call them, so the conventions are understood in exactly one place. + +// unsupportedCapabilityOps are the Nova extra-spec comparison operators the +// capabilities filter does not implement. A capability value containing any of +// these means the request uses an advanced match the pipeline can't honor, so +// SplitCapabilities drops capabilities entirely (the filter then becomes a +// no-op), matching Nova's "skip" behavior. +// See: https://github.com/sapcc/nova/blob/3ebf80/nova/scheduler/filters/extra_specs_ops.py#L23 +var unsupportedCapabilityOps = []string{ + "=", "", "", "==", "!=", ">=", "<=", + "s==", "s!=", "s<", "s<=", "s>", "s>=", "", +} + +// SplitTraits splits a flavor's "trait:" extra specs into required and +// forbidden trait names, classified by the "required"/"forbidden" value. +func SplitTraits(extraSpecs map[string]string) (required, forbidden []string) { + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "trait:") + if !ok { + continue + } + switch value { + case "required": + required = append(required, name) + case "forbidden": + forbidden = append(forbidden, name) + } + } + return required, forbidden +} + +// SplitCapabilities extracts a flavor's "capabilities:" extra specs as typed +// compute.Capability equalities (prefix stripped). The "hypervisor_type" value is +// canonicalized (qemu->QEMU, ch->CH) so it matches what hypervisors advertise +// (filter_capabilities compares case-sensitively); all other values pass through +// unchanged. If any value uses an unsupported comparison operator it returns nil so +// the capabilities filter is skipped entirely (matching Nova behavior). +func SplitCapabilities(extraSpecs map[string]string) []compute.Capability { + var caps []compute.Capability + for key, value := range extraSpecs { + name, ok := strings.CutPrefix(key, "capabilities:") + if !ok { + continue + } + for _, op := range unsupportedCapabilityOps { + if strings.Contains(value, op) { + return nil // Unsupported operator: skip the whole filter. + } + } + if name == "hypervisor_type" { + value = CanonicalHypervisorType(value) + } + caps = append(caps, compute.Capability{Name: name, Value: value}) + } + return caps +} + +// CanonicalHypervisorType normalizes a flavor's capabilities:hypervisor_type value +// to the casing hypervisors advertise (QEMU/CH), since filter_capabilities compares +// values case-sensitively. Unrecognized values pass through unchanged. +func CanonicalHypervisorType(value string) string { + switch strings.ToLower(value) { + case "qemu": + return "QEMU" + case "ch": + return "CH" + default: + return value + } +} + +// RequestsAccelerators reports whether a flavor requests a hardware accelerator, +// resolved from the presence of the "accel:device_profile" extra spec. +func RequestsAccelerators(extraSpecs map[string]string) bool { + _, ok := extraSpecs["accel:device_profile"] + return ok +} diff --git a/internal/scheduling/nova/adapter_test.go b/internal/scheduling/nova/adapter_test.go new file mode 100644 index 000000000..d45a6659e --- /dev/null +++ b/internal/scheduling/nova/adapter_test.go @@ -0,0 +1,91 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package nova + +import ( + "slices" + "strings" + "testing" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" +) + +func TestSplitTraits(t *testing.T) { + required, forbidden := SplitTraits(map[string]string{ + "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required", + "trait:CUSTOM_NO_GPU": "forbidden", + "trait:IGNORED": "preferred", // neither required nor forbidden + "capabilities:hypervisor_type": "qemu", // not a trait + }) + if !slices.Equal(required, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}) { + t.Errorf("required = %v, want [CUSTOM_HANA_EXCLUSIVE_HOST]", required) + } + if !slices.Equal(forbidden, []string{"CUSTOM_NO_GPU"}) { + t.Errorf("forbidden = %v, want [CUSTOM_NO_GPU]", forbidden) + } +} + +func TestSplitCapabilities(t *testing.T) { + sortCaps := func(c []compute.Capability) { + slices.SortFunc(c, func(a, b compute.Capability) int { return strings.Compare(a.Name, b.Name) }) + } + cases := []struct { + name string + extraSpecs map[string]string + want []compute.Capability + }{ + { + name: "canonicalizes lowercase hypervisor_type to match hosts", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, + }, + { + name: "ch canonicalized", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "ch"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, + }, + { + name: "non-hypervisor_type capabilities preserved verbatim", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "QEMU", + "capabilities:cpu_arch": "aarch64", + }, + want: []compute.Capability{ + {Name: "cpu_arch", Value: "aarch64"}, + {Name: "hypervisor_type", Value: "QEMU"}, + }, + }, + { + name: "unsupported operator drops all capabilities", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "QEMU", + "capabilities:cpu_arch": " aarch64 x86_64", + }, + want: nil, + }, + { + name: "non-capability extra specs ignored", + extraSpecs: map[string]string{"trait:CUSTOM_X": "required", "quota:hw_version": "2101"}, + want: nil, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := SplitCapabilities(c.extraSpecs) + sortCaps(got) + if !slices.Equal(got, c.want) { + t.Errorf("got %v, want %v", got, c.want) + } + }) + } +} + +func TestRequestsAccelerators(t *testing.T) { + if !RequestsAccelerators(map[string]string{"accel:device_profile": "gpu-v100"}) { + t.Error("expected accelerators requested when accel:device_profile is present") + } + if RequestsAccelerators(map[string]string{"capabilities:cpu_arch": "x86_64"}) { + t.Error("expected no accelerators requested without accel:device_profile") + } +} diff --git a/internal/scheduling/nova/candidate_gatherer.go b/internal/scheduling/nova/candidate_gatherer.go index a493aaf42..1da100cc1 100644 --- a/internal/scheduling/nova/candidate_gatherer.go +++ b/internal/scheduling/nova/candidate_gatherer.go @@ -6,9 +6,8 @@ package nova import ( "context" "fmt" - "strings" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -17,32 +16,28 @@ import ( // for a given Nova scheduling request. type CandidateGatherer interface { // Gather all placement candidates and mutate the request accordingly. - MutateWithAllCandidates(ctx context.Context, request *api.ExternalSchedulerRequest) error + MutateWithAllCandidates(ctx context.Context, request *compute.PlacementRequest) error } // candidateGatherer is the default implementation of CandidateGatherer // for Nova scheduling requests. type candidateGatherer struct{ client.Client } +// NewCandidateGatherer builds the default Nova candidate gatherer backed by the +// given client. It is handed to the compute.Placement, which invokes it when a +// request opts into gathering all placement candidates. +func NewCandidateGatherer(c client.Client) CandidateGatherer { + return &candidateGatherer{Client: c} +} + // MutateWithAllCandidates gathers all placement candidates and mutates // the request accordingly. -func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request *api.ExternalSchedulerRequest) error { +func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.PlacementRequest) error { // Currently we can only get candidates for kvm placements. - hvType, ok := request.Spec.Data.Flavor.Data.ExtraSpecs["capabilities:hypervisor_type"] - if !ok { - return fmt.Errorf( - "missing hypervisor_type in flavor extra specs: %v", - request.Spec.Data.Flavor.Data.ExtraSpecs, - ) - } - switch strings.ToLower(hvType) { - case "qemu", "ch": - // Supported hypervisor type. - default: - // Unsupported hypervisor type, do nothing. + if request.HypervisorFamily != compute.HypervisorFamilyKVM { return fmt.Errorf( - "cannot gather all placement candidates for hypervisor type %q", - request.Spec.Data.Flavor.Data.ExtraSpecs["capabilities:hypervisor_type"], + "cannot gather all placement candidates for hypervisor family %q", + request.HypervisorFamily, ) } @@ -51,10 +46,10 @@ func (g *candidateGatherer) MutateWithAllCandidates(ctx context.Context, request if err := g.List(ctx, hypervisorList); err != nil { return err } - hosts := make([]api.ExternalSchedulerHost, 0, len(hypervisorList.Items)) + hosts := make([]compute.Host, 0, len(hypervisorList.Items)) weights := make(map[string]float64, len(hypervisorList.Items)) for _, hv := range hypervisorList.Items { - host := api.ExternalSchedulerHost{ + host := compute.Host{ // For KVM hosts, compute host name and hypervisor hostname is identical. ComputeHost: hv.Name, HypervisorHostname: hv.Name, diff --git a/internal/scheduling/nova/candidate_gatherer_test.go b/internal/scheduling/nova/candidate_gatherer_test.go index 9f2b9a3b4..2d0fd14c7 100644 --- a/internal/scheduling/nova/candidate_gatherer_test.go +++ b/internal/scheduling/nova/candidate_gatherer_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -26,69 +26,41 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { tests := []struct { name string hypervisors []client.Object - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectError bool errorContains string expectedHosts []string expectedWeight float64 }{ { - name: "missing hypervisor_type in flavor extra specs", + name: "unresolved hypervisor family", hypervisors: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, + request: compute.PlacementRequest{ + HypervisorFamily: "", }, expectError: true, - errorContains: "missing hypervisor_type in flavor extra specs", + errorContains: "cannot gather all placement candidates for hypervisor family", }, { - name: "unsupported hypervisor type vmware", + name: "unsupported hypervisor family vmware", hypervisors: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "VMware vCenter Server", - }, - }, - }, - }, - }, + request: compute.PlacementRequest{ + HypervisorFamily: compute.HypervisorFamilyVMware, }, expectError: true, - errorContains: "cannot gather all placement candidates for hypervisor type", + errorContains: "cannot gather all placement candidates for hypervisor family", }, { - name: "unsupported hypervisor type unknown", + name: "unsupported hypervisor family unknown", hypervisors: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "unknown-type", - }, - }, - }, - }, - }, + request: compute.PlacementRequest{ + HypervisorFamily: "unknown-family", }, expectError: true, - errorContains: "cannot gather all placement candidates for hypervisor type", + errorContains: "cannot gather all placement candidates for hypervisor family", }, { - name: "successful gathering with qemu hypervisor type", + name: "successful gathering with kvm family (qemu)", hypervisors: []client.Object{ &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -106,25 +78,15 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "qemu", - }, - }, - }, - }, - }, + request: compute.PlacementRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, expectedHosts: []string{"host1", "host2", "host3"}, expectedWeight: 0.0, }, { - name: "successful gathering with QEMU uppercase", + name: "successful gathering with kvm family single host", hypervisors: []client.Object{ &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -132,25 +94,15 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - }, - }, - }, - }, - }, + request: compute.PlacementRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, expectedHosts: []string{"host1"}, expectedWeight: 0.0, }, { - name: "successful gathering with ch hypervisor type", + name: "successful gathering with kvm family (ch)", hypervisors: []client.Object{ &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -163,25 +115,15 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "ch", - }, - }, - }, - }, - }, + request: compute.PlacementRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, expectedHosts: []string{"ch-host1", "ch-host2"}, expectedWeight: 0.0, }, { - name: "successful gathering with CH uppercase", + name: "successful gathering with kvm family single ch host", hypervisors: []client.Object{ &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -189,18 +131,8 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, - }, + request: compute.PlacementRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, expectedHosts: []string{"ch-host1"}, @@ -209,49 +141,13 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { { name: "empty hypervisor list", hypervisors: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "qemu", - }, - }, - }, - }, - }, + request: compute.PlacementRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, }, expectError: false, expectedHosts: []string{}, expectedWeight: 0.0, }, - { - name: "mixed case Qemu", - hypervisors: []client.Object{ - &hv1.Hypervisor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "host1", - }, - }, - }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "Qemu", - }, - }, - }, - }, - }, - }, - expectError: false, - expectedHosts: []string{"host1"}, - expectedWeight: 0.0, - }, { name: "request with existing hosts and weights are overwritten", hypervisors: []client.Object{ @@ -261,19 +157,9 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { }, }, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "qemu", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: []compute.Host{ {ComputeHost: "old-host", HypervisorHostname: "old-hv"}, }, Weights: map[string]float64{ @@ -321,7 +207,7 @@ func TestCandidateGatherer_MutateWithAllCandidates(t *testing.T) { } // Verify each expected host is present - hostMap := make(map[string]api.ExternalSchedulerHost) + hostMap := make(map[string]compute.Host) for _, host := range tt.request.Hosts { hostMap[host.ComputeHost] = host } diff --git a/internal/scheduling/nova/cr_metrics_test.go b/internal/scheduling/nova/cr_metrics_test.go index 6aa1c74f4..a05013b7c 100644 --- a/internal/scheduling/nova/cr_metrics_test.go +++ b/internal/scheduling/nova/cr_metrics_test.go @@ -4,9 +4,12 @@ package nova import ( + "bytes" "context" "encoding/json" "fmt" + "net/http" + "net/http/httptest" "testing" "github.com/prometheus/client_golang/prometheus" @@ -21,6 +24,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -239,13 +243,10 @@ func TestLogNoHostFound(t *testing.T) { reg := prometheus.NewRegistry() reg.MustRegister(counter) - controller := &FilterWeigherPipelineController{ - CRRecorder: crs.Recorder{ - Client: fakeClient, - NoHostFoundCounter: counter, - }, + recorder := crs.Recorder{ + Client: fakeClient, + NoHostFoundCounter: counter, } - controller.Client = fakeClient requestFlavorName := flavorName if tt.payg { @@ -263,11 +264,7 @@ func TestLogNoHostFound(t *testing.T) { Context: api.NovaRequestContext{ProjectID: projectID}, Hosts: tt.requestHosts, } - decision := &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{Intent: api.CreateIntent}, - } - - controller.CRRecorder.RecordNoHostFound(context.Background(), decision, request) + recorder.RecordNoHostFound(context.Background(), api.CreateIntent, request) if tt.expectedCase == "" { total := testutil.ToFloat64(counter.WithLabelValues("no_cr", flavorGroup, string(api.CreateIntent))) + @@ -364,7 +361,6 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(flavorKnowledge). - WithStatusSubresource(&v1alpha1.Decision{}). Build() counter := crs.NewNoHostFoundCounter() @@ -372,16 +368,11 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { reg.MustRegister(counter) controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[sched.PlacementRequest]]{ Client: fakeClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, - FeatureGates: FeatureGates{CommittedResourceTracking: enabled}, - CRRecorder: crs.Recorder{ - Client: fakeClient, - NoHostFoundCounter: counter, - }, } controller.PipelineConfigs["test-pipeline"] = pipelineConf initResult := controller.InitPipeline(context.Background(), pipelineConf) @@ -390,18 +381,20 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { } controller.Pipelines["test-pipeline"] = initResult.Pipeline - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{Name: "test-decision", Namespace: "default"}, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - NovaRaw: &runtime.RawExtension{Raw: novaRaw}, - }, - } + // Drive the committed-resource tracking path through the real HTTP API + // handler, where the feature gate now lives (as a post-hook). + scheduler := sched.NewPlacement(controller, passthroughGatherer{}) + crRecorder := crs.Recorder{Client: fakeClient, NoHostFoundCounter: counter} + api := newTestAPIWithScheduler( + HTTPAPIConfig{}, + scheduler, + crRecorder, + FeatureGates{CommittedResourceTracking: enabled}, + ) - if err := controller.ProcessNewDecisionFromAPI(context.Background(), decision); err != nil { - t.Fatalf("ProcessNewDecisionFromAPI: %v", err) - } + req := httptest.NewRequest(http.MethodPost, "/scheduler/nova/external", bytes.NewReader(novaRaw)) + w := httptest.NewRecorder() + api.NovaExternalScheduler(w, req) count := testutil.CollectAndCount(counter) if enabled && count == 0 { diff --git a/internal/scheduling/nova/crs/recorder.go b/internal/scheduling/nova/crs/recorder.go index fc24f8345..31b1e0265 100644 --- a/internal/scheduling/nova/crs/recorder.go +++ b/internal/scheduling/nova/crs/recorder.go @@ -10,7 +10,8 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/prometheus/client_golang/prometheus" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,12 +37,12 @@ var errFlavorNotInGroup = errors.New("flavor not in any group") // Returns errFlavorNotInGroup (PAYG) if the flavor is not in any group. // Returns a different error for transient failures (Knowledge CRD unavailable, etc). func (r *Recorder) resolveFlavorGroup(ctx context.Context, flavorName string) (string, *compute.FlavorInGroup, error) { - fgClient := reservations.FlavorGroupKnowledgeClient{Client: r.Client} + fgClient := accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := fgClient.GetAllFlavorGroups(ctx, nil) if err != nil { return "", nil, err } - groupName, flavor, err := reservations.FindFlavorInGroups(flavorName, flavorGroups) + groupName, flavor, err := accounting.FindFlavorInGroups(flavorName, flavorGroups) if err != nil { return "", nil, errFlavorNotInGroup } @@ -50,14 +51,14 @@ func (r *Recorder) resolveFlavorGroup(ctx context.Context, flavorName string) (s // RecordPlacement writes the placed VM UUID into the matching Reservation slot // and emits placement metrics. Called after a successful Nova placement. -func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decision, request api.ExternalSchedulerRequest) { +func (r *Recorder) RecordPlacement(ctx context.Context, result sched.PlacementResult, intent v1alpha1.SchedulingIntent, request api.ExternalSchedulerRequest) { log := ctrl.LoggerFrom(ctx) instanceUUID := request.Spec.Data.InstanceUUID projectID := request.Context.ProjectID flavorName := request.Spec.Data.Flavor.Data.Name - selectedHost := *decision.Status.Result.TargetHost - intent := string(decision.Spec.Intent) + selectedHost := *result.TargetHost + intentStr := string(intent) flavorGroupName, flavorInGroup, err := r.resolveFlavorGroup(ctx, flavorName) if err != nil { @@ -67,7 +68,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis log.Error(err, "CR allocation: failed to resolve flavor group", "flavor", flavorName, "instanceUUID", instanceUUID) if r.PlacementCounter != nil { - r.PlacementCounter.WithLabelValues("unknown", intent, "error").Inc() + r.PlacementCounter.WithLabelValues("unknown", intentStr, "error").Inc() } } return @@ -92,7 +93,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis activeCRs = append(activeCRs, cr) } - candidateHosts := decision.Status.Result.OrderedHosts + candidateHosts := result.OrderedHosts crOutcome := ClassifyPlacement(evaluator, activeCRs, candidateHosts, projectID, flavorGroupName) log.V(1).Info("CR allocation: placement classified", @@ -103,7 +104,7 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis "flavorGroup", flavorGroupName, ) if r.PlacementCounter != nil { - r.PlacementCounter.WithLabelValues(flavorGroupName, intent, crOutcome).Inc() + r.PlacementCounter.WithLabelValues(flavorGroupName, intentStr, crOutcome).Inc() } if crOutcome != "slot_used" { @@ -159,13 +160,12 @@ func (r *Recorder) RecordPlacement(ctx context.Context, decision *v1alpha1.Decis } // RecordNoHostFound classifies a no-host-found result and emits a log line and metric. -func (r *Recorder) RecordNoHostFound(ctx context.Context, decision *v1alpha1.Decision, request api.ExternalSchedulerRequest) { +func (r *Recorder) RecordNoHostFound(ctx context.Context, intent v1alpha1.SchedulingIntent, request api.ExternalSchedulerRequest) { log := ctrl.LoggerFrom(ctx) projectID := request.Context.ProjectID flavorName := request.Spec.Data.Flavor.Data.Name instanceUUID := request.Spec.Data.InstanceUUID - intent := decision.Spec.Intent flavorGroupName, flavorInGroup, err := r.resolveFlavorGroup(ctx, flavorName) if err != nil { diff --git a/internal/scheduling/nova/crs/recorder_test.go b/internal/scheduling/nova/crs/recorder_test.go index 6e9aba55c..90ee32104 100644 --- a/internal/scheduling/nova/crs/recorder_test.go +++ b/internal/scheduling/nova/crs/recorder_test.go @@ -19,6 +19,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -110,15 +111,11 @@ func TestRecordCRAllocation(t *testing.T) { } } - makeDecision := func(host string, candidates ...string) *v1alpha1.Decision { + makeResult := func(host string, candidates ...string) sched.PlacementResult { h := host - return &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: &h, - OrderedHosts: candidates, - }, - }, + return sched.PlacementResult{ + TargetHost: &h, + OrderedHosts: candidates, } } @@ -159,33 +156,39 @@ func TestRecordCRAllocation(t *testing.T) { name string objects []client.Object request api.ExternalSchedulerRequest - decision *v1alpha1.Decision + result sched.PlacementResult + intent v1alpha1.SchedulingIntent expectedCRSlot string // if non-empty, asserts PlacementCounter cr_slot label expectedFG string // flavor_group label; defaults to "unknown" if empty + // annotateSlot, when non-empty, asserts that the named reservation slot + // was annotated with the candidate VM UUID (the #1438 recorder delegates + // the confirmed write to the reservation controller via this annotation). + annotateSlot string }{ { - name: "classifies slot_used and logs placement (recorder delegates write to reservation controller)", + name: "classifies slot_used and annotates reservation (recorder delegates write to reservation controller)", objects: []client.Object{ flavorKnowledge(), makeCRObject(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), expectedCRSlot: "slot_used", expectedFG: flavorGroup, + annotateSlot: "slot-1", }, { // With status.allocs set, ClassifyPlacement sees remaining=0 → "slot_missed". - // The recorder returns early at that point, so no write occurs (idempotency maintained via no-op). - name: "idempotent: UUID already confirmed in status.allocations — skips early", + // The recorder returns early at that point, so no annotation occurs (idempotency maintained via no-op). + name: "idempotent: UUID already confirmed in status.allocation — skips early", objects: []client.Object{ flavorKnowledge(), makeCRObject(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, vmAlloc())), }, - request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, flavorName), + result: makeResult(selectedHost, selectedHost), // No expectedCRSlot: with status.allocs set, slot appears exhausted (remaining=0), // crOutcome="slot_missed" — counter not checked for this idempotent path. }, @@ -195,8 +198,8 @@ func TestRecordCRAllocation(t *testing.T) { flavorKnowledge(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, - request: makeRequest(instanceUUID, projectID, "unknown-flavor"), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, "unknown-flavor"), + result: makeResult(selectedHost, selectedHost), }, { name: "no matching reservation: host mismatch", @@ -205,8 +208,8 @@ func TestRecordCRAllocation(t *testing.T) { makeCRObject(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, "other-host", nil)), }, - request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, flavorName), + result: makeResult(selectedHost, selectedHost), }, { name: "no slot fits: all capacity confirmed (status.allocs non-empty → 0 remaining memory)", @@ -223,8 +226,8 @@ func TestRecordCRAllocation(t *testing.T) { }, })), }, - request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, flavorName), + result: makeResult(selectedHost, selectedHost), }, { name: "inactive CR (pending state) is filtered by IsActive(): no slot_used", @@ -237,8 +240,8 @@ func TestRecordCRAllocation(t *testing.T) { }(), setReady(makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil)), }, - request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + request: makeRequest(instanceUUID, projectID, flavorName), + result: makeResult(selectedHost, selectedHost), }, { name: "no knowledge CRD: logs error, no allocation", @@ -246,7 +249,7 @@ func TestRecordCRAllocation(t *testing.T) { makeReservation("slot-1", 8192, 8, projectID, flavorGroup, selectedHost, nil), }, request: makeRequest(instanceUUID, projectID, flavorName), - decision: makeDecision(selectedHost, selectedHost), + result: makeResult(selectedHost, selectedHost), expectedCRSlot: "error", }, } @@ -267,10 +270,10 @@ func TestRecordCRAllocation(t *testing.T) { PlacementCounter: counter, } - recorder.RecordPlacement(context.Background(), tt.decision, tt.request) + recorder.RecordPlacement(context.Background(), tt.result, tt.intent, tt.request) if tt.expectedCRSlot != "" { - intent := string(tt.decision.Spec.Intent) + intent := string(tt.intent) fg := tt.expectedFG if fg == "" { fg = "unknown" @@ -281,6 +284,17 @@ func TestRecordCRAllocation(t *testing.T) { fg, intent, tt.expectedCRSlot, got) } } + + if tt.annotateSlot != "" { + var res v1alpha1.Reservation + if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: tt.annotateSlot}, &res); err != nil { + t.Fatalf("Get reservation %q: %v", tt.annotateSlot, err) + } + if got := res.Annotations[v1alpha1.AnnotationCandidateAllocation]; got != instanceUUID { + t.Errorf("annotation %q on %q = %q, want %q", + v1alpha1.AnnotationCandidateAllocation, tt.annotateSlot, got, instanceUUID) + } + } }) } } diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index 9e89f5f2a..1e14a4ac5 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -12,16 +12,14 @@ import ( "io" "math/rand" "net/http" + "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -46,27 +44,45 @@ func (c HTTPAPIConfig) forcedDestinationEnabled() bool { return c.ForcedDestinationEnabled == nil || *c.ForcedDestinationEnabled } -type HTTPAPIDelegate interface { - // Process the decision from the API. Should create and return the updated decision. - ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error -} - type HTTPAPI interface { // Bind the server handlers. Init(*http.ServeMux) } +// runner is the subset of compute.Placement the HTTP API depends on. It is an +// interface so the handler can be unit-tested with a mock. *compute.Placement +// satisfies it. +type runner interface { + Run(ctx context.Context, req compute.PlacementRequest, pre compute.PreHook, post compute.PostHook) (compute.PlacementResult, error) +} + type httpAPI struct { - monitor scheduling.APIMonitor - delegate HTTPAPIDelegate - config HTTPAPIConfig + monitor scheduling.APIMonitor + scheduler runner + vmHandler *VirtualMachineHandler + crRecorder crs.Recorder + featureGates FeatureGates + config HTTPAPIConfig } -func NewAPI(config HTTPAPIConfig, delegate HTTPAPIDelegate) HTTPAPI { +// NewAPI builds the Nova external-scheduler HTTP API. It calls the shared +// compute.Placement directly; VM-resource creation and committed-resource +// tracking run as post-hooks after the pipeline, under the scheduler's lock. +func NewAPI( + config HTTPAPIConfig, + scheduler *compute.Placement, + vmHandler *VirtualMachineHandler, + crRecorder crs.Recorder, + featureGates FeatureGates, +) HTTPAPI { + return &httpAPI{ - monitor: scheduling.NewSchedulerMonitor(), - delegate: delegate, - config: config, + monitor: scheduling.NewSchedulerMonitor(), + scheduler: scheduler, + vmHandler: vmHandler, + crRecorder: crRecorder, + featureGates: featureGates, + config: config, } } @@ -202,7 +218,6 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req c.Respond(logr.Logger{}, http.StatusInternalServerError, err, "failed to read request body") return } - raw := runtime.RawExtension{Raw: body} var requestData api.ExternalSchedulerRequest // Copy the raw body to a io.Reader for json deserialization. cp := body @@ -257,43 +272,55 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req intent = api.CreateIntent } - decision := &v1alpha1.Decision{ - TypeMeta: metav1.TypeMeta{ - Kind: "Decision", - APIVersion: "cortex.cloud/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "nova-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: requestData.Pipeline, - }, - ResourceID: requestData.Spec.Data.InstanceUUID, - NovaRaw: &raw, - Intent: intent, - }, - } ctx := r.Context() - if err := httpAPI.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { - c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) - return - } - // Check if the decision contains status conditions indicating an error. - if meta.IsStatusConditionFalse(decision.Status.Conditions, v1alpha1.DecisionConditionReady) { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision contains error condition"), "decision failed") - return + + // Translate the Nova wire request into the typed pipeline request. + schedReq := ToSchedulerRequest(requestData, intent, requestData.Pipeline) + + // Side effects run as a single post-hook after the pipeline, under the + // scheduler lock. It operates on the Nova wire request (which carries the + // fields the VM resource and CR tracking need) plus the typed result. + postHook := func(ctx context.Context, _ compute.PlacementRequest, result compute.PlacementResult) error { + // VM-resource creation. + switch { + case requestData.Options.ReadOnly: + logger.V(1).Info("skipping virtual machine creation, request is read-only", "instanceUUID", requestData.Spec.Data.InstanceUUID) + case !httpAPI.featureGates.VirtualMachineCreation: + logger.V(1).Info("skipping virtual machine creation, feature gate disabled", "instanceUUID", requestData.Spec.Data.InstanceUUID) + case !slices.Contains(api.NovaIntents, intent): + logger.V(1).Info("skipping virtual machine creation, intent not triggered by Nova", "instanceUUID", requestData.Spec.Data.InstanceUUID, "intent", intent) + case schedReq.HypervisorFamily != compute.HypervisorFamilyKVM: + logger.V(1).Info("skipping virtual machine creation, unsupported hypervisor type", "instanceUUID", requestData.Spec.Data.InstanceUUID) + default: + if upsertErr := httpAPI.vmHandler.Upsert(ctx, result, requestData, intent); upsertErr != nil { + if httpAPI.featureGates.VirtualMachineCreationMandatory { + return fmt.Errorf("failed to upsert virtual machine resource: %w", upsertErr) + } + // Non-fatal: the placement decision itself succeeded. + logger.Error(upsertErr, "failed to upsert virtual machine resource", "instanceUUID", requestData.Spec.Data.InstanceUUID) + } + } + // Committed-resource tracking. + if httpAPI.featureGates.CommittedResourceTracking && !requestData.Options.SkipCommittedResourceTracking { + if result.TargetHost != nil { + httpAPI.crRecorder.RecordPlacement(ctx, result, intent, requestData) + } else { + httpAPI.crRecorder.RecordNoHostFound(ctx, intent, requestData) + } + } + return nil } - if decision.Status.Result == nil { - c.Respond(logger, http.StatusInternalServerError, errors.New("decision didn't produce a result"), "decision failed") + + result, err := httpAPI.scheduler.Run(ctx, schedReq, nil, postHook) + if err != nil { + c.Respond(logger, http.StatusInternalServerError, err, fmt.Sprintf("failed to process scheduling decision: %v", err)) return } - hosts := decision.Status.Result.OrderedHosts + hosts := result.OrderedHosts if httpAPI.config.NovaLimitHostsToRequest { hosts = limitHostsToRequest(requestData, hosts) logger.V(1).Info("limited hosts to request", - "hosts", hosts, "originalHosts", decision.Status.Result.OrderedHosts) + "hosts", hosts, "originalHosts", result.OrderedHosts) } // This is a hack to address the problem that Nova only uses the first host in hosts for evacuation requests. // Only for evacuation we shuffle the first k hosts to ensure that we do not get stuck on a single host diff --git a/internal/scheduling/nova/external_scheduler_api_test.go b/internal/scheduling/nova/external_scheduler_api_test.go index f3dd2c82b..5a2014b27 100644 --- a/internal/scheduling/nova/external_scheduler_api_test.go +++ b/internal/scheduling/nova/external_scheduler_api_test.go @@ -14,48 +14,72 @@ import ( "testing" novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" ) -type mockHTTPAPIDelegate struct { - processDecisionFunc func(ctx context.Context, decision *v1alpha1.Decision) error +// mockRunner is a test double for the scheduler the HTTP API depends on. runFunc +// receives the mapped request and may invoke the post-hook to exercise the +// handler's side-effect path; it returns the result/error the handler sees. +type mockRunner struct { + runFunc func(ctx context.Context, req compute.PlacementRequest, post compute.PostHook) (compute.PlacementResult, error) } -func (m *mockHTTPAPIDelegate) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { - if m.processDecisionFunc != nil { - return m.processDecisionFunc(ctx, decision) +func (m *mockRunner) Run(ctx context.Context, req compute.PlacementRequest, _ compute.PreHook, post compute.PostHook) (compute.PlacementResult, error) { + if m.runFunc != nil { + return m.runFunc(ctx, req, post) } - return nil + return compute.PlacementResult{}, nil } -func TestNewAPI(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - - api := NewAPI(HTTPAPIConfig{}, delegate) - - if api == nil { - t.Fatal("NewAPI returned nil") +// newTestAPI builds an *httpAPI with the given config and mock runner, leaving +// the side-effect handlers nil (the handler tests that need them set runFunc to +// not invoke the post-hook, or provide their own). +func newTestAPI(config HTTPAPIConfig, r runner) *httpAPI { + return &httpAPI{ + monitor: scheduling.NewSchedulerMonitor(), + scheduler: r, + config: config, } +} - httpAPI, ok := api.(*httpAPI) - if !ok { - t.Fatal("NewAPI did not return httpAPI type") +// newTestAPIWithScheduler builds an *httpAPI over a real compute.Placement plus +// the CR recorder and feature gates, for tests that exercise the committed- +// resource post-hook end-to-end. +func newTestAPIWithScheduler(config HTTPAPIConfig, scheduler runner, crRecorder crs.Recorder, featureGates FeatureGates) *httpAPI { + return &httpAPI{ + monitor: scheduling.NewSchedulerMonitor(), + scheduler: scheduler, + crRecorder: crRecorder, + featureGates: featureGates, + config: config, } +} - if httpAPI.delegate != delegate { - t.Error("Delegate not set correctly") - } +// passthroughGatherer is a CandidateGatherer that leaves the request's hosts +// unchanged (tests seed their own host sets). +type passthroughGatherer struct{} +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *compute.PlacementRequest) error { + return nil +} + +func TestNewAPI(t *testing.T) { + r := &mockRunner{} + + httpAPI := newTestAPI(HTTPAPIConfig{}, r) + + if httpAPI.scheduler != r { + t.Error("scheduler (runner) not set correctly") + } if httpAPI.monitor.ApiRequestsTimer == nil { t.Error("Monitor not initialized") } } func TestHTTPAPI_Init(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(HTTPAPIConfig{}, delegate) + var api HTTPAPI = newTestAPI(HTTPAPIConfig{}, &mockRunner{}) mux := http.NewServeMux() api.Init(mux) @@ -72,8 +96,7 @@ func TestHTTPAPI_Init(t *testing.T) { } func TestHTTPAPI_canRunScheduler(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, &mockRunner{}) tests := []struct { name string @@ -146,7 +169,7 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { method string body string processDecisionErr error - decisionResult *v1alpha1.Decision + result *compute.PlacementResult expectedStatus int expectedHosts []string }{ @@ -185,12 +208,8 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { } return string(data) }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - }, - }, + result: &compute.PlacementResult{ + OrderedHosts: []string{"host1"}, }, expectedStatus: http.StatusOK, expectedHosts: []string{"host1"}, @@ -246,37 +265,26 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { } return string(data) }(), - decisionResult: &v1alpha1.Decision{ - Status: v1alpha1.DecisionStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "SchedulingError", - }, - }, - }, - }, - expectedStatus: http.StatusInternalServerError, + processDecisionErr: errors.New("scheduling failed"), + expectedStatus: http.StatusInternalServerError, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { + r := &mockRunner{ + runFunc: func(_ context.Context, _ compute.PlacementRequest, _ compute.PostHook) (compute.PlacementResult, error) { if tt.processDecisionErr != nil { - return tt.processDecisionErr + return compute.PlacementResult{}, tt.processDecisionErr } - if tt.decisionResult != nil { - decision.Status = tt.decisionResult.Status - return nil + if tt.result != nil { + return *tt.result, nil } - return nil + return compute.PlacementResult{}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) var body *strings.Reader if tt.body != "" { @@ -318,15 +326,12 @@ func TestHTTPAPI_NovaExternalScheduler_EmptyHostsSerializedAsArray(t *testing.T) // When the pipeline filters out all hosts, OrderedHosts is nil. The // response must serialize hosts as an empty array, not null, since Nova's // schema requires an array. - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - decision.Status = v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{OrderedHosts: nil}, - } - return nil + r := &mockRunner{ + runFunc: func(context.Context, compute.PlacementRequest, compute.PostHook) (compute.PlacementResult, error) { + return compute.PlacementResult{OrderedHosts: nil}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) req := novaapi.ExternalSchedulerRequest{ Spec: novaapi.NovaObject[novaapi.NovaSpec]{ @@ -357,20 +362,16 @@ func TestHTTPAPI_NovaExternalScheduler_EmptyHostsSerializedAsArray(t *testing.T) } } -func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) { - var capturedDecision *v1alpha1.Decision - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - capturedDecision = decision - // Set a successful result to avoid "decision didn't produce a result" error - decision.Status.Result = &v1alpha1.DecisionResult{ - OrderedHosts: []string{"host1"}, - } - return nil +func TestHTTPAPI_NovaExternalScheduler_RequestMapping(t *testing.T) { + var captured compute.PlacementRequest + r := &mockRunner{ + runFunc: func(_ context.Context, req compute.PlacementRequest, _ compute.PostHook) (compute.PlacementResult, error) { + captured = req + return compute.PlacementResult{OrderedHosts: []string{"host1"}}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) requestData := novaapi.ExternalSchedulerRequest{ Spec: novaapi.NovaObject[novaapi.NovaSpec]{ @@ -400,29 +401,16 @@ func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) { t.Errorf("Expected status %d, got %d", http.StatusOK, w.Code) } - if capturedDecision == nil { - t.Fatal("Decision was not captured") + // Verify the handler mapped the wire request into the typed scheduler request. + if captured.Pipeline != "test-pipeline" { + t.Errorf("Expected pipeline 'test-pipeline', got %s", captured.Pipeline) } - - // Verify decision fields - if capturedDecision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainNova { - t.Errorf("Expected scheduling domain %s, got %s", v1alpha1.SchedulingDomainNova, capturedDecision.Spec.SchedulingDomain) - } - - if capturedDecision.Spec.PipelineRef.Name != "test-pipeline" { - t.Errorf("Expected pipeline 'test-pipeline', got %s", capturedDecision.Spec.PipelineRef.Name) - } - - if capturedDecision.Spec.ResourceID != "test-uuid-123" { - t.Errorf("Expected resource ID 'test-uuid-123', got %s", capturedDecision.Spec.ResourceID) - } - - if capturedDecision.GenerateName != "nova-" { - t.Errorf("Expected generate name 'nova-', got %s", capturedDecision.GenerateName) + if captured.InstanceUUID != "test-uuid-123" { + t.Errorf("Expected instance UUID 'test-uuid-123', got %s", captured.InstanceUUID) } - - if capturedDecision.Spec.NovaRaw == nil { - t.Error("NovaRaw should not be nil") + // No explicit _nova_check_type hint means the handler defaults to the create intent. + if captured.Intent != novaapi.CreateIntent { + t.Errorf("Expected intent %s, got %s", novaapi.CreateIntent, captured.Intent) } } @@ -445,14 +433,14 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestination(t *testing.T) { }, } - delegateCalled := false - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - delegateCalled = true - return nil + schedulerCalled := false + r := &mockRunner{ + runFunc: func(context.Context, compute.PlacementRequest, compute.PostHook) (compute.PlacementResult, error) { + schedulerCalled = true + return compute.PlacementResult{}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) body, err := json.Marshal(requestData) if err != nil { @@ -466,8 +454,8 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestination(t *testing.T) { if w.Code != http.StatusOK { t.Fatalf("Expected status %d, got %d", http.StatusOK, w.Code) } - if delegateCalled { - t.Error("delegate should not be called for forced destination requests") + if schedulerCalled { + t.Error("scheduler should not be called for forced destination requests") } var response novaapi.ExternalSchedulerResponse if err := json.NewDecoder(w.Body).Decode(&response); err != nil { @@ -495,14 +483,14 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationNoWeights(t *testing.T) // for forced-destination requests. } - delegateCalled := false - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - delegateCalled = true - return nil + schedulerCalled := false + r := &mockRunner{ + runFunc: func(context.Context, compute.PlacementRequest, compute.PostHook) (compute.PlacementResult, error) { + schedulerCalled = true + return compute.PlacementResult{}, nil }, } - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, r) body, err := json.Marshal(requestData) if err != nil { @@ -516,8 +504,8 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationNoWeights(t *testing.T) if w.Code != http.StatusOK { t.Fatalf("Expected status %d, got %d (forced destination with no weights must not be rejected)", http.StatusOK, w.Code) } - if delegateCalled { - t.Error("delegate should not be called for forced destination requests") + if schedulerCalled { + t.Error("scheduler should not be called for forced destination requests") } var response novaapi.ExternalSchedulerResponse if err := json.NewDecoder(w.Body).Decode(&response); err != nil { @@ -548,15 +536,15 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationDisabled(t *testing.T) { Pipeline: "test-pipeline", } - delegateCalled := false - delegate := &mockHTTPAPIDelegate{ - processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { - delegateCalled = true - return nil + schedulerCalled := false + r := &mockRunner{ + runFunc: func(context.Context, compute.PlacementRequest, compute.PostHook) (compute.PlacementResult, error) { + schedulerCalled = true + return compute.PlacementResult{}, nil }, } disabled := false - api := NewAPI(HTTPAPIConfig{ForcedDestinationEnabled: &disabled}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{ForcedDestinationEnabled: &disabled}, r) body, err := json.Marshal(requestData) if err != nil { @@ -567,8 +555,8 @@ func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationDisabled(t *testing.T) { api.NovaExternalScheduler(w, req) - if !delegateCalled { - t.Error("delegate should be called when forced destination is disabled") + if !schedulerCalled { + t.Error("scheduler should be called when forced destination is disabled") } } @@ -804,8 +792,7 @@ func TestShuffleTopHosts(t *testing.T) { } func TestHTTPAPI_inferPipelineName(t *testing.T) { - delegate := &mockHTTPAPIDelegate{} - api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + api := newTestAPI(HTTPAPIConfig{}, &mockRunner{}) tests := []struct { name string diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index a81074072..7ee25cf55 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -5,20 +5,11 @@ package nova import ( "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sync" - "time" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" - "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/weighers" "github.com/cobaltcore-dev/cortex/pkg/multicluster" @@ -30,32 +21,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) -// The decision pipeline controller takes decision resources containing a -// placement request spec and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. -// -// Additionally, the controller watches for pipeline and step changes to -// reconfigure the pipelines as needed. +// FilterWeigherPipelineController owns the Nova filter-weigher pipelines: it +// watches Pipeline and Knowledge resources to (re)configure the pipelines, and +// exposes them via GetPipeline for the compute.Placement to run. Placement +// requests no longer flow through this controller — the compute.Placement, +// driven by the external-scheduler HTTP API and the reservation controllers, is +// the entrypoint. type FilterWeigherPipelineController struct { // Toolbox shared between all pipeline controllers. - lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]] + lib.BasePipelineController[lib.FilterWeigherPipeline[compute.PlacementRequest]] - // Mutex to only allow one process at a time. - // Read-only runs (opts.ReadOnly == true) acquire a read lock; write runs acquire the full lock. - processMu sync.RWMutex - - // FeatureGates holds feature flags for this controller. - FeatureGates FeatureGates // Monitor to pass down to all pipelines. Monitor lib.FilterWeigherPipelineMonitor - // Candidate gatherer to get all placement candidates if needed. - gatherer CandidateGatherer - - // vmHandler maps processed decisions into vm-operator VirtualMachine resources. - vmHandler *VirtualMachineHandler - - // CRRecorder receives scheduling events and updates CR reservations and metrics. - CRRecorder crs.Recorder } // The type of pipeline this controller manages. @@ -63,183 +40,10 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -// Callback executed when kubernetes asks to reconcile a decision resource. -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - // Peek at the decision before acquiring the lock so we can choose the right lock type. - // Read-only runs can proceed concurrently; write runs need the exclusive lock. - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - if c.peekReadOnly(decision) { - c.processMu.RLock() - defer c.processMu.RUnlock() - } else { - c.processMu.Lock() - defer c.processMu.Unlock() - // Re-fetch after acquiring the exclusive lock to see consistent state. - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - } - old := decision.DeepCopy() - if _, err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} - -// Process the decision from the API. Should create and return the updated decision. -func (c *FilterWeigherPipelineController) ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error { - // Read-only runs share the cached decision state; no re-fetch needed because they - // don't observe writes from concurrent exclusive-lock runs. - if c.peekReadOnly(decision) { - c.processMu.RLock() - defer c.processMu.RUnlock() - } else { - c.processMu.Lock() - defer c.processMu.Unlock() - } - - request, err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - if err == nil && decision.Status.Result != nil && request != nil && c.FeatureGates.CommittedResourceTracking { - if decision.Status.Result.TargetHost != nil { - if request.Options.SkipCommittedResourceTracking { - ctrl.LoggerFrom(ctx).V(1).Info("CR allocation tracking skipped (SkipCommittedResourceTracking=true)", - "instanceUUID", request.Spec.Data.InstanceUUID, - "targetHost", *decision.Status.Result.TargetHost, - "intent", decision.Spec.Intent, - ) - } else { - c.CRRecorder.RecordPlacement(ctx, decision, *request) - } - } - if decision.Status.Result.TargetHost == nil { - c.CRRecorder.RecordNoHostFound(ctx, decision, *request) - } - } - return err -} - -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) (*api.ExternalSchedulerRequest, error) { - log := ctrl.LoggerFrom(ctx) - startedAt := time.Now() // So we can measure sync duration. - - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) - if !ok { - log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return nil, fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) - } - if decision.Spec.NovaRaw == nil { - log.Error(nil, "skipping decision, no novaRaw spec defined") - return nil, errors.New("no novaRaw spec defined") - } - var request api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.NovaRaw.Raw, &request); err != nil { - log.Error(err, "failed to unmarshal novaRaw spec") - return nil, err - } - - // If necessary gather all placement candidates before filtering. - // This will override the hosts and weights in the nova request. - pipelineConf, ok := c.GetPipelineConfig(decision.Spec.PipelineRef.Name) - if !ok { - log.Error(nil, "pipeline config not found", "pipelineName", decision.Spec.PipelineRef.Name) - return &request, fmt.Errorf("pipeline config not found: %q", decision.Spec.PipelineRef.Name) - } - if pipelineConf.Spec.IgnorePreselection { - log.V(1).Info("gathering all placement candidates before filtering") - if err := c.gatherer.MutateWithAllCandidates(ctx, &request); err != nil { - log.Error(err, "failed to gather all placement candidates") - return &request, err - } - log.V(1).Info("gathered all placement candidates", "numHosts", len(request.Hosts)) - } - - result, err := pipeline.Run(request) - if err == nil { - decision.Status.Result = &result - } - // Record a human-readable explanation of the decision so it propagates to - // the VM CortexSpec.Explanation. On error the result is nil and the - // explanation carries the failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) - // We create a virtual machine resource based on the scheduling decision, - // logging the specific reason whenever we skip creation so it can be traced. - instanceUUID := request.Spec.Data.InstanceUUID - switch { - case err != nil: - log.V(1).Info("skipping virtual machine creation, pipeline run failed", "instanceUUID", instanceUUID) - case request.Options.ReadOnly: - log.V(1).Info("skipping virtual machine creation, request is read-only", "instanceUUID", instanceUUID) - case !c.FeatureGates.VirtualMachineCreation: - log.V(1).Info("skipping virtual machine creation, feature gate disabled", "instanceUUID", instanceUUID) - case !slices.Contains(api.NovaIntents, decision.Spec.Intent): - log.V(1).Info("skipping virtual machine creation, intent not triggered by Nova", "instanceUUID", instanceUUID, "intent", decision.Spec.Intent) - case !request.IsKVMRequest(): - log.V(1).Info("skipping virtual machine creation, unsupported hypervisor type", "instanceUUID", instanceUUID) - default: - vmUpsertErr := c.vmHandler.Upsert(ctx, decision, request) - if c.FeatureGates.VirtualMachineCreationMandatory && vmUpsertErr != nil { - log.Error(vmUpsertErr, "failed to upsert virtual machine resource and it is mandatory", "instanceUUID", instanceUUID) - return &request, vmUpsertErr - } - if vmUpsertErr != nil { - // Non-fatal: the placement decision itself succeeded. - log.Error(vmUpsertErr, "failed to upsert virtual machine resource", "instanceUUID", instanceUUID) - } - } - - if err != nil { - log.Error(err, "failed to run pipeline") - return &request, err - } - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - log.Info("decision processed successfully", "duration", time.Since(startedAt)) - return &request, nil -} - -// peekReadOnly determines whether a decision should use a read lock instead of -// the exclusive write lock. Defaults to false (exclusive) on any parse error. -func (c *FilterWeigherPipelineController) peekReadOnly(decision *v1alpha1.Decision) bool { - if decision.Spec.NovaRaw == nil { - return false - } - var request api.ExternalSchedulerRequest - if err := json.Unmarshal(decision.Spec.NovaRaw.Raw, &request); err != nil { - return false - } - return request.Options.ReadOnly -} - func (c *FilterWeigherPipelineController) InitPipeline( ctx context.Context, p v1alpha1.Pipeline, -) lib.PipelineInitResult[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]] { +) lib.PipelineInitResult[lib.FilterWeigherPipeline[compute.PlacementRequest]] { return lib.InitNewFilterWeigherPipeline( ctx, c.Client, p.Name, @@ -252,8 +56,6 @@ func (c *FilterWeigherPipelineController) InitPipeline( func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainNova - c.gatherer = &candidateGatherer{Client: mcl} - c.vmHandler = &VirtualMachineHandler{Client: mcl} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { return err } @@ -310,25 +112,14 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainNova { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } - return bldr.Named("cortex-nova-decisions"). + return bldr.Named("cortex-nova-pipelines"). Complete(c) } + +// Reconcile is a no-op: this controller reconfigures pipelines through its +// Pipeline/Knowledge watch handlers, and placement runs through the +// compute.Placement, not through object reconciliation. A Reconcile method is +// required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { + return ctrl.Result{}, nil +} diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go index f7c4f5a41..26576100c 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go @@ -5,22 +5,12 @@ package nova import ( "context" - "encoding/json" - "errors" - "strings" "testing" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - api "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -28,10 +18,10 @@ import ( type mockCandidateGatherer struct { called bool err error - gatheredHosts []api.ExternalSchedulerHost + gatheredHosts []compute.Host } -func (m *mockCandidateGatherer) MutateWithAllCandidates(ctx context.Context, request *api.ExternalSchedulerRequest) error { +func (m *mockCandidateGatherer) MutateWithAllCandidates(ctx context.Context, request *compute.PlacementRequest) error { m.called = true if m.err != nil { return m.err @@ -46,239 +36,6 @@ func (m *mockCandidateGatherer) MutateWithAllCandidates(ctx context.Context, req return nil } -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - novaRequest := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Name: "RequestSpec", - Namespace: "nova_object", - Version: "1.19", - Data: api.NovaSpec{ - ProjectID: "test-project", - UserID: "test-user", - InstanceUUID: "test-instance-uuid", - NumInstances: 1, - }, - }, - Context: api.NovaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: func() *string { s := "global-req-123"; return &s }(), - }, - Hosts: []api.ExternalSchedulerHost{ - {ComputeHost: "compute-1", HypervisorHostname: "hv-1"}, - {ComputeHost: "compute-2", HypervisorHostname: "hv-2"}, - }, - Weights: map[string]float64{"compute-1": 1.0, "compute-2": 0.5}, - Pipeline: "test-pipeline", - Options: scheduling.Options{}, - } - - novaRaw, err := json.Marshal(novaRequest) - if err != nil { - t.Fatalf("Failed to marshal nova request: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - pipeline *v1alpha1.Pipeline - expectError bool - expectResult bool - }{ - { - name: "successful nova decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "decision without novaRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: nil, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - { - name: "pipeline not found", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: nil, - expectError: true, - expectResult: false, - }, - { - name: "invalid novaRaw JSON", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-invalid-json", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: []byte("invalid json"), - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - expectError: true, - expectResult: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{tt.decision} - if tt.pipeline != nil { - objects = append(objects, tt.pipeline) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - - if tt.pipeline != nil { - initResult := controller.InitPipeline(t.Context(), v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.pipeline.Name, - }, - Spec: tt.pipeline.Spec, - }) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to initialize pipeline: filter errors: %v, weigher errors: %v", initResult.FilterErrors, initResult.WeigherErrors) - } - controller.Pipelines[tt.pipeline.Name] = initResult.Pipeline - controller.PipelineConfigs[tt.pipeline.Name] = *tt.pipeline - } - - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - Namespace: tt.decision.Namespace, - }, - } - - result, err := controller.Reconcile(context.Background(), req) - - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - - if result.RequeueAfter > 0 { - t.Error("Expected no requeue") - } - - var updatedDecision v1alpha1.Decision - if err := client.Get(context.Background(), req.NamespacedName, &updatedDecision); err != nil { - if !tt.expectError { - t.Fatalf("Failed to get updated decision: %v", err) - } - return - } - - if tt.expectResult && updatedDecision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - if !tt.expectResult && updatedDecision.Status.Result != nil { - t.Error("Expected result to be nil but was set") - } - }) - } -} - func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { controller := &FilterWeigherPipelineController{ Monitor: lib.FilterWeigherPipelineMonitor{}, @@ -355,609 +112,3 @@ func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { }) } } - -func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - novaRequest := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Name: "RequestSpec", - Namespace: "nova_object", - Version: "1.19", - Data: api.NovaSpec{ - ProjectID: "test-project", - UserID: "test-user", - InstanceUUID: "test-instance-uuid", - NumInstances: 1, - }, - }, - Context: api.NovaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: func() *string { s := "global-req-123"; return &s }(), - }, - Hosts: []api.ExternalSchedulerHost{ - {ComputeHost: "compute-1", HypervisorHostname: "hv-1"}, - {ComputeHost: "compute-2", HypervisorHostname: "hv-2"}, - }, - Weights: map[string]float64{"compute-1": 1.0, "compute-2": 0.5}, - Pipeline: "test-pipeline", - } - - novaRaw, err := json.Marshal(novaRequest) - if err != nil { - t.Fatalf("Failed to marshal nova request: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - pipeline *v1alpha1.Pipeline - pipelineConf *v1alpha1.Pipeline - setupPipelineConfigs bool - expectError bool - expectResult bool - expectUpdatedStatus bool - errorContains string - }{ - { - name: "successful processing with decision creation enabled", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-api", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "test-uuid-1", - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: false, - expectResult: true, - expectUpdatedStatus: true, - }, - { - name: "successful processing with decision creation disabled", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-create", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "test-uuid-2", - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline-no-create", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline-no-create", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline-no-create", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: false, - expectResult: true, - expectUpdatedStatus: false, - }, - { - name: "pipeline not configured", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-config", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "nonexistent-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: nil, - pipelineConf: nil, - setupPipelineConfigs: false, - expectError: true, - expectResult: false, - expectUpdatedStatus: false, - errorContains: "pipeline not found or not ready", - }, - { - name: "decision without novaRaw spec", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-raw-api", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: nil, - }, - }, - pipeline: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: true, - expectResult: false, - expectUpdatedStatus: false, - errorContains: "no novaRaw spec defined", - }, - { - name: "processing fails after decision creation", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-process-fail", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: nil, // This will cause processing to fail after creation - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: true, - expectResult: false, - expectUpdatedStatus: false, - errorContains: "pipeline not found or not ready", - }, - { - name: "pipeline not found in runtime map", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-runtime-pipeline", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "missing-runtime-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - }, - pipeline: nil, - pipelineConf: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "missing-runtime-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - setupPipelineConfigs: true, - expectError: true, - expectResult: false, - expectUpdatedStatus: false, - errorContains: "pipeline not found or not ready", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{} - if tt.pipeline != nil { - objects = append(objects, tt.pipeline) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: client, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - - // Setup pipeline configurations if needed - if tt.setupPipelineConfigs && tt.pipelineConf != nil { - controller.PipelineConfigs[tt.pipelineConf.Name] = *tt.pipelineConf - } - - // Setup runtime pipeline if needed - if tt.pipeline != nil { - initResult := controller.InitPipeline(context.Background(), v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.pipeline.Name, - }, - Spec: tt.pipeline.Spec, - }) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to initialize pipeline: filter errors: %v, weigher errors: %v", initResult.FilterErrors, initResult.WeigherErrors) - } - controller.Pipelines[tt.pipeline.Name] = initResult.Pipeline - } - - if tt.decision.Spec.NovaRaw != nil { - req := novaRequest - req.Options = scheduling.Options{} - raw, marshalErr := json.Marshal(req) - if marshalErr != nil { - t.Fatalf("Failed to marshal request with options: %v", marshalErr) - } - tt.decision.Spec.NovaRaw = &runtime.RawExtension{Raw: raw} - } - - // Call the method under test - err := controller.ProcessNewDecisionFromAPI(context.Background(), tt.decision) - - // Validate error expectations - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - if tt.errorContains != "" && (err == nil || !strings.Contains(err.Error(), tt.errorContains)) { - t.Errorf("Expected error to contain %q, got: %v", tt.errorContains, err) - } - - // Validate result expectations - if tt.expectResult && tt.decision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - if !tt.expectResult && tt.decision.Status.Result != nil { - t.Error("Expected result to be nil but was set") - } - }) - } -} - -func TestFilterWeigherPipelineController_IgnorePreselection(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - - // Create a request with initial hosts - novaRequest := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Name: "RequestSpec", - Namespace: "nova_object", - Version: "1.19", - Data: api.NovaSpec{ - ProjectID: "test-project", - UserID: "test-user", - InstanceUUID: "test-instance-uuid", - NumInstances: 1, - }, - }, - Context: api.NovaRequestContext{ - ProjectID: "test-project", - UserID: "test-user", - RequestID: "req-123", - GlobalRequestID: func() *string { s := "global-req-123"; return &s }(), - }, - Hosts: []api.ExternalSchedulerHost{ - {ComputeHost: "original-host-1", HypervisorHostname: "hv-1"}, - {ComputeHost: "original-host-2", HypervisorHostname: "hv-2"}, - }, - Weights: map[string]float64{"original-host-1": 1.0, "original-host-2": 0.5}, - Pipeline: "test-pipeline", - Options: scheduling.Options{}, - } - - novaRaw, err := json.Marshal(novaRequest) - if err != nil { - t.Fatalf("Failed to marshal nova request: %v", err) - } - - tests := []struct { - name string - ignorePreselection bool - gathererErr error - gatheredHosts []api.ExternalSchedulerHost - expectGathererCall bool - expectError bool - errorContains string - }{ - { - name: "IgnorePreselection disabled - gatherer not called", - ignorePreselection: false, - gathererErr: nil, - gatheredHosts: nil, - expectGathererCall: false, - expectError: false, - }, - { - name: "IgnorePreselection enabled - gatherer called and succeeds", - ignorePreselection: true, - gathererErr: nil, - gatheredHosts: []api.ExternalSchedulerHost{ - {ComputeHost: "gathered-host-1", HypervisorHostname: "gathered-host-1"}, - {ComputeHost: "gathered-host-2", HypervisorHostname: "gathered-host-2"}, - {ComputeHost: "gathered-host-3", HypervisorHostname: "gathered-host-3"}, - }, - expectGathererCall: true, - expectError: false, - }, - { - name: "IgnorePreselection enabled - gatherer returns error", - ignorePreselection: true, - gathererErr: errGathererFailed, - gatheredHosts: nil, - expectGathererCall: true, - expectError: true, - errorContains: "gatherer failed", - }, - { - name: "IgnorePreselection enabled - gatherer returns empty hosts", - ignorePreselection: true, - gathererErr: nil, - gatheredHosts: []api.ExternalSchedulerHost{}, - expectGathererCall: true, - expectError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockGatherer := &mockCandidateGatherer{ - err: tt.gathererErr, - gatheredHosts: tt.gatheredHosts, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]]{ - Client: fakeClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), - PipelineConfigs: make(map[string]v1alpha1.Pipeline), - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - gatherer: mockGatherer, - } - - // Setup pipeline config with IgnorePreselection flag - pipelineConf := v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pipeline", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainNova, - IgnorePreselection: tt.ignorePreselection, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - } - controller.PipelineConfigs["test-pipeline"] = pipelineConf - - // Initialize the pipeline - initResult := controller.InitPipeline(context.Background(), pipelineConf) - if len(initResult.FilterErrors) > 0 || len(initResult.WeigherErrors) > 0 { - t.Fatalf("Failed to initialize pipeline: filter errors: %v, weigher errors: %v", initResult.FilterErrors, initResult.WeigherErrors) - } - controller.Pipelines["test-pipeline"] = initResult.Pipeline - - // Create decision - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-preselection", - Namespace: "default", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: "test-pipeline", - }, - NovaRaw: &runtime.RawExtension{ - Raw: novaRaw, - }, - }, - } - - // Process the decision - err := controller.ProcessNewDecisionFromAPI(context.Background(), decision) - - // Verify gatherer was called (or not) as expected - if tt.expectGathererCall && !mockGatherer.called { - t.Error("Expected gatherer to be called but it was not") - } - if !tt.expectGathererCall && mockGatherer.called { - t.Error("Expected gatherer not to be called but it was") - } - - // Verify error expectations - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - if tt.errorContains != "" && (err == nil || !strings.Contains(err.Error(), tt.errorContains)) { - t.Errorf("Expected error to contain %q, got: %v", tt.errorContains, err) - } - - // Verify result is set when no error - if !tt.expectError && decision.Status.Result == nil { - t.Error("Expected result to be set but was nil") - } - }) - } -} - -// Error variable for testing -var errGathererFailed = errors.New("gatherer failed") - -func TestFilterWeigherPipelineController_PeekReadOnly(t *testing.T) { - makeRaw := func(readOnly bool) []byte { - r := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: api.NovaSpec{NumInstances: 1}}, - Options: scheduling.Options{ReadOnly: readOnly}, - } - raw, err := json.Marshal(r) - if err != nil { - panic(err) - } - return raw - } - - c := &FilterWeigherPipelineController{} - - tests := []struct { - name string - decision *v1alpha1.Decision - want bool - }{ - { - name: "nil NovaRaw defaults to exclusive lock", - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - }, - }, - want: false, - }, - { - name: "invalid JSON defaults to exclusive lock", - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - NovaRaw: &runtime.RawExtension{Raw: []byte("not-json")}, - }, - }, - want: false, - }, - { - name: "ReadOnly=false uses exclusive lock", - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - NovaRaw: &runtime.RawExtension{Raw: makeRaw(false)}, - }, - }, - want: false, - }, - { - name: "ReadOnly=true uses read lock", - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - PipelineRef: corev1.ObjectReference{Name: "test-pipeline"}, - NovaRaw: &runtime.RawExtension{Raw: makeRaw(true)}, - }, - }, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := c.peekReadOnly(tt.decision) - if got != tt.want { - t.Errorf("expected peekReadOnly = %v, got %v", tt.want, got) - } - }) - } -} diff --git a/internal/scheduling/nova/integration_test.go b/internal/scheduling/nova/integration_test.go index 157427e9b..2ff129ff6 100644 --- a/internal/scheduling/nova/integration_test.go +++ b/internal/scheduling/nova/integration_test.go @@ -15,7 +15,9 @@ import ( novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/weighers" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -247,9 +249,9 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec // Create the pipeline controller with shared monitor to avoid duplicate registration controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.PlacementRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: getSharedMonitor(), @@ -281,12 +283,17 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec controller.Pipelines[testPipeline.Name] = result.Pipeline controller.PipelineConfigs[testPipeline.Name] = testPipeline - // Create the HTTP API with the controller as delegate - skip metrics registration - // Set EvacuationShuffleK=0 to disable shuffle for deterministic test results + // Create the HTTP API backed by a real compute.Placement over the controller. + // Set EvacuationShuffleK=0 to disable shuffle for deterministic test results. + // KVM requests trigger candidate gathering; use a passthrough gatherer + // (nil gatheredHosts) so the test's preselected hosts are preserved. + scheduler := compute.NewPlacement(controller, &mockCandidateGatherer{}) api := &httpAPI{ - config: HTTPAPIConfig{EvacuationShuffleK: 0}, - monitor: lib.NewSchedulerMonitor(), // Create new monitor but don't register - delegate: controller, + config: HTTPAPIConfig{EvacuationShuffleK: 0}, + monitor: lib.NewSchedulerMonitor(), // Create new monitor but don't register + scheduler: scheduler, + vmHandler: &VirtualMachineHandler{Client: k8sClient}, + crRecorder: crs.Recorder{Client: k8sClient}, } // Create test server diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go index d223dfe5e..7a2aa17bc 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go @@ -10,23 +10,24 @@ import ( "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterAggregateMetadata struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Restrict hosts to specific projects if they are in an aggregate that has // the "filter_tenant_id" metadata key set. -func (s *FilterAggregateMetadata) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterAggregateMetadata) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Capacity probe calls are synthetic and not placed on behalf of a tenant // project, so they must ignore tenant pinning. Failover reservations DO respect // tenant pinning: a reservation on a host the VM's project cannot use is dead // weight and would be deleted at validation time. - if intent, err := request.GetIntent(); err == nil && intent == api.CapacityProbeIntent { + if request.Intent == api.CapacityProbeIntent { return result, nil } @@ -71,20 +72,20 @@ func (s *FilterAggregateMetadata) Run(traceLog logr.Logger, request api.External } for host, restrictedProjects := range restrictedProjectsByHost { - if !slices.Contains(restrictedProjects, request.Spec.Data.ProjectID) { + if !slices.Contains(restrictedProjects, request.ProjectID) { // Project is not allowed on this hypervisor, filter it out. delete(result.Activations, host) traceLog.V(1).Info( "filtering host not allowing project based on aggregate metadata", "host", host, - "project", request.Spec.Data.ProjectID, + "project", request.ProjectID, "restricted_projects", restrictedProjects, ) } else { traceLog.V(1).Info( "host allows project based on aggregate metadata, keeping", "host", host, - "project", request.Spec.Data.ProjectID, + "project", request.ProjectID, "restricted_projects", restrictedProjects, ) } diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go index 769d06729..c23479858 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go @@ -9,6 +9,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -20,20 +22,16 @@ import ( func TestFilterAggregateMetadata_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string }{ { name: "No aggregates with filter_tenant_id - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -64,13 +62,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host with filter_tenant_id matching project - host passes", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -94,13 +88,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host with filter_tenant_id not matching project - host filtered", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -124,13 +114,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host in multiple aggregates with different filter_tenant_id - project matches one", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -150,13 +136,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host in multiple aggregates - project matches none", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-c", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-c", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -176,13 +158,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Mixed hosts - some restricted, some open", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -220,13 +198,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{}, }, hypervisors: []hv1.Hypervisor{ { @@ -241,13 +215,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "All hosts filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-nonexistent", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-nonexistent", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -271,13 +241,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Host not in hypervisor list - passes (no restrictions found)", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, @@ -295,13 +261,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Aggregate with nil metadata - treated as no restriction", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -318,13 +280,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id - project matches one of many", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -341,13 +299,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id - project matches none", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-d", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-d", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -364,13 +318,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Comma-separated filter_tenant_id with surrounding whitespace - project matches", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -387,13 +337,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys - project matches one of the numbered keys", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-c", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-c", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -414,13 +360,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys with comma-separated values - project matches", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-e", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-e", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -441,13 +383,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { }, { name: "Numbered filter_tenant_id keys - project matches none", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-z", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-z", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -510,13 +448,9 @@ func TestFilterAggregateMetadata_Run(t *testing.T) { } func TestFilterAggregateMetadata_Run_ClientError(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request := compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, } @@ -575,14 +509,10 @@ func TestFilterAggregateMetadata_SkipsForCapacityProbeIntent(t *testing.T) { step := &FilterAggregateMetadata{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-y", - SchedulerHints: map[string]any{"_nova_check_type": "capacity_probe"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.PlacementRequest{ + ProjectID: "project-y", + Intent: api.CapacityProbeIntent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { @@ -612,18 +542,14 @@ func TestFilterAggregateMetadata_AppliesForFailoverIntents(t *testing.T) { &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, } - for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation"} { - t.Run(intent, func(t *testing.T) { + for _, intent := range []v1alpha1.SchedulingIntent{api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent} { + t.Run(string(intent), func(t *testing.T) { step := &FilterAggregateMetadata{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-y", - SchedulerHints: map[string]any{"_nova_check_type": intent}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.PlacementRequest{ + ProjectID: "project-y", + Intent: intent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go index 5e1e51b5a..ee0cac37f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go @@ -9,26 +9,27 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterAllowedProjectsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Lock certain hosts for certain projects, based on the hypervisor spec. // Note that hosts without specified projects are still accessible. -func (s *FilterAllowedProjectsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterAllowedProjectsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Capacity probe calls are synthetic and not placed on behalf of a tenant // project, so they must ignore tenant pinning. Failover reservations DO respect // tenant pinning: a reservation on a host the VM's project cannot use is dead // weight and would be deleted at validation time. - if intent, err := request.GetIntent(); err == nil && intent == api.CapacityProbeIntent { + if request.Intent == api.CapacityProbeIntent { return result, nil } - if request.Spec.Data.ProjectID == "" { + if request.ProjectID == "" { traceLog.V(1).Info("no project ID in request, skipping filter") return result, nil } @@ -48,13 +49,13 @@ func (s *FilterAllowedProjectsStep) Run(traceLog logr.Logger, request api.Extern traceLog.V(1).Info("host allows all projects, keeping", "host", hv.Name) continue } - if !slices.Contains(hv.Spec.AllowedProjects, request.Spec.Data.ProjectID) { + if !slices.Contains(hv.Spec.AllowedProjects, request.ProjectID) { // Project is not allowed on this hypervisor, filter it out. delete(result.Activations, hv.Name) traceLog.V(1).Info( "filtering host not allowing project", "host", hv.Name, - "project", request.Spec.Data.ProjectID, + "project", request.ProjectID, ) } } diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go index 755ea9903..3b4e9d60a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go @@ -8,6 +8,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -67,19 +69,15 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No project ID in request - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -90,13 +88,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches allowed projects on host2", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -107,13 +101,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches allowed projects on host3 only", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -124,13 +114,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project does not match any allowed projects", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-d", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-d", + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -141,13 +127,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Hosts without allowed projects are accessible", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-x", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-x", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -157,13 +139,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Mixed hosts - some with empty allowed projects, some with matching project", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -175,13 +153,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "All hosts filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-nonexistent", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-nonexistent", + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -192,26 +166,18 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Project matches on multiple hosts with allowed projects", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-a", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-a", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -224,13 +190,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Project matches second item in allowed projects list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-b", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-b", + Hosts: []compute.Host{ {ComputeHost: "host2"}, }, }, @@ -239,13 +201,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "All hosts have empty allowed projects", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "any-project", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "any-project", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -255,13 +213,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Mixed allowed projects configuration", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-c", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "project-c", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -274,13 +228,9 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }, { name: "Case sensitive project matching", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "Project-A", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ProjectID: "Project-A", + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, }, @@ -340,14 +290,10 @@ func TestFilterAllowedProjectsStep_SkipsForCapacityProbeIntent(t *testing.T) { step := &FilterAllowedProjectsStep{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-y", - SchedulerHints: map[string]any{"_nova_check_type": "capacity_probe"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.PlacementRequest{ + ProjectID: "project-y", + Intent: api.CapacityProbeIntent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { @@ -372,18 +318,14 @@ func TestFilterAllowedProjectsStep_AppliesForFailoverIntents(t *testing.T) { }, } - for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation"} { - t.Run(intent, func(t *testing.T) { + for _, intent := range []v1alpha1.SchedulingIntent{api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent} { + t.Run(string(intent), func(t *testing.T) { step := &FilterAllowedProjectsStep{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-y", - SchedulerHints: map[string]any{"_nova_check_type": intent}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.PlacementRequest{ + ProjectID: "project-y", + Intent: intent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_capabilities.go b/internal/scheduling/nova/plugins/filters/filter_capabilities.go index 31277eeaa..392ef4d06 100644 --- a/internal/scheduling/nova/plugins/filters/filter_capabilities.go +++ b/internal/scheduling/nova/plugins/filters/filter_capabilities.go @@ -7,15 +7,14 @@ import ( "context" "fmt" "github.com/go-logr/logr" - "strings" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterCapabilitiesStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Get the provided capabilities of a hypervisor resource in the format Nova expects. @@ -45,46 +44,14 @@ func hvToNovaCapabilities(hv hv1.Hypervisor) (map[string]string, error) { // Check the capabilities of each host and if they match the extra spec provided // in the request spec flavor. -func (s *FilterCapabilitiesStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterCapabilitiesStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - extraSpecs := request.Spec.Data.Flavor.Data.ExtraSpecs - if len(extraSpecs) == 0 { - traceLog.V(1).Info("no flavor extra spec capabilities in request, skipping filter") - return result, nil - } - - // Extract only capability-related extra specs into a separate map. - // We must not modify the original ExtraSpecs map as it's shared across filters. - requestedCapabilities := make(map[string]string) - for key, value := range extraSpecs { - if strings.HasPrefix(key, "capabilities:") { - requestedCapabilities[key] = value - } - } + requestedCapabilities := request.RequestedCapabilities if len(requestedCapabilities) == 0 { - traceLog.V(1).Info("no capabilities in flavor extra specs, skipping filter") + traceLog.V(1).Info("no capabilities requested, skipping filter") return result, nil } - // Note: currently none of the advanced operators for capabilities are - // supported because they are not used by any of our flavors in production. - // Ops: https://github.com/sapcc/nova/blob/3ebf80/nova/scheduler/filters/extra_specs_ops.py#L23 - unsupportedOps := []string{ - "=", "", "", "==", "!=", ">=", "<=", - "s==", "s!=", "s<", "s<=", "s>", "s>=", "", // or is special - } - for key, expr := range requestedCapabilities { - for _, op := range unsupportedOps { - if strings.Contains(expr, op) { - traceLog.V(1).Info( - "unsupported extra spec operator in capabilities filter, skipping filter", - "key", key, "expr", expr, "flavor", request.Spec.Data.Flavor, - ) - return result, nil - } - } - } - hvs := &hv1.HypervisorList{} if err := s.Client.List(context.Background(), hvs); err != nil { traceLog.Error(err, "failed to list hypervisors") @@ -118,11 +85,12 @@ func (s *FilterCapabilitiesStep) Run(traceLog logr.Logger, request api.ExternalS continue } // Check if the provided capabilities match the requested ones. - for keyRequested, valueRequested := range requestedCapabilities { - if providedValue, ok := provided[keyRequested]; !ok || providedValue != valueRequested { + for _, requested := range requestedCapabilities { + key := "capabilities:" + requested.Name + if providedValue, ok := provided[key]; !ok || providedValue != requested.Value { traceLog.V(1).Info( "filtering host with mismatched capabilities", "host", host, - "wantKey", keyRequested, "wantValue", valueRequested, + "wantKey", key, "wantValue", requested.Value, "haveKey?", ok, "haveValue", providedValue, ) delete(result.Activations, host) diff --git a/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go b/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go index 622932bb8..ce3000d95 100644 --- a/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -225,24 +225,15 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string expectedEvents []lib.FilterWeigherPipelineStepEvent }{ { name: "No extra specs in request - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -253,20 +244,8 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Non-capability extra specs - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "hw:mem_page_size": "large", - "hw:cpu_policy": "dedicated", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -276,19 +255,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match CH hypervisor type", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -299,19 +270,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match QEMU hypervisor type", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "QEMU"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -323,19 +286,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match x86_64 CPU architecture", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:cpu_arch": "x86_64", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "cpu_arch", Value: "x86_64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -347,19 +302,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match aarch64 CPU architecture", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:cpu_arch": "aarch64", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "cpu_arch", Value: "aarch64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -371,20 +318,12 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match both hypervisor type and CPU architecture", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - "capabilities:cpu_arch": "x86_64", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, + {Name: "cpu_arch", Value: "x86_64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -396,20 +335,12 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Match QEMU with aarch64", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "QEMU", - "capabilities:cpu_arch": "aarch64", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "QEMU"}, + {Name: "cpu_arch", Value: "aarch64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -421,19 +352,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Ignore hypervisors not in request even with unknown type", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -443,19 +366,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Candidate hypervisor with empty hypervisor type emits event and is filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host-empty-type"}, }, }, @@ -467,19 +382,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Candidate hypervisor with unknown hypervisor type emits event and is filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host-unknown-type"}, }, }, @@ -491,19 +398,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "No matching hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "KVM", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "KVM"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -513,21 +412,12 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Mixed capability and non-capability extra specs", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - "hw:mem_page_size": "large", - "capabilities:cpu_arch": "x86_64", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, + {Name: "cpu_arch", Value: "x86_64"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -538,38 +428,22 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "CH", - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "CH"}, }, - Hosts: []api.ExternalSchedulerHost{}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Case sensitive capability matching", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "ch", // lowercase should not match - }, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedCapabilities: []compute.Capability{ + {Name: "hypervisor_type", Value: "ch"}, // lowercase should not match }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -579,19 +453,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Unsupported operator in extra specs - skip filter", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:cpu_arch": "s>=x86_64", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + // An unsupported comparison operator is dropped by the adapter, + // so the filter sees no requested capabilities and is a no-op. + RequestedCapabilities: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -602,19 +468,11 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "Unsupported operator - skip filter", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": " CH QEMU", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + // An unsupported comparison operator is dropped by the adapter, + // so the filter sees no requested capabilities and is a no-op. + RequestedCapabilities: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -624,17 +482,8 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, { name: "All hosts match when no capabilities requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -701,78 +550,3 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }) } } - -// TestFilterCapabilitiesStep_DoesNotMutateExtraSpecs verifies that the filter -// does not modify the original ExtraSpecs map from the request. This is important -// because subsequent filters (like filter_has_requested_traits) also need to access -// the full ExtraSpecs including non-capability keys like trait:*. -func TestFilterCapabilitiesStep_DoesNotMutateExtraSpecs(t *testing.T) { - scheme := runtime.NewScheme() - err := hv1.AddToScheme(scheme) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - hvs := []client.Object{ - &hv1.Hypervisor{ - ObjectMeta: v1.ObjectMeta{Name: "host1"}, - Status: hv1.HypervisorStatus{ - DomainCapabilities: hv1.DomainCapabilities{HypervisorType: "ch"}, - Capabilities: hv1.Capabilities{HostCpuArch: "x86_64"}, - }, - }, - } - - // Create a request with mixed extra specs (capabilities + traits + other) - originalExtraSpecs := map[string]string{ - "capabilities:hypervisor_type": "CH", - "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "forbidden", - "trait:CUSTOM_HW_SAPPHIRE_RAPIDS": "required", - "hw:mem_page_size": "large", - "hw:numa_nodes": "1", - } - - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: originalExtraSpecs, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}}, - } - - step := &FilterCapabilitiesStep{} - step.Client = fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(hvs...). - Build() - - _, err = step.Run(logr.Discard(), request) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - // Verify that the original ExtraSpecs map was NOT mutated - expectedKeys := []string{ - "capabilities:hypervisor_type", - "trait:CUSTOM_HANA_EXCLUSIVE_HOST", - "trait:CUSTOM_HW_SAPPHIRE_RAPIDS", - "hw:mem_page_size", - "hw:numa_nodes", - } - - if len(request.Spec.Data.Flavor.Data.ExtraSpecs) != len(expectedKeys) { - t.Errorf("ExtraSpecs was mutated: expected %d keys, got %d", - len(expectedKeys), len(request.Spec.Data.Flavor.Data.ExtraSpecs)) - } - - for _, key := range expectedKeys { - if _, ok := request.Spec.Data.Flavor.Data.ExtraSpecs[key]; !ok { - t.Errorf("ExtraSpecs was mutated: key %q was deleted", key) - } - } -} diff --git a/internal/scheduling/nova/plugins/filters/filter_correct_az.go b/internal/scheduling/nova/plugins/filters/filter_correct_az.go index 882cf261c..730ca968d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_correct_az.go +++ b/internal/scheduling/nova/plugins/filters/filter_correct_az.go @@ -7,20 +7,20 @@ import ( "context" "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" ) type FilterCorrectAZStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Only get hosts in the requested az. -func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - if request.Spec.Data.AvailabilityZone == "" { + if request.AvailabilityZone == "" { traceLog.V(1).Info("no availability zone requested, skipping filter_correct_az step") return result, nil } @@ -42,7 +42,7 @@ func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request api.ExternalSche traceLog.V(1).Info("host missing zone label, keeping", "host", hv.Name) continue } - if az == request.Spec.Data.AvailabilityZone { + if az == request.AvailabilityZone { // We always assume the name of the resource corresponds // to the compute host name. computeHostsInAZ[hv.Name] = struct{}{} @@ -51,7 +51,7 @@ func (s *FilterCorrectAZStep) Run(traceLog logr.Logger, request api.ExternalSche traceLog.V(1).Info( "hosts inside requested az", - "availabilityZone", request.Spec.Data.AvailabilityZone, + "availabilityZone", request.AvailabilityZone, "hosts", computeHostsInAZ, ) for host := range result.Activations { diff --git a/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go b/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go index 135766806..8901fa81d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_correct_az_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -57,19 +57,15 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "Filter hosts in az-1", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-1", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AvailabilityZone: "az-1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -82,13 +78,9 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Filter hosts in az-2", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-2", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AvailabilityZone: "az-2", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -100,13 +92,9 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Filter hosts in az-3", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-3", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AvailabilityZone: "az-3", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -118,13 +106,9 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "No hosts in requested AZ", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-nonexistent", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AvailabilityZone: "az-nonexistent", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -135,26 +119,18 @@ func TestFilterCorrectAZStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-1", - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + AvailabilityZone: "az-1", + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Host not in database", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - AvailabilityZone: "az-1", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AvailabilityZone: "az-1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go index c35a63cff..de1241fa8 100644 --- a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go +++ b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts.go @@ -6,7 +6,7 @@ package filters import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -14,7 +14,7 @@ import ( // This step is useful to circumvent scheduling issues on specific hosts by // excluding them from all scheduling decisions. type FilterExcludeHostsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterExcludeHostsStepOpts] + lib.BaseFilter[compute.PlacementRequest, FilterExcludeHostsStepOpts] } type FilterExcludeHostsStepOpts struct { @@ -29,7 +29,7 @@ func (opts FilterExcludeHostsStepOpts) Validate() error { return nil } func (s *FilterExcludeHostsStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) diff --git a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go index 45676b731..6a4256f4b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_exclude_hosts_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -15,15 +15,15 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { tests := []struct { name string excludedHosts []string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No excluded hosts - all hosts pass", excludedHosts: []string{}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -35,8 +35,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single excluded host", excludedHosts: []string{"host2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -48,8 +48,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Multiple excluded hosts", excludedHosts: []string{"host1", "host3"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -62,8 +62,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "All hosts excluded", excludedHosts: []string{"host1", "host2", "host3"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -75,8 +75,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Excluded host not in request - no effect", excludedHosts: []string{"host-nonexistent"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -87,8 +87,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Some excluded hosts in request, some not", excludedHosts: []string{"host1", "host-nonexistent", "host3"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -100,8 +100,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Empty host list in request", excludedHosts: []string{"host1", "host2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, @@ -109,8 +109,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Nil excluded hosts list", excludedHosts: nil, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -121,8 +121,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Duplicate excluded hosts", excludedHosts: []string{"host1", "host1", "host2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -134,8 +134,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single host request - excluded", excludedHosts: []string{"host1"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -145,8 +145,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Single host request - not excluded", excludedHosts: []string{"host2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -156,8 +156,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Case sensitive matching", excludedHosts: []string{"Host1", "HOST2"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "Host1"}, @@ -169,8 +169,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Hosts with special characters", excludedHosts: []string{"nova-compute-bb123.region.example.com"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "nova-compute-bb123.region.example.com"}, {ComputeHost: "nova-compute-bb124.region.example.com"}, }, @@ -181,8 +181,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Empty string in excluded hosts", excludedHosts: []string{"", "host1"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -193,8 +193,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { { name: "Large number of excluded hosts", excludedHosts: []string{"host1", "host2", "host3", "host4", "host5", "host6", "host7", "host8", "host9", "host10"}, - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, {ComputeHost: "host10"}, @@ -210,8 +210,8 @@ func TestFilterExcludeHostsStep_Run(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { step := &FilterExcludeHostsStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, FilterExcludeHostsStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, FilterExcludeHostsStepOpts]{}, + BaseFilter: lib.BaseFilter[compute.PlacementRequest, FilterExcludeHostsStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, FilterExcludeHostsStepOpts]{}, }, } step.Options = FilterExcludeHostsStepOpts{ diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer.go b/internal/scheduling/nova/plugins/filters/filter_external_customer.go index 19a412797..e724e6c4d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer.go @@ -12,6 +12,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -29,29 +30,29 @@ func (opts FilterExternalCustomerStepOpts) Validate() error { } type FilterExternalCustomerStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterExternalCustomerStepOpts] + lib.BaseFilter[compute.PlacementRequest, FilterExternalCustomerStepOpts] } // Prefix-match the domain name for external customer domains and filter out hosts // that are not intended for external customers. -func (s *FilterExternalCustomerStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterExternalCustomerStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Failover and capacity probe calls are not associated with a user domain; // domain-based host restrictions don't apply. CR scheduling does carry a real domain // and must be checked. - if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + if slices.Contains([]v1alpha1.SchedulingIntent{ api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent, api.CapacityProbeIntent, - }, intent) { - traceLog.V(1).Info("skipping external customer filter for non-placement intent", "intent", intent) + }, request.Intent) { + traceLog.V(1).Info("skipping external customer filter for non-placement intent", "intent", request.Intent) return result, nil } - domainName, err := request.Spec.Data.GetSchedulerHintStr("domain_name") - if err != nil { - traceLog.Error(err, "failed to get domain_name scheduler hint, skipping filter") + domainName := request.DomainName + if domainName == "" { + traceLog.V(1).Info("no domain name in request, skipping filter") return result, nil } if slices.Contains(s.Options.CustomerIgnoredDomainNames, domainName) { diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go index 4d2ebe784..fe765695e 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go @@ -8,6 +8,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -77,7 +79,7 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { tests := []struct { name string opts FilterExternalCustomerStepOpts - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string expectError bool @@ -87,15 +89,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "ext-customer1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -110,15 +106,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "internal-customer", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "internal-customer", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -133,15 +123,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"external-", "ext-", "customer-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "customer-abc", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "customer-abc", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -156,15 +140,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { CustomerDomainNamePrefixes: []string{"ext-"}, CustomerIgnoredDomainNames: []string{"ext-special-domain"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-special-domain", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "ext-special-domain", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -178,15 +156,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer2", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "ext-customer2", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host5"}, @@ -200,15 +172,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer3", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "ext-customer3", + Hosts: []compute.Host{ {ComputeHost: "host3"}, {ComputeHost: "host4"}, }, @@ -221,15 +187,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + DomainName: "ext-customer", + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, @@ -239,15 +199,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": []any{"ext-customer", "other"}, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "ext-customer", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -260,13 +214,8 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -279,13 +228,8 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -298,15 +242,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "EXT-customer", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "EXT-customer", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -320,15 +258,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "ext", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -342,15 +274,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { CustomerDomainNamePrefixes: []string{"ext-"}, CustomerIgnoredDomainNames: []string{"ext-test", "ext-dev", "ext-staging"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-dev", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "ext-dev", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -363,16 +289,10 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForFailoverIntent), - "domain_name": "ext-customer1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.ReserveForFailoverIntent, + DomainName: "ext-customer1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -385,15 +305,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForCommittedResourceIntent), - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.ReserveForCommittedResourceIntent, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -406,16 +320,10 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForCommittedResourceIntent), - "domain_name": "ext-customer1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.ReserveForCommittedResourceIntent, + DomainName: "ext-customer1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -429,15 +337,9 @@ func TestFilterExternalCustomerStep_Run(t *testing.T) { opts: FilterExternalCustomerStepOpts{ CustomerDomainNamePrefixes: []string{"ext-"}, }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "ext-customer1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + DomainName: "ext-customer1", + Hosts: []compute.Host{ {ComputeHost: "host-custom-trait"}, {ComputeHost: "host4"}, }, @@ -565,16 +467,10 @@ func TestFilterExternalCustomerStep_SkipsForNonPlacementIntent(t *testing.T) { for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { t.Run(intent, func(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "domain_name": "iaas-customer", - "_nova_check_type": intent, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + request := compute.PlacementRequest{ + Intent: v1alpha1.SchedulingIntent(intent), + DomainName: "iaas-customer", + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go b/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go index ad3c20288..8b27f50f1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go @@ -8,20 +8,19 @@ import ( "github.com/go-logr/logr" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterHasAcceleratorsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // If requested, only get hosts with accelerators. -func (s *FilterHasAcceleratorsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHasAcceleratorsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - extraSpecs := request.Spec.Data.Flavor.Data.ExtraSpecs - if _, ok := extraSpecs["accel:device_profile"]; !ok { + if !request.AcceleratorsRequested { traceLog.V(1).Info("no accelerators requested, skipping filter") return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go b/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go index 2c92cf81b..57bef26fc 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_accelerators_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -67,23 +67,14 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No accelerators requested - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -95,19 +86,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - filter to hosts with COMPUTE_ACCELERATORS trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -119,19 +100,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested with different device profile value", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "fpga-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, {ComputeHost: "host5"}, @@ -142,19 +113,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - all hosts have the trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host5"}, @@ -165,19 +126,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested - no hosts have the trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host3"}, {ComputeHost: "host4"}, }, @@ -187,20 +138,8 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Other extra specs present but no accelerator request - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "hw:cpu_policy": "dedicated", - "quota:vif_inbound": "100000", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -211,19 +150,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators requested with empty device profile value", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -234,55 +163,26 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Empty host list with accelerators requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Empty host list without accelerators requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Mixed hosts with and without accelerators trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "custom-accelerator", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -295,21 +195,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Accelerators with additional extra specs", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - "hw:cpu_policy": "dedicated", - "hw:mem_page_size": "large", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -319,19 +207,9 @@ func TestFilterHasAcceleratorsStep_Run(t *testing.T) { }, { name: "Host not in database with accelerators requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "accel:device_profile": "gpu-profile", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + AcceleratorsRequested: true, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index 6ba228408..345c59b90 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -12,8 +12,9 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" - resv "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + resv "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -36,7 +37,7 @@ type FilterHasEnoughCapacityOpts struct { func (FilterHasEnoughCapacityOpts) Validate() error { return nil } type FilterHasEnoughCapacity struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterHasEnoughCapacityOpts] + lib.BaseFilter[compute.PlacementRequest, FilterHasEnoughCapacityOpts] } // Filter hosts that don't have enough capacity to run the requested flavor. @@ -63,7 +64,7 @@ type FilterHasEnoughCapacity struct { // known at this point. // // Please also note that disk space is currently not considered by this filter. -func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { opts := request.GetOptions() result := s.IncludeAllHostsFromRequest(request) @@ -157,22 +158,21 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // Check if this is a CR reservation scheduling request. // If so, we should NOT unlock any CR reservations to prevent overbooking. // CR capacity should only be unlocked for actual VM scheduling. - intent, err := request.GetIntent() switch { - case err == nil && intent == api.ReserveForCommittedResourceIntent: + case request.Intent == api.ReserveForCommittedResourceIntent: traceLog.V(1).Info("keeping CR reservation locked for CR reservation scheduling", "reservation", reservation.Name, - "intent", intent) + "intent", request.Intent) // Don't continue - fall through to block the resources case !s.Options.LockReserved && !opts.LockReservations && // For committed resource reservations: unlock resources only if: // 1. Project ID matches // 2. ResourceGroup matches the flavor's hw_version - reservation.Spec.CommittedResourceReservation.MatchesGroup(request.Spec.Data.ProjectID, request.Spec.Data.Flavor.Data.ExtraSpecs["quota:hw_version"]): + reservation.Spec.CommittedResourceReservation.MatchesGroup(request.ProjectID, request.HardwareVersion): traceLog.V(1).Info("unlocking resources reserved by matching committed resource reservation with allocation", "reservation", reservation.Name, - "instanceUUID", request.Spec.Data.InstanceUUID, - "projectID", request.Spec.Data.ProjectID, + "instanceUUID", request.InstanceUUID, + "projectID", request.ProjectID, "resourceGroup", reservation.Spec.CommittedResourceReservation.ResourceGroup) continue } @@ -184,25 +184,22 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // 1. Failover reservations are specifically for HA/evacuation scenarios. // 2. During live migrations or other operations, we don't want to use failover capacity. // Note: we cannot use failover reservations from other VMs, as that can invalidate our HA guarantees. - intent, err := request.GetIntent() - if err == nil { - switch intent { - case api.EvacuateIntent: - if reservation.Status.FailoverReservation != nil { - if _, contained := reservation.Status.FailoverReservation.Allocations[request.Spec.Data.InstanceUUID]; contained { - traceLog.V(1).Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", - "reservation", reservation.Name, - "instanceUUID", request.Spec.Data.InstanceUUID) - continue - } + switch request.Intent { + case api.EvacuateIntent: + if reservation.Status.FailoverReservation != nil { + if _, contained := reservation.Status.FailoverReservation.Allocations[request.InstanceUUID]; contained { + traceLog.V(1).Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", + "reservation", reservation.Name, + "instanceUUID", request.InstanceUUID) + continue } - case api.ReuseFailoverReservationIntent: - // Reuse check: the reservation already pre-blocks the right capacity for this VM. - // Don't subtract it from free capacity to avoid double-counting. - traceLog.V(1).Info("skipping failover reservation block for reuse compatibility check", - "reservation", reservation.Name) - continue } + case api.ReuseFailoverReservationIntent: + // Reuse check: the reservation already pre-blocks the right capacity for this VM. + // Don't subtract it from free capacity to avoid double-counting. + traceLog.V(1).Info("skipping failover reservation block for reuse compatibility check", + "reservation", reservation.Name) + continue } traceLog.V(1).Info("processing failover reservation", "reservation", reservation.Name) } @@ -276,7 +273,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External hostsEncountered[host] = struct{}{} // Check cpu capacity. - if request.Spec.Data.Flavor.Data.VCPUs == 0 { + if request.Flavor.VCPUs == 0 { return nil, errors.New("flavor has 0 vcpus") } freeCPU, ok := free["cpu"] @@ -291,11 +288,11 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // Calculate how many instances can fit on this host, based on cpu. //nolint:gosec // We're checking for underflows above (< 0). vcpuSlots := uint64(freeCPU.Value()) / - request.Spec.Data.Flavor.Data.VCPUs - if vcpuSlots < request.Spec.Data.NumInstances { + request.Flavor.VCPUs + if vcpuSlots < request.NumInstances { traceLog.V(1).Info( "filtering host due to insufficient CPU capacity", - "host", host, "requested", request.Spec.Data.Flavor.Data.VCPUs, + "host", host, "requested", request.Flavor.VCPUs, "available", freeCPU.String(), ) delete(result.Activations, host) @@ -303,7 +300,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External } // Check memory capacity. - if request.Spec.Data.Flavor.Data.MemoryMB == 0 { + if request.Flavor.MemoryMB == 0 { return nil, errors.New("flavor has 0 memory") } freeMemory, ok := free["memory"] @@ -320,11 +317,11 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External // are in mebibytes (1 MiB = 2^20 bytes). Multiply by 1<<20 to convert to bytes. //nolint:gosec // We're checking for underflows above (< 0). memorySlots := uint64(freeMemory.Value()) / - (request.Spec.Data.Flavor.Data.MemoryMB * (1 << 20)) - if memorySlots < request.Spec.Data.NumInstances { + (request.Flavor.MemoryMB * (1 << 20)) + if memorySlots < request.NumInstances { traceLog.V(1).Info( "filtering host due to insufficient RAM capacity", - "host", host, "requested_mb", request.Spec.Data.Flavor.Data.MemoryMB, + "host", host, "requested_mb", request.Flavor.MemoryMB, "available_mb", freeMemory.String(), ) delete(result.Activations, host) @@ -332,9 +329,9 @@ func (s *FilterHasEnoughCapacity) Run(traceLog logr.Logger, request api.External } traceLog.V(1).Info( "host has enough capacity", "host", host, - "requested_cpus", request.Spec.Data.Flavor.Data.VCPUs, + "requested_cpus", request.Flavor.VCPUs, "available_cpus", freeCPU.String(), - "requested_memory_mb", request.Spec.Data.Flavor.Data.MemoryMB, + "requested_memory_mb", request.Flavor.MemoryMB, "available_memory", freeMemory.String(), ) } diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go index c4b4cec2c..169f845f7 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go @@ -10,6 +10,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -229,58 +230,46 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { - return newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup, vcpus, memory, "", evacuation, hosts) +func newNovaRequest(instanceUUID, projectID, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) compute.PlacementRequest { + return newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup, vcpus, memory, "", evacuation, hosts) } // newNovaRequestWithIntent creates a nova request with a specific intent. // intentHint can be: "evacuate", "reserve_for_committed_resource", "reserve_for_failover", or "" for create. -func newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory, intentHint string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newNovaRequestWithIntent(instanceUUID, projectID, flavorGroup string, vcpus int, memory, intentHint string, evacuation bool, hosts []string) compute.PlacementRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + hostList[i] = compute.Host{ComputeHost: h} } - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", - "quota:hw_version": flavorGroup, - } - - var schedulerHints map[string]any - if evacuation { - schedulerHints = map[string]any{ - "_nova_check_type": []any{"evacuate"}, - } - } else if intentHint != "" { - schedulerHints = map[string]any{ - "_nova_check_type": intentHint, - } + var intent v1alpha1.SchedulingIntent + switch { + case evacuation: + intent = api.EvacuateIntent + case intentHint == "reserve_for_committed_resource": + intent = api.ReserveForCommittedResourceIntent + case intentHint == "reserve_for_failover": + intent = api.ReserveForFailoverIntent } memoryMB := parseMemoryToMB(memory) - spec := api.NovaSpec{ - ProjectID: projectID, - InstanceUUID: instanceUUID, - NumInstances: 1, - SchedulerHints: schedulerHints, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: flavorName, - VCPUs: uint64(vcpus), //nolint:gosec // test code - MemoryMB: memoryMB, - ExtraSpecs: extraSpecs, - }, - }, - } - weights := make(map[string]float64) for _, h := range hosts { weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.PlacementRequest{ + ProjectID: projectID, + InstanceUUID: instanceUUID, + NumInstances: 1, + Intent: intent, + HardwareVersion: flavorGroup, + HypervisorFamily: compute.HypervisorFamilyKVM, + Flavor: compute.FlavorSpec{ + VCPUs: uint64(vcpus), //nolint:gosec // test code + MemoryMB: memoryMB, + }, Hosts: hostList, Weights: weights, } @@ -343,7 +332,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.PlacementRequest opts FilterHasEnoughCapacityOpts expectedHosts []string filteredHosts []string @@ -351,7 +340,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { { name: "No reservations - all hosts with capacity pass", reservations: []*v1alpha1.Reservation{}, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -362,7 +351,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-1", "host1", "project-A", "m1.large", "gp-1", "8", "16Gi", nil), newCommittedReservation("res-2", "host2", "project-A", "m1.large", "gp-1", "4", "8Gi", nil), }, - request: newNovaRequest("instance-123", "project-B", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-B", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -373,7 +362,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-1 half used", "host1", "project-A", "m1.large", "gp-1", "8", "16Gi", map[string]v1alpha1.CommittedResourceAllocation{"vm-1": makeAlloc("vm-1", "4", "8Gi")}), newCommittedReservation("res-2 fully used", "host2", "project-A", "m1.large", "gp-1", "4", "8Gi", map[string]v1alpha1.CommittedResourceAllocation{"vm-2": makeAlloc("vm-2", "4", "8Gi")}), }, - request: newNovaRequest("instance-123", "project-B", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-B", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -392,7 +381,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { }, ), }, - request: newNovaRequest("instance-123", "project-B", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-B", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, // host1: 5cpu/10Gi free → 4cpu/8Gi request passes filteredHosts: []string{"host4"}, @@ -404,7 +393,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newMigratingReservation("res-2", "host1", "host2", "project-A", "m1.large", "gp-1", "2", "4Gi", nil), // migration reservation from host1 to host2 newMigratingReservation("res-3", "host2", "host1", "project-A", "m1.large", "gp-1", "2", "4Gi", map[string]v1alpha1.CommittedResourceAllocation{"vm-1": makeAlloc("vm-1", "2", "4Gi")}), // migration reservation from host2 to host1 }, - request: newNovaRequest("instance-123", "project-B", "m1.small", "gp-1", 2, "4Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-B", "gp-1", 2, "4Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -418,7 +407,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-3-used", "host1", "project-A", "some flavor", "gp-1", "2", "4Gi", map[string]v1alpha1.CommittedResourceAllocation{"vm-2": makeAlloc("vm-2", "2", "4Gi")}), // used reservation newCommittedReservation("res-4", "host2", "project-A", "some flavor", "gp-2", "4", "8Gi", nil), // different flavor group, should still block }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host3"}, filteredHosts: []string{"host4", "host2"}, @@ -429,7 +418,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-1", "host1", "project-A", "m1.large", "gp-1", "8", "16Gi", nil), newCommittedReservation("res-2", "host3", "project-A", "m1.large", "gp-1", "16", "32Gi", nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: true}, expectedHosts: []string{"host2"}, filteredHosts: []string{"host1", "host3", "host4"}, @@ -439,7 +428,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newCommittedReservation("legacy-res", "host1", "project-A", "m1.large", "gp-1", "4", "8Gi", nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, @@ -451,7 +440,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newCommittedReservation("res-2", "host2", "project-X", "m1.xlarge", "gp-1", "4", "8Gi", nil), newCommittedReservation("res-3", "host3", "project-X", "m1.xlarge", "gp-1", "16", "32Gi", nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{}, filteredHosts: []string{"host1", "host2", "host3", "host4"}, @@ -461,7 +450,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newMigratingReservation("pending-res", "host1", "", "project-X", "m1.large", "gp-1", "8", "16Gi", nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2", "host3"}, // host1 blocked by pending reservation filteredHosts: []string{"host1", "host4"}, @@ -472,7 +461,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newMigratingReservation("pending-res", "host1", "", "project-X", "m1.large", "gp-1", "8", "16Gi", nil), newMigratingReservation("placed-res", "host2", "host3", "project-X", "m1.large", "gp-1", "4", "8Gi", nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, // host1 blocked by pending, host2 blocked by placed, host3 still has capacity filteredHosts: []string{"host1", "host2", "host4"}, @@ -482,7 +471,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newCommittedReservation("no-host-res", "", "project-X", "m1.large", "gp-1", "8", "16Gi", nil), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -498,7 +487,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { map[string]v1alpha1.CommittedResourceAllocation{"vm-1": makeAlloc("vm-1", "6", "10Gi")}, ), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 9, "17Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 9, "17Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, // host1: 8 free CPU < 9, 16Gi < 17Gi → filtered; host2: 4 < 9 → filtered; host4: no capacity → filtered expectedHosts: []string{"host3"}, @@ -514,7 +503,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { map[string]v1alpha1.CommittedResourceAllocation{"vm-1": makeAlloc("vm-1", "8", "16Gi")}, ), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 3, "5Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 3, "5Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -525,7 +514,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newFailoverReservation("failover-1", "host1", "8", "16Gi", map[string]string{"instance-123": "host5"}), newFailoverReservation("failover-2", "host2", "4", "8Gi", map[string]string{"instance-123": "host6"}), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -536,7 +525,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newFailoverReservation("failover-1", "host1", "4", "8Gi", map[string]string{"instance-123": "host5"}), newFailoverReservation("failover-2", "host2", "4", "8Gi", map[string]string{"instance-123": "host6"}), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -547,7 +536,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newFailoverReservation("failover-1", "host1", "8", "16Gi", map[string]string{"other-instance": "host5"}), newFailoverReservation("failover-2", "host2", "4", "8Gi", map[string]string{"another-instance": "host6"}), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -557,7 +546,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newFailoverReservation("failover-1", "host1", "8", "16Gi", map[string]string{}), }, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2", "host3"}, filteredHosts: []string{"host1"}, @@ -567,7 +556,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newFailoverReservation("failover-1", "host1", "4", "8Gi", map[string]string{"instance-111": "host5", "instance-222": "host6", "instance-333": "host7"}), }, - request: newNovaRequest("instance-222", "project-A", "m1.large", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-222", "project-A", "gp-1", 4, "8Gi", true, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{"host4"}, @@ -580,7 +569,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { reservations: []*v1alpha1.Reservation{ newUnconfirmedReservation("unconfirmed-res", "host1", "project-X", "gp-1", "8", "16Gi"), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2", "host3"}, filteredHosts: []string{"host1", "host4"}, @@ -593,7 +582,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { newUnconfirmedReservation("unconfirmed-1", "host1", "project-X", "gp-1", "8", "16Gi"), newUnconfirmedReservation("unconfirmed-2", "host2", "project-X", "gp-1", "4", "8Gi"), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2", "host4"}, @@ -618,7 +607,7 @@ func TestFilterHasEnoughCapacity_ReservationTypes(t *testing.T) { Status: v1alpha1.ReservationStatus{}, }, }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2", "host3"}, filteredHosts: []string{"host1", "host4"}, @@ -708,7 +697,7 @@ func TestFilterHasEnoughCapacity_LockReservations(t *testing.T) { }, } - request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}) + request := newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { step := &FilterHasEnoughCapacity{} @@ -752,7 +741,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name string hypervisors []*hv1.Hypervisor reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.PlacementRequest ignoredReservationTypes []v1alpha1.ReservationType expectedHosts []string filteredHosts []string @@ -763,7 +752,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Two hosts: No ignore - both hosts blocked by reservations", hypervisors: twoHostHVs, reservations: twoHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), ignoredReservationTypes: nil, expectedHosts: []string{}, filteredHosts: []string{"host1", "host2"}, @@ -772,7 +761,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Two hosts: Ignore CR only - host1 passes, host2 blocked by failover", hypervisors: twoHostHVs, reservations: twoHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, expectedHosts: []string{"host1"}, filteredHosts: []string{"host2"}, @@ -781,7 +770,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Two hosts: Ignore Failover only - host2 passes, host1 blocked by CR", hypervisors: twoHostHVs, reservations: twoHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host2"}, filteredHosts: []string{"host1"}, @@ -790,7 +779,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Two hosts: Ignore both - both hosts pass", hypervisors: twoHostHVs, reservations: twoHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource, v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, @@ -803,7 +792,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Large VM (12 CPU): No ignore - blocked (6 free < 12 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 12, "24Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 12, "24Gi", false, []string{"host1"}), ignoredReservationTypes: nil, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -812,7 +801,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Large VM (12 CPU): Ignore CR - blocked (10 free < 12 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 12, "24Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 12, "24Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -821,7 +810,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Large VM (12 CPU): Ignore Failover - blocked (8 free < 12 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 12, "24Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 12, "24Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -830,7 +819,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Large VM (12 CPU): Ignore both - passes (12 free = 12 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 12, "24Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 12, "24Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource, v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -841,7 +830,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Failover-size VM (10 CPU): No ignore - blocked (6 free < 10 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), ignoredReservationTypes: nil, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -850,7 +839,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Failover-size VM (10 CPU): Ignore CR - passes (10 free = 10 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -859,7 +848,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Failover-size VM (10 CPU): Ignore Failover - blocked (8 free < 10 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -868,7 +857,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, Failover-size VM (10 CPU): Ignore both - passes (12 free > 10 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource, v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -879,7 +868,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, CR-size VM (8 CPU): No ignore - blocked (6 free < 8 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1"}), ignoredReservationTypes: nil, expectedHosts: []string{}, filteredHosts: []string{"host1"}, @@ -888,7 +877,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, CR-size VM (8 CPU): Ignore CR - passes (10 free > 8 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -897,7 +886,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, CR-size VM (8 CPU): Ignore Failover - passes (8 free = 8 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -906,7 +895,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { name: "Single host, CR-size VM (8 CPU): Ignore both - passes (12 free > 8 needed)", hypervisors: singleHostHVs, reservations: singleHostRes, - request: newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1"}), ignoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource, v1alpha1.ReservationTypeFailover}, expectedHosts: []string{"host1"}, filteredHosts: []string{}, @@ -947,7 +936,7 @@ func TestFilterHasEnoughCapacity_AssumeEmptyHosts(t *testing.T) { hypervisors := []*hv1.Hypervisor{ newHypervisor("host1", "8", "6", "32Gi", "0"), } - request := newNovaRequest("vm", "proj", "m1.large", "gp", 4, "1Gi", false, []string{"host1"}) + request := newNovaRequest("vm", "proj", "gp", 4, "1Gi", false, []string{"host1"}) objects := make([]client.Object, 0, len(hypervisors)) for _, h := range hypervisors { @@ -987,7 +976,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) newCommittedReservation("cr-res", "host1", "project-X", "m1.large", "gp-1", "4", "8Gi", nil), newFailoverReservation("failover-res", "host2", "4", "8Gi", map[string]string{"other-vm": "host3"}), } - request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) + request := newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) for _, h := range hypervisors { @@ -1026,7 +1015,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTimeMerge(t *testin newCommittedReservation("cr-res", "host1", "project-X", "m1.large", "gp-1", "4", "8Gi", nil), newFailoverReservation("failover-res", "host2", "4", "8Gi", map[string]string{"other-vm": "host3"}), } - request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) + request := newNovaRequest("instance-123", "project-A", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) for _, h := range hypervisors { @@ -1065,7 +1054,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) name string hypervisors []*hv1.Hypervisor reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.PlacementRequest opts FilterHasEnoughCapacityOpts expectedHosts []string filteredHosts []string @@ -1082,7 +1071,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Request with reserve_for_committed_resource intent (scheduling a new CR reservation). // The intent causes the filter to keep CR reservations locked (prevents overbooking). - request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "m1.large", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1", "host2"}), + request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1", "host2"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2"}, // host1 blocked because existing-cr stays locked filteredHosts: []string{"host1"}, @@ -1098,7 +1087,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) newCommittedReservation("existing-cr", "host1", "project-A", "m1.large", "gp-1", "8", "16Gi", nil), }, // Normal VM create request (no special intent) - CR reservation should be unlocked - request: newNovaRequest("vm-instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("vm-instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1", "host2"}, // host1 passes because existing-cr is unlocked for matching project+flavor filteredHosts: []string{}, @@ -1114,7 +1103,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) newCommittedReservation("other-project-cr", "host1", "project-B", "m1.large", "gp-1", "8", "16Gi", nil), }, // Request with reserve_for_committed_resource intent - request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "m1.large", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1", "host2"}), + request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1", "host2"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host2"}, filteredHosts: []string{"host1"}, // host1 blocked by other project's reservation (would be blocked anyway) @@ -1132,7 +1121,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Request with reserve_for_committed_resource intent, needs 10 CPU // After blocking all 3 reservations (24 CPU), only 8 CPU free -> should fail - request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "m1.large", "gp-1", 10, "20Gi", "reserve_for_committed_resource", false, []string{"host1"}), + request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "gp-1", 10, "20Gi", "reserve_for_committed_resource", false, []string{"host1"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{}, filteredHosts: []string{"host1"}, // All reservations stay locked (intent prevents unlock), not enough capacity @@ -1150,7 +1139,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Normal VM create request, needs 10 CPU // All 3 reservations unlocked for matching project+flavor -> 32 CPU free -> should pass - request: newNovaRequest("vm-instance-123", "project-A", "m1.large", "gp-1", 10, "20Gi", false, []string{"host1"}), + request: newNovaRequest("vm-instance-123", "project-A", "gp-1", 10, "20Gi", false, []string{"host1"}), opts: FilterHasEnoughCapacityOpts{LockReserved: false}, expectedHosts: []string{"host1"}, // All reservations unlocked, enough capacity filteredHosts: []string{}, @@ -1166,7 +1155,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Request with reserve_for_committed_resource intent. // IgnoredReservationTypes is a YAML-level safety override that bypasses the intent check. - request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "m1.large", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1"}), + request: newNovaRequestWithIntent("new-reservation-uuid", "project-A", "gp-1", 4, "8Gi", "reserve_for_committed_resource", false, []string{"host1"}), opts: FilterHasEnoughCapacityOpts{ LockReserved: false, IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, @@ -1185,7 +1174,7 @@ func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) }, // Normal VM create request (different project, so unlocking via project match won't work) // But IgnoredReservationTypes should make it work - request: newNovaRequest("vm-instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1"}), + request: newNovaRequest("vm-instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1"}), opts: FilterHasEnoughCapacityOpts{ LockReserved: false, IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, @@ -1258,7 +1247,7 @@ func TestFilterHasEnoughCapacity_PlannedCRDoesNotBlock(t *testing.T) { step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() step.Options = FilterHasEnoughCapacityOpts{LockReserved: false} - request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 4, "8Gi", false, []string{"host1"}) + request := newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1"}) result, err := step.Run(logr.Discard(), request) if err != nil { t.Fatalf("expected no error, got %v", err) @@ -1274,7 +1263,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ @@ -1286,7 +1275,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { newHypervisorWithCapacityOnly("host3", "2", "4Gi"), // nil EffectiveCapacity, uses Capacity: 2 CPU free (not enough) newHypervisorWithCapacityOnly("host4", "16", "32Gi"), // nil EffectiveCapacity, uses Capacity: 16 CPU free, 32Gi free }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2", "host3", "host4"}), expectedHosts: []string{"host1", "host2", "host4"}, filteredHosts: []string{"host3"}, }, @@ -1296,7 +1285,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { newHypervisorWithCapacityOnly("host1", "8", "16Gi"), newHypervisorWithCapacityOnly("host2", "4", "8Gi"), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 4, "8Gi", false, []string{"host1", "host2"}), expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, }, @@ -1315,7 +1304,7 @@ func TestFilterHasEnoughCapacity_NilEffectiveCapacityFallback(t *testing.T) { // Both: 4 free -> filtered (need 5) newHypervisorWithBothCapacities("host3", "4", "4", "8Gi", "8Gi"), }, - request: newNovaRequest("instance-123", "project-A", "m1.small", "gp-1", 5, "8Gi", false, []string{"host1", "host2", "host3"}), + request: newNovaRequest("instance-123", "project-A", "gp-1", 5, "8Gi", false, []string{"host1", "host2", "host3"}), expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{"host3"}, }, @@ -1447,7 +1436,7 @@ func TestFilterHasEnoughCapacity_CapacityBoundaries(t *testing.T) { step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.hv.DeepCopy()).Build() step.Options = FilterHasEnoughCapacityOpts{LockReserved: false} - req := newNovaRequest("inst-1", "project-A", "m1.small", "gp-1", flavorVCPUs, flavorMem, false, []string{"h"}) + req := newNovaRequest("inst-1", "project-A", "gp-1", flavorVCPUs, flavorMem, false, []string{"h"}) result, err := step.Run(logr.Discard(), req) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -1638,7 +1627,7 @@ func TestFilterHasEnoughCapacity_VMInterReservationMigration(t *testing.T) { step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() step.Options = FilterHasEnoughCapacityOpts{LockReserved: false} - request := newNovaRequest("instance-new", thirdParty, "m1.small", flavorGroup, 3, "6Gi", false, []string{"hv-a", "hv-b"}) + request := newNovaRequest("instance-new", thirdParty, flavorGroup, 3, "6Gi", false, []string{"hv-a", "hv-b"}) result, err := step.Run(logr.Discard(), request) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go index d6d322707..a8c00db7c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go @@ -7,43 +7,23 @@ import ( "context" "github.com/go-logr/logr" "slices" - "strings" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterHasRequestedTraits struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } -// Filter hosts that do not have the requested traits given by the extra spec: -// - "trait:": "forbidden" means the host must not have the specified trait. -// - "trait:": "required" means the host must have the specified trait. -func (s *FilterHasRequestedTraits) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +// Filter hosts based on the traits requested in the flavor: +// - a forbidden trait means the host must not have the specified trait. +// - a required trait means the host must have the specified trait. +func (s *FilterHasRequestedTraits) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - var requiredTraits, forbiddenTraits []string - for key, value := range request.Spec.Data.Flavor.Data.ExtraSpecs { - if !strings.HasPrefix(key, "trait:") { - traceLog.V(1).Info("ignoring extra spec that does not start with trait:", "key", key) - continue - } - traitName := strings.TrimPrefix(key, "trait:") - switch value { - case "forbidden": - forbiddenTraits = append(forbiddenTraits, traitName) - traceLog.V(1).Info("trait marked forbidden, will filter out hosts with this trait", - "trait", traitName) - case "required": - requiredTraits = append(requiredTraits, traitName) - traceLog.V(1).Info("trait marked required, will filter out hosts without this trait", - "trait", traitName) - default: - traceLog.V(1).Info("ignoring trait with unsupported value", - "trait", traitName, "value", value) - } - } + requiredTraits := request.RequiredTraits + forbiddenTraits := request.ForbiddenTraits if len(requiredTraits) == 0 && len(forbiddenTraits) == 0 { traceLog.V(1).Info("no traits requested, skipping filter") return result, nil diff --git a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go index 719aa37fd..e5276054a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -75,23 +75,14 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No traits requested - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -102,19 +93,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Single required trait - filter hosts with trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -126,20 +107,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Multiple required traits - filter hosts with all traits", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - "trait:CUSTOM_TRAIT_B": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -151,19 +121,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Single forbidden trait - filter hosts without trait", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForbiddenTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -176,20 +136,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Multiple forbidden traits - filter hosts without any of them", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "forbidden", - "trait:CUSTOM_TRAIT_B": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForbiddenTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -202,20 +151,10 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Mixed required and forbidden traits", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - "trait:CUSTOM_TRAIT_D": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + ForbiddenTraits: []string{"CUSTOM_TRAIT_D"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host4"}, @@ -227,19 +166,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Required trait that no host has", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_NONEXISTENT": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_NONEXISTENT"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -250,19 +179,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Forbidden trait that no host has - all pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_NONEXISTENT": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForbiddenTraits: []string{"CUSTOM_TRAIT_NONEXISTENT"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -273,19 +192,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Host with no traits - required trait filters it out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -295,19 +204,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Host with no traits - forbidden trait lets it pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForbiddenTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -317,20 +216,9 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Non-trait extra specs are ignored", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "hw:cpu_policy": "dedicated", - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host3"}, }, @@ -340,19 +228,8 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Invalid trait value (not required or forbidden) - ignored", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "invalid", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -363,38 +240,18 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Host not in database", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, @@ -404,21 +261,10 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "Complex scenario with many traits", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - "trait:CUSTOM_TRAIT_B": "required", - "trait:CUSTOM_TRAIT_D": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A", "CUSTOM_TRAIT_B"}, + ForbiddenTraits: []string{"CUSTOM_TRAIT_D"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -432,20 +278,10 @@ func TestFilterHasRequestedTraits_Run(t *testing.T) { }, { name: "All hosts match required and forbidden traits", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - ExtraSpecs: map[string]string{ - "trait:CUSTOM_TRAIT_A": "required", - "trait:CUSTOM_TRAIT_E": "forbidden", - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequiredTraits: []string{"CUSTOM_TRAIT_A"}, + ForbiddenTraits: []string{"CUSTOM_TRAIT_E"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_host_instructions.go b/internal/scheduling/nova/plugins/filters/filter_host_instructions.go index c4121c22d..88b1b48b6 100644 --- a/internal/scheduling/nova/plugins/filters/filter_host_instructions.go +++ b/internal/scheduling/nova/plugins/filters/filter_host_instructions.go @@ -7,30 +7,30 @@ import ( "github.com/go-logr/logr" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) type FilterHostInstructionsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Filter hosts based on instructions given in the request spec. Supported are: // - spec.ignore_hosts: Filter out all hosts in this list. // - spec.force_hosts: Include only hosts in this list. -func (s *FilterHostInstructionsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterHostInstructionsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - if request.Spec.Data.IgnoreHosts != nil { - for _, host := range *request.Spec.Data.IgnoreHosts { + if len(request.IgnoreHosts) > 0 { + for _, host := range request.IgnoreHosts { delete(result.Activations, host) traceLog.V(1).Info("filtering host which is ignored", "host", host) } } else { traceLog.V(1).Info("no hosts to ignore specified") } - if request.Spec.Data.ForceHosts != nil { + if len(request.ForceHosts) > 0 { for host := range result.Activations { - if !slices.Contains(*request.Spec.Data.ForceHosts, host) { + if !slices.Contains(request.ForceHosts, host) { delete(result.Activations, host) traceLog.V(1).Info("filtering host which is not forced", "host", host) } diff --git a/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go b/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go index 349932c3c..973b40dc1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_host_instructions_test.go @@ -7,26 +7,20 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) func TestFilterHostInstructionsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No host instructions - no filtering", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -38,14 +32,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore single host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host2"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -57,14 +46,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore multiple hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1", "host3", "host4"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host1", "host3", "host4"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -76,14 +60,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore all hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1", "host2", "host3", "host4"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host1", "host2", "host3", "host4"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -95,14 +74,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Ignore non-existent host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host-nonexistent"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host-nonexistent"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -113,14 +87,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force single host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{"host2"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForceHosts: []string{"host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -132,14 +101,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force multiple hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{"host1", "host3"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForceHosts: []string{"host1", "host3"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -151,14 +115,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force non-existent host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{"host-nonexistent"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForceHosts: []string{"host-nonexistent"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -169,14 +128,10 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force and ignore same host - ignore takes precedence", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host2"}, - ForceHosts: &[]string{"host2"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host2"}, + ForceHosts: []string{"host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -187,14 +142,10 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Force and ignore different hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1"}, - ForceHosts: &[]string{"host2", "host3"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host1"}, + ForceHosts: []string{"host2", "host3"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -206,14 +157,10 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Complex scenario - multiple ignore and force", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host2", "host5"}, - ForceHosts: &[]string{"host1", "host3", "host4"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host2", "host5"}, + ForceHosts: []string{"host1", "host3", "host4"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -227,28 +174,19 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1"}, - ForceHosts: &[]string{"host2"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host1"}, + ForceHosts: []string{"host2"}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Empty ignore list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -258,48 +196,36 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Empty force list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForceHosts: []string{}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, }, - expectedHosts: []string{}, - filteredHosts: []string{"host1", "host2"}, + // An empty force list is no constraint in the typed spec (it cannot be + // distinguished from "unset"), so all hosts are kept. + expectedHosts: []string{"host1", "host2"}, + filteredHosts: []string{}, }, { name: "Both lists empty", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{}, - ForceHosts: &[]string{}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{}, + ForceHosts: []string{}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, }, - expectedHosts: []string{}, - filteredHosts: []string{"host1", "host2"}, + expectedHosts: []string{"host1", "host2"}, + filteredHosts: []string{}, }, { name: "Duplicate hosts in ignore list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: &[]string{"host1", "host1", "host2"}, - ForceHosts: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + IgnoreHosts: []string{"host1", "host1", "host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -310,14 +236,9 @@ func TestFilterHostInstructionsStep_Run(t *testing.T) { }, { name: "Duplicate hosts in force list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - IgnoreHosts: nil, - ForceHosts: &[]string{"host1", "host1", "host2"}, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + ForceHosts: []string{"host1", "host1", "host2"}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties.go b/internal/scheduling/nova/plugins/filters/filter_image_properties.go index f4467c828..96d595d64 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties.go @@ -10,55 +10,47 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterImagePropertiesStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Filter hosts based on image properties given in the request spec. -func (s *FilterImagePropertiesStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterImagePropertiesStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - // Resolve the scheduling intent once and fall back to a readable default - // when it cannot be determined. - intent, intentErr := request.GetIntent() - intentLabel := "unknown" - if intentErr == nil { - intentLabel = string(intent) - } - // Apply this filter to all requests, unless we know from the request's // intent that image metadata is expected to not be set. - if intentErr == nil { - intentsExpectedToNotHaveImageMeta := []v1alpha1.SchedulingIntent{ - // Cortex-internal intents in which scheduling requests are sent - // mainly based on flavor-related metadata, independent of the - // image that'll land there. - api.ReserveForFailoverIntent, - api.ReuseFailoverReservationIntent, - api.ReserveForCommittedResourceIntent, - api.CapacityProbeIntent, - } - if slices.Contains(intentsExpectedToNotHaveImageMeta, intent) { - traceLog.V(1).Info("skipping filter: expected to have no image metadata", - "intent", intent) - return result, nil - } + intentsExpectedToNotHaveImageMeta := []v1alpha1.SchedulingIntent{ + // Cortex-internal intents in which scheduling requests are sent + // mainly based on flavor-related metadata, independent of the + // image that'll land there. + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + } + if slices.Contains(intentsExpectedToNotHaveImageMeta, request.Intent) { + traceLog.V(1).Info("skipping filter: expected to have no image metadata", + "intent", request.Intent) + return result, nil } - // If the image properties indicate any other hypervisor type than kvm, - // we filter out all known kvm hypervisors. - hvType, err := request.Spec.Data.Image.Data.GetHypervisorType() - if err != nil { - // Hypervisor type could not be determined. In this case, just - // ignore the filter and return all hosts. - traceLog.Error(err, "could not determine hypervisor type from image properties") - // Expose this event through the step monitor to alert on high-frequency - // occurrences of this situation. Include the request intent as a dynamic - // label so the alert can pinpoint which scheduling intents are affected. + hvType := request.ImageHypervisorType + + // The image declared a hypervisor type that could not be parsed. Ignore the + // filter and keep all hosts, but surface an event so high-frequency + // occurrences can be alerted on (labeled by intent). + if hvType == compute.ImageHypervisorTypeUnparseable { + traceLog.Error(nil, "could not determine hypervisor type from image properties") + intentLabel := string(request.Intent) + if intentLabel == "" { + intentLabel = "unknown" + } result.Events = append(result.Events, lib.FilterWeigherPipelineStepEvent{ Name: "image_properties_hv_type_undetermined", Labels: map[string]string{"intent": intentLabel}, @@ -66,7 +58,10 @@ func (s *FilterImagePropertiesStep) Run(traceLog logr.Logger, request api.Extern return result, nil } - if hvType != api.NovaImageMetaHVTypeKVM { + // If the image properties indicate any other hypervisor type than kvm, + // we filter out all known kvm hypervisors. An unset image hypervisor type + // means the image declared none, so no image constraint applies. + if hvType != compute.ImageHypervisorTypeUnset && hvType != compute.ImageHypervisorTypeKVM { traceLog.V(1).Info("filtering out all known kvm hypervisors since image properties indicate a different hypervisor type", "image_hypervisor_type", hvType) hvs := new(hv1.HypervisorList) diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go index 0c836a9e2..be98bb96c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go @@ -8,6 +8,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,24 +18,25 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -// requestWith builds an ExternalSchedulerRequest with the given hosts and image -// properties. -func requestWith(hosts []string, properties map[string]any) api.ExternalSchedulerRequest { - schedHosts := make([]api.ExternalSchedulerHost, 0, len(hosts)) +// requestWith builds a SchedulerRequest with the given hosts and image +// hypervisor type. +func requestWith(hosts []string, hvType compute.ImageHypervisorType) compute.PlacementRequest { + schedHosts := make([]compute.Host, 0, len(hosts)) for _, h := range hosts { - schedHosts = append(schedHosts, api.ExternalSchedulerHost{ComputeHost: h}) + schedHosts = append(schedHosts, compute.Host{ComputeHost: h}) + } + return compute.PlacementRequest{ + Hosts: schedHosts, + ImageHypervisorType: hvType, } - request := api.ExternalSchedulerRequest{Hosts: schedHosts} - request.Spec.Data.Image.Data.Properties = api.NovaObject[map[string]any]{Data: properties} - return request } -// requestWithIntent builds an ExternalSchedulerRequest like requestWith, but -// additionally sets the _nova_check_type scheduler hint so GetIntent resolves -// to the given intent. -func requestWithIntent(hosts []string, properties map[string]any, checkType string) api.ExternalSchedulerRequest { - request := requestWith(hosts, properties) - request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": checkType} +// requestWithIntent builds a SchedulerRequest with the given intent. The image +// hypervisor type is irrelevant to the intent-skip paths these callers exercise, +// so it is fixed to VMware. +func requestWithIntent(hosts []string, intent v1alpha1.SchedulingIntent) compute.PlacementRequest { + request := requestWith(hosts, compute.ImageHypervisorTypeVMware) + request.Intent = intent return request } @@ -51,32 +54,32 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string expectedEvents []lib.FilterWeigherPipelineStepEvent }{ { name: "kvm image keeps all hosts", - request: requestWith([]string{"host1", "host2", "host3"}, map[string]any{"img_hv_type": "kvm"}), + request: requestWith([]string{"host1", "host2", "host3"}, compute.ImageHypervisorTypeKVM), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "vmware image filters out known kvm hypervisors", - request: requestWith([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}), + request: requestWith([]string{"host1", "host2", "host3"}, compute.ImageHypervisorTypeVMware), expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2"}, }, { name: "baremetal image filters out known kvm hypervisors", - request: requestWith([]string{"host1", "host3"}, map[string]any{"img_hv_type": "baremetal"}), + request: requestWith([]string{"host1", "host3"}, compute.ImageHypervisorTypeBaremetal), expectedHosts: []string{"host3"}, filteredHosts: []string{"host1"}, }, { name: "undeterminable hypervisor type keeps all hosts", - request: requestWith([]string{"host1", "host2", "host3"}, nil), + request: requestWith([]string{"host1", "host2", "host3"}, compute.ImageHypervisorTypeUnparseable), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, expectedEvents: []lib.FilterWeigherPipelineStepEvent{ @@ -85,7 +88,7 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { }, { name: "unsupported hypervisor type keeps all hosts", - request: requestWith([]string{"host1", "host2"}, map[string]any{"img_hv_type": "xen"}), + request: requestWith([]string{"host1", "host2"}, compute.ImageHypervisorTypeUnparseable), expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, expectedEvents: []lib.FilterWeigherPipelineStepEvent{ @@ -94,37 +97,37 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { }, { name: "empty host list", - request: requestWith([]string{}, map[string]any{"hypervisor_type": "vmware"}), + request: requestWith([]string{}, compute.ImageHypervisorTypeVMware), expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "reserve_for_failover intent skips filter and keeps all hosts", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "reserve_for_failover"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.ReserveForFailoverIntent), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "reuse_failover_reservation intent skips filter and keeps all hosts", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "reuse_failover_reservation"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.ReuseFailoverReservationIntent), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "reserve_for_committed_resource intent skips filter and keeps all hosts", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "reserve_for_committed_resource"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.ReserveForCommittedResourceIntent), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "capacity_probe intent skips filter and keeps all hosts", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "capacity_probe"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.CapacityProbeIntent), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, }, { name: "create intent still filters out known kvm hypervisors", - request: requestWithIntent([]string{"host1", "host2", "host3"}, map[string]any{"hypervisor_type": "vmware"}, "create"), + request: requestWithIntent([]string{"host1", "host2", "host3"}, api.CreateIntent), expectedHosts: []string{"host3"}, filteredHosts: []string{"host1", "host2"}, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go index a233bd37a..4430139d9 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go @@ -9,49 +9,50 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) type FilterInstanceGroupAffinityStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Select hosts in spec.instance_group. func (s *FilterInstanceGroupAffinityStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Instance group hints are only present in user-initiated VM placement requests; - if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + if slices.Contains([]v1alpha1.SchedulingIntent{ api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent, api.ReserveForCommittedResourceIntent, api.CapacityProbeIntent, - }, intent) { + }, request.Intent) { return result, nil } - ig := request.Spec.Data.InstanceGroup + ig := request.InstanceGroup if ig == nil { traceLog.V(1).Info("no instance group in request, skipping filter") return result, nil } - policy := ig.Data.Policy + policy := ig.Policy if policy != "affinity" { traceLog.V(1).Info("instance group policy is not 'affinity', skipping filter") return result, nil } - if len(ig.Data.Hosts) == 0 { + if len(ig.Hosts) == 0 { // Nothing to do. traceLog.V(1).Info("instance group has no hosts, skipping filter") return result, nil } for host := range result.Activations { - if slices.Contains(ig.Data.Hosts, host) { + if slices.Contains(ig.Hosts, host) { traceLog.V(1).Info("host is in instance group, keeping", "host", host) continue } diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go index 5a955d515..66c773bf2 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go @@ -7,25 +7,22 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" ) func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No instance group - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -36,18 +33,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Instance group with anti-affinity policy - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Hosts: []string{"host1", "host2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Hosts: []string{"host1", "host2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -58,18 +46,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Instance group with soft-affinity policy - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "soft-affinity", - Hosts: []string{"host1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "soft-affinity", Hosts: []string{"host1"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -79,18 +58,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy with empty hosts list - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -101,18 +71,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - only hosts in instance group pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1", "host3"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host3"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -123,18 +84,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - single host in group", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -146,18 +98,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - all hosts filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host5", "host6"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host5", "host6"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -168,18 +111,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - all hosts in group pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1", "host2", "host3"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2", "host3"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -190,18 +124,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - partial overlap", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host2", "host3", "host5"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host2", "host3", "host5"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -213,36 +138,18 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - empty request hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1", "host2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2"}}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Affinity policy - case sensitive host matching", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"Host1", "HOST2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"Host1", "HOST2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -253,23 +160,11 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - with instance UUID and project ID", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-123", - ProjectID: "project-abc", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - UUID: "ig-uuid-456", - Policy: "affinity", - Hosts: []string{"host1", "host2"}, - Members: []string{"vm-uuid-789"}, - ProjectID: "project-abc", - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-123", + ProjectID: "project-abc", + InstanceGroup: &compute.InstanceGroupSpec{UUID: "ig-uuid-456", Policy: "affinity", Hosts: []string{"host1", "host2"}, Members: []string{"vm-uuid-789"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -281,18 +176,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - duplicate hosts in group list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1", "host2", "host1", "host2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1", "host2", "host1", "host2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -303,18 +189,9 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }, { name: "Affinity policy - single host scenario", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Hosts: []string{"host1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, }, @@ -357,11 +234,9 @@ func TestFilterInstanceGroupAffinityStep_SkipsForNonPlacementIntent(t *testing.T // Affinity group on host1 only — host2 would normally be filtered. for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { t.Run(intent, func(t *testing.T) { - request := newNovaRequest("vm", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) - request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{Policy: "affinity", Hosts: []string{"host1"}}, - } - request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": intent} + request := newNovaRequest("vm", "proj", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request.InstanceGroup = &compute.InstanceGroupSpec{Policy: "affinity", Hosts: []string{"host1"}} + request.Intent = v1alpha1.SchedulingIntent(intent) step := &FilterInstanceGroupAffinityStep{} result, err := step.Run(logr.Discard(), request) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go index ad17d8ab2..2840d3f13 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go @@ -10,55 +10,52 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterInstanceGroupAntiAffinityStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Select hosts not in spec_obj.instance_group but only until // max_server_per_host is reached (by default = 1). func (s *FilterInstanceGroupAntiAffinityStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Instance group hints are only present in user-initiated VM placement requests; - if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + if slices.Contains([]v1alpha1.SchedulingIntent{ api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent, api.ReserveForCommittedResourceIntent, api.CapacityProbeIntent, - }, intent) { + }, request.Intent) { return result, nil } - ig := request.Spec.Data.InstanceGroup + ig := request.InstanceGroup if ig == nil { traceLog.V(1).Info("no instance group in request, skipping filter") return result, nil } - policy := ig.Data.Policy + policy := ig.Policy if policy != "anti-affinity" { traceLog.V(1).Info("instance group policy is not 'anti-affinity', skipping filter") return result, nil } - memberVMs := ig.Data.Members + memberVMs := ig.Members if len(memberVMs) == 0 { // Nothing to do. traceLog.V(1).Info("instance group has no members, skipping filter") return result, nil } - maxServersPerHost := 1 - if ig.Data.Rules != nil { - if maxServersPerHostAny, ok := ig.Data.Rules["max_server_per_host"]; ok { - if maxServersPerHostInt, ok := maxServersPerHostAny.(int); ok { - maxServersPerHost = maxServersPerHostInt - } - } + maxServersPerHost := ig.MaxServerPerHost + if maxServersPerHost == 0 { + maxServersPerHost = 1 } hvs := &hv1.HypervisorList{} @@ -84,7 +81,7 @@ func (s *FilterInstanceGroupAntiAffinityStep) Run( // Check if this host is already running the same vm (resizes). // In this case we should not filter it out. if slices.ContainsFunc(hv.Status.Instances, func(inst hv1.Instance) bool { - return inst.ID == request.Spec.Data.InstanceUUID + return inst.ID == request.InstanceUUID }) { traceLog.V(1).Info("host is running the same VM, not filtering out", "host", host) continue diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go index bfbd1c475..87ef4ef6c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go @@ -7,7 +7,8 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -79,20 +80,16 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "No instance group - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -103,19 +100,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Instance group with affinity policy - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -126,19 +114,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Instance group with soft-anti-affinity policy - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "soft-anti-affinity", - Members: []string{"vm-uuid-1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "soft-anti-affinity", Members: []string{"vm-uuid-1"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -148,19 +127,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy with empty members list - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -171,19 +141,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - default max_server_per_host=1", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -195,22 +156,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=2", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - Rules: map[string]any{ - "max_server_per_host": 2, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 2}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -222,22 +171,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=3", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-4", "vm-uuid-5", "vm-uuid-6"}, - Rules: map[string]any{ - "max_server_per_host": 3, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-4", "vm-uuid-5", "vm-uuid-6"}, MaxServerPerHost: 3}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host5"}, }, @@ -247,19 +184,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - host running same VM (resize scenario)", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-1", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-1", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host4"}, @@ -270,19 +198,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - all hosts filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}}, + Hosts: []compute.Host{ {ComputeHost: "host2"}, {ComputeHost: "host3"}, {ComputeHost: "host4"}, @@ -293,19 +212,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - mixed hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3", "vm-uuid-4", "vm-uuid-5"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3", "vm-uuid-4", "vm-uuid-5"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -318,38 +228,20 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - empty request hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Anti-affinity policy - host with non-member instances", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-100", "vm-uuid-101"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-100", "vm-uuid-101"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -362,22 +254,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - max_server_per_host=2 with mixed instances", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2"}, - Rules: map[string]any{ - "max_server_per_host": 2, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2"}, MaxServerPerHost: 2}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -390,22 +270,11 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - with instance UUID and project ID", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - ProjectID: "project-abc", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - UUID: "ig-uuid-456", - Policy: "anti-affinity", - Members: []string{"vm-uuid-1"}, - ProjectID: "project-abc", - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + ProjectID: "project-abc", + InstanceGroup: &compute.InstanceGroupSpec{UUID: "ig-uuid-456", Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -416,22 +285,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - multiple members on same host with max=2", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - Rules: map[string]any{ - "max_server_per_host": 2, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 2}, + Hosts: []compute.Host{ {ComputeHost: "host3"}, {ComputeHost: "host4"}, }, @@ -441,19 +298,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - resize with VM on multiple hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-2", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-2", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -465,19 +313,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - single host scenario", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1"}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1"}}, + Hosts: []compute.Host{ {ComputeHost: "host2"}, }, }, @@ -486,22 +325,10 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }, { name: "Anti-affinity policy - high max_server_per_host value", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: "vm-uuid-new", - InstanceGroup: &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, - Rules: map[string]any{ - "max_server_per_host": 10, - }, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + InstanceUUID: "vm-uuid-new", + InstanceGroup: &compute.InstanceGroupSpec{Policy: "anti-affinity", Members: []string{"vm-uuid-1", "vm-uuid-2", "vm-uuid-3"}, MaxServerPerHost: 10}, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -566,15 +393,13 @@ func TestFilterInstanceGroupAntiAffinityStep_SkipsForNonPlacementIntent(t *testi for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { t.Run(intent, func(t *testing.T) { - request := newNovaRequest("vm-new", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) - request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - Policy: "anti-affinity", - Members: []string{"vm-existing"}, - Rules: map[string]any{"max_server_per_host": 1}, - }, + request := newNovaRequest("vm-new", "proj", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request.InstanceGroup = &compute.InstanceGroupSpec{ + Policy: "anti-affinity", + Members: []string{"vm-existing"}, + MaxServerPerHost: 1, } - request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": intent} + request.Intent = v1alpha1.SchedulingIntent(intent) result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_live_migratable.go b/internal/scheduling/nova/plugins/filters/filter_live_migratable.go index 94c525f53..a7cbc4f13 100644 --- a/internal/scheduling/nova/plugins/filters/filter_live_migratable.go +++ b/internal/scheduling/nova/plugins/filters/filter_live_migratable.go @@ -10,12 +10,13 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterLiveMigratableStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Check if a vm can be live migrated from a source to a given target host. @@ -50,25 +51,19 @@ func (s *FilterLiveMigratableStep) checkHasSufficientFeatures( // Ensure the target host of a live migration can accept the migrating VM. func (s *FilterLiveMigratableStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - intent, err := request.GetIntent() - if err != nil { - traceLog.V(1).Info("failed to determine request intent, skipping filter", "error", err) - return result, nil - } - if intent != api.LiveMigrationIntent { + if request.Intent != api.LiveMigrationIntent { traceLog.V(1).Info("not a live migration request, skipping filter") return result, nil } - sourceHost, err := request.Spec.Data.GetSchedulerHintStr("source_host") - if err != nil || sourceHost == "" { - traceLog.V(1).Info("no source_host scheduler hint, skipping filter") - //nolint:nilerr // Not an error we want to fail the scheduling for. + sourceHost := request.SourceHost + if sourceHost == "" { + traceLog.V(1).Info("no source host set, skipping filter") return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go b/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go index 8f866b309..5bae46da5 100644 --- a/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go @@ -9,6 +9,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -200,7 +201,7 @@ func TestFilterLiveMigratableStep_checkHasSufficientFeatures(t *testing.T) { func TestFilterLiveMigratableStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string @@ -208,15 +209,9 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }{ { name: "Not a live migration - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "rebuild", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.EvacuateIntent, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -241,15 +236,9 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration without source_host - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -274,16 +263,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with compatible hosts", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -330,16 +313,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with incompatible architecture", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -380,16 +357,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing CPU modes", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -436,16 +407,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing features", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -476,16 +441,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with missing devices", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -516,16 +475,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with non-existent target host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host-nonexistent"}, @@ -551,16 +504,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with all hosts filtered", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -592,16 +539,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { }, { name: "Live migration with empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{}, }, hypervisors: []hv1.Hypervisor{ { @@ -634,8 +575,8 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -677,16 +618,10 @@ func TestFilterLiveMigratableStep_Run(t *testing.T) { } func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "nonexistent-host", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request := compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "nonexistent-host", + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -721,8 +656,8 @@ func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -738,16 +673,10 @@ func TestFilterLiveMigratableStep_Run_SourceHostNotFound(t *testing.T) { } func TestFilterLiveMigratableStep_Run_ClientError(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - "source_host": "host1", - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request := compute.PlacementRequest{ + Intent: api.LiveMigrationIntent, + SourceHost: "host1", + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, } @@ -767,8 +696,8 @@ func TestFilterLiveMigratableStep_Run_ClientError(t *testing.T) { Build() step := &FilterLiveMigratableStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 2f4eca9bb..697701b6a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -6,13 +6,15 @@ package filters import ( "context" "fmt" - "github.com/go-logr/logr" "math" "slices" "time" + "github.com/go-logr/logr" + api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -96,7 +98,7 @@ func (FilterQuotaEnforcementOpts) Validate() error { return nil } // // Disabled by default; activated by adding "filter_quota_enforcement" to the pipeline config. type FilterQuotaEnforcement struct { - lib.BaseFilter[api.ExternalSchedulerRequest, FilterQuotaEnforcementOpts] + lib.BaseFilter[compute.PlacementRequest, FilterQuotaEnforcementOpts] } // recordDecision appends an accept/reject/skip outcome as a named pipeline @@ -117,7 +119,7 @@ func recordDecision(result *lib.FilterWeigherPipelineStepResult, mode, decision, }) } -func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) mode := "shadow" @@ -130,34 +132,31 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS // external non-consuming intents (evacuate / live-migrate — VM already exists, // being moved) apart from cortex-internal reservation intents (failover / // committed-resource reservations — new schedules driven by cortex itself). - intent, err := request.GetIntent() - if err == nil { - switch intent { - case api.EvacuateIntent, api.LiveMigrationIntent: - traceLog.V(1).Info("skipping quota enforcement for non-consuming intent", "intent", intent) - recordDecision(result, mode, "accept_skipped_intent_migration", "", "", "") - return result, nil - case api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent: - traceLog.V(1).Info("skipping quota enforcement for failover reservation intent") - recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") - return result, nil - case api.ReserveForCommittedResourceIntent: - traceLog.V(1).Info("skipping quota enforcement for committed resource reservation intent") - recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") - return result, nil - case api.CapacityProbeIntent: - traceLog.V(1).Info("skipping quota enforcement for capacity probe intent") - recordDecision(result, mode, "accept_skipped", "", "", "") - return result, nil - } + switch request.Intent { + case api.EvacuateIntent, api.LiveMigrationIntent: + traceLog.V(1).Info("skipping quota enforcement for non-consuming intent", "intent", request.Intent) + recordDecision(result, mode, "accept_skipped_intent_migration", "", "", "") + return result, nil + case api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent: + traceLog.V(1).Info("skipping quota enforcement for failover reservation intent") + recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") + return result, nil + case api.ReserveForCommittedResourceIntent: + traceLog.V(1).Info("skipping quota enforcement for committed resource reservation intent") + recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") + return result, nil + case api.CapacityProbeIntent: + traceLog.V(1).Info("skipping quota enforcement for capacity probe intent") + recordDecision(result, mode, "accept_skipped", "", "", "") + return result, nil } // Step 2: Extract project, AZ, and hw_version from the request. // Missing-data skips share a single decision bucket; the specific missing // field is in the trace log. - projectID := request.Spec.Data.ProjectID - az := request.Spec.Data.AvailabilityZone - hwVersion := request.Spec.Data.Flavor.Data.ExtraSpecs["quota:hw_version"] + projectID := request.ProjectID + az := request.AvailabilityZone + hwVersion := request.HardwareVersion if projectID == "" { traceLog.V(1).Info("no project ID in request, skipping quota enforcement") @@ -176,12 +175,12 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS } // Step 3: Compute resource demands from the request. - numInstances := request.Spec.Data.NumInstances + numInstances := request.NumInstances if numInstances == 0 { numInstances = 1 } - memoryMB := request.Spec.Data.Flavor.Data.MemoryMB - vcpus := request.Spec.Data.Flavor.Data.VCPUs + memoryMB := request.Flavor.MemoryMB + vcpus := request.Flavor.VCPUs // MB to GiB: using 1024 as divisor is slightly conservative (overestimates GiB needed). requestRAM := int64(math.Ceil(float64(memoryMB*numInstances) / 1024.0)) requestCores := int64(vcpus * numInstances) //nolint:gosec // VCPUs from flavor specs, realistically bounded diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 2c492bb38..bb9b3d856 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -9,6 +9,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/resource" @@ -18,27 +19,44 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) api.ExternalSchedulerRequest { - flavor := api.NovaFlavor{ - MemoryMB: memoryMB, - VCPUs: 4, +// intentFromHint maps an old "_nova_check_type" scheduler hint value to the +// typed scheduling intent the plugin now reads. The hints map carries at most +// the "_nova_check_type" key in these tests. +func intentFromHint(hints map[string]any) v1alpha1.SchedulingIntent { + raw, ok := hints["_nova_check_type"] + if !ok { + return "" } - if hwVersion != "" { - flavor.ExtraSpecs = map[string]string{"quota:hw_version": hwVersion} - } else { - flavor.ExtraSpecs = map[string]string{} + switch raw { + case "evacuate": + return api.EvacuateIntent + case "live_migrate": + return api.LiveMigrationIntent + case "reserve_for_failover": + return api.ReserveForFailoverIntent + case "reserve_for_committed_resource": + return api.ReserveForCommittedResourceIntent + case "resize": + return api.ResizeIntent + case "capacity_probe": + return api.CapacityProbeIntent + default: + return "" } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: projectID, - AvailabilityZone: az, - NumInstances: numInstances, - SchedulerHints: hints, - Flavor: api.NovaObject[api.NovaFlavor]{Data: flavor}, - }, +} + +func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) compute.PlacementRequest { + return compute.PlacementRequest{ + ProjectID: projectID, + AvailabilityZone: az, + NumInstances: numInstances, + Intent: intentFromHint(hints), + HardwareVersion: hwVersion, + Flavor: compute.FlavorSpec{ + VCPUs: 4, + MemoryMB: memoryMB, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -55,7 +73,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { type tc struct { name string objects []client.Object - request api.ExternalSchedulerRequest + request compute.PlacementRequest enforce bool // crStateFilter sets Options.CRStateFilter; nil exercises the default (confirmed). crStateFilter []v1alpha1.CommitmentStatus @@ -461,7 +479,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { // Fully used, so it provides no CR headroom itself — the request // falls through to the PAYG check where its amount is deducted. Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}, + Billing: &v1alpha1.BillingStatus{UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}}, }, }, &v1alpha1.ProjectQuota{ @@ -500,7 +518,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("100Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}, + Billing: &v1alpha1.BillingStatus{UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}}, }, }, &v1alpha1.ProjectQuota{ diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go index e4fa2af5d..397cfdd88 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go @@ -11,12 +11,13 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) type FilterRequestedDestinationStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // processRequestedAggregates filters hosts based on the requested aggregates. @@ -101,23 +102,23 @@ func (s *FilterRequestedDestinationStep) processRequestedHost( // The requested destination can include a specific host, aggregates, or both. // When both are specified, aggregate filtering is applied first, followed by // host filtering. -func (s *FilterRequestedDestinationStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterRequestedDestinationStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // The requested_destination hint is only set by Nova for user-directed placement; - if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + if slices.Contains([]v1alpha1.SchedulingIntent{ api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent, api.ReserveForCommittedResourceIntent, api.CapacityProbeIntent, - }, intent) { + }, request.Intent) { return result, nil } - rd := request.Spec.Data.RequestedDestination + rd := request.RequestedDestination if rd == nil { traceLog.V(1).Info("no requested_destination in request, skipping filter") return result, nil } - if len(rd.Data.Aggregates) == 0 && rd.Data.Host == "" { + if len(rd.Aggregates) == 0 && rd.Host == "" { traceLog.V(1).Info("requested_destination has no host or aggregates, skipping filter") return result, nil } @@ -133,8 +134,8 @@ func (s *FilterRequestedDestinationStep) Run(traceLog logr.Logger, request api.E } hvsByName[hv.Name] = hv } - s.processRequestedAggregates(traceLog, rd.Data.Aggregates, hvsByName, result.Activations) - s.processRequestedHost(traceLog, rd.Data.Host, result.Activations) + s.processRequestedAggregates(traceLog, rd.Aggregates, hvsByName, result.Activations) + s.processRequestedHost(traceLog, rd.Host, result.Activations) return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go index a145190b9..f1442e648 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go @@ -9,6 +9,8 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,7 +23,7 @@ import ( func TestFilterRequestedDestinationStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest hypervisors []hv1.Hypervisor expectedHosts []string filteredHosts []string @@ -29,13 +31,9 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }{ { name: "No requested_destination - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: nil, - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + RequestedDestination: nil, + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -61,18 +59,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Empty requested_destination - all hosts pass", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{}, - Host: "", - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{}, + Host: "", }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -93,17 +85,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by specific host", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Host: "host2", - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Host: "host2", }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -129,17 +115,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by single aggregate - hosts in spec aggregates", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -165,17 +145,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by single aggregate - hosts in status aggregates", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -201,18 +175,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by multiple aggregates with OR syntax (comma-separated)", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - // "aggregate1,aggregate3" means host must be in aggregate1 OR aggregate3 - Aggregates: []string{"aggregate1,aggregate3"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + // "aggregate1,aggregate3" means host must be in aggregate1 OR aggregate3 + Aggregates: []string{"aggregate1,aggregate3"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -243,18 +211,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by multiple aggregates with AND logic", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - // ["aggregate1", "aggregate2"] means host must be in aggregate1 AND aggregate2 - Aggregates: []string{"aggregate1", "aggregate2"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + // ["aggregate1", "aggregate2"] means host must be in aggregate1 AND aggregate2 + Aggregates: []string{"aggregate1", "aggregate2"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -280,18 +242,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by mixed AND/OR aggregates", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - // Host must be in (aggregate1 OR aggregate2) AND aggregate3 - Aggregates: []string{"aggregate1,aggregate2", "aggregate3"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + // Host must be in (aggregate1 OR aggregate2) AND aggregate3 + Aggregates: []string{"aggregate1,aggregate2", "aggregate3"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -322,18 +278,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by aggregates with OR syntax - hosts in both spec and status", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - // "aggregate1,aggregate2" means host must be in aggregate1 OR aggregate2 - Aggregates: []string{"aggregate1,aggregate2"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + // "aggregate1,aggregate2" means host must be in aggregate1 OR aggregate2 + Aggregates: []string{"aggregate1,aggregate2"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -359,18 +309,12 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Filter by both host and aggregates - host takes precedence", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - Host: "host2", - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, + Host: "host2", }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -396,17 +340,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Host not found in hypervisors - filtered out", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-nonexistent"}, {ComputeHost: "host2"}, @@ -428,17 +366,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Host with no aggregates - filtered out when aggregates requested", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -464,17 +396,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "All hosts filtered by aggregate", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"nonexistent-aggregate"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"nonexistent-aggregate"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -495,17 +421,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Requested host not in candidates", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Host: "host-not-in-list", - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Host: "host-not-in-list", }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, }, @@ -530,17 +450,11 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request: compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{}, + Hosts: []compute.Host{}, }, hypervisors: []hv1.Hypervisor{ { @@ -571,8 +485,8 @@ func TestFilterRequestedDestinationStep_Run(t *testing.T) { Build() step := &FilterRequestedDestinationStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -740,17 +654,11 @@ func TestFilterRequestedDestinationStep_processRequestedHost(t *testing.T) { } func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{ - Aggregates: []string{"aggregate1"}, - }, - }, - }, + request := compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{ + Aggregates: []string{"aggregate1"}, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, }, } @@ -770,8 +678,8 @@ func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { Build() step := &FilterRequestedDestinationStep{ - BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ - BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilter: lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ Client: fakeClient, }, }, @@ -796,18 +704,17 @@ func TestFilterRequestedDestinationStep_SkipsForNonPlacementIntent(t *testing.T) step := &FilterRequestedDestinationStep{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { - t.Run(intent, func(t *testing.T) { - request := api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ - Data: api.NovaRequestedDestination{Host: "host1"}, - }, - SchedulerHints: map[string]any{"_nova_check_type": intent}, - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + for _, intent := range []v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + } { + t.Run(string(intent), func(t *testing.T) { + request := compute.PlacementRequest{ + RequestedDestination: &compute.RequestedDestination{Host: "host1"}, + Intent: intent, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, } result, err := step.Run(logr.Discard(), request) if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_status_conditions.go b/internal/scheduling/nova/plugins/filters/filter_status_conditions.go index 310a238c2..5bcbfd75f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_status_conditions.go +++ b/internal/scheduling/nova/plugins/filters/filter_status_conditions.go @@ -7,7 +7,7 @@ import ( "context" "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -15,12 +15,12 @@ import ( ) type FilterStatusConditionsStep struct { - lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseFilter[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } // Check that all status conditions meet the expected values, for example, // that the hypervisor is ready and not disabled. -func (s *FilterStatusConditionsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *FilterStatusConditionsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) hvs := &hv1.HypervisorList{} diff --git a/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go b/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go index 7862b6544..b38bebf12 100644 --- a/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_status_conditions_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -214,14 +214,14 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedHosts []string filteredHosts []string }{ { name: "Filter hosts with all conditions met", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -233,8 +233,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host not ready should be filtered", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host2"}, }, }, @@ -243,8 +243,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Terminating host should be filtered", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host3"}, }, }, @@ -253,8 +253,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Tainted host should not be filtered", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host4"}, }, }, @@ -263,8 +263,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Disabled hypervisor should be filtered", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host8"}, }, }, @@ -273,8 +273,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host with optional conditions in any state should pass", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host5"}, }, }, @@ -283,8 +283,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host missing Ready condition should be kept", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host6"}, }, }, @@ -293,8 +293,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Host with no conditions should be kept", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host7"}, }, }, @@ -303,16 +303,16 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Empty host list", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{}, + request: compute.PlacementRequest{ + Hosts: []compute.Host{}, }, expectedHosts: []string{}, filteredHosts: []string{}, }, { name: "Host not in database", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host-unknown"}, }, @@ -322,8 +322,8 @@ func TestFilterStatusConditionsStep_Run(t *testing.T) { }, { name: "Mixed condition states", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: compute.PlacementRequest{ + Hosts: []compute.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, diff --git a/internal/scheduling/nova/plugins/filters/zz_index.go b/internal/scheduling/nova/plugins/filters/zz_index.go index 0f63f8842..b72812ffd 100644 --- a/internal/scheduling/nova/plugins/filters/zz_index.go +++ b/internal/scheduling/nova/plugins/filters/zz_index.go @@ -4,11 +4,11 @@ package filters import ( - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type NovaFilter = lib.Filter[api.ExternalSchedulerRequest] +type NovaFilter = lib.Filter[compute.PlacementRequest] // Configuration of filters supported by the nova scheduler. var Index = map[string]func() NovaFilter{} diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go index a314dc5e8..9f19b47d1 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go @@ -9,7 +9,7 @@ import ( "fmt" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/go-logr/logr" @@ -65,11 +65,11 @@ func (o KVMBinpackStepOpts) Validate() error { // Explanation of the algorithm: https://volcano.sh/en/docs/plugins/#binpack type KVMBinpackStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMBinpackStepOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMBinpackStepOpts] } // Run this weigher in the pipeline after filters have been executed. -func (s *KVMBinpackStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMBinpackStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["binpack score"] = s.PrepareStats(request, "float") @@ -141,14 +141,14 @@ func (s *KVMBinpackStep) Run(traceLog logr.Logger, request api.ExternalScheduler } // calcVMResources calculates the total resource requests for the VM to be scheduled. -func (s *KVMBinpackStep) calcVMResources(req api.ExternalSchedulerRequest) map[hv1.ResourceName]resource.Quantity { +func (s *KVMBinpackStep) calcVMResources(req compute.PlacementRequest) map[hv1.ResourceName]resource.Quantity { resources := make(map[hv1.ResourceName]resource.Quantity) - resourcesMemBytes := int64(req.Spec.Data.Flavor.Data.MemoryMB) * (1024 * 1024) //nolint:gosec // memory values are bounded by Nova - resourcesMemBytes *= int64(req.Spec.Data.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesMemBytes := int64(req.Flavor.MemoryMB) * (1024 * 1024) //nolint:gosec // memory values are bounded by Nova + resourcesMemBytes *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[hv1.ResourceMemory] = *resource. NewQuantity(resourcesMemBytes, resource.BinarySI) - resourcesCPU := int64(req.Spec.Data.Flavor.Data.VCPUs) //nolint:gosec // vCPU values are bounded by Nova - resourcesCPU *= int64(req.Spec.Data.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesCPU := int64(req.Flavor.VCPUs) //nolint:gosec // vCPU values are bounded by Nova + resourcesCPU *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[hv1.ResourceCPU] = *resource. NewQuantity(resourcesCPU, resource.DecimalSI) return resources diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go index 421b53042..d7df0dc22 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,28 +34,10 @@ func newHypervisor(name, capacityCPU, capacityMem, allocationCPU, allocationMem } } -func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) compute.PlacementRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} - } - - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: "instance-123", - NumInstances: numInstances, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: vcpus, - MemoryMB: memoryMB, - ExtraSpecs: extraSpecs, - }, - }, + hostList[i] = compute.Host{ComputeHost: h} } weights := make(map[string]float64) @@ -63,8 +45,15 @@ func newBinpackRequest(memoryMB, vcpus, numInstances uint64, hosts []string) api weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.PlacementRequest{ + ProjectID: "project-A", + InstanceUUID: "instance-123", + NumInstances: numInstances, + HypervisorFamily: compute.HypervisorFamilyKVM, + Flavor: compute.FlavorSpec{ + VCPUs: vcpus, + MemoryMB: memoryMB, + }, Hosts: hostList, Weights: weights, } @@ -183,7 +172,7 @@ func TestKVMBinpackStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request api.ExternalSchedulerRequest + request compute.PlacementRequest opts KVMBinpackStepOpts expectedWeights map[string]float64 wantErr bool @@ -485,7 +474,7 @@ func TestKVMBinpackStep_Run(t *testing.T) { func TestKVMBinpackStep_calcVMResources(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedMemBytes int64 expectedCPU int64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go index 63cd032f9..2c626b7ad 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation.go @@ -11,6 +11,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -46,31 +47,31 @@ func (o KVMCommittedResourceReservationOpts) GetDefaultHostWeight() float64 { // Hosts that have a ready Reservation matching the request's project and flavor group, // with enough free memory capacity for the requested VM, receive a higher weight. type KVMCommittedResourceReservationStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMCommittedResourceReservationOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMCommittedResourceReservationOpts] } // Run the weigher step. // Hosts with a matching CommittedResourceReservation (project + flavor group + free capacity) get a higher weight. -func (s *KVMCommittedResourceReservationStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMCommittedResourceReservationStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) // Skip for failover reservation scheduling — CR reservations should not influence // where failover capacity is placed. - if intent, err := request.GetIntent(); err == nil && intent == api.ReserveForFailoverIntent { + if request.Intent == api.ReserveForFailoverIntent { traceLog.V(1).Info("skipping committed resource reservation weigher for failover reservation intent") return result, nil } - projectID := request.Spec.Data.ProjectID - az := request.Spec.Data.AvailabilityZone - resourceGroup, err := request.Spec.Data.GetSchedulerHintStr(api.HintKeyResourceGroup) - if err != nil || resourceGroup == "" { + projectID := request.ProjectID + az := request.AvailabilityZone + resourceGroup := request.ResourceGroup + if resourceGroup == "" { traceLog.V(1).Info("skipping committed resource reservation weigher: no resource group in request") - return result, nil //nolint:nilerr + return result, nil } // Flavor memory in bytes for free-capacity check. - flavorMemoryBytes := int64(request.Spec.Data.Flavor.Data.MemoryMB) * 1024 * 1024 //nolint:gosec + flavorMemoryBytes := int64(request.Flavor.MemoryMB) * 1024 * 1024 //nolint:gosec var reservations v1alpha1.ReservationList if err := s.Client.List(context.Background(), &reservations); err != nil { diff --git a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go index 5134f9776..84a0befe4 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_committed_resource_reservation_test.go @@ -9,6 +9,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -59,28 +60,22 @@ func newCRReservation(name, host, projectID, resourceGroup, az, totalMemory stri return res } -func newCRRequest(hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newCRRequest(hosts []string) compute.PlacementRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} - } - hints := map[string]any{ - api.HintKeyResourceGroup: []any{"group-2101"}, - } - spec := api.NovaSpec{ - ProjectID: "project-A", - AvailabilityZone: "qa-de-1a", - SchedulerHints: hints, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{MemoryMB: 8 * 1024}, - }, + hostList[i] = compute.Host{ComputeHost: h} } weights := make(map[string]float64, len(hosts)) for _, h := range hosts { weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.PlacementRequest{ + ProjectID: "project-A", + AvailabilityZone: "qa-de-1a", + ResourceGroup: "group-2101", + Flavor: compute.FlavorSpec{ + MemoryMB: 8 * 1024, + }, Hosts: hostList, Weights: weights, } @@ -94,7 +89,7 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.PlacementRequest opts KVMCommittedResourceReservationOpts expectedWeights map[string]float64 }{ @@ -177,9 +172,9 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { { name: "no resource group hint skips weigher", reservations: []*v1alpha1.Reservation{newCRReservation("res-1", "host1", "project-A", "group-2101", "qa-de-1a", "16Gi")}, - request: func() api.ExternalSchedulerRequest { + request: func() compute.PlacementRequest { r := newCRRequest([]string{"host1", "host2"}) - r.Spec.Data.SchedulerHints = nil + r.ResourceGroup = "" return r }(), opts: KVMCommittedResourceReservationOpts{ReservationHostWeight: floatPtr(1.0), DefaultHostWeight: floatPtr(0.1)}, @@ -216,19 +211,15 @@ func TestKVMCommittedResourceReservationStep_Run(t *testing.T) { reservations: []*v1alpha1.Reservation{ newCRReservation("res-1", "host1", "project-A", "group-2101", "qa-de-1a", "16Gi"), }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-A", - AvailabilityZone: "qa-de-1a", - Flavor: api.NovaObject[api.NovaFlavor]{Data: api.NovaFlavor{MemoryMB: 4096}}, - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForFailoverIntent), - api.HintKeyResourceGroup: "group-2101", - }, - }, + request: compute.PlacementRequest{ + ProjectID: "project-A", + AvailabilityZone: "qa-de-1a", + Intent: api.ReserveForFailoverIntent, + ResourceGroup: "group-2101", + Flavor: compute.FlavorSpec{ + MemoryMB: 4096, }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + Hosts: []compute.Host{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, }, opts: KVMCommittedResourceReservationOpts{ReservationHostWeight: floatPtr(1.0), DefaultHostWeight: floatPtr(0.1)}, expectedWeights: map[string]float64{"host1": 0.0, "host2": 0.0}, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go index 3b821a1a7..92ac4c057 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go @@ -11,6 +11,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -58,24 +59,23 @@ func (o KVMCRMigrationSlotOpts) GetDefaultHostWeight() float64 { // // Only activates for LiveMigrationIntent. type KVMCRMigrationSlotStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMCRMigrationSlotOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMCRMigrationSlotOpts] } func (s *KVMCRMigrationSlotStep) Run( traceLog logr.Logger, - request api.ExternalSchedulerRequest, + request compute.PlacementRequest, ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - intent, err := request.GetIntent() - if err != nil || intent != api.LiveMigrationIntent { + if request.Intent != api.LiveMigrationIntent { traceLog.V(1).Info("not a live migration, skipping CR migration slot weigher") - return result, nil //nolint:nilerr + return result, nil } - instanceUUID := request.Spec.Data.InstanceUUID - projectID := request.Spec.Data.ProjectID + instanceUUID := request.InstanceUUID + projectID := request.ProjectID var allReservations v1alpha1.ReservationList if err := s.Client.List(context.Background(), &allReservations, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go index 6003073e3..d3f73416d 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go @@ -9,6 +9,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -35,22 +36,16 @@ func newCRMigrationSlotWeigher(t *testing.T, opts KVMCRMigrationSlotOpts, objs . } // migrationRequest builds a live-migration request from the given candidate hosts. -func migrationRequest(hosts ...string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func migrationRequest(hosts ...string) compute.PlacementRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + hostList[i] = compute.Host{ComputeHost: h} } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: testInstanceUUID, - ProjectID: testProjectID, - SchedulerHints: map[string]any{ - "_nova_check_type": "live_migrate", - }, - }, - }, - Hosts: hostList, + return compute.PlacementRequest{ + InstanceUUID: testInstanceUUID, + ProjectID: testProjectID, + Intent: api.LiveMigrationIntent, + Hosts: hostList, } } @@ -143,7 +138,7 @@ func TestKVMCRMigrationSlotStep_Run(t *testing.T) { tests := []struct { name string objects []client.Object - request api.ExternalSchedulerRequest + request compute.PlacementRequest opts KVMCRMigrationSlotOpts expectedWeights map[string]float64 }{ @@ -152,15 +147,11 @@ func TestKVMCRMigrationSlotStep_Run(t *testing.T) { objects: []client.Object{ confirmedSourceSlot("16Gi"), }, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: testInstanceUUID, - ProjectID: testProjectID, - // no _nova_check_type → CreateIntent - }, - }, - Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host-a"}, {ComputeHost: "host-b"}}, + request: compute.PlacementRequest{ + InstanceUUID: testInstanceUUID, + ProjectID: testProjectID, + // no Intent → CreateIntent + Hosts: []compute.Host{{ComputeHost: "host-a"}, {ComputeHost: "host-b"}}, }, opts: defaultOpts, expectedWeights: map[string]float64{"host-a": 0.0, "host-b": 0.0}, diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go index 3088496b0..f213ae40b 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go @@ -10,6 +10,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -45,22 +46,21 @@ func (o KVMFailoverEvacuationOpts) GetDefaultHostWeight() float64 { // Hosts that match a failover reservation for the VM being scheduled get a higher weight, // encouraging placement on pre-reserved failover capacity. type KVMFailoverEvacuationStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMFailoverEvacuationOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMFailoverEvacuationOpts] } // Run the weigher step. // For evacuation requests, hosts matching a failover reservation where the VM is in Allocations get a higher weight. // For non-evacuation requests (e.g., live migration, rebuild), this weigher has no effect. -func (s *KVMFailoverEvacuationStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMFailoverEvacuationStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - intent, err := request.GetIntent() - if err != nil || intent != api.EvacuateIntent { + if request.Intent != api.EvacuateIntent { traceLog.V(1).Info("skipping failover weigher for non-evacuation request") - return result, nil //nolint:nilerr // intentionally skip weigher on error + return result, nil } - instanceUUID := request.Spec.Data.InstanceUUID + instanceUUID := request.InstanceUUID // Fetch all reservations var reservations v1alpha1.ReservationList diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go index 16299100b..d4de183d6 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation_test.go @@ -9,6 +9,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -112,48 +113,34 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { //nolint:unparam // instanceUUID varies in real usage - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newNovaRequest(instanceUUID string, evacuation bool, hosts []string) compute.PlacementRequest { //nolint:unparam // instanceUUID varies in real usage + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + hostList[i] = compute.Host{ComputeHost: h} } - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", + req := compute.PlacementRequest{ + ProjectID: "project-A", + InstanceUUID: instanceUUID, + NumInstances: 1, + HypervisorFamily: compute.HypervisorFamilyKVM, + Flavor: compute.FlavorSpec{ + VCPUs: 4, + MemoryMB: parseMemoryToMB("8Gi"), + }, + Hosts: hostList, } - - var schedulerHints map[string]any if evacuation { - schedulerHints = map[string]any{ - "_nova_check_type": []any{"evacuate"}, - } - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: instanceUUID, - NumInstances: 1, - SchedulerHints: schedulerHints, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: 4, - MemoryMB: parseMemoryToMB("8Gi"), - ExtraSpecs: extraSpecs, - }, - }, + req.Intent = api.EvacuateIntent } weights := make(map[string]float64) for _, h := range hosts { weights[h] = 1.0 } + req.Weights = weights - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, - Hosts: hostList, - Weights: weights, - } + return req } // ============================================================================ @@ -166,7 +153,7 @@ func TestKVMFailoverEvacuationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.PlacementRequest opts KVMFailoverEvacuationOpts expectedWeights map[string]float64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go index 270d3c34c..f6b10ee04 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go @@ -11,6 +11,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -73,25 +74,24 @@ func (o KVMFailoverReservationConsolidationOpts) GetSameSpecPenalty() float64 { // // This produces bounded output (~0 to 1) that plays nicely with other weighers. type KVMFailoverReservationConsolidationStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMFailoverReservationConsolidationOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMFailoverReservationConsolidationOpts] } // Run the weigher step. // For reserve_for_failover requests, hosts are scored based on existing failover reservation density // and same-spec diversity. For all other request types, this weigher has no effect. -func (s *KVMFailoverReservationConsolidationStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMFailoverReservationConsolidationStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - intent, err := request.GetIntent() - if err != nil || intent != api.ReserveForFailoverIntent { + if request.Intent != api.ReserveForFailoverIntent { traceLog.V(1).Info("skipping failover reservation consolidation weigher for non-failover-reservation request") - return result, nil //nolint:nilerr // intentionally skip weigher on error + return result, nil } - // Extract the resource group from the scheduler hint. + // Extract the resource group from the request. // This identifies which "spec group" the incoming reservation belongs to. - // If the hint is missing, requestResourceGroup will be empty and the same-group penalty is skipped. - requestResourceGroup, _ := request.Spec.Data.GetSchedulerHintStr(api.HintKeyResourceGroup) //nolint:errcheck // missing hint is fine, same-group penalty is simply skipped + // If it is empty, the same-group penalty is skipped. + requestResourceGroup := request.ResourceGroup // Fetch all reservations. var reservations v1alpha1.ReservationList diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go index 44736e6a4..c2bdf5996 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go @@ -10,6 +10,7 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -50,30 +51,10 @@ func newFailoverReservationWithGroup(name, targetHost, resourceGroup string) *v1 } } -func newFailoverReservationRequest(resourceGroup string, hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newFailoverReservationRequest(resourceGroup string, hosts []string) compute.PlacementRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: "test-instance", - NumInstances: 1, - SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForFailoverIntent), - api.HintKeyResourceGroup: resourceGroup, - }, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: 4, - MemoryMB: 8192, - ExtraSpecs: map[string]string{ - "capabilities:hypervisor_type": "qemu", - }, - }, - }, + hostList[i] = compute.Host{ComputeHost: h} } weights := make(map[string]float64) @@ -81,8 +62,17 @@ func newFailoverReservationRequest(resourceGroup string, hosts []string) api.Ext weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.PlacementRequest{ + ProjectID: "project-A", + InstanceUUID: "test-instance", + NumInstances: 1, + Intent: api.ReserveForFailoverIntent, + ResourceGroup: resourceGroup, + HypervisorFamily: compute.HypervisorFamilyKVM, + Flavor: compute.FlavorSpec{ + VCPUs: 4, + MemoryMB: 8192, + }, Hosts: hostList, Weights: weights, } @@ -98,7 +88,7 @@ func TestKVMFailoverReservationConsolidationStep_Run(t *testing.T) { tests := []struct { name string reservations []*v1alpha1.Reservation - request api.ExternalSchedulerRequest + request compute.PlacementRequest opts KVMFailoverReservationConsolidationOpts expectedWeights map[string]float64 }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go index c95ff1b3c..d26947c7c 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go @@ -7,7 +7,7 @@ import ( "context" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/go-logr/logr" @@ -23,19 +23,19 @@ import ( // respectively, for the scheduler to choose that host for new instances of // the same group. type KVMInstanceGroupSoftAffinityStep struct { - lib.BaseWeigher[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] + lib.BaseWeigher[compute.PlacementRequest, lib.EmptyFilterWeigherPipelineStepOpts] } -func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["affinity"] = s.PrepareStats(request, "float") - ig := request.Spec.Data.InstanceGroup + ig := request.InstanceGroup if ig == nil { traceLog.V(1).Info("no instance group in request, skipping weigher") return result, nil } - policy := ig.Data.Policy + policy := ig.Policy var factor float64 switch policy { case "soft-anti-affinity": @@ -46,7 +46,7 @@ func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request api traceLog.V(1).Info("instance group policy is not 'soft-affinity' or 'soft-anti-affinity', skipping weigher", "policy", policy) return result, nil } - if len(ig.Data.Members) == 0 { + if len(ig.Members) == 0 { traceLog.V(1).Info("instance group has no members, skipping weigher") return result, nil } @@ -72,7 +72,7 @@ func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog logr.Logger, request api } count := 0 for _, instance := range hv.Status.Instances { - if slices.Contains(ig.Data.Members, instance.ID) { + if slices.Contains(ig.Members, instance.ID) { count++ } } diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go index b839388b2..ec360c361 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,39 +33,30 @@ func newHypervisorWithInstances(name string, instanceIDs ...string) *hv1.Hypervi } } -func newInstanceGroupRequest(policy string, members, hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newInstanceGroupRequest(policy string, members, hosts []string) compute.PlacementRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + hostList[i] = compute.Host{ComputeHost: h} } - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: "new-instance-uuid", - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: 4, - MemoryMB: 8192, - ExtraSpecs: extraSpecs, - }, + req := compute.PlacementRequest{ + ProjectID: "project-A", + InstanceUUID: "new-instance-uuid", + NumInstances: 1, + HypervisorFamily: compute.HypervisorFamilyKVM, + Flavor: compute.FlavorSpec{ + VCPUs: 4, + MemoryMB: 8192, }, + Hosts: hostList, } // Only set InstanceGroup if policy is provided if policy != "" { - spec.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ - Data: api.NovaInstanceGroup{ - UUID: "instance-group-uuid", - Name: "test-instance-group", - Policy: policy, - Members: members, - }, + req.InstanceGroup = &compute.InstanceGroupSpec{ + UUID: "instance-group-uuid", + Policy: policy, + Members: members, } } @@ -73,12 +64,9 @@ func newInstanceGroupRequest(policy string, members, hosts []string) api.Externa for _, h := range hosts { weights[h] = 1.0 } + req.Weights = weights - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, - Hosts: hostList, - Weights: weights, - } + return req } func TestKVMInstanceGroupSoftAffinityStep_Run(t *testing.T) { @@ -87,7 +75,7 @@ func TestKVMInstanceGroupSoftAffinityStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request api.ExternalSchedulerRequest + request compute.PlacementRequest expectedWeights map[string]float64 wantErr bool }{ diff --git a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go index 52632df77..87bf495e4 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go @@ -9,7 +9,7 @@ import ( "fmt" "slices" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/go-logr/logr" @@ -53,11 +53,11 @@ func (o KVMPreferSmallerHostsStepOpts) Validate() error { // accommodate larger VMs when they need to be scheduled. type KVMPreferSmallerHostsStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[api.ExternalSchedulerRequest, KVMPreferSmallerHostsStepOpts] + lib.BaseWeigher[compute.PlacementRequest, KVMPreferSmallerHostsStepOpts] } // Run this weigher in the pipeline after filters have been executed. -func (s *KVMPreferSmallerHostsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *KVMPreferSmallerHostsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["small host score"] = s.PrepareStats(request, "float") diff --git a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go index 1f5a07677..c6902a101 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,28 +30,10 @@ func newHypervisorWithCapacity(name, capacityCPU, capacityMem string) *hv1.Hyper } } -func newPreferSmallerHostsRequest(hosts []string) api.ExternalSchedulerRequest { - hostList := make([]api.ExternalSchedulerHost, len(hosts)) +func newPreferSmallerHostsRequest(hosts []string) compute.PlacementRequest { + hostList := make([]compute.Host, len(hosts)) for i, h := range hosts { - hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} - } - - extraSpecs := map[string]string{ - "capabilities:hypervisor_type": "qemu", - } - - spec := api.NovaSpec{ - ProjectID: "project-A", - InstanceUUID: "instance-123", - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: "m1.large", - VCPUs: 4, - MemoryMB: 8192, - ExtraSpecs: extraSpecs, - }, - }, + hostList[i] = compute.Host{ComputeHost: h} } weights := make(map[string]float64) @@ -59,8 +41,15 @@ func newPreferSmallerHostsRequest(hosts []string) api.ExternalSchedulerRequest { weights[h] = 1.0 } - return api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + return compute.PlacementRequest{ + ProjectID: "project-A", + InstanceUUID: "instance-123", + NumInstances: 1, + HypervisorFamily: compute.HypervisorFamilyKVM, + Flavor: compute.FlavorSpec{ + VCPUs: 4, + MemoryMB: 8192, + }, Hosts: hostList, Weights: weights, } @@ -181,7 +170,7 @@ func TestKVMPreferSmallerHostsStep_Run(t *testing.T) { tests := []struct { name string hypervisors []*hv1.Hypervisor - request api.ExternalSchedulerRequest + request compute.PlacementRequest opts KVMPreferSmallerHostsStepOpts expectedWeights map[string]float64 wantErr bool diff --git a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go index 426526e2d..3b775977b 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects.go @@ -9,9 +9,9 @@ import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,7 +38,7 @@ func (o VMwareAntiAffinityNoisyProjectsStepOpts) Validate() error { // Step to avoid noisy projects by downvoting the hosts they are running on. type VMwareAntiAffinityNoisyProjectsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[api.ExternalSchedulerRequest, VMwareAntiAffinityNoisyProjectsStepOpts] + lib.BaseWeigher[sched.PlacementRequest, VMwareAntiAffinityNoisyProjectsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -53,7 +53,7 @@ func (s *VMwareAntiAffinityNoisyProjectsStep) Init(ctx context.Context, client c } // Downvote the hosts a project is currently running on if it's noisy. -func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request sched.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu usage of this project"] = s.PrepareStats(request, "%") @@ -72,7 +72,7 @@ func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request return nil, err } for _, p := range projectNoisinessOnHosts { - if p.Project != request.Spec.Data.ProjectID { + if p.Project != request.ProjectID { continue } // Only modify the weight if the host is in the scenario. @@ -92,7 +92,7 @@ func (s *VMwareAntiAffinityNoisyProjectsStep) Run(traceLog logr.Logger, request } func init() { - Index["vmware_anti_affinity_noisy_projects"] = func() lib.Weigher[api.ExternalSchedulerRequest] { + Index["vmware_anti_affinity_noisy_projects"] = func() lib.Weigher[sched.PlacementRequest] { return &VMwareAntiAffinityNoisyProjectsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go index 5005bae45..38a28de94 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_anti_affinity_noisy_projects_test.go @@ -9,9 +9,9 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -219,18 +219,14 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.PlacementRequest downvotedHosts map[string]struct{} }{ { name: "Noisy project", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project1", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: sched.PlacementRequest{ + ProjectID: "project1", + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -243,13 +239,9 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { }, { name: "Non-noisy project", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project2", - }, - }, - Hosts: []api.ExternalSchedulerHost{ + request: sched.PlacementRequest{ + ProjectID: "project2", + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -259,8 +251,8 @@ func TestVMwareAntiAffinityNoisyProjectsStep_Run(t *testing.T) { }, { name: "No noisy project data", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.PlacementRequest{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go index a12c35fe1..c52cd921a 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts.go @@ -9,9 +9,9 @@ import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,7 +47,7 @@ func (o VMwareAvoidLongTermContendedHostsStepOpts) Validate() error { // Step to avoid long term contended hosts by downvoting them. type VMwareAvoidLongTermContendedHostsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[api.ExternalSchedulerRequest, VMwareAvoidLongTermContendedHostsStepOpts] + lib.BaseWeigher[sched.PlacementRequest, VMwareAvoidLongTermContendedHostsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -62,7 +62,7 @@ func (s *VMwareAvoidLongTermContendedHostsStep) Init(ctx context.Context, client } // Downvote hosts that are highly contended. -func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, request sched.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu contention"] = s.PrepareStats(request, "%") @@ -110,7 +110,7 @@ func (s *VMwareAvoidLongTermContendedHostsStep) Run(traceLog logr.Logger, reques } func init() { - Index["vmware_avoid_long_term_contended_hosts"] = func() lib.Weigher[api.ExternalSchedulerRequest] { + Index["vmware_avoid_long_term_contended_hosts"] = func() lib.Weigher[sched.PlacementRequest] { return &VMwareAvoidLongTermContendedHostsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go index 701cef92e..25f697a4d 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_long_term_contended_hosts_test.go @@ -9,9 +9,9 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -218,13 +218,13 @@ func TestVMwareAvoidLongTermContendedHostsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.PlacementRequest expected map[string]float64 }{ { name: "Avoid contended hosts", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.PlacementRequest{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -240,8 +240,8 @@ func TestVMwareAvoidLongTermContendedHostsStep_Run(t *testing.T) { }, { name: "Missing data", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.PlacementRequest{ + Hosts: []sched.Host{ {ComputeHost: "host4"}, {ComputeHost: "host5"}, }, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go index 209eb97ba..6d2f0fc37 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts.go @@ -9,9 +9,9 @@ import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + extractorcompute "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,7 +47,7 @@ func (o VMwareAvoidShortTermContendedHostsStepOpts) Validate() error { // Step to avoid recently contended hosts by downvoting them. type VMwareAvoidShortTermContendedHostsStep struct { // BaseStep is a helper struct that provides common functionality for all steps. - lib.BaseWeigher[api.ExternalSchedulerRequest, VMwareAvoidShortTermContendedHostsStepOpts] + lib.BaseWeigher[compute.PlacementRequest, VMwareAvoidShortTermContendedHostsStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -62,7 +62,7 @@ func (s *VMwareAvoidShortTermContendedHostsStep) Init(ctx context.Context, clien } // Downvote hosts that are highly contended. -func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, request compute.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["avg cpu contention"] = s.PrepareStats(request, "%") @@ -77,7 +77,7 @@ func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, reque return nil, err } highlyContendedHosts, err := v1alpha1. - UnboxFeatureList[compute.VROpsHostsystemContentionShortTerm](knowledge.Status.Raw) + UnboxFeatureList[extractorcompute.VROpsHostsystemContentionShortTerm](knowledge.Status.Raw) if err != nil { return nil, err } @@ -110,7 +110,7 @@ func (s *VMwareAvoidShortTermContendedHostsStep) Run(traceLog logr.Logger, reque } func init() { - Index["vmware_avoid_short_term_contended_hosts"] = func() lib.Weigher[api.ExternalSchedulerRequest] { + Index["vmware_avoid_short_term_contended_hosts"] = func() lib.Weigher[compute.PlacementRequest] { return &VMwareAvoidShortTermContendedHostsStep{} } } diff --git a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go index 624ee7e14..768973f1c 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_avoid_short_term_contended_hosts_test.go @@ -9,9 +9,9 @@ import ( "strings" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -218,13 +218,13 @@ func TestVMwareAvoidShortTermContendedHostsStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.PlacementRequest expected map[string]float64 }{ { name: "Avoid contended hosts", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.PlacementRequest{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -240,8 +240,8 @@ func TestVMwareAvoidShortTermContendedHostsStep_Run(t *testing.T) { }, { name: "Missing data", - request: api.ExternalSchedulerRequest{ - Hosts: []api.ExternalSchedulerHost{ + request: sched.PlacementRequest{ + Hosts: []sched.Host{ {ComputeHost: "host4"}, {ComputeHost: "host5"}, // No data for host5 }, diff --git a/internal/scheduling/nova/plugins/weighers/vmware_binpack.go b/internal/scheduling/nova/plugins/weighers/vmware_binpack.go index 3f8b01fd8..27bc33f39 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_binpack.go @@ -11,9 +11,9 @@ import ( "github.com/go-logr/logr" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -69,7 +69,7 @@ func (o VMwareBinpackStepOpts) Validate() error { // Explanation of the algorithm: https://volcano.sh/en/docs/plugins/#binpack type VMwareBinpackStep struct { // Base weigher providing common functionality. - lib.BaseWeigher[api.ExternalSchedulerRequest, VMwareBinpackStepOpts] + lib.BaseWeigher[sched.PlacementRequest, VMwareBinpackStepOpts] } // Initialize the step and validate that all required knowledges are ready. @@ -86,7 +86,7 @@ func (s *VMwareBinpackStep) Init(ctx context.Context, client client.Client, weig } // Run this weigher in the pipeline after filters have been executed. -func (s *VMwareBinpackStep) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { +func (s *VMwareBinpackStep) Run(traceLog logr.Logger, request sched.PlacementRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) result.Statistics["binpack score"] = s.PrepareStats(request, "float") @@ -185,14 +185,14 @@ func (s *VMwareBinpackStep) calcHostAllocation(hostUtilization compute.HostUtili } // calcVMResources calculates the total resource requests for the VM to be scheduled. -func (s *VMwareBinpackStep) calcVMResources(req api.ExternalSchedulerRequest) map[corev1.ResourceName]resource.Quantity { +func (s *VMwareBinpackStep) calcVMResources(req sched.PlacementRequest) map[corev1.ResourceName]resource.Quantity { resources := make(map[corev1.ResourceName]resource.Quantity) - resourcesMemBytes := int64(req.Spec.Data.Flavor.Data.MemoryMB) * (1 << 20) //nolint:gosec // memory values are bounded by Nova - resourcesMemBytes *= int64(req.Spec.Data.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesMemBytes := int64(req.Flavor.MemoryMB) * (1 << 20) //nolint:gosec // memory values are bounded by Nova + resourcesMemBytes *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[corev1.ResourceMemory] = *resource. NewQuantity(resourcesMemBytes, resource.BinarySI) - resourcesCPU := int64(req.Spec.Data.Flavor.Data.VCPUs) //nolint:gosec // vCPU values are bounded by Nova - resourcesCPU *= int64(req.Spec.Data.NumInstances) //nolint:gosec // instance count is bounded by Nova + resourcesCPU := int64(req.Flavor.VCPUs) //nolint:gosec // vCPU values are bounded by Nova + resourcesCPU *= int64(req.NumInstances) //nolint:gosec // instance count is bounded by Nova resources[corev1.ResourceCPU] = *resource. NewQuantity(resourcesCPU, resource.DecimalSI) return resources diff --git a/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go b/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go index 404722f43..7d3226256 100644 --- a/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go +++ b/internal/scheduling/nova/plugins/weighers/vmware_binpack_test.go @@ -7,9 +7,9 @@ import ( "github.com/go-logr/logr" "testing" - api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -156,25 +156,19 @@ func TestVMwareBinpackStep_Run(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.PlacementRequest expectedHigherScore string expectedLowerScore string }{ { name: "binpack prefers hosts with higher utilization", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 1000, // 1GB - VCPUs: 1, - }, - }, - }, + request: sched.PlacementRequest{ + NumInstances: 1, + Flavor: sched.FlavorSpec{ + MemoryMB: 1000, // 1GB + VCPUs: 1, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "host2"}, {ComputeHost: "host3"}, @@ -187,19 +181,13 @@ func TestVMwareBinpackStep_Run(t *testing.T) { }, { name: "handles missing host data gracefully", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 1000, - VCPUs: 1, - }, - }, - }, + request: sched.PlacementRequest{ + NumInstances: 1, + Flavor: sched.FlavorSpec{ + MemoryMB: 1000, + VCPUs: 1, }, - Hosts: []api.ExternalSchedulerHost{ + Hosts: []sched.Host{ {ComputeHost: "host1"}, {ComputeHost: "unknown_host"}, // No utilization data }, @@ -292,23 +280,17 @@ func TestVMwareBinpackStep_CalcVMResources(t *testing.T) { tests := []struct { name string - request api.ExternalSchedulerRequest + request sched.PlacementRequest expectedMemory int64 expectedCPU int64 }{ { name: "single instance", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 2048, - VCPUs: 4, - }, - }, - }, + request: sched.PlacementRequest{ + NumInstances: 1, + Flavor: sched.FlavorSpec{ + MemoryMB: 2048, + VCPUs: 4, }, }, expectedMemory: 2048 * (1 << 20), // 2048 MiB in bytes @@ -316,17 +298,11 @@ func TestVMwareBinpackStep_CalcVMResources(t *testing.T) { }, { name: "multiple instances", - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 3, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 1024, - VCPUs: 2, - }, - }, - }, + request: sched.PlacementRequest{ + NumInstances: 3, + Flavor: sched.FlavorSpec{ + MemoryMB: 1024, + VCPUs: 2, }, }, expectedMemory: 3 * 1024 * (1 << 20), // 3 instances * 1024 MiB in bytes diff --git a/internal/scheduling/nova/plugins/weighers/zz_index.go b/internal/scheduling/nova/plugins/weighers/zz_index.go index d3136e724..45bef3727 100644 --- a/internal/scheduling/nova/plugins/weighers/zz_index.go +++ b/internal/scheduling/nova/plugins/weighers/zz_index.go @@ -4,11 +4,11 @@ package weighers import ( - api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) -type NovaWeigher = lib.Weigher[api.ExternalSchedulerRequest] +type NovaWeigher = lib.Weigher[compute.PlacementRequest] // Configuration of weighers supported by the nova scheduler. var Index = map[string]func() NovaWeigher{} diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index 3548122f3..bbc22d635 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -12,6 +12,8 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -33,21 +35,26 @@ type VirtualMachineHandler struct { } // Upsert creates or patches the VirtualMachine resource described by the -// decision and request. Region and AZ are immutable on the CRD and only written -// on create; mutable fields (target host, cortex spec, labels) are patched on -// updates. -func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.Decision, request api.ExternalSchedulerRequest) error { +// scheduling result and request. Region and AZ are immutable on the CRD and only +// written on create; mutable fields (target host, cortex spec, labels) are +// patched on updates. A human-readable explanation of the result and the intent +// are recorded on the VM's cortex spec. +func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.PlacementResult, request api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent) error { log := ctrl.LoggerFrom(ctx) - if decision.Status.Result == nil || decision.Status.Result.TargetHost == nil { - log.V(1).Info("skipping virtual machine upsert, no target host in decision") + if result.TargetHost == nil { + log.V(1).Info("skipping virtual machine upsert, no target host in result") return nil } + // The explanation is a VM-creation concern, so it is generated here from the + // pipeline result and recorded on the VM's cortex spec. + explanation := lib.GenerateExplanation(&result, nil) + // We derive hierarchical location information (region and az) from the // target host (i.e. the first host in the list). This assumes that all // hosts in the list share the same region and availability zone. - targetHost := *decision.Status.Result.TargetHost + targetHost := *result.TargetHost targetHv := &hv1.Hypervisor{} if err := h.Get(ctx, client.ObjectKey{Name: targetHost}, targetHv); err != nil { return fmt.Errorf("failed to get hypervisor %q: %w", targetHost, err) @@ -136,20 +143,18 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D } } - // Read intent from the decision, which was already resolved by the API layer - // (including a graceful fallback to CreateIntent for initial placements). - intent := decision.Spec.Intent - - // Write the cortex spec (our scheduling result). + // Write the cortex spec (our scheduling result). The intent was resolved by + // the API layer (with a graceful fallback to CreateIntent for initial + // placements) and passed in. vm.Spec.Cortex = &vmv1alpha1.CortexSpec{} - for _, host := range decision.Status.Result.OrderedHosts { + for _, host := range result.OrderedHosts { vm.Spec.Cortex.HostCandidates = append(vm.Spec.Cortex.HostCandidates, vmv1alpha1.HostRef{ APIVersion: hv1.GroupVersion.String(), Kind: "Hypervisor", Name: host, }) } - vm.Spec.Cortex.Explanation = decision.Status.Explanation + vm.Spec.Cortex.Explanation = explanation vm.Spec.Cortex.Intent = string(intent) // Write the ConditionTypeScheduled condition to indicate that the VM has diff --git a/internal/scheduling/nova/vm_handler_test.go b/internal/scheduling/nova/vm_handler_test.go index 8c1499c07..6c81f6f60 100644 --- a/internal/scheduling/nova/vm_handler_test.go +++ b/internal/scheduling/nova/vm_handler_test.go @@ -8,7 +8,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" vmopv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -56,15 +56,8 @@ func vmTestRequest() api.ExternalSchedulerRequest { return req } -func vmTestDecision(targetHost *string) *v1alpha1.Decision { - return &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - Intent: api.CreateIntent, - }, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{TargetHost: targetHost}, - }, - } +func vmTestResult(targetHost *string) compute.PlacementResult { + return compute.PlacementResult{TargetHost: targetHost} } func vmTestHypervisor(region, az string) *hv1.Hypervisor { @@ -88,7 +81,7 @@ func TestVirtualMachineHandlerCreate(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestDecision(&targetHost), vmTestRequest()); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert: %v", err) } @@ -137,7 +130,7 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestDecision(&targetHost), vmTestRequest()); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert: %v", err) } @@ -162,7 +155,7 @@ func TestVirtualMachineHandlerNoTargetHost(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() h := &VirtualMachineHandler{Client: fakeClient} - if err := h.Upsert(context.Background(), vmTestDecision(nil), vmTestRequest()); err != nil { + if err := h.Upsert(context.Background(), vmTestResult(nil), vmTestRequest(), api.CreateIntent); err != nil { t.Fatalf("Upsert with nil target host should be a no-op, got: %v", err) } var vmList vmopv1alpha1.VirtualMachineList @@ -183,7 +176,7 @@ func TestVirtualMachineHandlerMissingRegion(t *testing.T) { h := &VirtualMachineHandler{Client: fakeClient} targetHost := "compute-1" - if err := h.Upsert(context.Background(), vmTestDecision(&targetHost), vmTestRequest()); err == nil { + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.CreateIntent); err == nil { t.Fatal("expected error for missing region label, got nil") } vm := &vmopv1alpha1.VirtualMachine{} diff --git a/internal/scheduling/pods/filter_weigher_pipeline_controller.go b/internal/scheduling/pods/filter_weigher_pipeline_controller.go index 50afde197..ee140526c 100644 --- a/internal/scheduling/pods/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/pods/filter_weigher_pipeline_controller.go @@ -18,7 +18,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/pods/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/pods/plugins/weighers" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" @@ -30,9 +29,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -// The decision pipeline controller takes decision resources containing a -// pod ref and runs the scheduling pipeline to make a decision. -// This decision is then written back to the decision resource status. +// The pod pipeline controller watches for new pods and runs the scheduling +// pipeline to assign each a node. The result is written back directly via a +// Binding. // // Additionally, the controller watches for pipeline and step changes to // reconfigure the pipelines as needed. @@ -52,87 +51,30 @@ func (c *FilterWeigherPipelineController) PipelineType() v1alpha1.PipelineType { return v1alpha1.PipelineTypeFilterWeigher } -func (c *FilterWeigherPipelineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - c.processMu.Lock() - defer c.processMu.Unlock() - - // Determine if this is a decision or pod reconciliation. - decision := &v1alpha1.Decision{} - if err := c.Get(ctx, req.NamespacedName, decision); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - old := decision.DeepCopy() - if err := c.process(ctx, decision); err != nil { - return ctrl.Result{}, err - } - patch := client.MergeFrom(old) - if err := c.Status().Patch(ctx, decision, patch); err != nil { - return ctrl.Result{}, err - } +// Reconcile is a no-op: this controller schedules pods through its Pod watch +// handler and reconfigures pipelines through its Pipeline watch handler. A +// Reconcile method is required to satisfy the controller builder. +func (c *FilterWeigherPipelineController) Reconcile(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } func (c *FilterWeigherPipelineController) ProcessNewPod(ctx context.Context, pod *corev1.Pod) error { c.processMu.Lock() defer c.processMu.Unlock() - - // Create a decision resource to schedule the pod. - decision := &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pod-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainPods, - ResourceID: pod.Namespace + "--" + pod.Name, - PipelineRef: corev1.ObjectReference{ - Name: "pods-scheduler", - }, - PodRef: &corev1.ObjectReference{ - Name: pod.Name, - Namespace: pod.Namespace, - }, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - } - - err := c.process(ctx, decision) - if err != nil { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionFalse, - Reason: "PipelineRunFailed", - Message: "pipeline run failed: " + err.Error(), - }) - } else { - meta.SetStatusCondition(&decision.Status.Conditions, metav1.Condition{ - Type: v1alpha1.DecisionConditionReady, - Status: metav1.ConditionTrue, - Reason: "PipelineRunSucceeded", - Message: "pipeline run succeeded", - }) - } - return err + return c.process(ctx, pod) } -func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) error { +func (c *FilterWeigherPipelineController) process(ctx context.Context, pod *corev1.Pod) error { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. - pipeline, ok := c.GetPipeline(decision.Spec.PipelineRef.Name) + const pipelineName = "pods-scheduler" + pipeline, ok := c.GetPipeline(pipelineName) if !ok { - log.Error(nil, "pipeline not found or not ready", "pipelineName", decision.Spec.PipelineRef.Name) - return fmt.Errorf("pipeline not found or not ready: %q", decision.Spec.PipelineRef.Name) + log.Error(nil, "pipeline not found or not ready", "pipelineName", pipelineName) + return fmt.Errorf("pipeline not found or not ready: %q", pipelineName) } - // Check if the pod is already assigned to a node. - pod := &corev1.Pod{} - if err := c.Get(ctx, client.ObjectKey{ - Name: decision.Spec.PodRef.Name, - Namespace: decision.Spec.PodRef.Namespace, - }, pod); err != nil { - log.Error(err, "failed to fetch pod for decision") - return err - } if pod.Spec.NodeName != "" { log.V(1).Info("pod is already assigned to a node", "node", pod.Spec.NodeName) return nil @@ -150,14 +92,6 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision // Execute the scheduling pipeline. request := pods.PodPipelineRequest{Nodes: nodes.Items, Pod: *pod} result, err := pipeline.Run(request) - - if err == nil { - decision.Status.Result = &result - } - // Record a human-readable explanation of the decision for downstream - // consumers. On error the result is nil and the explanation carries the - // failure reason. - decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.V(1).Error(err, "failed to run scheduler pipeline") return errors.New("failed to run scheduler pipeline") @@ -167,8 +101,8 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision // Assign the first node returned by the pipeline using a Binding. binding := &corev1.Binding{ ObjectMeta: metav1.ObjectMeta{ - Name: decision.Spec.PodRef.Name, - Namespace: decision.Spec.PodRef.Namespace, + Name: pod.Name, + Namespace: pod.Namespace, }, Target: corev1.ObjectReference{ Kind: "Node", @@ -263,25 +197,6 @@ func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, if err != nil { return err } - // Watch decision changes across all clusters. - bldr, err = bldr.WatchesMulticluster( - &v1alpha1.Decision{}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(func(obj client.Object) bool { - decision := obj.(*v1alpha1.Decision) - if decision.Spec.SchedulingDomain != v1alpha1.SchedulingDomainPods { - return false - } - // Ignore already decided schedulings. - if decision.Status.Result != nil { - return false - } - return true - }), - ) - if err != nil { - return err - } return bldr.Named("cortex-pod-scheduler"). Complete(c) } diff --git a/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go b/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go index 0c9ad6fb8..e9a8f93c3 100644 --- a/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go @@ -15,166 +15,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheduling scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add corev1 scheme: %v", err) - } - - tests := []struct { - name string - decision *v1alpha1.Decision - nodes []corev1.Node - pod *corev1.Pod - expectError bool - expectDecision bool - expectTargetHost string - }{ - { - name: "successful pod decision processing", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainPods, - ResourceID: "test-pod", - PipelineRef: corev1.ObjectReference{ - Name: "pods-scheduler", - }, - PodRef: &corev1.ObjectReference{ - Name: "test-pod", - Namespace: "default", - }, - }, - }, - nodes: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "node2"}, - }, - }, - pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Spec: corev1.PodSpec{ - SchedulerName: "", - }, - }, - expectError: false, - expectDecision: true, - expectTargetHost: "node1", // NoopFilter returns first node - }, - { - name: "no nodes available", - decision: &v1alpha1.Decision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-decision-no-nodes", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainPods, - ResourceID: "test-pod", - PipelineRef: corev1.ObjectReference{ - Name: "pods-scheduler", - }, - PodRef: &corev1.ObjectReference{ - Name: "test-pod", - Namespace: "default", - }, - }, - }, - nodes: []corev1.Node{}, - expectError: true, - expectDecision: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := []runtime.Object{tt.decision} - for i := range tt.nodes { - objects = append(objects, &tt.nodes[i]) - } - if tt.pod != nil { - objects = append(objects, tt.pod) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). - Build() - - controller := &FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[pods.PodPipelineRequest]]{ - Pipelines: map[string]lib.FilterWeigherPipeline[pods.PodPipelineRequest]{ - "pods-scheduler": createMockPodPipeline(), - }, - }, - Monitor: lib.FilterWeigherPipelineMonitor{}, - } - controller.Client = client - - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: tt.decision.Name, - }, - } - - result, err := controller.Reconcile(context.Background(), req) - - if tt.expectError && err == nil { - t.Error("expected error but got none") - return - } - - if !tt.expectError && err != nil { - t.Errorf("expected no error, got: %v", err) - return - } - - if result.RequeueAfter > 0 { - t.Errorf("unexpected requeue: %v", result.RequeueAfter) - } - - // Verify decision status if expected - if tt.expectDecision { - var updatedDecision v1alpha1.Decision - err := client.Get(context.Background(), req.NamespacedName, &updatedDecision) - if err != nil { - t.Errorf("Failed to get updated decision: %v", err) - return - } - - if updatedDecision.Status.Result == nil { - t.Error("expected decision result to be set") - return - } - - if updatedDecision.Status.Result.TargetHost == nil { - t.Error("expected target host to be set") - return - } - - if *updatedDecision.Status.Result.TargetHost != tt.expectTargetHost { - t.Errorf("expected target host %q, got %q", tt.expectTargetHost, *updatedDecision.Status.Result.TargetHost) - } - } - }) - } -} - func TestFilterWeigherPipelineController_InitPipeline(t *testing.T) { controller := &FilterWeigherPipelineController{ Monitor: lib.FilterWeigherPipelineMonitor{}, @@ -393,7 +236,6 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { client := fake.NewClientBuilder(). WithScheme(scheme). WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ @@ -452,14 +294,14 @@ func createMockPodPipeline() lib.FilterWeigherPipeline[pods.PodPipelineRequest] type mockPodPipeline struct{} -func (m *mockPodPipeline) Run(request pods.PodPipelineRequest) (v1alpha1.DecisionResult, error) { +func (m *mockPodPipeline) Run(request pods.PodPipelineRequest) (lib.PipelineResult, error) { if len(request.Nodes) == 0 { - return v1alpha1.DecisionResult{}, nil + return lib.PipelineResult{}, nil } // Return the first node as the target host targetHost := request.Nodes[0].Name - return v1alpha1.DecisionResult{ + return lib.PipelineResult{ TargetHost: &targetHost, }, nil } diff --git a/internal/scheduling/reservations/capacity_accounting.go b/internal/scheduling/reservations/accounting/accounting.go similarity index 94% rename from internal/scheduling/reservations/capacity_accounting.go rename to internal/scheduling/reservations/accounting/accounting.go index 752d1b241..f896800e9 100644 --- a/internal/scheduling/reservations/capacity_accounting.go +++ b/internal/scheduling/reservations/accounting/accounting.go @@ -1,7 +1,12 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package reservations +// Package accounting holds the reservation capacity-accounting math and flavor +// group lookup. Both are pure, self-contained helpers over Reservation/Knowledge +// CRDs with no dependency on the scheduling pipeline, so they can live below both +// the reservation controllers and the Nova scheduler tree without forming an +// import cycle. +package accounting import ( hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" diff --git a/internal/scheduling/reservations/capacity_accounting_test.go b/internal/scheduling/reservations/accounting/accounting_test.go similarity index 99% rename from internal/scheduling/reservations/capacity_accounting_test.go rename to internal/scheduling/reservations/accounting/accounting_test.go index 448609868..1d00239e0 100644 --- a/internal/scheduling/reservations/capacity_accounting_test.go +++ b/internal/scheduling/reservations/accounting/accounting_test.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package reservations +package accounting import ( "testing" diff --git a/internal/scheduling/reservations/flavor_groups.go b/internal/scheduling/reservations/accounting/flavor_groups.go similarity index 99% rename from internal/scheduling/reservations/flavor_groups.go rename to internal/scheduling/reservations/accounting/flavor_groups.go index b2ce7d859..9d115592d 100644 --- a/internal/scheduling/reservations/flavor_groups.go +++ b/internal/scheduling/reservations/accounting/flavor_groups.go @@ -1,7 +1,7 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -package reservations +package accounting import ( "context" diff --git a/internal/scheduling/reservations/capacity/config.go b/internal/scheduling/reservations/capacity/config.go index 4b9292042..beab45642 100644 --- a/internal/scheduling/reservations/capacity/config.go +++ b/internal/scheduling/reservations/capacity/config.go @@ -29,9 +29,6 @@ type Config struct { // PlaceablePipeline is the scheduler pipeline used for the current-state probe. // This pipeline considers current VM allocations to determine remaining placement capacity. PlaceablePipeline string `json:"capacityPlaceablePipeline"` - - // SchedulerURL is the endpoint of the nova external scheduler. - SchedulerURL string `json:"schedulerURL"` } // ApplyDefaults fills in any unset values with defaults. @@ -49,9 +46,6 @@ func (c *Config) ApplyDefaults() { if c.PlaceablePipeline == "" { c.PlaceablePipeline = defaults.PlaceablePipeline } - if c.SchedulerURL == "" { - c.SchedulerURL = defaults.SchedulerURL - } } // Validate checks that the config is internally consistent after defaults are applied. @@ -69,6 +63,5 @@ func DefaultConfig() Config { MinReconcileInterval: metav1.Duration{Duration: 30 * time.Second}, TotalPipeline: "kvm-general-purpose-load-balancing", PlaceablePipeline: "kvm-general-purpose-load-balancing", - SchedulerURL: "http://localhost:8080/scheduler/nova/external", } } diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index caeab9556..90c9c2b01 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -29,7 +29,10 @@ import ( "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/go-logr/logr" ) @@ -102,20 +105,20 @@ func capacityMapsEqual(a, b map[hv1.ResourceName]resource.Quantity) bool { // periodic floor timer. All watch events are coalesced into a single reconcile key so // rapid changes produce at most one queued reconcile. type Reconciler struct { - client client.Client - vmSource reservations.VMSource - schedulerClient *reservations.SchedulerClient - config Config + client client.Client + vmSource reservations.VMSource + scheduler *sched.Placement + config Config lastReconcileAt time.Time } -func NewController(c client.Client, config Config, vmSource reservations.VMSource) *Reconciler { +func NewController(c client.Client, config Config, vmSource reservations.VMSource, scheduler *sched.Placement) *Reconciler { return &Reconciler{ - client: c, - vmSource: vmSource, - schedulerClient: reservations.NewSchedulerClient(config.SchedulerURL), - config: config, + client: c, + vmSource: vmSource, + scheduler: scheduler, + config: config, } } @@ -223,7 +226,7 @@ func (c *Reconciler) reconcileAll(ctx context.Context) error { logger := LoggerFromContext(ctx) startTime := time.Now() - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { return fmt.Errorf("failed to get flavor groups: %w", err) @@ -814,29 +817,50 @@ func (c *Reconciler) probeScheduler( } } - resp, err := c.schedulerClient.ScheduleReservation(ctx, reservations.ScheduleReservationRequest{ - InstanceUUID: "capacity-" + flavor.Name, - ProjectID: "cortex-capacity-probe", - FlavorName: flavor.Name, - MemoryMB: flavor.MemoryMB, - VCPUs: flavor.VCPUs, - FlavorExtraSpecs: flavor.ExtraSpecs, - AvailabilityZone: az, - Pipeline: pipeline, - EligibleHosts: eligibleHosts, - SchedulerHints: map[string]any{"_nova_check_type": string(schedulerapi.CapacityProbeIntent)}, - }, scheduling.Options{ - ReadOnly: true, - AssumeEmptyHosts: ignoreAllocations, - IgnoredReservationTypes: ignoredReservationTypes, - SkipInflight: true, - SkipCommittedResourceTracking: true, - }) + probeHosts := make([]sched.Host, 0, len(eligibleHosts)) + probeWeights := make(map[string]float64, len(eligibleHosts)) + for _, h := range eligibleHosts { + probeHosts = append(probeHosts, sched.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) + probeWeights[h.ComputeHost] = 0.0 + } + // Honor the flavor's trait and hypervisor-type constraints so the probe only + // counts compatible hosts (the typed request is built directly, so + // filter_has_requested_traits/filter_capabilities would otherwise see empty + // slices and become no-ops). No hypervisor-type default — only match when the + // flavor names one. + requiredTraits, forbiddenTraits := nova.SplitTraits(flavor.ExtraSpecs) + out, err := c.scheduler.RunWithoutLock(ctx, sched.PlacementRequest{ + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: sched.HypervisorFamilyKVM, + Hosts: probeHosts, + Weights: probeWeights, + Intent: schedulerapi.CapacityProbeIntent, + InstanceUUID: "capacity-" + flavor.Name, + NumInstances: 1, // probe a single flavor slot; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free + ProjectID: "cortex-capacity-probe", + AvailabilityZone: az, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + RequestedCapabilities: reservations.HypervisorTypeCapability(flavor.ExtraSpecs, false), + AcceleratorsRequested: nova.RequestsAccelerators(flavor.ExtraSpecs), + Flavor: sched.FlavorSpec{ + VCPUs: flavor.VCPUs, + MemoryMB: flavor.MemoryMB, + }, + Options: scheduling.Options{ + ReadOnly: true, + AssumeEmptyHosts: ignoreAllocations, + IgnoredReservationTypes: ignoredReservationTypes, + SkipInflight: true, + SkipCommittedResourceTracking: true, + }, + }, nil, nil) if err != nil { return 0, 0, nil, fmt.Errorf("scheduler call failed (pipeline=%s): %w", pipeline, err) } - for _, hostName := range resp.Hosts { + for _, hostName := range out.OrderedHosts { hv, ok := hvByName[hostName] if !ok { continue @@ -897,7 +921,7 @@ func (c *Reconciler) blockedResourcesByHost(ctx context.Context) (map[string]map continue } - resourcesToBlock := reservations.UnusedReservationCapacity(res, false) + resourcesToBlock := accounting.UnusedReservationCapacity(res, false) for host := range hostsToBlock { if blocked[host] == nil { blocked[host] = make(map[string]int64) @@ -968,7 +992,7 @@ func (c *Reconciler) reservedResourcesByGroupAZ( "reservation", res.Name, "type", res.Spec.Type, "az", az) continue } - blockedRes := reservations.UnusedReservationCapacity(res, false) + blockedRes := accounting.UnusedReservationCapacity(res, false) // Route each reservation to exactly one bucket by type; the two are disjoint. bucket := committed if res.Spec.Type == v1alpha1.ReservationTypeFailover { diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 1894c859e..5501166ec 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -5,9 +5,7 @@ package capacity import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" + "errors" "regexp" "sort" "testing" @@ -25,6 +23,8 @@ import ( schedulerapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) @@ -99,38 +99,98 @@ func newHypervisor(name, az string, memoryBytes int64) *hv1.Hypervisor { return hv } -// newMockSchedulerServer creates an httptest server that always returns the given host list. -func newMockSchedulerServer(t *testing.T, hosts []string) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - resp := schedulerapi.ExternalSchedulerResponse{Hosts: hosts} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Errorf("mock scheduler: failed to encode response: %v", err) - } - })) +// fakePipeline returns a fixed ordered-host list (modelling a scheduler run). +type fakePipeline struct{ hosts []string } + +func (p fakePipeline) Run(sched.PlacementRequest) (sched.PlacementResult, error) { + return sched.PlacementResult{OrderedHosts: p.hosts}, nil } -// newMockSchedulerServerPerPipeline returns different candidate hosts depending on the request's -// pipeline, so a test can model a host that passes the total (empty-datacenter) probe but drops -// out of the placeable probe because it is fully occupied. -func newMockSchedulerServerPerPipeline(t *testing.T, hostsByPipeline map[string][]string) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req schedulerapi.ExternalSchedulerRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("mock scheduler: failed to decode request: %v", err) - } - resp := schedulerapi.ExternalSchedulerResponse{Hosts: hostsByPipeline[req.Pipeline]} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Errorf("mock scheduler: failed to encode response: %v", err) - } - })) +// fakePipelineProvider serves a fakePipeline per pipeline name, returning the +// per-pipeline host list configured for the test. When allPipelines is set it +// returns that host list for every pipeline name regardless of hostsByPipeline. +type fakePipelineProvider struct { + hostsByPipeline map[string][]string + allPipelines []string + useAll bool +} + +func (f fakePipelineProvider) GetPipeline(name string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { + if f.useAll { + return fakePipeline{hosts: f.allPipelines}, true + } + return fakePipeline{hosts: f.hostsByPipeline[name]}, true +} + +// fakeGatherer is a no-op gatherer (tests preselect their own hosts). +type fakeGatherer struct{} + +func (fakeGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { + return nil +} + +// newMockScheduler builds an in-process compute.Placement whose pipelines return +// the given per-pipeline host lists. Replaces the former httptest mock server. +func newMockScheduler(hostsByPipeline map[string][]string) *sched.Placement { + return sched.NewPlacement(fakePipelineProvider{hostsByPipeline: hostsByPipeline}, fakeGatherer{}) +} + +// newMockSchedulerHosts is a convenience for a scheduler that returns the same +// host list regardless of pipeline. +func newMockSchedulerHosts(hosts []string) *sched.Placement { + return sched.NewPlacement(fakePipelineProvider{allPipelines: hosts, useAll: true}, fakeGatherer{}) +} + +// fakeErrPipeline models a scheduler run that fails (replaces the former +// HTTP 500 mock server). +type fakeErrPipeline struct{} + +func (fakeErrPipeline) Run(sched.PlacementRequest) (sched.PlacementResult, error) { + return sched.PlacementResult{}, errors.New("boom") +} + +type fakeErrPipelineProvider struct{} + +func (fakeErrPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { + return fakeErrPipeline{}, true +} + +// newErrScheduler builds a scheduler whose pipeline always returns an error. +func newErrScheduler() *sched.Placement { + return sched.NewPlacement(fakeErrPipelineProvider{}, fakeGatherer{}) +} + +// capturePipeline records the request it is run with and returns a fixed host list. +type capturePipeline struct { + captured *sched.PlacementRequest + hosts []string +} + +func (p capturePipeline) Run(req sched.PlacementRequest) (sched.PlacementResult, error) { + *p.captured = req + return sched.PlacementResult{OrderedHosts: p.hosts}, nil +} + +type capturePipelineProvider struct { + captured *sched.PlacementRequest + hosts []string +} + +func (f capturePipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { + return capturePipeline(f), true } -// newController is a test helper that creates a Reconciler with a nil VMSource. -func newController(t *testing.T, c client.Client, cfg Config) *Reconciler { +// captureScheduler builds a scheduler that records the request it is run with +// into captured and returns the given hosts. +func captureScheduler(captured *sched.PlacementRequest, hosts []string) *sched.Placement { + return sched.NewPlacement(capturePipelineProvider{captured: captured, hosts: hosts}, fakeGatherer{}) +} + +// newController is a test helper that creates a Reconciler with a nil VMSource +// and the given scheduler. +func newController(t *testing.T, c client.Client, cfg Config, scheduler *sched.Placement) *Reconciler { t.Helper() - return NewController(c, cfg, nil) + return NewController(c, cfg, nil, scheduler) } // --- unit tests for pure helper functions --- @@ -224,14 +284,12 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { Build() // Both probes return host-1 → total capacity = floor(4GiB/4GiB) = 1, placeable = 1. - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) ctrl := newController(t, fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -311,17 +369,15 @@ func TestReconcileAZ_RawCapacityIncludesOccupiedHosts(t *testing.T) { // Total probe sees both hosts (empty-datacenter); placeable probe sees only host-1 because // host-2 is fully occupied and the smallest flavor no longer fits. - schedulerServer := newMockSchedulerServerPerPipeline(t, map[string][]string{ + scheduler := newMockScheduler(map[string][]string{ totalPipe: {"host-1", "host-2"}, placeablePipe: {"host-1"}, }) - defer schedulerServer.Close() ctrl := newController(t, fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: totalPipe, PlaceablePipeline: placeablePipe, - }) + }, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -370,14 +426,12 @@ func TestReconcileAZ_WritesReservedCapacity(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) ctrl := newController(t, fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: flavorCPUs} groupData := compute.FlavorGroupFeature{SmallestFlavor: smallFlavor, Flavors: []compute.FlavorInGroup{smallFlavor}} @@ -495,7 +549,7 @@ func TestReservedResourcesByGroupAZ(t *testing.T) { WithObjects(committedA, failoverB, failoverAByFlavor, pending, inflight). Build() - c := newController(t, fakeClient, Config{}) + c := newController(t, fakeClient, Config{}, newMockSchedulerHosts(nil)) flavorA := compute.FlavorInGroup{Name: "group-a-small", MemoryMB: 4096, VCPUs: flavorCPUs} flavorB := compute.FlavorInGroup{Name: "group-b-small", MemoryMB: 4096, VCPUs: flavorCPUs} @@ -546,17 +600,11 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - // Scheduler returns 500 to simulate error. - failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer failServer.Close() - + // Scheduler returns an error to simulate a failed probe. ctrl := newController(t, fakeClient, Config{ - SchedulerURL: failServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, newErrScheduler()) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -616,17 +664,11 @@ func TestReconcileAZ_MarksExistingCRDNotReadyOnSchedulerError(t *testing.T) { t.Fatalf("failed to set Ready=True: %v", err) } - // Scheduler returns 500 to simulate error. - failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer failServer.Close() - + // Scheduler returns an error to simulate a failed probe. ctrl := newController(t, fakeClient, Config{ - SchedulerURL: failServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, newErrScheduler()) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -685,14 +727,12 @@ func TestReconcileAZ_IdempotentUpdate(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) ctrl := newController(t, fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -731,10 +771,9 @@ func TestReconcileAll_SkipsGroupsWithNoAZs(t *testing.T) { Build() ctrl := newController(t, fakeClient, Config{ - SchedulerURL: "http://localhost:9999", // unreachable; not called TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, newMockSchedulerHosts(nil)) // scheduler not called (no AZs) if err := ctrl.reconcileAll(context.Background()); err != nil { t.Errorf("reconcileAll with no hypervisors returned error: %v", err) @@ -760,10 +799,9 @@ func TestProbeScheduler_CapacityCalculation(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() // Scheduler returns both hosts. - srv := newMockSchedulerServer(t, []string{"host-1", "host-2"}) - defer srv.Close() + scheduler := newMockSchedulerHosts([]string{"host-1", "host-2"}) - c := newController(t, fakeClient, Config{SchedulerURL: srv.URL}) + c := newController(t, fakeClient, Config{}, scheduler) hvByName := map[string]hv1.Hypervisor{ "host-1": *hv1Obj, "host-2": *hv2Obj, @@ -810,10 +848,9 @@ func TestProbeScheduler_SubtractsAllocationsWhenNotIgnored(t *testing.T) { } fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() - srv := newMockSchedulerServer(t, []string{"host-1"}) - defer srv.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) - c := newController(t, fakeClient, Config{SchedulerURL: srv.URL}) + c := newController(t, fakeClient, Config{}, scheduler) hvByName := map[string]hv1.Hypervisor{"host-1": *hv} flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: memMB} @@ -853,14 +890,12 @@ func TestReconcileAll_MultipleGroupsAndAZs(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - srv := newMockSchedulerServer(t, []string{}) - defer srv.Close() + scheduler := newMockSchedulerHosts([]string{}) c := newController(t, fakeClient, Config{ - SchedulerURL: srv.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, scheduler) if err := c.reconcileAll(context.Background()); err != nil { t.Fatalf("reconcileAll failed: %v", err) @@ -908,10 +943,9 @@ func TestReconcileAll_FlavorGroupsKnowledgeNotReady(t *testing.T) { Build() c := newController(t, fakeClient, Config{ - SchedulerURL: "http://localhost:9999", TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }) + }, newMockSchedulerHosts(nil)) // scheduler not called (knowledge not ready) // Should return an error when knowledge is not ready. if err := c.reconcileAll(context.Background()); err == nil { @@ -925,7 +959,7 @@ func TestReconcileAZ_ZeroMemoryFlavorSkipped(t *testing.T) { WithScheme(scheme). WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}). Build() - c := newController(t, fakeClient, Config{}) + c := newController(t, fakeClient, Config{}, newMockSchedulerHosts(nil)) groupData := compute.FlavorGroupFeature{ SmallestFlavor: compute.FlavorInGroup{Name: "bad-flavor", MemoryMB: 0}, @@ -1075,12 +1109,6 @@ func TestComputeTotalCapacity(t *testing.T) { } } -// Verify that the module-level log variable from reservations package doesn't -// collide with the one in this package. -func TestPackageLogVar(t *testing.T) { - _ = reservations.NewSchedulerClient("http://localhost") -} - func TestSumCommittedCapacity(t *testing.T) { const ( groupName = "hana-v2" @@ -1131,7 +1159,7 @@ func TestSumCommittedCapacity(t *testing.T) { WithObjects(objects...). Build() - c := newController(t, fakeClient, Config{}) + c := newController(t, fakeClient, Config{}, newMockSchedulerHosts(nil)) // smallestFlavorBytes = 4GiB → cr1 = 8GiB/4GiB = 2 slots, cr2 = 4GiB/4GiB = 1 slot → total = 3. got, err := c.sumCommittedCapacity(context.Background(), groupName, az, memBytes) if err != nil { @@ -1217,17 +1245,15 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { } // Scheduler returns host-1 so probes succeed. - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) // VMSource returns empty list (all VMs removed). vmSource := &mockVMSource{vms: []reservations.VM{}} ctrl := NewController(fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", - }, vmSource) + }, vmSource, scheduler) smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} groupData := compute.FlavorGroupFeature{ @@ -1284,10 +1310,9 @@ func TestProbeScheduler_SubtractsReservationBlocksWhenNotIgnored(t *testing.T) { } fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() - srv := newMockSchedulerServer(t, []string{"host-1"}) - defer srv.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) - c := newController(t, fakeClient, Config{SchedulerURL: srv.URL}) + c := newController(t, fakeClient, Config{}, scheduler) hvByName := map[string]hv1.Hypervisor{"host-1": *hv} flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: memMB} @@ -1317,32 +1342,23 @@ func TestProbeScheduler_SetsCapacityProbeIntent(t *testing.T) { scheme := newTestScheme(t) hv := newHypervisor("host-1", "az-a", 4096*1024*1024) - var capturedReq schedulerapi.ExternalSchedulerRequest - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - json.NewEncoder(w).Encode(schedulerapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}}) //nolint:errcheck - })) - defer srv.Close() + // Capture the request the scheduler receives so we can assert the probe + // intent and project are set. + var capturedReq sched.PlacementRequest + captured := captureScheduler(&capturedReq, []string{"host-1"}) - c := NewController(fake.NewClientBuilder().WithScheme(scheme).Build(), Config{SchedulerURL: srv.URL}, nil) + c := NewController(fake.NewClientBuilder().WithScheme(scheme).Build(), Config{}, nil, captured) hvByName := map[string]hv1.Hypervisor{"host-1": *hv} flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: 4096} if _, _, _, err := c.probeScheduler(context.Background(), flavor, "az-a", "test-pipeline", hvByName, true, nil); err != nil { t.Fatalf("probeScheduler failed: %v", err) } - hint, err := capturedReq.Spec.Data.GetSchedulerHintStr("_nova_check_type") - if err != nil { - t.Fatalf("failed to get _nova_check_type hint: %v", err) - } - if hint != string(schedulerapi.CapacityProbeIntent) { - t.Errorf("capacity probe must set _nova_check_type=%q, got %q", schedulerapi.CapacityProbeIntent, hint) + if capturedReq.Intent != schedulerapi.CapacityProbeIntent { + t.Errorf("capacity probe must set Intent=%q, got %q", schedulerapi.CapacityProbeIntent, capturedReq.Intent) } - if capturedReq.Spec.Data.ProjectID != "cortex-capacity-probe" { - t.Errorf("capacity probe must send ProjectID cortex-capacity-probe, got %q", capturedReq.Spec.Data.ProjectID) + if capturedReq.ProjectID != "cortex-capacity-probe" { + t.Errorf("capacity probe must send ProjectID cortex-capacity-probe, got %q", capturedReq.ProjectID) } } @@ -1367,16 +1383,14 @@ func TestReconcile_ReactsToKnowledgeChange(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) r := NewController(fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, MinReconcileInterval: metav1.Duration{Duration: 30 * time.Second}, - }, nil) + }, nil, scheduler) req := ctrl.Request{NamespacedName: types.NamespacedName{Name: coalescedKey}} result, err := r.Reconcile(context.Background(), req) @@ -1418,17 +1432,15 @@ func TestReconcile_MinIntervalEarlyReturn(t *testing.T) { WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). Build() - schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) - defer schedulerServer.Close() + scheduler := newMockSchedulerHosts([]string{"host-1"}) minInterval := 30 * time.Second r := NewController(fakeClient, Config{ - SchedulerURL: schedulerServer.URL, TotalPipeline: "kvm-report-capacity", PlaceablePipeline: "kvm-general-purpose", ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, MinReconcileInterval: metav1.Duration{Duration: minInterval}, - }, nil) + }, nil, scheduler) req := ctrl.Request{NamespacedName: types.NamespacedName{Name: coalescedKey}} diff --git a/internal/scheduling/reservations/commitments/api/change_commitments.go b/internal/scheduling/reservations/commitments/api/change_commitments.go index 41df5ea9d..df981e452 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments.go @@ -15,6 +15,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/go-logr/logr" @@ -180,7 +181,7 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque } { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} if knowledgeCRD, err := knowledge.Get(ctx); err == nil && knowledgeCRD != nil { var currentVersion int64 = -1 if !knowledgeCRD.Status.LastContentChange.IsZero() { @@ -231,7 +232,7 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque } func (api *HTTPAPI) processCommitmentChanges(ctx context.Context, w http.ResponseWriter, logger logr.Logger, req liquid.CommitmentChangeRequest, resp *liquid.CommitmentChangeResponse) error { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Info("failed to get flavor groups from knowledge extractor", "error", err) @@ -640,7 +641,7 @@ func rollbackCR(ctx context.Context, logger logr.Logger, k8sClient client.Client // performDryRun checks whether the net capacity delta in req can be placed by running // temporary probe CommittedResource CRDs through the controller, then always cleaning them up. func (api *HTTPAPI) performDryRun(ctx context.Context, logger logr.Logger, req liquid.CommitmentChangeRequest, resp *liquid.CommitmentChangeResponse) { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { resp.RejectionReason = "dry run: caches not ready, please retry later" diff --git a/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go b/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go index d344d5aca..f4963ef1b 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go @@ -12,7 +12,6 @@ package api import ( "context" - "encoding/json" "net/http" "net/http/httptest" "os" @@ -20,8 +19,9 @@ import ( "testing" "time" - schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/prometheus/client_golang/prometheus" @@ -46,19 +46,18 @@ const ( // e2eEnv is a full end-to-end test environment: real controllers, fake k8s client, // mock scheduler, and a background reconcile driver goroutine. type e2eEnv struct { - t *testing.T - k8sClient client.Client - httpServer *httptest.Server - schedulerSrv *httptest.Server - crCtrl *commitments.CommittedResourceController - resCtrl *commitments.CommitmentReservationController - cancelBg context.CancelFunc - bgDone chan struct{} + t *testing.T + k8sClient client.Client + httpServer *httptest.Server + crCtrl *commitments.CommittedResourceController + resCtrl *commitments.CommitmentReservationController + cancelBg context.CancelFunc + bgDone chan struct{} } // newE2EEnv creates an e2eEnv with the given flavors and scheduler handler. // The scheduler handler controls what the mock Nova scheduler returns. -func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, schedulerHandler http.HandlerFunc) *e2eEnv { +func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler *sched.Placement) *e2eEnv { t.Helper() log.SetLogger(zap.New(zap.WriteTo(os.Stderr), zap.UseDevMode(true))) @@ -98,8 +97,6 @@ func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler }). Build() - schedulerSrv := httptest.NewServer(schedulerHandler) - crCtrl := &commitments.CommittedResourceController{ Client: k8sClient, Scheme: scheme, @@ -107,10 +104,10 @@ func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler } resCtrl := &commitments.CommitmentReservationController{ - Client: k8sClient, - Scheme: scheme, + Client: k8sClient, + Scheme: scheme, + Scheduler: scheduler, Conf: commitments.ReservationControllerConfig{ - SchedulerURL: schedulerSrv.URL, RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, RequeueIntervalRetry: metav1.Duration{Duration: 100 * time.Millisecond}, }, @@ -133,14 +130,13 @@ func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, scheduler ctx, cancel := context.WithCancel(context.Background()) env := &e2eEnv{ - t: t, - k8sClient: k8sClient, - httpServer: httpServer, - schedulerSrv: schedulerSrv, - crCtrl: crCtrl, - resCtrl: resCtrl, - cancelBg: cancel, - bgDone: make(chan struct{}), + t: t, + k8sClient: k8sClient, + httpServer: httpServer, + crCtrl: crCtrl, + resCtrl: resCtrl, + cancelBg: cancel, + bgDone: make(chan struct{}), } go env.driveReconciles(ctx) return env @@ -150,7 +146,6 @@ func (e *e2eEnv) close() { e.cancelBg() <-e.bgDone e.httpServer.Close() - e.schedulerSrv.Close() } // asCRTestEnv wraps e2eEnv as a CRTestEnv to reuse its HTTP-call and assertion helpers. @@ -273,27 +268,35 @@ func e2eIsTerminalReservation(res v1alpha1.Reservation) bool { // Scheduler handlers // ============================================================================ -func e2eAcceptScheduler(t *testing.T) http.HandlerFunc { - t.Helper() - return func(w http.ResponseWriter, r *http.Request) { - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Errorf("scheduler encode: %v", err) - } - } +// e2eFakePipeline models a scheduler run returning a fixed host list. +type e2eFakePipeline struct{ hosts []string } + +func (p e2eFakePipeline) Run(sched.PlacementRequest) (sched.PlacementResult, error) { + return sched.PlacementResult{OrderedHosts: p.hosts}, nil } -func e2eRejectScheduler(t *testing.T) http.HandlerFunc { - t.Helper() - return func(w http.ResponseWriter, r *http.Request) { - // Return an empty hosts list — the reservation controller treats this as NoHostsFound. - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{}} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Errorf("scheduler encode: %v", err) - } - } +type e2eFakeProvider struct{ hosts []string } + +func (f e2eFakeProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { + return e2eFakePipeline(f), true +} + +type e2eFakeGatherer struct{} + +func (e2eFakeGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { + return nil } +func e2eScheduler(hosts []string) *sched.Placement { + return sched.NewPlacement(e2eFakeProvider{hosts: hosts}, e2eFakeGatherer{}) +} + +// e2eAcceptScheduler returns a scheduler that always places on host-1. +func e2eAcceptScheduler() *sched.Placement { return e2eScheduler([]string{"host-1"}) } + +// e2eRejectScheduler returns a scheduler that finds no host (NoHostsFound). +func e2eRejectScheduler() *sched.Placement { return e2eScheduler(nil) } + // assertNoDryRunProbes verifies that all probe CommittedResource CRDs and their child // Reservations have been cleaned up. It only checks objects associated with dry-run probes // (name prefix "commitment-dryrun-") rather than asserting zero total objects, so the @@ -333,7 +336,7 @@ var e2eFlavor = &TestFlavor{Name: "m1.small", Group: "hana_1", MemoryMB: 1024, V func TestE2EChangeCommitments(t *testing.T) { testCases := []struct { Name string - Scheduler func(*testing.T) http.HandlerFunc + Scheduler func() *sched.Placement ReqJSON string WantResp APIResponseExpectation WantAbsent []string @@ -521,7 +524,7 @@ func TestE2EChangeCommitments(t *testing.T) { for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { - env := newE2EEnv(t, []*TestFlavor{e2eFlavor}, e2eInfoVersion, tc.Scheduler(t)) + env := newE2EEnv(t, []*TestFlavor{e2eFlavor}, e2eInfoVersion, tc.Scheduler()) defer env.close() te := env.asCRTestEnv() diff --git a/internal/scheduling/reservations/commitments/api/info.go b/internal/scheduling/reservations/commitments/api/info.go index 9950ab610..26993422e 100644 --- a/internal/scheduling/reservations/commitments/api/info.go +++ b/internal/scheduling/reservations/commitments/api/info.go @@ -14,6 +14,7 @@ import ( "time" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" "github.com/go-logr/logr" "github.com/google/uuid" @@ -105,7 +106,7 @@ func mibToGiB(v *uint64) *uint64 { // All flavor groups report usage; only those with fixed RAM/core ratio accept commitments. func (api *HTTPAPI) buildServiceInfo(ctx context.Context, logger logr.Logger) (liquid.ServiceInfo, error) { // Get all flavor groups from Knowledge CRDs - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { // Return -1 as version when knowledge is not ready diff --git a/internal/scheduling/reservations/commitments/api/quota.go b/internal/scheduling/reservations/commitments/api/quota.go index 59d759a9a..ff989fd32 100644 --- a/internal/scheduling/reservations/commitments/api/quota.go +++ b/internal/scheduling/reservations/commitments/api/quota.go @@ -13,7 +13,7 @@ import ( "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/google/uuid" @@ -97,7 +97,7 @@ func (api *HTTPAPI) HandleQuota(w http.ResponseWriter, r *http.Request) { // Fetch flavor groups to determine per-resource RAM unit. // The ProjectQuota CRD stores RAM values in GiB; Limes sends in the declared unit // (slots for fixed-ratio groups, GiB for variable-ratio). Convert on receipt. - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(r.Context(), nil) if err != nil { log.Info("flavor groups not available for quota unit conversion", "error", err.Error()) diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index 78381d0f5..b9f01f8c0 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -15,7 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" ) // ErrCapacityNotReady is returned by CalculateCapacity when one or more FlavorGroupCapacity @@ -64,7 +64,7 @@ func NewCapacityCalculator(client client.Client, conf APIConfig) *CapacityCalcul // - Usage: always None. Limes derives project usage from the separate Report-Usage endpoint, so // a usage value here is unused and would only invite misinterpretation of reserved capacity. func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.ServiceCapacityRequest) (liquid.ServiceCapacityReport, error) { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { return liquid.ServiceCapacityReport{}, fmt.Errorf("failed to get flavor groups: %w", err) diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go index 9a589bd10..9d15cd2ce 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go @@ -24,6 +24,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" ) @@ -440,7 +441,7 @@ func (r *CommittedResourceController) reconcileCoresHeadroom(ctx context.Context } func (r *CommittedResourceController) applyReservationState(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (*ApplyResult, error) { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { return nil, fmt.Errorf("flavor knowledge not ready: %w", err) @@ -639,7 +640,7 @@ func (r *CommittedResourceController) rollbackToAccepted(ctx context.Context, lo if cr.Status.AcceptedSpec == nil { return r.deleteChildReservations(ctx, cr) } - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { // Can't compute the rollback target — fall back to full delete rather than leaving diff --git a/internal/scheduling/reservations/commitments/config.go b/internal/scheduling/reservations/commitments/config.go index 787bb8ef1..34e41ab9d 100644 --- a/internal/scheduling/reservations/commitments/config.go +++ b/internal/scheduling/reservations/commitments/config.go @@ -72,8 +72,6 @@ type ReservationControllerConfig struct { // When false, violations are detected and exposed via metrics but no eviction is performed. // Has no effect when EnableOversubscriptionCheck is false. EnableOversubscriptionReservationEviction bool `json:"enableOversubscriptionReservationEviction,omitempty"` - // SchedulerURL is the endpoint of the nova external scheduler. - SchedulerURL string `json:"schedulerURL"` // PipelineDefault is the fallback pipeline when no FlavorGroupPipelines entry matches. PipelineDefault string `json:"pipelineDefault"` // FlavorGroupPipelines maps flavor group IDs to pipeline names; "*" acts as catch-all. diff --git a/internal/scheduling/reservations/commitments/host_oversubscription_controller.go b/internal/scheduling/reservations/commitments/host_oversubscription_controller.go index 7259664a4..b116465f7 100644 --- a/internal/scheduling/reservations/commitments/host_oversubscription_controller.go +++ b/internal/scheduling/reservations/commitments/host_oversubscription_controller.go @@ -22,6 +22,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -267,7 +268,7 @@ func (r *HostOversubscriptionController) SetupWithManager(mgr ctrl.Manager, mcl // computeViolations returns the per-resource excess (positive quantity) for any resource where // the host is over-subscribed. Returns nil if capacity data is unavailable, empty map if no violation. func computeViolations(allReservations []v1alpha1.Reservation, hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity { - free := reservations.HostFreeCapacity(allReservations, hv) + free := accounting.HostFreeCapacity(allReservations, hv) if free == nil { return nil } @@ -333,7 +334,7 @@ func selectEvictionTarget( } entries := make([]allocatedEntry, len(allocated)) for i, res := range allocated { - unusedCap := reservations.UnusedReservationCapacity(res, false) + unusedCap := accounting.UnusedReservationCapacity(res, false) buckets := make(map[hv1.ResourceName]int64) for rn, total := range res.Spec.Resources { if t := total.Value(); t != 0 { @@ -381,7 +382,7 @@ func evictReservation( logger := LoggerFromContext(ctx).WithValues("component", "host-oversubscription", "reservation", res.Name) - freed := reservations.UnusedReservationCapacity(res, false) + freed := accounting.UnusedReservationCapacity(res, false) old := res.DeepCopy() res.Spec.TargetHost = "" diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go index 1ca03a624..9bc0ee94b 100644 --- a/internal/scheduling/reservations/commitments/integration_test.go +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -19,10 +19,7 @@ package commitments import ( "context" - "encoding/json" "fmt" - "net/http" - "net/http/httptest" "strings" "sync/atomic" "testing" @@ -30,7 +27,10 @@ import ( schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -406,10 +406,30 @@ type intgEnv struct { crController *CommittedResourceController resController *CommitmentReservationController hvController *HostOversubscriptionController - schedulerSrv *httptest.Server } -func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.HandlerFunc, vmSource reservations.VMSource) *intgEnv { +// intgSchedulerFn models a scheduler run: given the request it returns the ordered +// hosts (an empty list models NoHostsFound). It replaces the former httptest mock +// Nova scheduler handler. +type intgSchedulerFn func(sched.PlacementRequest) sched.PlacementResult + +type intgPipeline struct{ fn intgSchedulerFn } + +func (p intgPipeline) Run(req sched.PlacementRequest) (sched.PlacementResult, error) { + return p.fn(req), nil +} + +type intgPipelineProvider struct{ fn intgSchedulerFn } + +func (p intgPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { + return intgPipeline(p), true +} + +func newIntgScheduler(fn intgSchedulerFn) *sched.Placement { + return sched.NewPlacement(intgPipelineProvider{fn: fn}, fakeGatherer{}) +} + +func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn intgSchedulerFn, vmSource reservations.VMSource) *intgEnv { t.Helper() scheme := newCRTestScheme(t) @@ -466,8 +486,6 @@ func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.H }). Build() - schedulerSrv := httptest.NewServer(schedulerFn) - crCtrl := &CommittedResourceController{ Client: k8sClient, Scheme: scheme, @@ -479,11 +497,11 @@ func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.H } monitor := NewReservationControllerMonitor() resCtrl := &CommitmentReservationController{ - Client: k8sClient, - Scheme: scheme, - VMSource: vmSource, + Client: k8sClient, + Scheme: scheme, + Scheduler: newIntgScheduler(schedulerFn), + VMSource: vmSource, Conf: ReservationControllerConfig{ - SchedulerURL: schedulerSrv.URL, RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, EnableOversubscriptionCheck: true, EnableOversubscriptionReservationEviction: true, @@ -497,10 +515,10 @@ func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.H Monitor: &monitor, Conf: resCtrl.Conf, } - return &intgEnv{k8sClient: k8sClient, crController: crCtrl, resController: resCtrl, hvController: hvCtrl, schedulerSrv: schedulerSrv} + return &intgEnv{k8sClient: k8sClient, crController: crCtrl, resController: resCtrl, hvController: hvCtrl} } -func (e *intgEnv) close() { e.schedulerSrv.Close() } +func (e *intgEnv) close() {} func newDefaultIntgEnv(t *testing.T) *intgEnv { t.Helper() @@ -699,42 +717,38 @@ func intgAssertCRCondition(t *testing.T, k8sClient client.Client, crNames []stri } // ============================================================================ -// Scheduler handlers +// Scheduler functions // ============================================================================ -func intgAcceptScheduler(w http.ResponseWriter, r *http.Request) { - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} - json.NewEncoder(w).Encode(resp) //nolint:errcheck +func intgAcceptScheduler(sched.PlacementRequest) sched.PlacementResult { + return sched.PlacementResult{OrderedHosts: []string{"host-1"}} } -func intgRejectScheduler(w http.ResponseWriter, r *http.Request) { - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{}} - json.NewEncoder(w).Encode(resp) //nolint:errcheck +func intgRejectScheduler(sched.PlacementRequest) sched.PlacementResult { + return sched.PlacementResult{OrderedHosts: []string{}} } -// intgAcceptFirstScheduler returns a handler that accepts the first count placement calls +// intgAcceptFirstScheduler returns a scheduler fn that accepts the first count placement calls // and rejects all subsequent ones. Uses an atomic counter so concurrent calls are safe. -func intgAcceptFirstScheduler(count int) http.HandlerFunc { +func intgAcceptFirstScheduler(count int) intgSchedulerFn { var calls atomic.Int32 - return func(w http.ResponseWriter, r *http.Request) { + return func(req sched.PlacementRequest) sched.PlacementResult { if int(calls.Add(1)) <= count { - intgAcceptScheduler(w, r) - } else { - intgRejectScheduler(w, r) + return intgAcceptScheduler(req) } + return intgRejectScheduler(req) } } -// intgRejectFirstScheduler returns a handler that rejects the first count placement calls +// intgRejectFirstScheduler returns a scheduler fn that rejects the first count placement calls // and accepts all subsequent ones. Used to test AllowRejection=false retry-until-success paths. -func intgRejectFirstScheduler(count int) http.HandlerFunc { +func intgRejectFirstScheduler(count int) intgSchedulerFn { var calls atomic.Int32 - return func(w http.ResponseWriter, r *http.Request) { + return func(req sched.PlacementRequest) sched.PlacementResult { if int(calls.Add(1)) <= count { - intgRejectScheduler(w, r) - } else { - intgAcceptScheduler(w, r) + return intgRejectScheduler(req) } + return intgAcceptScheduler(req) } } @@ -1238,17 +1252,13 @@ func TestCRScheduling_SetsReserveForCommittedResourceIntent(t *testing.T) { // CR slot scheduling has a real project ID and must run tenant-context filters // (filter_allowed_projects, filter_aggregate_metadata, etc.). Verify the intent is // reserve_for_committed_resource so filters can identify the call type correctly. - var capturedReq schedulerdelegationapi.ExternalSchedulerRequest + var capturedReq sched.PlacementRequest var schedulerCalled bool - captureScheduler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + captureScheduler := func(req sched.PlacementRequest) sched.PlacementResult { schedulerCalled = true - if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} - json.NewEncoder(w).Encode(resp) //nolint:errcheck - }) + capturedReq = req + return sched.PlacementResult{OrderedHosts: []string{"host-1"}} + } env := newIntgEnv(t, []client.Object{newTestFlavorKnowledge(), intgHypervisor("host-1")}, captureScheduler, nil) defer env.close() @@ -1263,12 +1273,8 @@ func TestCRScheduling_SetsReserveForCommittedResourceIntent(t *testing.T) { if !schedulerCalled { t.Fatal("scheduler was never called — test did not exercise the scheduling path") } - hint, err := capturedReq.Spec.Data.GetSchedulerHintStr("_nova_check_type") - if err != nil { - t.Fatalf("failed to get _nova_check_type hint: %v", err) - } - if hint != string(schedulerdelegationapi.ReserveForCommittedResourceIntent) { - t.Errorf("CR slot scheduling must set _nova_check_type=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, hint) + if capturedReq.Intent != schedulerdelegationapi.ReserveForCommittedResourceIntent { + t.Errorf("CR slot scheduling must set Intent=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, capturedReq.Intent) } } @@ -1342,9 +1348,9 @@ func TestCROversubscriptionRemediation(t *testing.T) { } var schedulerCalls atomic.Int32 - schedulerFn := func(w http.ResponseWriter, r *http.Request) { + schedulerFn := func(req sched.PlacementRequest) sched.PlacementResult { schedulerCalls.Add(1) - intgRejectScheduler(w, r) // reject so evicted slots don't land back on host-1 + return intgRejectScheduler(req) // reject so evicted slots don't land back on host-1 } env := newIntgEnv(t, append(objects, newTestFlavorKnowledge()), schedulerFn, nil) defer env.close() @@ -1382,7 +1388,7 @@ func TestCROversubscriptionRemediation(t *testing.T) { placedSlots = append(placedSlots, r) } } - free := reservations.HostFreeCapacity(placedSlots, *hv) + free := accounting.HostFreeCapacity(placedSlots, *hv) zero := resource.MustParse("0") for rn, f := range free { if f.Cmp(zero) < 0 { diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 966c4434c..ddd889805 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -32,7 +32,10 @@ import ( schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/cobaltcore-dev/cortex/pkg/multicluster" "github.com/cobaltcore-dev/cortex/pkg/sso" @@ -49,8 +52,8 @@ type CommitmentReservationController struct { Scheme *runtime.Scheme // Configuration for the controller. Conf ReservationControllerConfig - // SchedulerClient for making scheduler API calls. - SchedulerClient *reservations.SchedulerClient + // Scheduler is the in-process scheduler used to place reservations. + Scheduler *compute.Placement // DomainResolver resolves OpenStack domain IDs to domain names so the // domain_name scheduler hint can be populated for filter_external_customer. // Nil when KeystoneSecretRef is not configured; hint is omitted in that case. @@ -218,27 +221,25 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr availabilityZone = res.Spec.AvailabilityZone } - // Resolve domain name for the domain_name scheduler hint consumed by - // filter_external_customer to enforce host restrictions for external customers. - // Resolution is best-effort: if the DomainResolver is not configured (no - // KeystoneSecretRef) or the lookup fails, we log and proceed without the hint. - // filter_external_customer already handles a missing hint by skipping the filter, - // so omitting it degrades gracefully rather than blocking scheduling. - schedulerHints := map[string]any{ - "_nova_check_type": string(schedulerdelegationapi.ReserveForCommittedResourceIntent), - } + // Resolve domain name so filter_external_customer can enforce host + // restrictions for external customers. Resolution is best-effort: if the + // DomainResolver is not configured (no KeystoneSecretRef) or the lookup + // fails, we log and proceed without it. filter_external_customer handles a + // missing domain name by skipping the filter, so omitting it degrades + // gracefully rather than blocking scheduling. + domainName := "" if r.DomainResolver != nil && res.Spec.CommittedResourceReservation != nil && res.Spec.CommittedResourceReservation.DomainID != "" { - domainName, err := r.DomainResolver.ResolveDomainName(ctx, res.Spec.CommittedResourceReservation.DomainID) + resolved, err := r.DomainResolver.ResolveDomainName(ctx, res.Spec.CommittedResourceReservation.DomainID) if err != nil { logger.Error(err, "failed to resolve domain name for scheduler hint, proceeding without it", "domainID", res.Spec.CommittedResourceReservation.DomainID) } else { - schedulerHints["domain_name"] = domainName + domainName = resolved } } // Get flavor details from flavor group knowledge CRD - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { logger.V(1).Info("flavor knowledge not ready, requeueing", @@ -248,7 +249,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr } // Search for the flavor across all flavor groups - flavorGroupName, flavorDetails, err := reservations.FindFlavorInGroups(resourceName, flavorGroups) + flavorGroupName, flavorDetails, err := accounting.FindFlavorInGroups(resourceName, flavorGroups) if err != nil { logger.Error(err, "flavor not found in any flavor group", "resourceName", resourceName) @@ -301,36 +302,56 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr return "initial" }()) - // Use the SchedulerClient to schedule the reservation - scheduleReq := reservations.ScheduleReservationRequest{ - InstanceUUID: res.Name, - ProjectID: projectID, - FlavorName: flavorDetails.Name, - FlavorExtraSpecs: flavorDetails.ExtraSpecs, - MemoryMB: flavorDetails.MemoryMB, - VCPUs: flavorDetails.VCPUs, - EligibleHosts: eligibleHosts, - Pipeline: pipelineName, - AvailabilityZone: availabilityZone, - SchedulerHints: schedulerHints, - } - scheduleOpts := scheduling.Options{ - ReadOnly: false, // mutates state (reservation placement) - LockReservations: true, // don't unlock CR reservations; finding a slot, not placing a VM - AssumeEmptyHosts: false, - IgnoredReservationTypes: nil, - MaxCandidates: 1, - SkipInflight: false, // TODO pessimistic blocking needed, will be addressed in follow up ticket - SkipCommittedResourceTracking: true, // CR slot scheduling, not a VM placement - } - - scheduleResp, err := r.SchedulerClient.ScheduleReservation(ctx, scheduleReq, scheduleOpts) + // Build the typed scheduler request directly (in-process; no HTTP). KVM + // reservations gather all candidates, so the eligible-host list is a starting + // set; GatherAllCandidates mirrors the previous ignore-preselection behavior. + hosts := make([]compute.Host, 0, len(eligibleHosts)) + weights := make(map[string]float64, len(eligibleHosts)) + for _, h := range eligibleHosts { + hosts = append(hosts, compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) + weights[h.ComputeHost] = 0.0 + } + // Honor the flavor's trait and hypervisor-type constraints so a + // committed-resource reservation lands on a compatible host (the typed + // request is built directly, so filter_has_requested_traits/filter_capabilities + // would otherwise see empty slices and become no-ops). No hypervisor-type + // default here — only match when the flavor names one. + requiredTraits, forbiddenTraits := nova.SplitTraits(flavorDetails.ExtraSpecs) + schedReq := compute.PlacementRequest{ + Pipeline: pipelineName, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: hosts, + Weights: weights, + Intent: schedulerdelegationapi.ReserveForCommittedResourceIntent, + InstanceUUID: res.Name, + NumInstances: 1, // one reservation slot; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free + ProjectID: projectID, + AvailabilityZone: availabilityZone, + DomainName: domainName, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + RequestedCapabilities: reservations.HypervisorTypeCapability(flavorDetails.ExtraSpecs, false), + AcceleratorsRequested: nova.RequestsAccelerators(flavorDetails.ExtraSpecs), + Flavor: compute.FlavorSpec{ + VCPUs: flavorDetails.VCPUs, + MemoryMB: flavorDetails.MemoryMB, + }, + Options: scheduling.Options{ + ReadOnly: false, // mutates state (reservation placement) + LockReservations: true, // don't unlock CR reservations; finding a slot, not placing a VM + MaxCandidates: 1, + SkipCommittedResourceTracking: true, // CR slot scheduling, not a VM placement + }, + } + + out, err := r.Scheduler.Run(ctx, schedReq, nil, nil) if err != nil { logger.Error(err, "failed to schedule reservation") return ctrl.Result{}, err } - if len(scheduleResp.Hosts) == 0 { + if len(out.OrderedHosts) == 0 { logger.Info("no hosts found for reservation, will retry", "reservation", res.Name, "flavorName", resourceName) old := res.DeepCopy() meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ @@ -356,7 +377,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // Update the reservation Spec with the found host (idx 0) // Only update Spec here - the Status will be synced in the next reconcile cycle // This avoids race conditions from doing two patches in one reconcile - host := scheduleResp.Hosts[0] + host := out.OrderedHosts[0] logger.Info("found host for reservation", "host", host, "flavorName", resourceName) old := res.DeepCopy() @@ -663,9 +684,6 @@ func (r *CommitmentReservationController) hypervisorToReservations(ctx context.C // Init initializes the reconciler with required clients and DB connection. func (r *CommitmentReservationController) Init(ctx context.Context, conf ReservationControllerConfig) error { - r.SchedulerClient = reservations.NewSchedulerClient(conf.SchedulerURL) - logf.FromContext(ctx).Info("scheduler client initialized for commitment reservation controller", "url", conf.SchedulerURL) - if conf.KeystoneSecretRef.Name != "" { var authenticatedHTTP *http.Client if conf.SSOSecretRef != nil { diff --git a/internal/scheduling/reservations/commitments/reservation_controller_test.go b/internal/scheduling/reservations/commitments/reservation_controller_test.go index d7b015a1f..d5439852f 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller_test.go +++ b/internal/scheduling/reservations/commitments/reservation_controller_test.go @@ -5,10 +5,7 @@ package commitments import ( "context" - "encoding/json" "errors" - "net/http" - "net/http/httptest" "testing" "time" @@ -24,9 +21,39 @@ import ( schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) +// fakePipeline returns a fixed ordered-host list (modelling a scheduler run). +type fakePipeline struct{ hosts []string } + +func (p fakePipeline) Run(sched.PlacementRequest) (sched.PlacementResult, error) { + return sched.PlacementResult{OrderedHosts: p.hosts}, nil +} + +// fakePipelineProvider serves a fakePipeline for every pipeline name, returning +// the configured host list. +type fakePipelineProvider struct{ hosts []string } + +func (f fakePipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { + return fakePipeline(f), true +} + +// fakeGatherer is a no-op gatherer (tests supply their own hosts). +type fakeGatherer struct{} + +func (fakeGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { + return nil +} + +// newMockScheduler builds an in-process compute.Placement whose pipeline returns +// the given host list. Replaces the former httptest mock scheduler server. +func newMockScheduler(hosts []string) *sched.Placement { + return sched.NewPlacement(fakePipelineProvider{hosts: hosts}, fakeGatherer{}) +} + // mockDomainResolver is a test double for DomainResolver. type mockDomainResolver struct { // names maps domainID → name returned on success. @@ -744,44 +771,11 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t k8sClient := newCRTestClient(scheme, reservation, newTestFlavorKnowledge(), hypervisor1, hypervisor2) - // Create a mock server that returns a successful response - mockResponse := &schedulerdelegationapi.ExternalSchedulerResponse{ - Hosts: []string{"test-host-1", "test-host-2"}, - } - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Verify the request body - var req schedulerdelegationapi.ExternalSchedulerRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("Failed to decode request: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - - // Verify request structure - if req.Spec.Data.NumInstances != 1 { - t.Errorf("Expected NumInstances to be 1, got %d", req.Spec.Data.NumInstances) - } - - if err := json.NewEncoder(w).Encode(mockResponse); err != nil { - t.Fatalf("failed to write response: %v", err) - } - })) - defer server.Close() - - config := ReservationControllerConfig{ - SchedulerURL: server.URL, - } - reconciler := &CommitmentReservationController{ - Client: k8sClient, - Scheme: scheme, - Conf: config, - } - - // Initialize the reconciler (this sets up SchedulerClient) - if err := reconciler.Init(context.Background(), config); err != nil { - t.Fatalf("Failed to initialize reconciler: %v", err) + Client: k8sClient, + Scheme: scheme, + Conf: ReservationControllerConfig{}, + Scheduler: newMockScheduler([]string{"test-host-1", "test-host-2"}), } req := ctrl.Request{ @@ -840,20 +834,19 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t // Test: domain_name scheduler hint // ============================================================================ -// newTestSchedulerServer captures the decoded request and returns a single host. -func newTestSchedulerServer(t *testing.T, captured *schedulerdelegationapi.ExternalSchedulerRequest) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(captured); err != nil { - t.Errorf("failed to decode scheduler request: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - resp := schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"test-host-1"}} - if err := json.NewEncoder(w).Encode(resp); err != nil { - t.Fatalf("failed to write scheduler response: %v", err) - } - })) +// capturingPipeline records the SchedulerRequest it was run with and returns a +// single host, so tests can assert on the request the reconciler built. +type capturingPipeline struct{ captured *sched.PlacementRequest } + +func (p capturingPipeline) Run(req sched.PlacementRequest) (sched.PlacementResult, error) { + *p.captured = req + return sched.PlacementResult{OrderedHosts: []string{"test-host-1"}}, nil +} + +type capturingPipelineProvider struct{ captured *sched.PlacementRequest } + +func (f capturingPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { + return capturingPipeline(f), true } func TestCommitmentReservationController_DomainNameHint(t *testing.T) { @@ -902,16 +895,13 @@ func TestCommitmentReservationController_DomainNameHint(t *testing.T) { k8sClient := newCRTestClient(scheme, reservation, newTestFlavorKnowledge(), &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "test-host-1"}}) - var captured schedulerdelegationapi.ExternalSchedulerRequest - server := newTestSchedulerServer(t, &captured) - defer server.Close() - + var captured sched.PlacementRequest reconciler := &CommitmentReservationController{ - Client: k8sClient, - Scheme: scheme, - Conf: ReservationControllerConfig{SchedulerURL: server.URL}, - SchedulerClient: reservations.NewSchedulerClient(server.URL), - DomainResolver: tt.resolver, + Client: k8sClient, + Scheme: scheme, + Conf: ReservationControllerConfig{}, + Scheduler: sched.NewPlacement(capturingPipelineProvider{captured: &captured}, fakeGatherer{}), + DomainResolver: tt.resolver, } if _, err := reconciler.Reconcile(context.Background(), ctrl.Request{ @@ -928,25 +918,14 @@ func TestCommitmentReservationController_DomainNameHint(t *testing.T) { } } - // Verify domain_name hint. - gotDomain, err := captured.Spec.Data.GetSchedulerHintStr("domain_name") - if tt.wantDomainName == "" { - if err == nil { - t.Errorf("expected domain_name hint absent, got %q", gotDomain) - } - } else { - if err != nil { - t.Fatalf("expected domain_name hint %q, got error: %v", tt.wantDomainName, err) - } - if gotDomain != tt.wantDomainName { - t.Errorf("expected domain_name hint %q, got %q", tt.wantDomainName, gotDomain) - } + // Verify domain_name hint (now the typed request field). + if captured.DomainName != tt.wantDomainName { + t.Errorf("expected domain name %q, got %q", tt.wantDomainName, captured.DomainName) } - // _nova_check_type must always be present regardless of resolver outcome. - gotIntent, err := captured.Spec.Data.GetSchedulerHintStr("_nova_check_type") - if err != nil || gotIntent != string(schedulerdelegationapi.ReserveForCommittedResourceIntent) { - t.Errorf("expected _nova_check_type=%q, got %q (err=%v)", schedulerdelegationapi.ReserveForCommittedResourceIntent, gotIntent, err) + // Intent must always be set regardless of resolver outcome. + if captured.Intent != schedulerdelegationapi.ReserveForCommittedResourceIntent { + t.Errorf("expected Intent=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, captured.Intent) } }) } diff --git a/internal/scheduling/reservations/commitments/syncer.go b/internal/scheduling/reservations/commitments/syncer.go index c40ec0a0e..9e0175ff7 100644 --- a/internal/scheduling/reservations/commitments/syncer.go +++ b/internal/scheduling/reservations/commitments/syncer.go @@ -13,6 +13,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/go-logr/logr" "github.com/sapcc/go-api-declarations/liquid" corev1 "k8s.io/api/core/v1" @@ -248,7 +249,7 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { }() // Check if flavor group knowledge is ready - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: s.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: s.Client} knowledgeCRD, err := knowledge.Get(ctx) if err != nil { logger.Error(err, "failed to check flavor group knowledge readiness") diff --git a/internal/scheduling/reservations/commitments/usage.go b/internal/scheduling/reservations/commitments/usage.go index 34a909f72..6fcdd789c 100644 --- a/internal/scheduling/reservations/commitments/usage.go +++ b/internal/scheduling/reservations/commitments/usage.go @@ -14,6 +14,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/go-logr/logr" "github.com/sapcc/go-api-declarations/liquid" @@ -99,7 +100,7 @@ func (c *UsageCalculator) CalculateUsage( allAZs []liquid.AvailabilityZone, ) (liquid.ServiceUsageReport, error) { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { return liquid.ServiceUsageReport{}, fmt.Errorf("failed to get flavor groups: %w", err) diff --git a/internal/scheduling/reservations/commitments/usage_reconciler.go b/internal/scheduling/reservations/commitments/usage_reconciler.go index 843bd8647..ad3068dc5 100644 --- a/internal/scheduling/reservations/commitments/usage_reconciler.go +++ b/internal/scheduling/reservations/commitments/usage_reconciler.go @@ -25,6 +25,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" ) @@ -117,7 +118,7 @@ func (r *UsageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl logger := log logger.V(1).Info("usage reconcile starting", "trigger", trigger, "generation", cr.Generation) - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: r.Client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { r.Monitor.reconcileDuration.WithLabelValues("error").Observe(time.Since(start).Seconds()) diff --git a/internal/scheduling/reservations/extra_specs.go b/internal/scheduling/reservations/extra_specs.go new file mode 100644 index 000000000..66e568585 --- /dev/null +++ b/internal/scheduling/reservations/extra_specs.go @@ -0,0 +1,34 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package reservations + +import ( + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" +) + +// The reservation controllers build the typed compute.PlacementRequest directly +// (not via the Nova wire adapter), yet their flavors still carry Nova's extra-spec +// string conventions. HypervisorTypeCapability reuses the shared parser in +// internal/scheduling/nova and layers the reservation-specific defaultQEMU seeding +// on top, so reservation scheduling honors the same capability constraints as +// regular placement with the conventions understood in one place. + +// HypervisorTypeCapability returns the required (key=value) hypervisor capabilities +// for a reservation scheduler request, so filter_capabilities keeps a workload off +// incompatible hosts. It extracts ALL of the flavor's "capabilities:" extra +// specs (via nova.SplitCapabilities, which canonicalizes the hypervisor_type casing +// and drops everything on an unsupported operator) — not just hypervisor_type. +// +// When the flavor names no hypervisor type and defaultQEMU is set (the failover +// controller seeds from running VMs whose flavor may omit it), the canonical "QEMU" +// hypervisor_type is added. Without defaultQEMU, a missing hypervisor type adds no +// hypervisor_type constraint. +func HypervisorTypeCapability(extraSpecs map[string]string, defaultQEMU bool) []compute.Capability { + caps := nova.SplitCapabilities(extraSpecs) + if extraSpecs["capabilities:hypervisor_type"] == "" && defaultQEMU { + caps = append(caps, compute.Capability{Name: "hypervisor_type", Value: "QEMU"}) + } + return caps +} diff --git a/internal/scheduling/reservations/extra_specs_test.go b/internal/scheduling/reservations/extra_specs_test.go new file mode 100644 index 000000000..c268b243e --- /dev/null +++ b/internal/scheduling/reservations/extra_specs_test.go @@ -0,0 +1,69 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package reservations + +import ( + "slices" + "strings" + "testing" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" +) + +func TestHypervisorTypeCapability(t *testing.T) { + tests := []struct { + name string + extraSpecs map[string]string + defaultQEMU bool + want []compute.Capability + }{ + { + name: "value passed through", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, + }, + { + name: "lowercase value canonicalized to match hosts", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}, + want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, + }, + { + name: "missing without default -> nil", + extraSpecs: map[string]string{}, + want: nil, + }, + { + name: "missing with default -> canonical QEMU", + extraSpecs: map[string]string{}, + defaultQEMU: true, + want: []compute.Capability{{Name: "hypervisor_type", Value: "QEMU"}}, + }, + { + name: "explicit value wins over default", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}, + defaultQEMU: true, + want: []compute.Capability{{Name: "hypervisor_type", Value: "CH"}}, + }, + { + name: "non-hypervisor_type capabilities preserved", + extraSpecs: map[string]string{ + "capabilities:hypervisor_type": "qemu", + "capabilities:cpu_arch": "aarch64", + }, + want: []compute.Capability{ + {Name: "cpu_arch", Value: "aarch64"}, + {Name: "hypervisor_type", Value: "QEMU"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := HypervisorTypeCapability(tt.extraSpecs, tt.defaultQEMU) + slices.SortFunc(got, func(a, b compute.Capability) int { return strings.Compare(a.Name, b.Name) }) + if !slices.Equal(got, tt.want) { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/scheduling/reservations/failover/config.go b/internal/scheduling/reservations/failover/config.go index b8bde49a6..3c2f9d3f1 100644 --- a/internal/scheduling/reservations/failover/config.go +++ b/internal/scheduling/reservations/failover/config.go @@ -27,10 +27,6 @@ type FailoverConfig struct { // This is used to read VM data from the Nova database. DatasourceName string `json:"datasourceName"` - // SchedulerURL is the URL of the nova external scheduler API. - // Example: "http://localhost:8080/scheduler/nova/external" - SchedulerURL string `json:"schedulerURL"` - // MaxVMsToProcess limits the number of VMs to process per reconciliation cycle. // Set to negative to process all VMs (default behavior). // Useful for debugging and testing with large VM counts. @@ -99,9 +95,6 @@ func (c *FailoverConfig) ApplyDefaults() { if c.DatasourceName == "" { c.DatasourceName = defaults.DatasourceName } - if c.SchedulerURL == "" { - c.SchedulerURL = defaults.SchedulerURL - } if c.ReconcileInterval.Duration == 0 { c.ReconcileInterval = defaults.ReconcileInterval } @@ -142,7 +135,6 @@ func DefaultConfig() FailoverConfig { MaxVMsToProcess: 30, Creator: "cortex-failover-controller", DatasourceName: "nova-servers", // we have the server and flavor data source (both store in same postgres and same secret but still) - SchedulerURL: "http://localhost:8080/scheduler/nova/external", TrustHypervisorLocation: false, RevalidationInterval: metav1.Duration{Duration: 30 * time.Minute}, LimitOneNewReservationPerHypervisor: true, diff --git a/internal/scheduling/reservations/failover/controller.go b/internal/scheduling/reservations/failover/controller.go index c15e85427..0d26b8c37 100644 --- a/internal/scheduling/reservations/failover/controller.go +++ b/internal/scheduling/reservations/failover/controller.go @@ -14,7 +14,9 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" "github.com/cobaltcore-dev/cortex/pkg/multicluster" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/google/uuid" @@ -40,21 +42,21 @@ var log = ctrl.Log.WithName("failover-reservation-controller").WithValues("modul // 2. Watch-based per-reservation reconciliation (Reconcile) - handles acknowledgment and validation of individual reservations type FailoverReservationController struct { client.Client - VMSource reservations.VMSource - Config FailoverConfig - SchedulerClient *reservations.SchedulerClient - Recorder events.EventRecorder // Event recorder for emitting Kubernetes events - Monitor *FailoverMonitor - reconcileCount int64 // Track reconciliation count for rotating VM selection + VMSource reservations.VMSource + Config FailoverConfig + Scheduler *sched.Placement + Recorder events.EventRecorder // Event recorder for emitting Kubernetes events + Monitor *FailoverMonitor + reconcileCount int64 // Track reconciliation count for rotating VM selection } -func NewFailoverReservationController(c client.Client, vmSource reservations.VMSource, config FailoverConfig, schedulerClient *reservations.SchedulerClient, monitor *FailoverMonitor) *FailoverReservationController { +func NewFailoverReservationController(c client.Client, vmSource reservations.VMSource, config FailoverConfig, scheduler *sched.Placement, monitor *FailoverMonitor) *FailoverReservationController { return &FailoverReservationController{ - Client: c, - VMSource: vmSource, - Config: config, - SchedulerClient: schedulerClient, - Monitor: monitor, + Client: c, + VMSource: vmSource, + Config: config, + Scheduler: scheduler, + Monitor: monitor, } } @@ -346,7 +348,7 @@ func (c *FailoverReservationController) ReconcilePeriodic(ctx context.Context) ( // 6. Fetch flavor groups for reservation sizing (if configured) var flavorGroups map[string]compute.FlavorGroupFeature if c.Config.UseFlavorGroupResources { - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + knowledge := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} fg, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Info("flavor group knowledge not available, will fall back to VM resources for sizing", @@ -881,7 +883,6 @@ func (c *FailoverReservationController) Start(ctx context.Context) error { "maxFailuresForShortInterval", c.Config.MaxFailuresForShortInterval, "creator", c.Config.Creator, "datasourceName", c.Config.DatasourceName, - "schedulerURL", c.Config.SchedulerURL, "flavorFailoverRequirements", c.Config.FlavorFailoverRequirements, "maxVMsToProcess", c.Config.MaxVMsToProcess, "trustHypervisorLocation", c.Config.TrustHypervisorLocation, diff --git a/internal/scheduling/reservations/failover/helpers.go b/internal/scheduling/reservations/failover/helpers.go index 0bd28bd9e..1a2ee46e6 100644 --- a/internal/scheduling/reservations/failover/helpers.go +++ b/internal/scheduling/reservations/failover/helpers.go @@ -12,6 +12,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -85,7 +86,7 @@ func resolveVMSpecForScheduling( logger := LoggerFromContext(ctx) if useFlavorGroupResources && flavorGroups != nil { - groupName, _, err := reservations.FindFlavorInGroups(vm.FlavorName, flavorGroups) + groupName, _, err := accounting.FindFlavorInGroups(vm.FlavorName, flavorGroups) if err == nil { fg := flavorGroups[groupName] largest := fg.LargestFlavor diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index 0ba8df232..04f9f92e7 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -18,6 +18,7 @@ import ( novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" @@ -32,6 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +// passthroughGatherer is a nova.CandidateGatherer that leaves the request's +// preselected hosts unchanged (the tests seed their own eligible hosts). +type passthroughGatherer struct{} + +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *compute.PlacementRequest) error { + return nil +} + // ============================================================================ // Integration Tests // ============================================================================ @@ -695,7 +704,9 @@ func (env *IntegrationTestEnv) LogStateSummary() { func (env *IntegrationTestEnv) TriggerFailoverReconcile(flavorRequirements map[string]int) error { env.T.Helper() - schedulerClient := reservations.NewSchedulerClient(env.SchedulerBaseURL + "/scheduler/nova/external") + // Drive the same in-process pipeline the testHTTPAPI uses, so the failover + // reservation flow exercises the real scheduler over the env's nova controller. + scheduler := compute.NewPlacement(env.NovaController, passthroughGatherer{}) config := FailoverConfig{ ReconcileInterval: metav1.Duration{Duration: time.Minute}, @@ -708,7 +719,7 @@ func (env *IntegrationTestEnv) TriggerFailoverReconcile(flavorRequirements map[s env.K8sClient, env.VMSource, config, - schedulerClient, + scheduler, nil, ) @@ -1142,9 +1153,9 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h Build() novaController := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.PlacementRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: getSharedMonitor(), @@ -1223,7 +1234,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h novaController.PipelineConfigs[pipeline.Name] = pipeline } - api := &testHTTPAPI{delegate: novaController} + api := &testHTTPAPI{scheduler: compute.NewPlacement(novaController, passthroughGatherer{})} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) server := httptest.NewServer(mux) @@ -1242,7 +1253,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h // testHTTPAPI is a simplified HTTP API for testing that delegates to the controller. // It does NOT shuffle evacuation hosts, ensuring deterministic test results. type testHTTPAPI struct { - delegate nova.HTTPAPIDelegate + scheduler *compute.Placement } // NovaExternalScheduler handles the POST request from the Nova scheduler. @@ -1260,49 +1271,26 @@ func (api *testHTTPAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req return } - rawBytes, err := json.Marshal(requestData) - if err != nil { - http.Error(w, "failed to marshal request", http.StatusInternalServerError) - return - } - raw := runtime.RawExtension{Raw: rawBytes} - pipelineName := requestData.Pipeline if pipelineName == "" { pipelineName = "test-filter-pipeline" } - decision := &v1alpha1.Decision{ - TypeMeta: metav1.TypeMeta{ - Kind: "Decision", - APIVersion: "cortex.cloud/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "nova-", - }, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{ - Name: pipelineName, - }, - ResourceID: requestData.Spec.Data.InstanceUUID, - NovaRaw: &raw, - }, + // Resolve the scheduling intent from the request hints, mirroring the + // production handler (external_scheduler_api.go). + intent, err := requestData.GetIntent() + if err != nil { + intent = novaapi.CreateIntent } - ctx := r.Context() - if err := api.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { + req := nova.ToSchedulerRequest(requestData, intent, pipelineName) + out, err := api.scheduler.Run(r.Context(), req, nil, nil) + if err != nil { http.Error(w, "failed to process scheduling decision", http.StatusInternalServerError) return } - if decision.Status.Result == nil { - http.Error(w, "decision didn't produce a result", http.StatusInternalServerError) - return - } - - hosts := decision.Status.Result.OrderedHosts - response := novaapi.ExternalSchedulerResponse{Hosts: hosts} + response := novaapi.ExternalSchedulerResponse{Hosts: out.OrderedHosts} w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(response); err != nil { http.Error(w, "failed to encode response", http.StatusInternalServerError) @@ -1342,9 +1330,9 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, Build() novaController := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[compute.PlacementRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[compute.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: getSharedMonitor(), @@ -1424,7 +1412,7 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, novaController.PipelineConfigs[pipeline.Name] = pipeline } - api := &testHTTPAPI{delegate: novaController} + api := &testHTTPAPI{scheduler: compute.NewPlacement(novaController, passthroughGatherer{})} mux := http.NewServeMux() mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) server := httptest.NewServer(mux) diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 122456520..dd6fceb03 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -13,6 +13,8 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -31,9 +33,9 @@ var DefaultFailoverOptions = scheduling.Options{ SkipCommittedResourceTracking: true, } -// inferFailoverPipeline returns the standard pipeline for a failover scheduling call based on +// InferFailoverPipeline returns the standard pipeline for a failover scheduling call based on // the VM's flavor extra specs — the same HANA vs general-purpose split used by Nova placement. -func inferFailoverPipeline(extraSpecs map[string]string) string { +func InferFailoverPipeline(extraSpecs map[string]string) string { if strings.ToLower(extraSpecs["trait:CUSTOM_HANA_EXCLUSIVE_HOST"]) == "required" { return "kvm-hana-bin-packing" } @@ -94,38 +96,40 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex ignoreHypervisors := []string{vm.CurrentHypervisor} - // Build flavor extra specs from VM's extra specs - // Start with the VM's actual extra specs, then ensure required defaults are set - flavorExtraSpecs := make(map[string]string) - for k, v := range vm.FlavorExtraSpecs { - flavorExtraSpecs[k] = v - } - // Ensure hypervisor_type is set for KVM scheduling if not already present - if _, ok := flavorExtraSpecs["capabilities:hypervisor_type"]; !ok { - flavorExtraSpecs["capabilities:hypervisor_type"] = "qemu" + // Build the typed scheduler request directly (in-process; no HTTP). KVM + // failover reservations gather all candidates; the eligible list is a + // starting set. The filter_correct_az filter excludes hosts outside the VM's + // AZ. ImageHypervisorType is irrelevant here: filter_image_properties skips + // the failover/reuse reservation intents. + hosts := make([]compute.Host, 0, len(eligibleHypervisors)) + weights := make(map[string]float64, len(eligibleHypervisors)) + for _, h := range eligibleHypervisors { + hosts = append(hosts, compute.Host{ComputeHost: h.ComputeHost, HypervisorHostname: h.HypervisorHostname}) + weights[h.ComputeHost] = 0.0 } - - // Schedule the reservation using the SchedulerClient. - // Note: We pass all hypervisors (from all AZs) in EligibleHosts. The scheduler pipeline's - // filter_correct_az filter will exclude hosts that are not in the VM's availability zone. - // Use resSpec.FlavorName and reservation spec resources so the scheduler checks capacity for the - // correct flavor size (which may be the LargestFlavor from the flavor group). - scheduleReq := reservations.ScheduleReservationRequest{ - InstanceUUID: vm.UUID, - ProjectID: vm.ProjectID, - FlavorName: resSpec.FlavorName, - FlavorExtraSpecs: flavorExtraSpecs, - MemoryMB: resSpec.MemoryMB, - VCPUs: resSpec.VCPUs, - EligibleHosts: eligibleHypervisors, - IgnoreHosts: ignoreHypervisors, - Pipeline: pipeline, - AvailabilityZone: vm.AvailabilityZone, - SchedulerHints: map[string]any{ - "_nova_check_type": string(intent), - api.HintKeyResourceGroup: resSpec.ResourceGroup(vm.FlavorName), + requiredTraits, forbiddenTraits := nova.SplitTraits(vm.FlavorExtraSpecs) + schedReq := compute.PlacementRequest{ + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: hosts, + Weights: weights, + Options: opts, + Intent: intent, + InstanceUUID: vm.UUID, + NumInstances: 1, // one reservation per VM; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free + ProjectID: vm.ProjectID, + AvailabilityZone: vm.AvailabilityZone, + IgnoreHosts: ignoreHypervisors, + ResourceGroup: resSpec.ResourceGroup(vm.FlavorName), + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + RequestedCapabilities: reservations.HypervisorTypeCapability(vm.FlavorExtraSpecs, true), + AcceleratorsRequested: nova.RequestsAccelerators(vm.FlavorExtraSpecs), + Flavor: compute.FlavorSpec{ + VCPUs: resSpec.VCPUs, + MemoryMB: resSpec.MemoryMB, }, - ImageProperties: imagePropertiesFromFlavor(flavorExtraSpecs), } logger.V(1).Info("scheduling failover reservation", @@ -134,7 +138,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex "eligibleHypervisors", len(eligibleHypervisors), "ignoreHypervisors", ignoreHypervisors) - scheduleResp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, opts) + out, err := c.runScheduler(ctx, schedReq) if err != nil { logger.Error(err, "failed to schedule failover reservation", "vmUUID", vm.UUID, "pipeline", pipeline) return nil, fmt.Errorf("failed to schedule failover reservation: %w", err) @@ -145,9 +149,19 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex "pipeline", pipeline, "eligibleHypervisors", len(eligibleHypervisors), "ignoreHypervisors", ignoreHypervisors, - "returnedHypervisors", scheduleResp.Hosts) + "returnedHypervisors", out.OrderedHosts) - return scheduleResp.Hosts, nil + return out.OrderedHosts, nil +} + +// runScheduler runs the shared scheduler, choosing the lock-free path for +// read-only runs (e.g. reuse/evacuate compatibility checks) and the locked path +// for runs that mutate shared reservation state. +func (c *FailoverReservationController) runScheduler(ctx context.Context, req compute.PlacementRequest) (compute.PlacementResult, error) { + if req.Options.ReadOnly { + return c.Scheduler.RunWithoutLock(ctx, req, nil, nil) + } + return c.Scheduler.Run(ctx, req, nil, nil) } // tryReuseExistingReservation finds an existing reservation that can be reused for a VM. @@ -166,7 +180,7 @@ func (c *FailoverReservationController) tryReuseExistingReservation( reuseOpts := DefaultFailoverOptions reuseOpts.ReadOnly = true - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, inferFailoverPipeline(vm.FlavorExtraSpecs), resSpec, api.ReuseFailoverReservationIntent, reuseOpts) + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, InferFailoverPipeline(vm.FlavorExtraSpecs), resSpec, api.ReuseFailoverReservationIntent, reuseOpts) if err != nil { logger.Error(err, "failed to get potential hypervisors for VM", "vmUUID", vm.UUID) return nil @@ -233,7 +247,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( vcpus = uint64(vcpusRes.Value()) //nolint:gosec // vcpus values won't overflow } - // Build flavor extra specs from VM's extra specs + // Infer the pipeline from the VM's flavor extra specs (HANA vs general purpose). flavorExtraSpecs := make(map[string]string) for k, v := range vm.FlavorExtraSpecs { flavorExtraSpecs[k] = v @@ -242,57 +256,61 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( flavorExtraSpecs["capabilities:hypervisor_type"] = "qemu" } - // Build a single-host request to validate the VM can use the reservation host - // Use vm.CurrentHypervisor directly instead of a separate parameter to avoid stale data - // Set _nova_check_type to "evacuate" so that filter_has_enough_capacity unlocks - // the failover reservation's resources for this VM (avoiding self-blocking). - // Use the actual VM UUID (not prefixed) so the filter can match it in the reservation's allocations. - scheduleReq := reservations.ScheduleReservationRequest{ - InstanceUUID: vm.UUID, - ProjectID: vm.ProjectID, - FlavorName: vm.FlavorName, - FlavorExtraSpecs: flavorExtraSpecs, - MemoryMB: memoryMB, - VCPUs: vcpus, - EligibleHosts: []api.ExternalSchedulerHost{{ComputeHost: reservationHost}}, - IgnoreHosts: []string{vm.CurrentHypervisor}, - Pipeline: inferFailoverPipeline(flavorExtraSpecs), - AvailabilityZone: vm.AvailabilityZone, - SchedulerHints: map[string]any{"_nova_check_type": string(api.EvacuateIntent)}, - ImageProperties: imagePropertiesFromFlavor(flavorExtraSpecs), + // Validate the VM can use the reservation host. Set the Evacuate intent so + // filter_has_enough_capacity unlocks the failover reservation's resources for + // this VM (avoiding self-blocking). Use the actual VM UUID so the filter can + // match it in the reservation's allocations. KVM gathers all candidates; the + // reservation host must come back ranked for the VM to be compatible. + pipeline := InferFailoverPipeline(flavorExtraSpecs) + requiredTraits, forbiddenTraits := nova.SplitTraits(flavorExtraSpecs) + evacuateOpts := DefaultFailoverOptions + evacuateOpts.ReadOnly = true + evacuateOpts.LockReservations = true + schedReq := compute.PlacementRequest{ + Pipeline: pipeline, + GatherAllCandidates: true, + HypervisorFamily: compute.HypervisorFamilyKVM, + Hosts: []compute.Host{{ComputeHost: reservationHost, HypervisorHostname: reservationHost}}, + Weights: map[string]float64{reservationHost: 0.0}, + Options: evacuateOpts, + Intent: api.EvacuateIntent, + InstanceUUID: vm.UUID, + NumInstances: 1, // validating one VM; NumInstances must be >= 1 or capacity/bin-pack math treats the host as free + ProjectID: vm.ProjectID, + AvailabilityZone: vm.AvailabilityZone, + IgnoreHosts: []string{vm.CurrentHypervisor}, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + RequestedCapabilities: reservations.HypervisorTypeCapability(flavorExtraSpecs, true), + AcceleratorsRequested: nova.RequestsAccelerators(flavorExtraSpecs), + Flavor: compute.FlavorSpec{ + VCPUs: vcpus, + MemoryMB: memoryMB, + }, } logger.V(1).Info("validating VM via scheduler evacuation", "vmUUID", vm.UUID, "reservationHost", reservationHost, "vmCurrentHost", vm.CurrentHypervisor, - "pipeline", scheduleReq.Pipeline) + "pipeline", pipeline) - evacuateOpts := DefaultFailoverOptions - evacuateOpts.ReadOnly = true - evacuateOpts.LockReservations = true - resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, evacuateOpts) + out, err := c.runScheduler(ctx, schedReq) if err != nil { logger.Error(err, "failed to validate VM for reservation host", "vmUUID", vm.UUID, "reservationHost", reservationHost) return false, fmt.Errorf("failed to validate VM for reservation host: %w", err) } + hosts := out.OrderedHosts // Handle empty response - no hosts returned - if len(resp.Hosts) < 1 { + if len(hosts) < 1 { logger.V(1).Info("scheduler returned no hosts for VM validation", "vmUUID", vm.UUID, "reservationHost", reservationHost) return false, nil } - // Log unexpected scheduler responses - if len(resp.Hosts) > 1 || resp.Hosts[0] != reservationHost { - logger.Error(nil, "scheduler returned unexpected hosts for single-host validation request", - "vmUUID", vm.UUID, - "reservationHost", reservationHost, - "returnedHosts", resp.Hosts) - } - - // If the reservation host is returned, the VM can use it - return resp.Hosts[0] == reservationHost, nil + // The reservation host must be among the returned hosts for the VM to be + // compatible with it. + return slices.Contains(hosts, reservationHost), nil } // scheduleAndBuildNewFailoverReservation schedules a failover reservation for a VM. diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go index a37957f00..eee4d21a8 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling_test.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -5,14 +5,13 @@ package failover import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" + "slices" "testing" - novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -475,22 +474,40 @@ func buildSchedulingTestVMWithResources(uuid, hypervisor string, memoryMB, vcpus } } -// captureSchedulerRequest spins up a test HTTP server that captures one scheduler request -// and returns a single host. The captured request is written into *out. -func captureSchedulerRequest(t *testing.T, host string, out *novaapi.ExternalSchedulerRequest) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(out); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - resp := novaapi.ExternalSchedulerResponse{Hosts: []string{host}} - _ = json.NewEncoder(w).Encode(resp) //nolint:errcheck - })) +// capturingPipeline records the last SchedulerRequest it ran and returns a +// single fixed host, so tests can assert the options the controller built. +type capturingPipeline struct { + host string + captured *sched.PlacementRequest +} + +func (p *capturingPipeline) Run(req sched.PlacementRequest) (sched.PlacementResult, error) { + *p.captured = req + return sched.PlacementResult{OrderedHosts: []string{p.host}}, nil +} + +// capturingProvider serves the same capturingPipeline for every pipeline name. +type capturingProvider struct{ pipeline *capturingPipeline } + +func (f capturingProvider) GetPipeline(string) (lib.FilterWeigherPipeline[sched.PlacementRequest], bool) { + return f.pipeline, true +} + +// capturingGatherer is a no-op gatherer (the tests preselect their own hosts). +type capturingGatherer struct{} + +func (capturingGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { + return nil +} + +// newCapturingScheduler builds an in-process scheduler whose pipeline returns +// the given host and records every request it ran into *out. +func newCapturingScheduler(host string, out *sched.PlacementRequest) *sched.Placement { + return sched.NewPlacement(capturingProvider{pipeline: &capturingPipeline{host: host, captured: out}}, capturingGatherer{}) } -// buildMinimalController returns a FailoverReservationController wired to a scheduler at the given URL. -func buildMinimalController(t *testing.T, schedulerURL string) *FailoverReservationController { +// buildMinimalController returns a FailoverReservationController wired to the given scheduler. +func buildMinimalController(t *testing.T, scheduler *sched.Placement) *FailoverReservationController { t.Helper() scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { @@ -500,10 +517,9 @@ func buildMinimalController(t *testing.T, schedulerURL string) *FailoverReservat t.Fatalf("add hv1 to scheme: %v", err) } k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - schedulerClient := reservations.NewSchedulerClient(schedulerURL + "/scheduler/nova/external") config := FailoverConfig{} config.ApplyDefaults() - return NewFailoverReservationController(k8sClient, nil, config, schedulerClient, nil) + return NewFailoverReservationController(k8sClient, nil, config, scheduler, nil) } // ============================================================================ @@ -559,11 +575,10 @@ func TestFailoverSchedulerOptions(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var capturedReq novaapi.ExternalSchedulerRequest - server := captureSchedulerRequest(t, "host-2", &capturedReq) - defer server.Close() + var capturedReq sched.PlacementRequest + scheduler := newCapturingScheduler("host-2", &capturedReq) - tt.call(buildMinimalController(t, server.URL), context.Background()) + tt.call(buildMinimalController(t, scheduler), context.Background()) if capturedReq.Options.ReadOnly != tt.wantReadOnly { t.Errorf("ReadOnly = %v, want %v", capturedReq.Options.ReadOnly, tt.wantReadOnly) @@ -575,6 +590,53 @@ func TestFailoverSchedulerOptions(t *testing.T) { } } +// ============================================================================ +// Test: trait and hypervisor_type constraints sent to the scheduler +// ============================================================================ + +// TestFailoverSchedulerFlavorConstraints verifies the scheduler request carries +// the flavor's traits and capabilities:hypervisor_type, so +// filter_has_requested_traits / filter_capabilities keep a workload off an +// incompatible host — including the qemu default when the flavor omits the +// hypervisor type. +func TestFailoverSchedulerFlavorConstraints(t *testing.T) { + tests := []struct { + name string + extraSpecs map[string]string + wantHVType string + wantTraits []string + }{ + {name: "flavor without hypervisor_type defaults to canonical QEMU", extraSpecs: nil, wantHVType: "QEMU"}, + {name: "explicit CH preserved", extraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}, wantHVType: "CH"}, + { + name: "lowercase qemu canonicalized and trait forwarded", + extraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu", "trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required"}, + wantHVType: "QEMU", + wantTraits: []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vm := buildSchedulingTestVM("vm-1", "host-1") + vm.FlavorExtraSpecs = tt.extraSpecs + resolved := resolveVMSpecForScheduling(context.Background(), vm, false, nil) + + var capturedReq sched.PlacementRequest + scheduler := newCapturingScheduler("host-2", &capturedReq) + c := buildMinimalController(t, scheduler) + _, _ = c.scheduleAndBuildNewFailoverReservation(context.Background(), vm, []string{"host-1", "host-2"}, nil, nil, resolved) //nolint:errcheck + + wantCap := []sched.Capability{{Name: "hypervisor_type", Value: tt.wantHVType}} + if len(capturedReq.RequestedCapabilities) != 1 || capturedReq.RequestedCapabilities[0] != wantCap[0] { + t.Errorf("RequestedCapabilities = %v, want %v", capturedReq.RequestedCapabilities, wantCap) + } + if !slices.Equal(capturedReq.RequiredTraits, tt.wantTraits) { + t.Errorf("RequiredTraits = %v, want %v", capturedReq.RequiredTraits, tt.wantTraits) + } + }) + } +} + // ============================================================================ // Test: imagePropertiesFromFlavor // ============================================================================ diff --git a/internal/scheduling/reservations/failoverhost/controller.go b/internal/scheduling/reservations/failoverhost/controller.go index 8f001304f..def981f08 100644 --- a/internal/scheduling/reservations/failoverhost/controller.go +++ b/internal/scheduling/reservations/failoverhost/controller.go @@ -59,11 +59,11 @@ import ( "context" "errors" "maps" - "net/http" "sync" "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -107,10 +107,6 @@ type ControllerConfig struct { // GenerationInterval is the interval within which failover partitions are // recalculated and failover host reservations are updated accordingly. GenerationInterval metav1.Duration `json:"generationInterval"` - - // ExternalSchedulerURL is the URL of the external scheduler service that - // will be used to request failover host placements. - ExternalSchedulerURL string `json:"externalSchedulerURL"` } // Controller keeps the cluster's failover host reservations in sync with the @@ -129,6 +125,11 @@ type Controller struct { Config ControllerConfig + // Scheduler is the shared in-process scheduler the path finder probes for + // placement candidates. Required (the controller cannot compute failover + // partitions without it). + Scheduler *compute.Placement + // Monitor records Prometheus metrics for the calculation loop and the // reconciler. It may be nil (e.g. in tests); every call site nil-guards it. Monitor *Monitor @@ -336,15 +337,16 @@ func (c *Controller) SetupWithManager(mgr ctrl.Manager) error { // Default the generator. Tests may set it before calling Setup to substitute // a fake; here we wire the production pipeline stages. if c.generator == nil { + if c.Scheduler == nil { + return errors.New("scheduler cannot be nil") + } c.generator = &pipelineGenerator{ Client: c.Client, Monitor: c.Monitor, seeder: &seeder{k8s: c.Client}, pathFinder: &evacuationPathFinder{ - // Timeout so a hung scheduler can't stall the calculation loop. - httpClient: &http.Client{Timeout: 30 * time.Second}, - schedulerURL: c.Config.ExternalSchedulerURL, - monitor: c.Monitor, + scheduler: c.Scheduler, + monitor: c.Monitor, }, partitionBuilder: cliquePartitionBuilder{}, reservationPlanner: ratioReservationPlanner{ratio: c.Config.Ratio, minimum: c.Config.Minimum}, diff --git a/internal/scheduling/reservations/failoverhost/generation_test.go b/internal/scheduling/reservations/failoverhost/generation_test.go index 074ea1a98..1d6191040 100644 --- a/internal/scheduling/reservations/failoverhost/generation_test.go +++ b/internal/scheduling/reservations/failoverhost/generation_test.go @@ -38,10 +38,9 @@ func TestGenerateProducesReservations(t *testing.T) { hvA := buildHV("hvA", 64, 32) hvB := buildHV("hvB", 64, 32) - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"hvA", "hvB"}, }, nil) - defer server.Close() g := &pipelineGenerator{ Client: newTestClient(t, hvA, hvB), @@ -84,10 +83,9 @@ func TestGenerateDeduplicatesSharedHost(t *testing.T) { // The path finder's edges are irrelevant here: the fake builder ignores them // and hands Generate the overlapping cliques directly. The stub still needs an // answer so FindPaths succeeds; one seed accepted on all three suffices. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"hvA", "hvB", "hvC"}, }, nil) - defer server.Close() g := &pipelineGenerator{ Client: newTestClient(t, hvA, hvB, hvC), @@ -131,10 +129,9 @@ func TestGenerateSharedHostKeptInSmallestPartition(t *testing.T) { hvC := buildHV("hvC", 64, 32) hvD := buildHV("hvD", 64, 32) - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"hvA", "hvB", "hvC", "hvD"}, }, nil) - defer server.Close() g := &pipelineGenerator{ Client: newTestClient(t, hvA, hvB, hvC, hvD), diff --git a/internal/scheduling/reservations/failoverhost/integration_test.go b/internal/scheduling/reservations/failoverhost/integration_test.go index 013d8e269..c34dae82d 100644 --- a/internal/scheduling/reservations/failoverhost/integration_test.go +++ b/internal/scheduling/reservations/failoverhost/integration_test.go @@ -5,16 +5,13 @@ package failoverhost import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" "slices" "sync" "testing" - novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + sched "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" @@ -22,10 +19,18 @@ import ( hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) +// passthroughGatherer is a nova.CandidateGatherer that leaves the request's +// preselected hosts unchanged. The integration tests seed their own eligible +// hosts, so gathering is a no-op here. +type passthroughGatherer struct{} + +func (passthroughGatherer) MutateWithAllCandidates(context.Context, *sched.PlacementRequest) error { + return nil +} + // ============================================================================ // Integration Tests // @@ -199,7 +204,6 @@ func TestIntegration(t *testing.T) { for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { env := newIntegrationTestEnv(t, tc.Hypervisors, tc.Flavors) - defer env.Close() ratio, minimum := tc.Ratio, tc.Minimum if ratio == 0 { @@ -234,19 +238,17 @@ func TestIntegration(t *testing.T) { // Test Environment // ============================================================================ -// integrationTestEnv wires a real Nova filter/weigher pipeline (served over -// HTTP) on top of a fake k8s client holding the fleet and the flavor-groups -// Knowledge CRD, so the generation pipeline can be driven end to end. +// integrationTestEnv wires a real Nova filter/weigher pipeline behind the shared +// in-process scheduler on top of a fake k8s client holding the fleet and the +// flavor-groups Knowledge CRD, so the generation pipeline can be driven end to +// end. type integrationTestEnv struct { - k8sClient client.Client - server *httptest.Server - schedulerURL string - flavors []compute.FlavorInGroup - hypervisors map[string]hv1.Hypervisor + k8sClient client.Client + scheduler *sched.Placement + flavors []compute.FlavorInGroup + hypervisors map[string]hv1.Hypervisor } -func (env *integrationTestEnv) Close() { env.server.Close() } - // newIntegrationTestEnv builds the fake cluster and the real scheduler pipeline. func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors []compute.FlavorInGroup) *integrationTestEnv { t.Helper() @@ -262,9 +264,9 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors k8sClient := newTestClient(t, objects...) controller := &nova.FilterWeigherPipelineController{ - BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[sched.PlacementRequest]]{ Client: k8sClient, - Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + Pipelines: make(map[string]lib.FilterWeigherPipeline[sched.PlacementRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), }, Monitor: sharedPipelineMonitor(), @@ -294,17 +296,15 @@ func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors controller.Pipelines[pipeline.Name] = result.Pipeline controller.PipelineConfigs[pipeline.Name] = pipeline - api := &integrationHTTPAPI{delegate: controller} - mux := http.NewServeMux() - mux.HandleFunc("/scheduler/nova/external", api.handle) - server := httptest.NewServer(mux) + // KVM requests trigger candidate gathering; the test seeds its own eligible + // hosts, so use a passthrough gatherer that preserves them. + scheduler := sched.NewPlacement(controller, passthroughGatherer{}) return &integrationTestEnv{ - k8sClient: k8sClient, - server: server, - schedulerURL: server.URL + "/scheduler/nova/external", - flavors: flavors, - hypervisors: hvsByName, + k8sClient: k8sClient, + scheduler: scheduler, + flavors: flavors, + hypervisors: hvsByName, } } @@ -319,7 +319,7 @@ func (env *integrationTestEnv) Generate(t *testing.T, ratio float64, minimum int g := &pipelineGenerator{ Client: env.k8sClient, seeder: &seeder{k8s: env.k8sClient}, - pathFinder: &evacuationPathFinder{httpClient: http.DefaultClient, schedulerURL: env.schedulerURL}, + pathFinder: &evacuationPathFinder{scheduler: env.scheduler}, partitionBuilder: cliquePartitionBuilder{}, reservationPlanner: ratioReservationPlanner{ratio: ratio, minimum: minimum}, } @@ -342,7 +342,7 @@ func (env *integrationTestEnv) AcceptsByHost(t *testing.T) map[string]map[string for _, hv := range env.hypervisors { zones[hv.Labels[corev1.LabelTopologyZone]] = true } - finder := &evacuationPathFinder{httpClient: http.DefaultClient, schedulerURL: env.schedulerURL} + finder := &evacuationPathFinder{scheduler: env.scheduler} accepts := map[string]map[string]bool{} for _, flavor := range env.flavors { for zone := range zones { @@ -429,49 +429,6 @@ func (env *integrationTestEnv) VerifySiblingSources(t *testing.T, desired map[co // probe exercises. const evacProbePipeline = "kvm-general-purpose-load-balancing" -// integrationHTTPAPI is a thin, deterministic HTTP front-end for the real Nova -// filter/weigher pipeline — the same shape the failover package's integration -// tests use. It does NOT shuffle, so the probe answers are stable. -type integrationHTTPAPI struct { - delegate nova.HTTPAPIDelegate -} - -func (a *integrationHTTPAPI) handle(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - var reqData novaapi.ExternalSchedulerRequest - if err := json.NewDecoder(r.Body).Decode(&reqData); err != nil { - http.Error(w, "decode", http.StatusBadRequest) - return - } - rawBytes, err := json.Marshal(reqData) - if err != nil { - http.Error(w, "marshal", http.StatusInternalServerError) - return - } - decision := &v1alpha1.Decision{ - TypeMeta: metav1.TypeMeta{Kind: "Decision", APIVersion: "cortex.cloud/v1alpha1"}, - ObjectMeta: metav1.ObjectMeta{GenerateName: "nova-"}, - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - PipelineRef: corev1.ObjectReference{Name: evacProbePipeline}, - ResourceID: reqData.Spec.Data.InstanceUUID, - NovaRaw: &runtime.RawExtension{Raw: rawBytes}, - }, - } - if err := a.delegate.ProcessNewDecisionFromAPI(r.Context(), decision); err != nil { - http.Error(w, "process", http.StatusInternalServerError) - return - } - if decision.Status.Result == nil { - http.Error(w, "no result", http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: decision.Status.Result.OrderedHosts}); err != nil { - http.Error(w, "encode", http.StatusInternalServerError) - } -} - // ============================================================================ // Object Creation Functions // ============================================================================ diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go index 5ca983e97..2e592d18e 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding.go @@ -4,20 +4,20 @@ package failoverhost import ( - "bytes" "context" - "encoding/json" "fmt" "maps" - "net/http" "slices" "sort" - "strconv" "time" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -71,21 +71,19 @@ type PathFinder interface { FindPaths(ctx context.Context, topo topology, seeds []Seed) ([]Path, error) } -// evacuationPathFinder finds failover paths by asking the external cortex -// scheduler, per seed, which hosts would accept that flavor, then pairing up the -// co-accepting hosts subject to gang-level fit. +// evacuationPathFinder finds failover paths by probing the shared in-process +// scheduler, per seed, for which hosts would accept that flavor, then pairing up +// the co-accepting hosts subject to gang-level fit. // // Asking the real scheduler (rather than reimplementing placement here) is // deliberate: it means failover paths are computed with exactly the same // filters, weighers and constraints that govern ordinary placement, so a // partition can never contain a target the scheduler would actually reject. type evacuationPathFinder struct { - // httpClient issues the scheduler requests. Give it a timeout so a hung - // scheduler cannot stall the whole calculation loop. Separately injected so - // tests can point it at a stub server. - httpClient *http.Client - // schedulerURL is the external cortex scheduler endpoint to POST to. - schedulerURL string + // scheduler is the shared in-process scheduler used to probe placement. The + // pathfinder issues read-only runs via RunWithoutLock. Separately injected so + // tests can point it at a fake pipeline. + scheduler *compute.Placement // monitor records scheduler request metrics. May be nil (e.g. in tests). monitor *Monitor } @@ -242,25 +240,53 @@ func hvCapacity(hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity { func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed Seed, hvsByName map[string]hv1.Hypervisor) ([]hv1.Hypervisor, error) { log := logf.FromContext(ctx) - // Build valid image properties. We set the image hypervisor type explicitly so - // the filter that checks flavor/image compatibility can evaluate it. Since we - // only handle KVM virtual machines, we set it to KVM. - imageProps := map[string]any{ - "img_hv_type": string(api.NovaImageMetaHVTypeKVM), - } // Offer every known host as a candidate. The scheduler filters/weighs over // exactly this host set — it does not gather the cluster itself — so leaving // Hosts empty would make it return no candidates. Each host also needs a // weight entry, which the scheduler requires to accompany the host set. - hosts := make([]api.ExternalSchedulerHost, 0, len(hvsByName)) + hosts := make([]compute.Host, 0, len(hvsByName)) weights := make(map[string]float64, len(hvsByName)) for name := range hvsByName { - hosts = append(hosts, api.ExternalSchedulerHost{ComputeHost: name}) + hosts = append(hosts, compute.Host{ComputeHost: name}) weights[name] = 0 } - requestObj := api.ExternalSchedulerRequest{ - Hosts: hosts, - Weights: weights, + // Honor the flavor's trait and hypervisor-type constraints so the probe only + // considers compatible hosts (filter_has_requested_traits/filter_capabilities + // would otherwise see empty slices and become no-ops). The seeder only emits + // seeds whose flavor names a supported hypervisor type, so no default is needed. + requiredTraits, forbiddenTraits := nova.SplitTraits(seed.Flavor.ExtraSpecs) + req := compute.PlacementRequest{ + // Same HANA vs general-purpose split as regular failover placement. + Pipeline: failover.InferFailoverPipeline(seed.Flavor.ExtraSpecs), + // We offer the host set ourselves; the scheduler must not gather the + // cluster itself and overwrite our candidates. + GatherAllCandidates: false, + HypervisorFamily: compute.HypervisorFamilyKVM, + // We only handle KVM virtual machines, so the flavor/image compatibility + // filter must see a KVM image hypervisor type. + ImageHypervisorType: compute.ImageHypervisorTypeKVM, + Hosts: hosts, + Weights: weights, + // Probe with evacuation intent. A failover cold-migrates (evacuates) the + // workload, and evacuation does not run the live-migratability feature + // check, so no source host is needed. This is deliberately weaker than a + // live-migration probe — see the PathFinder doc. + Intent: api.EvacuateIntent, + // Probe a single flavor slot; NumInstances must be >= 1 or capacity/ + // bin-pack math treats the host as free. + NumInstances: 1, + AvailabilityZone: seed.AvailabilityZone, + RequiredTraits: requiredTraits, + ForbiddenTraits: forbiddenTraits, + // Match the flavor's hypervisor type (QEMU vs CH). The seeder only emits + // seeds whose flavor names a supported hypervisor type, so this is always + // set (no default needed). + RequestedCapabilities: reservations.HypervisorTypeCapability(seed.Flavor.ExtraSpecs, false), + AcceleratorsRequested: nova.RequestsAccelerators(seed.Flavor.ExtraSpecs), + Flavor: compute.FlavorSpec{ + VCPUs: seed.Flavor.VCPUs, + MemoryMB: seed.Flavor.MemoryMB, + }, Options: scheduling.Options{ // Read-only probe: don't write any resources. ReadOnly additionally // requires SkipInflight and SkipCommittedResourceTracking, else the @@ -280,80 +306,23 @@ func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed See v1alpha1.ReservationTypeFailoverHost, }, }, - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - NumInstances: 1, - AvailabilityZone: seed.AvailabilityZone, - // Probe with evacuation intent. A failover cold-migrates (evacuates) - // the workload, and evacuation does not run the live-migratability - // feature check, so no source_host is needed. The check type must be - // the literal "evacuate" nova uses, which maps to EvacuateIntent in - // GetIntent. See the PathFinder doc for why this is deliberately - // weaker than a live-migration probe. - SchedulerHints: map[string]any{ - "_nova_check_type": "evacuate", - }, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: seed.Flavor.Name, - ExtraSpecs: seed.Flavor.ExtraSpecs, - MemoryMB: seed.Flavor.MemoryMB, - VCPUs: seed.Flavor.VCPUs, - }, - }, - Image: api.NovaObject[api.NovaImageMeta]{ - Data: api.NovaImageMeta{ - Properties: api.NovaObject[map[string]any]{ - Data: imageProps, - }, - }, - }, - }, - }, } - log.V(1).Info("external scheduler request prepared", "request", requestObj) - - reqBody, err := json.Marshal(requestObj) - if err != nil { - log.Error(err, "failed to marshal external scheduler request") - return nil, fmt.Errorf("failed to marshal external scheduler request: %w", err) - } - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, - f.schedulerURL, bytes.NewReader(reqBody)) - if err != nil { - log.Error(err, "failed to create HTTP request") - return nil, fmt.Errorf("failed to create HTTP request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") + log.V(1).Info("scheduler probe prepared", "request", req) startTime := time.Now() - response, err := f.httpClient.Do(httpReq) + result, err := f.scheduler.RunWithoutLock(ctx, req, nil, nil) duration := time.Since(startTime) if err != nil { f.monitor.RecordSchedulerRequest("error", duration) - log.Error(err, "failed to send external scheduler request") - return nil, fmt.Errorf("failed to send external scheduler request: %w", err) - } - defer response.Body.Close() - f.monitor.RecordSchedulerRequest(strconv.Itoa(response.StatusCode), duration) - log.V(1).Info("external scheduler request sent", - "duration", duration, "status", response.StatusCode) - - if response.StatusCode != http.StatusOK { - log.Error(nil, "external scheduler request failed", "status", response.StatusCode) - return nil, fmt.Errorf("external scheduler request failed with status: %d", response.StatusCode) - } - - var responseObj api.ExternalSchedulerResponse - if err := json.NewDecoder(response.Body).Decode(&responseObj); err != nil { - log.Error(err, "failed to decode external scheduler response") - return nil, fmt.Errorf("failed to decode external scheduler response: %w", err) + log.Error(err, "scheduler probe failed") + return nil, fmt.Errorf("scheduler probe failed: %w", err) } - log.V(1).Info("external scheduler response received", "response", responseObj) + f.monitor.RecordSchedulerRequest("200", duration) + log.V(1).Info("scheduler probe completed", "duration", duration, "hosts", result.OrderedHosts) candidates := []hv1.Hypervisor{} for _, hv := range hvsByName { - if slices.Contains(responseObj.Hosts, hv.Name) { + if slices.Contains(result.OrderedHosts, hv.Name) { candidates = append(candidates, hv) } } diff --git a/internal/scheduling/reservations/failoverhost/pathfinding_test.go b/internal/scheduling/reservations/failoverhost/pathfinding_test.go index e489e0ba4..72c451f38 100644 --- a/internal/scheduling/reservations/failoverhost/pathfinding_test.go +++ b/internal/scheduling/reservations/failoverhost/pathfinding_test.go @@ -5,66 +5,90 @@ package failoverhost import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" + "fmt" "slices" "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) -// stubScheduler returns a path finder wired to an httptest server whose +// stubScheduler returns a path finder wired to an in-process scheduler whose // per-flavor answers come from candidatesByFlavor (keyed by the seed's flavor -// name); a flavor in errFlavors gets an HTTP 500. The handler asserts the request -// carries valid options, a non-empty host set, a kvm img_hv_type, and evacuation -// intent with no source_host — mirroring what the new seeded probe sends, so a -// regression that drops any of them is caught here. The caller closes the server. -func stubScheduler(t *testing.T, candidatesByFlavor map[string][]string, errFlavors map[string]bool) (*evacuationPathFinder, *httptest.Server) { +// name); a flavor in errFlavors makes the probe fail. The fake pipeline asserts +// the request carries valid options, a non-empty host set, AssumeEmptyHosts, a +// KVM image hypervisor type, and evacuation intent with no source host — +// mirroring what the seeded probe sends, so a regression that drops any of them +// is caught here. +func stubScheduler(t *testing.T, candidatesByFlavor map[string][]string, errFlavors map[string]bool) *evacuationPathFinder { t.Helper() - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req api.ExternalSchedulerRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("decode request: %v", err) - } - if err := req.Options.Validate(); err != nil { - t.Errorf("invalid scheduling options: %v", err) - } - if !req.Options.AssumeEmptyHosts { - // A failover target is reserved empty, so candidates must be judged by - // total capacity. Dropping this would orphan every busy host. - t.Errorf("request for flavor %q must set AssumeEmptyHosts", req.Spec.Data.Flavor.Data.Name) - } - if len(req.Hosts) == 0 { - t.Errorf("request for flavor %q has no candidate hosts", req.Spec.Data.Flavor.Data.Name) - } - if got := req.Spec.Data.Image.Data.Properties.Data["img_hv_type"]; got != string(api.NovaImageMetaHVTypeKVM) { - // The hypervisor-type filter needs img_hv_type to judge flavor/image - // compatibility; dropping it would make that filter misbehave. - t.Errorf("request for flavor %q must set img_hv_type=%q, got %q", req.Spec.Data.Flavor.Data.Name, api.NovaImageMetaHVTypeKVM, got) - } - // The probe uses evacuation intent (cold migration) and sets no source_host: - // a failover evacuates rather than live-migrates, so the live-migratability - // feature check is deliberately skipped. Catch a regression to the stricter - // live-migration probe here. - if intent, err := req.GetIntent(); err != nil || intent != api.EvacuateIntent { - t.Errorf("request for flavor %q must use evacuate intent, got %v (err=%v)", req.Spec.Data.Flavor.Data.Name, intent, err) - } - if _, ok := req.Spec.Data.SchedulerHints["source_host"]; ok { - t.Errorf("request for flavor %q must not set source_host", req.Spec.Data.Flavor.Data.Name) - } - flavor := req.Spec.Data.Flavor.Data.Name - if errFlavors[flavor] { - w.WriteHeader(http.StatusInternalServerError) - return - } - if err := json.NewEncoder(w).Encode(api.ExternalSchedulerResponse{Hosts: candidatesByFlavor[flavor]}); err != nil { - t.Errorf("encode response: %v", err) - } - })) - finder := &evacuationPathFinder{httpClient: server.Client(), schedulerURL: server.URL} - return finder, server + provider := &stubPipelineProvider{pipeline: &stubPipeline{ + t: t, + candidatesByFlavor: candidatesByFlavor, + errFlavors: errFlavors, + }} + scheduler := compute.NewPlacement(provider, passthroughGatherer{}) + return &evacuationPathFinder{scheduler: scheduler} +} + +// stubPipelineProvider serves the same stub pipeline for every pipeline name the +// probe infers (always a KVM pipeline here). +type stubPipelineProvider struct { + pipeline *stubPipeline +} + +func (p *stubPipelineProvider) GetPipeline(string) (lib.FilterWeigherPipeline[compute.PlacementRequest], bool) { + return p.pipeline, true +} + +// stubPipeline is a fake filter/weigher pipeline that validates the probe's +// request shape and returns the preconfigured candidate hosts for the request's +// flavor, mirroring the old httptest stub. +type stubPipeline struct { + t *testing.T + candidatesByFlavor map[string][]string + errFlavors map[string]bool +} + +func (s *stubPipeline) Run(req compute.PlacementRequest) (compute.PlacementResult, error) { + // Identify which seed this probe is for by the trait the seed constructor + // puts on the flavor (see seed()), which the probe carries in RequiredTraits. + flavor := "" + if len(req.RequiredTraits) > 0 { + flavor = req.RequiredTraits[0] + } + if err := req.Options.Validate(); err != nil { + s.t.Errorf("invalid scheduling options: %v", err) + } + if !req.Options.AssumeEmptyHosts { + // A failover target is reserved empty, so candidates must be judged by + // total capacity. Dropping this would orphan every busy host. + s.t.Errorf("request for flavor %q must set AssumeEmptyHosts", flavor) + } + if len(req.Hosts) == 0 { + s.t.Errorf("request for flavor %q has no candidate hosts", flavor) + } + if req.ImageHypervisorType != compute.ImageHypervisorTypeKVM { + // The hypervisor-type filter needs the image hypervisor type to judge + // flavor/image compatibility; dropping it would make that filter misbehave. + s.t.Errorf("request for flavor %q must set ImageHypervisorType=%q, got %q", flavor, compute.ImageHypervisorTypeKVM, req.ImageHypervisorType) + } + // The probe uses evacuation intent (cold migration) and sets no source host: + // a failover evacuates rather than live-migrates, so the live-migratability + // feature check is deliberately skipped. Catch a regression to the stricter + // live-migration probe here. + if req.Intent != api.EvacuateIntent { + s.t.Errorf("request for flavor %q must use evacuate intent, got %v", flavor, req.Intent) + } + if req.SourceHost != "" { + s.t.Errorf("request for flavor %q must not set a source host", flavor) + } + if s.errFlavors[flavor] { + return compute.PlacementResult{}, fmt.Errorf("probe failed for flavor %q", flavor) + } + return compute.PlacementResult{OrderedHosts: s.candidatesByFlavor[flavor]}, nil } // pathNames collapses failover edges into sorted "source->target" strings, for @@ -89,9 +113,14 @@ func hostTopo(hostMemGi map[string]int64) topology { return topology{hypervisors: hypervisors} } -// seed is a tiny constructor for a flavor seed named name in az1. +// seed is a tiny constructor for a flavor seed named name in az1. The name is +// also set as a required trait so the stub pipeline can tell which seed a probe +// request belongs to (the probe carries the flavor's traits in RequiredTraits). func seed(name string) Seed { - return Seed{Flavor: Flavor{Name: name}, AvailabilityZone: "az1"} + return Seed{ + Flavor: Flavor{Name: name, ExtraSpecs: map[string]string{"trait:" + name: "required"}}, + AvailabilityZone: "az1", + } } func TestFindPathsErrorsOnLookupFailure(t *testing.T) { @@ -100,11 +129,10 @@ func TestFindPathsErrorsOnLookupFailure(t *testing.T) { // return an error to abort the pass rather than silently dropping paths — // otherwise the caller would publish an empty desired state and prune valid // reservations. - finder, server := stubScheduler(t, + finder := stubScheduler(t, map[string][]string{"good": {"a", "b"}}, map[string]bool{"bad": true}, ) - defer server.Close() // "bad" is probed and errors; the pass must abort regardless of "good". topo := hostTopo(map[string]int64{"a": 32, "b": 32}) @@ -116,10 +144,9 @@ func TestFindPathsErrorsOnLookupFailure(t *testing.T) { func TestFindPathsPairsCoAcceptingHosts(t *testing.T) { // A single seed accepted on a, b and c makes all three mutually compatible. // Equal-sized hosts yield edges both ways between every pair. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"a", "b", "c"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"a": 32, "b": 32, "c": 32}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) @@ -136,10 +163,9 @@ func TestFindPathsDropsTargetSmallerThanSource(t *testing.T) { // A seed is accepted on "big" (64Gi), "small" (16Gi) and "equal" (64Gi). The // gang-fit constraint keeps an edge only toward a target at least as large as // the source, so no edge points at "small", and big<->equal are mutual. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"big", "small", "equal"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"big": 64, "small": 16, "equal": 64}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) @@ -157,11 +183,10 @@ func TestFindPathsDropsTargetSmallerThanSource(t *testing.T) { func TestFindPathsDeduplicatesAcrossSeeds(t *testing.T) { // Two seeds both accepted on the same pair of equal hosts. Each directed edge // must appear once, not once per seed. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavorA": {"hvA", "hvB"}, "flavorB": {"hvA", "hvB"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"hvA": 64, "hvB": 64}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavorA"), seed("flavorB")}) @@ -175,10 +200,9 @@ func TestFindPathsDeduplicatesAcrossSeeds(t *testing.T) { func TestFindPathsSingleCandidateYieldsNoEdge(t *testing.T) { // A seed accepted on only one host can form no pair, so contributes no edge. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavor": {"only"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"only": 64, "other": 64}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) @@ -194,11 +218,10 @@ func TestFindPathsRecordsSupportingFlavors(t *testing.T) { // flavorA is accepted on both hosts, flavorB only on hvA. The edge hvA<->hvB is // produced solely by flavorA, so each directed edge must carry exactly // ["flavorA"] — flavorB co-accepts on no pair and leaves no trace. - finder, server := stubScheduler(t, map[string][]string{ + finder := stubScheduler(t, map[string][]string{ "flavorA": {"hvA", "hvB"}, "flavorB": {"hvA"}, }, nil) - defer server.Close() topo := hostTopo(map[string]int64{"hvA": 64, "hvB": 64}) got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavorA"), seed("flavorB")}) diff --git a/internal/scheduling/reservations/failoverhost/seeding.go b/internal/scheduling/reservations/failoverhost/seeding.go index 1af4c98fc..4dd9d7508 100644 --- a/internal/scheduling/reservations/failoverhost/seeding.go +++ b/internal/scheduling/reservations/failoverhost/seeding.go @@ -25,8 +25,8 @@ import ( // fields fed into api.NovaFlavor, so the rest of the flavor-group shape never // leaks into the pathfinder. type Flavor struct { - // Name is the flavor name, forwarded as the probe's flavor name so - // flavor-name-sensitive filters behave as in a real placement. + // Name is the flavor name, used for pipeline inference, trait/capability + // extraction and to label the failover edges a seed supports. Name string // MemoryMB and VCPUs size the probe. RAM is in MiB (despite the "MB" label), // matching what the scheduler expects on NovaFlavor.MemoryMB. diff --git a/internal/scheduling/reservations/quota/controller.go b/internal/scheduling/reservations/quota/controller.go index e6d6dcf0e..46ea8dd1e 100644 --- a/internal/scheduling/reservations/quota/controller.go +++ b/internal/scheduling/reservations/quota/controller.go @@ -13,6 +13,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/accounting" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -74,7 +75,7 @@ func (c *QuotaController) ReconcilePeriodic(ctx context.Context) error { logger.Info("starting full quota reconcile") // Fetch flavor groups from Knowledge CRD - flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + flavorGroupClient := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Error(err, "failed to get flavor groups") @@ -228,7 +229,7 @@ func (c *QuotaController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl } // Fetch flavor groups for CRUsage computation - flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + flavorGroupClient := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Error(err, "failed to get flavor groups") @@ -267,7 +268,7 @@ func (c *QuotaController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // a ProjectQuota is first created and has no persisted TotalUsage yet. func (c *QuotaController) computeTotalUsageForProject(ctx context.Context, projectID string) (map[string]map[string]int64, error) { // Fetch flavor groups from Knowledge CRD - flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + flavorGroupClient := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) if err != nil { return nil, fmt.Errorf("failed to get flavor groups: %w", err) @@ -362,7 +363,7 @@ func (c *QuotaController) ReconcileHVDiff(ctx context.Context, oldHV, newHV *hv1 logger.V(1).Info("HV instance diff detected", "added", len(added), "removed", len(removed)) // Get flavor groups for mapping - flavorGroupClient := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + flavorGroupClient := &accounting.FlavorGroupKnowledgeClient{Client: c.Client} flavorGroups, err := flavorGroupClient.GetAllFlavorGroups(ctx, nil) if err != nil { logger.Error(err, "failed to get flavor groups for incremental update") diff --git a/internal/scheduling/reservations/scheduler_client.go b/internal/scheduling/reservations/scheduler_client.go deleted file mode 100644 index e5ee2c3ee..000000000 --- a/internal/scheduling/reservations/scheduler_client.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package reservations - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - api "github.com/cobaltcore-dev/cortex/api/external/nova" - "github.com/cobaltcore-dev/cortex/api/scheduling" - "github.com/go-logr/logr" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -// DefaultHTTPTimeout is the default timeout for HTTP requests to the scheduler API. -const DefaultHTTPTimeout = 30 * time.Second - -var log = logf.Log.WithName("scheduler-client").WithValues("module", "reservations") - -// loggerFromContext returns a logger with greq and req values from the context. -func loggerFromContext(ctx context.Context) logr.Logger { - return log.WithValues( - "greq", GlobalRequestIDFromContext(ctx), - "req", RequestIDFromContext(ctx), - ) -} - -// NOTE+FIXME: we should not send ourselves REST API calls. This needs to be replaced by direct Go call (if possible) or communication via CRDs - -// SchedulerClient is a client for the external scheduler API. -// It can be used by both the ReservationReconciler and FailoverReservationController. -type SchedulerClient struct { - // URL of the external scheduler API. - URL string - // HTTP client to use for requests. - HTTPClient *http.Client -} - -// NewSchedulerClient creates a new SchedulerClient with a default timeout. -func NewSchedulerClient(url string) *SchedulerClient { - return &SchedulerClient{ - URL: url, - HTTPClient: &http.Client{ - Timeout: DefaultHTTPTimeout, - }, - } -} - -// ScheduleReservationRequest contains the parameters for scheduling a reservation. -type ScheduleReservationRequest struct { - // InstanceUUID is the unique identifier for the reservation (usually the reservation name). - InstanceUUID string - // ProjectID is the OpenStack project ID. - ProjectID string - // FlavorName is the name of the flavor. - FlavorName string - // FlavorExtraSpecs are extra specifications for the flavor. - FlavorExtraSpecs map[string]string - // MemoryMB is the memory in MB. - MemoryMB uint64 - // VCPUs is the number of virtual CPUs. - VCPUs uint64 - // EligibleHosts is the list of hosts that can be considered for placement. - EligibleHosts []api.ExternalSchedulerHost - // IgnoreHosts is a list of hosts to ignore during scheduling. - // This is used for failover reservations to avoid placing on the same host as the VMs. - IgnoreHosts []string - // Pipeline is the name of the pipeline to execute. - // If empty, the default pipeline will be used. - Pipeline string - // AvailabilityZone is the availability zone to schedule in. - // This is used by the filter_correct_az filter to ensure hosts are in the correct AZ. - AvailabilityZone string - // SchedulerHints are hints passed to the scheduler pipeline. - // Used to set _nova_check_type for evacuation intent detection. - SchedulerHints map[string]any - // ImageProperties are Glance-style image properties (e.g., "img_hv_type", "hypervisor_type") - // forwarded on Spec.Data.Image.Data.Properties.Data. Optional; when empty the outgoing - // request leaves the image field at its zero value. Used by FilterImagePropertiesStep. - ImageProperties map[string]any -} - -// ScheduleReservationResponse contains the result of scheduling a reservation. -type ScheduleReservationResponse struct { - // Hosts is the ordered list of hosts that the reservation can be placed on. - // The first host is the best choice. - Hosts []string -} - -// ScheduleReservation calls the external scheduler API to find a host for a reservation. -// The context should contain GlobalRequestID and RequestID for logging (use WithGlobalRequestID/WithRequestID). -func (c *SchedulerClient) ScheduleReservation(ctx context.Context, req ScheduleReservationRequest, opts scheduling.Options) (*ScheduleReservationResponse, error) { - logger := loggerFromContext(ctx) - - if err := opts.Validate(); err != nil { - return nil, fmt.Errorf("invalid scheduling options: %w", err) - } - - // Build weights map (all zero for reservations) - weights := make(map[string]float64, len(req.EligibleHosts)) - for _, host := range req.EligibleHosts { - weights[host.ComputeHost] = 0.0 - } - - // Build ignore hosts pointer - var ignoreHosts *[]string - if len(req.IgnoreHosts) > 0 { - ignoreHosts = &req.IgnoreHosts - } - - // Build the context with request IDs - var globalReqID *string - if greq := GlobalRequestIDFromContext(ctx); greq != "" { - globalReqID = &greq - } - - // Build the external scheduler request - externalSchedulerRequest := api.ExternalSchedulerRequest{ - Pipeline: req.Pipeline, - Hosts: req.EligibleHosts, - Weights: weights, - Options: opts, - Context: api.NovaRequestContext{ - RequestID: RequestIDFromContext(ctx), - GlobalRequestID: globalReqID, - ProjectID: req.ProjectID, - }, - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - InstanceUUID: req.InstanceUUID, - NumInstances: 1, // One for each reservation. - ProjectID: req.ProjectID, - AvailabilityZone: req.AvailabilityZone, - IgnoreHosts: ignoreHosts, - SchedulerHints: req.getSchedulerHints(), - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - Name: req.FlavorName, - ExtraSpecs: req.FlavorExtraSpecs, - MemoryMB: req.MemoryMB, - VCPUs: req.VCPUs, - // Disk is currently not considered. - }, - }, - Image: api.NovaObject[api.NovaImageMeta]{ - Data: api.NovaImageMeta{ - Properties: api.NovaObject[map[string]any]{ - Data: req.ImageProperties, - }, - }, - }, - }, - }, - } - - logger.V(1).Info("sending external scheduler request", - "url", c.URL, - "pipeline", req.Pipeline, - "instanceUUID", req.InstanceUUID, - "flavorName", req.FlavorName, - "flavorExtraSpecs", req.FlavorExtraSpecs, - "eligibleHostsCount", len(req.EligibleHosts), - "ignoreHostsCount", len(req.IgnoreHosts), - "hasSchedulerHints", len(req.SchedulerHints) > 0, - "availabilityZone", req.AvailabilityZone) - - // Marshal the request - reqBody, err := json.Marshal(externalSchedulerRequest) - if err != nil { - logger.Error(err, "failed to marshal external scheduler request") - return nil, fmt.Errorf("failed to marshal external scheduler request: %w", err) - } - - // Create HTTP request with context - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL, bytes.NewReader(reqBody)) - if err != nil { - logger.Error(err, "failed to create HTTP request") - return nil, fmt.Errorf("failed to create HTTP request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") - - // Send the request and measure duration - startTime := time.Now() - response, err := c.HTTPClient.Do(httpReq) - duration := time.Since(startTime) - if err != nil { - logger.Error(err, "failed to send external scheduler request") - return nil, fmt.Errorf("failed to send external scheduler request: %w", err) - } - defer response.Body.Close() - - // Check response status - if response.StatusCode != http.StatusOK { - body, err := io.ReadAll(io.LimitReader(response.Body, 1024)) - if err != nil { - logger.Error(err, "failed to read error response body", "statusCode", response.StatusCode) - return nil, fmt.Errorf("external scheduler returned status %d", response.StatusCode) - } - logger.Error(nil, "external scheduler returned non-OK status", "statusCode", response.StatusCode, "body", string(body)) - return nil, fmt.Errorf("external scheduler returned status %d: %s", response.StatusCode, string(body)) - } - - // Decode the response - var externalSchedulerResponse api.ExternalSchedulerResponse - if err := json.NewDecoder(response.Body).Decode(&externalSchedulerResponse); err != nil { - logger.Error(err, "failed to decode external scheduler response") - return nil, fmt.Errorf("failed to decode external scheduler response: %w", err) - } - - logger.V(1).Info("received external scheduler response", - "hostsCount", len(externalSchedulerResponse.Hosts), - "duration", duration.Round(time.Millisecond)) - - return &ScheduleReservationResponse{ - Hosts: externalSchedulerResponse.Hosts, - }, nil -} - -// getSchedulerHints returns the scheduler hints, or an empty map if nil. -func (req ScheduleReservationRequest) getSchedulerHints() map[string]any { - if req.SchedulerHints == nil { - return make(map[string]any) - } - return req.SchedulerHints -} diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index f31b5e6d3..2d0b7b06a 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -563,8 +563,8 @@ func TestGVKFromHomeScheme_Success(t *testing.T) { }{ {"ConfigMap", &corev1.ConfigMap{}, configMapGVK}, {"ConfigMapList", &corev1.ConfigMapList{}, configMapListGVK}, - {"Decision", &v1alpha1.Decision{}, schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Decision"}}, - {"DecisionList", &v1alpha1.DecisionList{}, schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "DecisionList"}}, + {"Pipeline", &v1alpha1.Pipeline{}, schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Pipeline"}}, + {"PipelineList", &v1alpha1.PipelineList{}, schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "PipelineList"}}, } for _, tt := range tests { @@ -1778,7 +1778,7 @@ func TestClient_InitFromConf_GVKFormatting(t *testing.T) { expectedStr string }{ {"core ConfigMap", configMapGVK, "v1/ConfigMap"}, - {"cortex Decision", schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Decision"}, "cortex.cloud/v1alpha1/Decision"}, + {"cortex Pipeline", schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Pipeline"}, "cortex.cloud/v1alpha1/Pipeline"}, } for _, tt := range tests {