feat: CR controller checks for host overload scenario and resolves via reservation re-placements

cmd/manager/main.go

@@ -668,6 +668,20 @@ func main() {
 				os.Exit(1)
 			}
 		}
+
+		if commitmentsConfig.ReservationController.EnableOversubscriptionCheck {
+			reservationControllerMonitor := commitments.NewReservationControllerMonitor()
+			metrics.Registry.MustRegister(&reservationControllerMonitor)
+
+			if err := (&commitments.HostOversubscriptionController{
+				Client:  multiclusterClient,
+				Conf:    commitmentsConfig.ReservationController,
+				Monitor: &reservationControllerMonitor,
+			}).SetupWithManager(mgr, multiclusterClient); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "HostOversubscription")
+				os.Exit(1)
+			}
+		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "datasource-controllers") {
 		setupLog.Info("enabling controller", "controller", "datasource-controllers")

helm/bundles/cortex-nova/templates/alerts.yaml

@@ -749,4 +749,74 @@ spec:
           resource router is mapping the same object to multiple clusters, or an
           object was created out-of-band on the wrong cluster. Investigate the
           affected resources and the routing configuration.
+
+    {{- if .Values.kvm.enabled }}
+    - alert: CortexNovaHostReservationsOversubscribed
+      # Fires when the sum of running VM allocations + reservation blocks (committed +
+      # failover) exceeds the host's effective capacity for CPU or memory.
+      # This can happen due to: concurrent slot creation with stale informer cache,
+      # operator-driven VM migrations where the slot stays on the old host, or
+      # capacity changes (e.g. hardware replacement changing EffectiveCapacity).
+      # The 10m hold-off tolerates the known migration window: after a VM departs,
+      # the slot remains on the old host until the usage reconciler cleans it up.
+      # Note: `reserved` only counts Ready reservations — violations during the
+      # initial unready window (slot just created) are not captured by this alert.
+      expr: |
+        (
+          max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="utilized"})
+          + max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="reserved"})
+          + max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="failover"})
+          - max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_total)
+        ) > 0
+      for: 15m
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "Host {{ "{{" }} $labels.compute_host {{ "}}" }} reservation blocks exceed capacity for {{ "{{" }} $labels.resource {{ "}}" }}"
+        description: >
+          The total of running VM allocations and reservation blocks (committed resource +
+          failover) on host {{ "{{" }} $labels.compute_host {{ "}}" }} exceeds its effective
+          capacity for {{ "{{" }} $labels.resource {{ "}}" }} by {{ "{{" }} $value | humanize {{ "}}" }}.
+          This means the host is over-subscribed and committed resource or failover guarantees may not be
+          satisfied. Common causes: VM migrations with slot not yet reclaimed, outdated/stale reservations, or out of sync issues. 
+          If problem remains, inspect the reservations on this host and check the CR/failover controller logs.
+    - alert: CortexCommittedResourceHostOversubscribed
+      expr: max by (host, az, resource) (cortex_committed_resource_host_oversubscribed) > 0
+      for: 15m
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "Host {{ "{{" }} $labels.host {{ "}}" }} CR slots exceed effective capacity for {{ "{{" }} $labels.resource {{ "}}" }}"
+        description: >
+          Committed resource reservation slots on host {{ "{{" }} $labels.host {{ "}}" }} exceed
+          its effective capacity for {{ "{{" }} $labels.resource {{ "}}" }} by {{ "{{" }} $value | humanize {{ "}}" }}.
+          The oversubscription controller was unable to resolve this by evicting unallocated slots.
+          Inspect the CR reservations on this host and check the oversubscription controller logs.
+    - alert: CortexCommittedResourceHostOversubscriptionEvictionsHigh
+      expr: sum by (az) (increase(cortex_committed_resource_host_oversubscribed_evicted_reservations_total{service="cortex-nova-metrics"}[1h])) > 10
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "High CR slot eviction rate in AZ {{ "{{" }} $labels.az {{ "}}" }}"
+        description: >
+          More than 10 committed resource reservation slots were evicted due to oversubscription
+          in AZ {{ "{{" }} $labels.az {{ "}}" }} within the last hour ({{ "{{" }} $value | humanize {{ "}}" }} evictions).
+          This indicates recurring oversubscription events — investigate capacity changes,
+          concurrent slot creation races, or VM migration patterns in this AZ.
+    {{- end }}
 {{- end }}

helm/bundles/cortex-nova/values.yaml

@@ -185,6 +185,16 @@ cortex-scheduling-controllers:
       # How long after a VM is allocated to a reservation before it is expected to appear
       # on the target host; allocations not confirmed within this window are removed
       allocationGracePeriod: "15m"
+      # How long to wait after detecting host over-subscription before evicting reservation slots. 
+      # Gives other controllers (e.g. failover) time to self-heal.
+      oversubscriptionGracePeriod: "3m"
+      # Minimum time between consecutive over-subscription checks for the same host.
+      # Should be shorter than oversubscriptionGracePeriod.
+      oversubscriptionMinCheckInterval: "1m"
+      # Enable host over-subscription detection
+      enableOversubscriptionCheck: true
+      # When false, violations are detected and exposed via metrics but no slots are evicted.
+      enableOversubscriptionReservationEviction: true
       # URL of the nova external scheduler API for placement decisions
       schedulerURL: "http://localhost:8080/scheduler/nova/external"
       # Keystone credentials used to resolve domain IDs to domain names for the

internal/scheduling/reservations/capacity_accounting.go

@@ -25,6 +25,7 @@ import (
 // res itself is excluded from step 3 to avoid subtracting its own block from free capacity.
 // Returns false when the hypervisor has no capacity data.
 func HostHasCapacityForReservation(allReservations []v1alpha1.Reservation, hv hv1.Hypervisor, res *v1alpha1.Reservation) bool {
+	// TODO consider refactor with HostFreeCapacity
 	effCap := hv.Status.EffectiveCapacity
 	if effCap == nil {
 		effCap = hv.Status.Capacity
@@ -75,6 +76,49 @@ func HostHasCapacityForReservation(allReservations []v1alpha1.Reservation, hv hv
 	return true
 }
 
+// HostFreeCapacity computes the remaining free capacity on hv after subtracting
+// hv.Status.Allocation and UnusedReservationCapacity for all reservations on this host.
+// Negative values indicate over-subscription for that resource.
+// Returns nil when the hypervisor has no capacity data.
+// Reservations not targeting this host (via Spec.TargetHost or Status.Host) are ignored.
+func HostFreeCapacity(hostReservations []v1alpha1.Reservation, hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity {
+	// TODO consider refactor with HostHasCapacityForReservation
+	effCap := hv.Status.EffectiveCapacity
+	if effCap == nil {
+		effCap = hv.Status.Capacity
+	}
+	if effCap == nil {
+		return nil
+	}
+
+	free := make(map[hv1.ResourceName]resource.Quantity, len(effCap))
+	for rn, qty := range effCap {
+		free[rn] = qty.DeepCopy()
+	}
+	for rn, allocated := range hv.Status.Allocation {
+		if f, ok := free[rn]; ok {
+			f.Sub(allocated)
+			free[rn] = f
+		}
+	}
+	for i := range hostReservations {
+		res := &hostReservations[i]
+		if res.Spec.TargetHost == "" {
+			continue // evicted; status.Host may lag until reconcile
+		}
+		if res.Spec.TargetHost != hv.Name && res.Status.Host != hv.Name {
+			continue
+		}
+		for rn, block := range UnusedReservationCapacity(res, false) {
+			if f, ok := free[rn]; ok {
+				f.Sub(block)
+				free[rn] = f
+			}
+		}
+	}
+	return free
+}
+
 // UnusedReservationCapacity returns the resources a Reservation should block on its host(s).
 // This is the single source of truth used by both the capacity controller and
 // filter_has_enough_capacity to ensure consistent accounting.

internal/scheduling/reservations/capacity_accounting_test.go

@@ -172,6 +172,190 @@ func TestUnusedReservationCapacity(t *testing.T) {
 	}
 }
 
+func TestHostFreeCapacity(t *testing.T) {
+	gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) }
+	cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) }
+
+	hvWithCap := func(name string, memGiB, cpuCores int64) hv1.Hypervisor {
+		return hv1.Hypervisor{
+			ObjectMeta: metav1.ObjectMeta{Name: name},
+			Status: hv1.HypervisorStatus{
+				EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
+					hv1.ResourceMemory: gib(memGiB),
+					hv1.ResourceCPU:    cpu(cpuCores),
+				},
+			},
+		}
+	}
+	crSlot := func(name, host string, memGiB, cpuCores int64) v1alpha1.Reservation {
+		return v1alpha1.Reservation{
+			ObjectMeta: metav1.ObjectMeta{Name: name},
+			Spec: v1alpha1.ReservationSpec{
+				Type:       v1alpha1.ReservationTypeCommittedResource,
+				TargetHost: host,
+				Resources: map[hv1.ResourceName]resource.Quantity{
+					hv1.ResourceMemory: gib(memGiB),
+					hv1.ResourceCPU:    cpu(cpuCores),
+				},
+			},
+			Status: v1alpha1.ReservationStatus{Host: host},
+		}
+	}
+	freeMemGiB := func(free map[hv1.ResourceName]resource.Quantity) int64 {
+		q := free[hv1.ResourceMemory]
+		return q.Value() / (1024 * 1024 * 1024)
+	}
+	freeCPU := func(free map[hv1.ResourceName]resource.Quantity) int64 {
+		q := free[hv1.ResourceCPU]
+		return q.Value()
+	}
+
+	t.Run("no capacity data returns nil", func(t *testing.T) {
+		hv := hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host"}}
+		if got := HostFreeCapacity(nil, hv); got != nil {
+			t.Errorf("expected nil, got %v", got)
+		}
+	})
+
+	t.Run("no reservations and no allocation: free = effective capacity", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		free := HostFreeCapacity(nil, hv)
+		if freeMemGiB(free) != 1024 {
+			t.Errorf("expected 1024 GiB free, got %d", freeMemGiB(free))
+		}
+		if freeCPU(free) != 256 {
+			t.Errorf("expected 256 CPU free, got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("allocation subtracted from capacity", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{
+			hv1.ResourceMemory: gib(512),
+			hv1.ResourceCPU:    cpu(128),
+		}
+		free := HostFreeCapacity(nil, hv)
+		if freeMemGiB(free) != 512 {
+			t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free))
+		}
+		if freeCPU(free) != 128 {
+			t.Errorf("expected 128 CPU free, got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("reservation blocks subtracted", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		slots := []v1alpha1.Reservation{
+			crSlot("slot-1", "host", 512, 128),
+		}
+		free := HostFreeCapacity(slots, hv)
+		if freeMemGiB(free) != 512 {
+			t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free))
+		}
+		if freeCPU(free) != 128 {
+			t.Errorf("expected 128 CPU free, got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("over-subscribed: negative free values", func(t *testing.T) {
+		// 5 x 1TiB slots on a 4TiB host — the production scenario
+		hv := hvWithCap("host", 4096, 256)
+		slots := []v1alpha1.Reservation{
+			crSlot("slot-0", "host", 1024, 128),
+			crSlot("slot-1", "host", 1024, 128),
+			crSlot("slot-2", "host", 1024, 128),
+			crSlot("slot-3", "host", 1024, 128),
+			crSlot("slot-4", "host", 1024, 128),
+		}
+		free := HostFreeCapacity(slots, hv)
+		if freeMemGiB(free) != -1024 {
+			t.Errorf("expected -1024 GiB (over-subscribed), got %d GiB", freeMemGiB(free))
+		}
+		if freeCPU(free) != -384 {
+			t.Errorf("expected -384 CPU (over-subscribed), got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("allocation + reservations combined", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{
+			hv1.ResourceMemory: gib(256),
+			hv1.ResourceCPU:    cpu(64),
+		}
+		slots := []v1alpha1.Reservation{
+			crSlot("slot-1", "host", 512, 128),
+		}
+		free := HostFreeCapacity(slots, hv)
+		// 1024 - 256 (alloc) - 512 (slot) = 256 GiB free
+		if freeMemGiB(free) != 256 {
+			t.Errorf("expected 256 GiB free, got %d", freeMemGiB(free))
+		}
+		// 256 - 64 (alloc) - 128 (slot) = 64 free
+		if freeCPU(free) != 64 {
+			t.Errorf("expected 64 CPU free, got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("confirmed VM reduces slot block (not double counted)", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		// 256 GiB confirmed VM already counted in Allocation
+		hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{
+			hv1.ResourceMemory: gib(256),
+		}
+		// 512 GiB slot with 256 GiB confirmed VM → block = 512-256 = 256 GiB
+		slot := v1alpha1.Reservation{
+			ObjectMeta: metav1.ObjectMeta{Name: "slot-1"},
+			Spec: v1alpha1.ReservationSpec{
+				Type:       v1alpha1.ReservationTypeCommittedResource,
+				TargetHost: "host",
+				Resources:  map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: gib(512)},
+				CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{
+					Allocations: map[string]v1alpha1.CommittedResourceAllocation{
+						"vm-1": {Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: gib(256)}},
+					},
+				},
+			},
+			Status: v1alpha1.ReservationStatus{
+				Host: "host",
+				CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{
+					Allocations: map[string]string{"vm-1": "host"},
+				},
+			},
+		}
+		free := HostFreeCapacity([]v1alpha1.Reservation{slot}, hv)
+		// 1024 - 256 (alloc/vm) - 256 (remaining slot block) = 512
+		if freeMemGiB(free) != 512 {
+			t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free))
+		}
+	})
+
+	t.Run("reservations on other hosts are ignored even if passed in", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		slots := []v1alpha1.Reservation{
+			crSlot("slot-other", "other-host", 1024, 256), // different host — must not block
+		}
+		free := HostFreeCapacity(slots, hv)
+		if freeMemGiB(free) != 1024 {
+			t.Errorf("expected 1024 GiB free (other host ignored), got %d", freeMemGiB(free))
+		}
+	})
+
+	t.Run("falls back to Capacity when EffectiveCapacity nil", func(t *testing.T) {
+		hv := hv1.Hypervisor{
+			ObjectMeta: metav1.ObjectMeta{Name: "host"},
+			Status: hv1.HypervisorStatus{
+				Capacity: map[hv1.ResourceName]resource.Quantity{
+					hv1.ResourceMemory: gib(512),
+				},
+			},
+		}
+		free := HostFreeCapacity(nil, hv)
+		if freeMemGiB(free) != 512 {
+			t.Errorf("expected 512 GiB, got %d", freeMemGiB(free))
+		}
+	})
+}
+
 func TestHostHasCapacityForReservation(t *testing.T) {
 	gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) }
 	cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) }