From 0fac36de66d39b9a2e4c5e02ef61732a7f264f77 Mon Sep 17 00:00:00 2001 From: Malte Viering Date: Mon, 18 May 2026 13:07:58 +0000 Subject: [PATCH 1/2] quota: improve status completeness and observability - Set Ready condition on ProjectQuota status after successful reconcile - Seed zero values for all spec.quota keys in status so tracked groups are always visible even with no active VMs - Show all resource types (c/i/r) in usage summaries including zeros - Add diagnostic logging: flavor group mapping stats, resource names in quota API log --- api/v1alpha1/project_quota_types.go | 5 ++ .../reservations/commitments/api/quota.go | 13 ++++- .../reservations/quota/controller.go | 50 +++++++++++++------ .../reservations/quota/summary_test.go | 16 +++--- 4 files changed, 59 insertions(+), 25 deletions(-) diff --git a/api/v1alpha1/project_quota_types.go b/api/v1alpha1/project_quota_types.go index 89c7d8f01..e1b325317 100644 --- a/api/v1alpha1/project_quota_types.go +++ b/api/v1alpha1/project_quota_types.go @@ -108,6 +108,11 @@ type ProjectQuotaStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } +const ( + // ProjectQuotaConditionReady indicates whether the quota usage has been successfully computed. + ProjectQuotaConditionReady = "Ready" +) + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster diff --git a/internal/scheduling/reservations/commitments/api/quota.go b/internal/scheduling/reservations/commitments/api/quota.go index 08f380a28..8d51de195 100644 --- a/internal/scheduling/reservations/commitments/api/quota.go +++ b/internal/scheduling/reservations/commitments/api/quota.go @@ -241,12 +241,21 @@ func (api *HTTPAPI) HandleQuota(w http.ResponseWriter, r *http.Request) { } } - // Collect AZ names for the success log + // Collect AZ names and resource group names for the success log azNames := make([]string, 0, len(activeAZs)) for az := range activeAZs { azNames = append(azNames, az) } - log.Info("quota request completed", "projectID", projectID, "azs", azNames) + groupNames := make(map[string]bool) + for resourceName := range req.Resources { + groupNames[string(resourceName)] = true + } + groups := make([]string, 0, len(groupNames)) + for g := range groupNames { + groups = append(groups, g) + } + log.Info("quota request completed", "projectID", projectID, "azs", azNames, + "resources", len(req.Resources), "resourceNames", groups) // Return 204 No Content as expected by the LIQUID API w.WriteHeader(http.StatusNoContent) diff --git a/internal/scheduling/reservations/quota/controller.go b/internal/scheduling/reservations/quota/controller.go index 9b6033a5b..745967624 100644 --- a/internal/scheduling/reservations/quota/controller.go +++ b/internal/scheduling/reservations/quota/controller.go @@ -16,6 +16,7 @@ import ( commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -85,6 +86,9 @@ func (c *QuotaController) ReconcilePeriodic(ctx context.Context) error { // Build flavorName → flavorGroup lookup flavorToGroup := buildFlavorToGroupMap(flavorGroups) + logger.Info("flavor group knowledge loaded", + "flavorGroups", len(flavorGroups), + "totalFlavorsInMap", len(flavorToGroup)) // Fetch all VMs using VMSource (reads from Postgres via DBVMSource) vms, err := c.VMSource.ListVMs(ctx) @@ -97,6 +101,17 @@ func (c *QuotaController) ReconcilePeriodic(ctx context.Context) error { // Compute totalUsage per project/AZ/resource totalUsageByProject := c.computeTotalUsage(vms, flavorToGroup, flavorGroups) + // Diagnostic: count how many VMs were actually mapped to a flavor group + mappedVMs := 0 + for _, vm := range vms { + if _, ok := flavorToGroup[vm.FlavorName]; ok { + mappedVMs++ + } + } + logger.Info("VM flavor mapping complete", + "totalVMs", len(vms), "mappedVMs", mappedVMs, "unmappedVMs", len(vms)-mappedVMs, + "projectsWithUsage", len(totalUsageByProject)) + // List all existing ProjectQuota CRDs var pqList v1alpha1.ProjectQuotaList if err := c.List(ctx, &pqList); err != nil { @@ -945,6 +960,16 @@ func (c *QuotaController) updateProjectQuotaStatusWithRetry( az := pq.Spec.AvailabilityZone pq.Status.TotalUsage = extractAZSlice(totalUsage, az) pq.Status.PaygUsage = extractAZSlice(paygUsage, az) + // Seed zero values for all resource keys defined in spec.quota so the status + // always shows which groups are tracked (even when no VMs exist for this project). + for key := range pq.Spec.Quota { + if _, ok := pq.Status.TotalUsage[key]; !ok { + pq.Status.TotalUsage[key] = 0 + } + if _, ok := pq.Status.PaygUsage[key]; !ok { + pq.Status.PaygUsage[key] = 0 + } + } pq.Status.TotalUsageSummary = buildUsageSummary(pq.Status.TotalUsage) pq.Status.PaygUsageSummary = buildUsageSummary(pq.Status.PaygUsage) // Limes unit summaries for debugging (converted from internal GiB to declared units) @@ -956,6 +981,14 @@ func (c *QuotaController) updateProjectQuotaStatusWithRetry( if fullReconcile { pq.Status.LastFullReconcileAt = &now } + // Set Ready condition to indicate successful reconciliation + meta.SetStatusCondition(&pq.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ProjectQuotaConditionReady, + Status: metav1.ConditionTrue, + ObservedGeneration: pq.Generation, + Reason: "Reconciled", + Message: "Quota usage successfully computed", + }) return c.Status().Update(ctx, &pq) }) } @@ -1021,24 +1054,11 @@ func buildUsageSummary(usage map[string]int64) string { } sort.Strings(groupNames) - // Build compact summary — only include resource types with non-zero values + // Build compact summary — include all resource types for each group (even zeros) var parts []string for _, name := range groupNames { gv := groups[name] - var vals []string - if gv.cores != 0 { - vals = append(vals, fmt.Sprintf("c=%d", gv.cores)) - } - if gv.instances != 0 { - vals = append(vals, fmt.Sprintf("i=%d", gv.instances)) - } - if gv.ram != 0 { - vals = append(vals, fmt.Sprintf("r=%d", gv.ram)) - } - if len(vals) == 0 { - continue // skip groups with all zeros - } - parts = append(parts, name+": "+strings.Join(vals, " ")) + parts = append(parts, fmt.Sprintf("%s: c=%d i=%d r=%d", name, gv.cores, gv.instances, gv.ram)) } return strings.Join(parts, "; ") } diff --git a/internal/scheduling/reservations/quota/summary_test.go b/internal/scheduling/reservations/quota/summary_test.go index a336628ef..b3810b301 100644 --- a/internal/scheduling/reservations/quota/summary_test.go +++ b/internal/scheduling/reservations/quota/summary_test.go @@ -49,21 +49,21 @@ func TestBuildUsageSummary(t *testing.T) { usage: map[string]int64{ "hw_version_222_ram": 22, }, - expected: "222: r=22", + expected: "222: c=0 i=0 r=22", }, { name: "only cores set", usage: map[string]int64{ "hw_version_3000_cores": 10, }, - expected: "3000: c=10", + expected: "3000: c=10 i=0 r=0", }, { name: "only instances set", usage: map[string]int64{ "hw_version_abc_instances": 5, }, - expected: "abc: i=5", + expected: "abc: c=0 i=5 r=0", }, { name: "partial resources across multiple groups", @@ -72,7 +72,7 @@ func TestBuildUsageSummary(t *testing.T) { "hw_version_2152_cores": 6, "hw_version_2152_instances": 3, }, - expected: "2101: r=14; 2152: c=6 i=3", + expected: "2101: c=0 i=0 r=14; 2152: c=6 i=3 r=0", }, { name: "unknown suffix is ignored", @@ -82,7 +82,7 @@ func TestBuildUsageSummary(t *testing.T) { "hw_version_2101_disks": 99, "hw_version_2101_network": 42, }, - expected: "2101: c=18 r=21", + expected: "2101: c=18 i=0 r=21", }, { name: "unknown prefix is ignored", @@ -91,7 +91,7 @@ func TestBuildUsageSummary(t *testing.T) { "other_prefix_cores": 50, "hw_version_2101_ram": 21, }, - expected: "2101: r=21", + expected: "2101: c=0 i=0 r=21", }, { name: "no hw_version_ prefix at all returns empty", @@ -115,13 +115,13 @@ func TestBuildUsageSummary(t *testing.T) { expected: "2101: c=18 i=7 r=21", }, { - name: "all zero values produce empty output", + name: "all zero values still show tracked groups", usage: map[string]int64{ "hw_version_2101_cores": 0, "hw_version_2101_instances": 0, "hw_version_2101_ram": 0, }, - expected: "", + expected: "2101: c=0 i=0 r=0", }, { name: "group name with underscores", From 50db759a700134d4f3cd95bafae8315e16d88452 Mon Sep 17 00:00:00 2001 From: Malte Viering Date: Mon, 18 May 2026 15:25:18 +0200 Subject: [PATCH 2/2] pr feedback --- .../reservations/commitments/api/quota.go | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/internal/scheduling/reservations/commitments/api/quota.go b/internal/scheduling/reservations/commitments/api/quota.go index 8d51de195..a4ce6e2e7 100644 --- a/internal/scheduling/reservations/commitments/api/quota.go +++ b/internal/scheduling/reservations/commitments/api/quota.go @@ -8,6 +8,7 @@ import ( "fmt" "math" "net/http" + "sort" "strconv" "time" @@ -242,20 +243,28 @@ func (api *HTTPAPI) HandleQuota(w http.ResponseWriter, r *http.Request) { } // Collect AZ names and resource group names for the success log - azNames := make([]string, 0, len(activeAZs)) - for az := range activeAZs { - azNames = append(azNames, az) + var storedAZs, skippedAZs []string + for az, isActive := range activeAZs { + if isActive { + storedAZs = append(storedAZs, az) + } else { + skippedAZs = append(skippedAZs, az) + } } - groupNames := make(map[string]bool) + sort.Strings(storedAZs) + sort.Strings(skippedAZs) + groups := make([]string, 0, len(req.Resources)) + seen := make(map[string]bool, len(req.Resources)) for resourceName := range req.Resources { - groupNames[string(resourceName)] = true - } - groups := make([]string, 0, len(groupNames)) - for g := range groupNames { - groups = append(groups, g) + name := string(resourceName) + if !seen[name] { + seen[name] = true + groups = append(groups, name) + } } - log.Info("quota request completed", "projectID", projectID, "azs", azNames, - "resources", len(req.Resources), "resourceNames", groups) + sort.Strings(groups) + log.Info("quota request completed", "projectID", projectID, "storedAZs", storedAZs, + "skippedAZs", skippedAZs, "resources", len(req.Resources), "resourceNames", groups) // Return 204 No Content as expected by the LIQUID API w.WriteHeader(http.StatusNoContent)