From 5fb763ef37d3dfe42b2ad99f5a789f0f98367570 Mon Sep 17 00:00:00 2001 From: Stefan Majewsky Date: Tue, 11 Aug 2026 13:24:19 +0200 Subject: [PATCH 01/12] remove go-bits/easypg (#1123) I am replacing go-bits/easypg with a clean-room reimplementation, gg/pgruntime, that features several improvements over the original API (listed e.g. in sapcc/keppel#766). None of those are really relevant for you since Cortex uses barely anything from easypg, but since I would like to get rid of go-bits/easypg eventually, this replaces your usage of the API with the equivalent API from gg/pgruntime. If you will allow me a side note (which is not relevant for the scope of this PR), I had to chuckle when I saw that you are using sqlite for your unit tests. I did the same in Limes and Castellum all those years ago, until I finally had enough of having to deal with two separate database dialects, and realized that spawning a PostgreSQL server is barely any slower, and in fact can make CI faster because it removes the compile time for the go-sqlite CGo stuff. If you are interested in getting rid of some useless code paths, have a look at https://pkg.go.dev/go.xyrillian.de/gg/pgruntime#WithTestDB. (You will also need to patch your Makefile and GitHub Actions workflows to install PostgreSQL etc.; you can find what to do by grepping for `if sr.UsesPostgres` in the go-makefile-maker source code.) Signed-off-by: Stefan Majewsky --- go.mod | 2 +- go.sum | 27 ++++----------------------- internal/knowledge/db/db.go | 9 ++++++--- internal/knowledge/db/testing/env.go | 9 ++++++--- 4 files changed, 17 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 129333abe..7e5757ea9 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/go-openapi/swag/stringutils v0.25.1 // indirect github.com/go-openapi/swag/typeutils v0.25.1 // indirect github.com/go-openapi/swag/yamlutils v0.25.1 // indirect + github.com/go-sql-driver/mysql v1.10.0 // indirect github.com/gofrs/uuid/v5 v5.5.1 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -62,7 +63,6 @@ require ( github.com/go-openapi/jsonreference v0.21.4 // indirect github.com/go-openapi/swag v0.25.1 // indirect github.com/go-openapi/swag/jsonname v0.25.4 // indirect - github.com/golang-migrate/migrate/v4 v4.19.1 // indirect github.com/google/cel-go v0.29.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect diff --git a/go.sum b/go.sum index de07f6930..ff9d4ea05 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= @@ -24,10 +26,6 @@ github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 h1:2KjrD+LzU8pgvm github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3/go.mod h1:DyQ6MO1TsC4kzeG3l8tnIXQVeA/PITm22RyitJhDaKI= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/databus23/goslo.policy v0.0.0-20250326134918-4afc2c56a903 h1:RiumxYxPww35QeXCGV9NTohc7eGQwlVdz+p3nNHIF28= github.com/databus23/goslo.policy v0.0.0-20250326134918-4afc2c56a903/go.mod h1:tRj172JgwQmUmEqZZJBWzYWFStitMFTtb95NtUnmpkw= @@ -35,12 +33,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= -github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -98,16 +90,12 @@ github.com/go-openapi/swag/yamlutils v0.25.1 h1:mry5ez8joJwzvMbaTGLhw8pXUnhDK91o github.com/go-openapi/swag/yamlutils v0.25.1/go.mod h1:cm9ywbzncy3y6uPm/97ysW8+wZ09qsks+9RS8fLWKqg= github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gofrs/uuid/v5 v5.5.1 h1:z1Ce19/JwNidXpy3tOQc3241lnJLKdKyq/xlNvlD4Ng= github.com/gofrs/uuid/v5 v5.5.1/go.mod h1:bbAA98EoIlxyRHIVg6ektCSsZ5n8mSbwgEhvhMYlZgg= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= -github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.29.0 h1:fEG+Ja3YRwNOqnQxTyJwoByAUAvTuxUGiro/jhrm4F4= @@ -155,23 +143,16 @@ github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w= github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= diff --git a/internal/knowledge/db/db.go b/internal/knowledge/db/db.go index 498bac742..d9ff32d7d 100644 --- a/internal/knowledge/db/db.go +++ b/internal/knowledge/db/db.go @@ -18,7 +18,8 @@ import ( "github.com/go-gorp/gorp" _ "github.com/lib/pq" "github.com/prometheus/client_golang/prometheus" - "github.com/sapcc/go-bits/easypg" + "github.com/sapcc/go-api-declarations/bininfo" + "go.xyrillian.de/gg/pgruntime" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -77,14 +78,16 @@ func (c Connector) FromSecretRef(ctx context.Context, ref corev1.SecretReference return nil, errors.New("missing port in secret data") } strip := func(s string) string { return strings.ReplaceAll(s, "\n", "") } - dbURL, err := easypg.URLFrom(easypg.URLParts{ + target := pgruntime.ConnectionTarget{ HostName: strip(string(host)), Port: strip(string(port)), UserName: strip(string(user)), Password: strip(string(password)), DatabaseName: strip(string(database)), ConnectionOptions: "sslmode=disable", - }) + ApplicationName: bininfo.Component(), + } + dbURL, err := target.IntoURL() if err != nil { return nil, err } diff --git a/internal/knowledge/db/testing/env.go b/internal/knowledge/db/testing/env.go index 4c88b55bc..eef6c6456 100644 --- a/internal/knowledge/db/testing/env.go +++ b/internal/knowledge/db/testing/env.go @@ -13,7 +13,8 @@ import ( "github.com/cobaltcore-dev/cortex/internal/knowledge/db/testing/containers" "github.com/go-gorp/gorp" _ "github.com/mattn/go-sqlite3" - "github.com/sapcc/go-bits/easypg" + "github.com/sapcc/go-api-declarations/bininfo" + "go.xyrillian.de/gg/pgruntime" ) type DBEnv struct { @@ -28,14 +29,16 @@ func SetupDBEnv(t *testing.T) DBEnv { slog.Info("Using real postgres container") container := containers.PostgresContainer{} container.Init(t) - dbURL, err := easypg.URLFrom(easypg.URLParts{ + target := pgruntime.ConnectionTarget{ HostName: "localhost", Port: container.GetPort(), UserName: "postgres", Password: "secret", ConnectionOptions: "sslmode=disable", DatabaseName: "postgres", - }) + ApplicationName: bininfo.Component(), + } + dbURL, err := target.IntoURL() if err != nil { t.Fatal(err) } From f04781283c98022eba78b0f9bff33c2bbf07dc40 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:24:44 +0200 Subject: [PATCH 02/12] Renovate: Update kube-prometheus-stack Docker tag to v88.2.0 (#1127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [kube-prometheus-stack](https://redirect.github.com/prometheus-operator/kube-prometheus) ([source](https://redirect.github.com/prometheus-community/helm-charts)) | minor | `88.1.6` → `88.2.0` | --- ### Release Notes
prometheus-community/helm-charts (kube-prometheus-stack) ### [`v88.2.0`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-88.2.0) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. #### What's Changed - \[kube-prometheus-stack] Update Helm release kube-state-metrics to v8.2.0 by [@​renovate](https://redirect.github.com/renovate)\[bot] in [#​7171](https://redirect.github.com/prometheus-community/helm-charts/pull/7171) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 51c0ca83c..77b9d79a6 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 88.1.6 + version: 88.2.0 From 5ad603b0c1331142493237bf6cb98e93f4ed29a5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:38:20 +0200 Subject: [PATCH 03/12] Renovate: Update github.com/sapcc/go-bits digest to ed453da (#1126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/sapcc/go-bits](https://redirect.github.com/sapcc/go-bits) | require | digest | `4bbc84d` → `ed453da` | --- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7e5757ea9..0cc21ac7e 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/ironcore-dev/ironcore v0.5.0 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 - github.com/sapcc/go-bits v0.0.0-20260806170240-4bbc84d224db + github.com/sapcc/go-bits v0.0.0-20260807114049-ed453da227d5 go.xyrillian.de/gg v1.13.3 k8s.io/api v0.36.3 k8s.io/apimachinery v0.36.3 diff --git a/go.sum b/go.sum index ff9d4ea05..c6553e6fd 100644 --- a/go.sum +++ b/go.sum @@ -185,8 +185,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.24.0 h1:sGBvOMVSM1olJlyvNoQSk7NX5uatXHKkztGDBPnTWMs= github.com/sapcc/go-api-declarations v1.24.0/go.mod h1:ZWRTijvgF8o8aHg5stgg7u4DF6jFrd0X97le/uGlZsA= -github.com/sapcc/go-bits v0.0.0-20260806170240-4bbc84d224db h1:DfseqB6CZzdpbSdL4sGlqEBSYu3JpUpwYFiNzoYTkog= -github.com/sapcc/go-bits v0.0.0-20260806170240-4bbc84d224db/go.mod h1:1wh2+fuMXNrJYttuaqiU7sNQgJhTmsZ/0S0SK1ttwQc= +github.com/sapcc/go-bits v0.0.0-20260807114049-ed453da227d5 h1:GNKzSme3B3DWJe+lrR/gwXNJDMYWmJ/q6HHzVpPx6jU= +github.com/sapcc/go-bits v0.0.0-20260807114049-ed453da227d5/go.mod h1:chN4NKgcfKa8+lmQxuG/ckDXNrQ60wC22vZtJyjhibo= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= From 0fd8207d32174bf6795bae11641d65d2c5246170 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:38:14 +0200 Subject: [PATCH 04/12] Renovate: Update kube-prometheus-stack Docker tag to v88.3.0 (#1132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [kube-prometheus-stack](https://redirect.github.com/prometheus-operator/kube-prometheus) ([source](https://redirect.github.com/prometheus-community/helm-charts)) | minor | `88.2.0` → `88.3.0` | --- ### Release Notes
prometheus-community/helm-charts (kube-prometheus-stack) ### [`v88.3.0`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-88.3.0) [Compare Source](https://redirect.github.com/prometheus-community/helm-charts/compare/kube-prometheus-stack-88.2.0...kube-prometheus-stack-88.3.0) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. #### What's Changed - \[kube-prometheus-stack] Update externalUrl to support TLS configuration by [@​Turgon37](https://redirect.github.com/Turgon37) in [#​7121](https://redirect.github.com/prometheus-community/helm-charts/pull/7121) #### New Contributors - [@​Turgon37](https://redirect.github.com/Turgon37) made their first contribution in [#​7121](https://redirect.github.com/prometheus-community/helm-charts/pull/7121) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 77b9d79a6..110034136 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 88.2.0 + version: 88.3.0 From 5e9eedbdef61959bb73e09012809370806c683ee Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 12 Aug 2026 13:59:33 +0200 Subject: [PATCH 05/12] fix: remove unused volumes and volumeMounts for perses-plugins (#1133) Perses fails to come up in tilt Signed-off-by: Markus Wieland --- tools/perses/values.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tools/perses/values.yaml b/tools/perses/values.yaml index 30e68380f..40a8987e9 100644 --- a/tools/perses/values.yaml +++ b/tools/perses/values.yaml @@ -4,14 +4,6 @@ sidecar: enabled: true -volumes: - - name: perses-plugins - emptyDir: {} - -volumeMounts: - - name: perses-plugins - mountPath: /etc/perses/plugins - config: provisioning: folders: From 38aba43bd134897e0facf79f0b92c310af8b6544 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:21:38 +0200 Subject: [PATCH 06/12] Renovate: Update postgres Docker tag to v18.6 (#1137) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [postgres](https://hub.docker.com/_/postgres) ([source](https://redirect.github.com/docker-library/postgres)) | minor | `18.4` → `18.6` | --- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 36a12142d..6aa9c65b1 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -73,7 +73,7 @@ RUN set -ex; \ ENV PG_MAJOR 18 ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin -ENV PG_VERSION 18.4-1.pgdg13+1 +ENV PG_VERSION 18.6-1.pgdg13+1 RUN set -ex; \ \ From 5809e5f3ad1b0e5e28314e59127b9c9795d7ca36 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:21:59 +0200 Subject: [PATCH 07/12] Renovate: Update github.com/sapcc/go-bits digest to ea1a144 (#1136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/sapcc/go-bits](https://redirect.github.com/sapcc/go-bits) | require | digest | `ed453da` → `ea1a144` | --- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 0cc21ac7e..39642d8e9 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/ironcore-dev/ironcore v0.5.0 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 - github.com/sapcc/go-bits v0.0.0-20260807114049-ed453da227d5 + github.com/sapcc/go-bits v0.0.0-20260813170327-ea1a14435d35 go.xyrillian.de/gg v1.13.3 k8s.io/api v0.36.3 k8s.io/apimachinery v0.36.3 @@ -113,7 +113,7 @@ require ( golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 // indirect golang.org/x/term v0.45.0 - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect diff --git a/go.sum b/go.sum index c6553e6fd..9c4e7ac38 100644 --- a/go.sum +++ b/go.sum @@ -185,8 +185,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.24.0 h1:sGBvOMVSM1olJlyvNoQSk7NX5uatXHKkztGDBPnTWMs= github.com/sapcc/go-api-declarations v1.24.0/go.mod h1:ZWRTijvgF8o8aHg5stgg7u4DF6jFrd0X97le/uGlZsA= -github.com/sapcc/go-bits v0.0.0-20260807114049-ed453da227d5 h1:GNKzSme3B3DWJe+lrR/gwXNJDMYWmJ/q6HHzVpPx6jU= -github.com/sapcc/go-bits v0.0.0-20260807114049-ed453da227d5/go.mod h1:chN4NKgcfKa8+lmQxuG/ckDXNrQ60wC22vZtJyjhibo= +github.com/sapcc/go-bits v0.0.0-20260813170327-ea1a14435d35 h1:pTo+tUdMlRJRGY+MsAPeicedx58QYdnne8uWWufHA68= +github.com/sapcc/go-bits v0.0.0-20260813170327-ea1a14435d35/go.mod h1:oxx7AA+bHudahaXHEcg/Xk4avBckWR/rOOa0SAsuSgg= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -241,8 +241,8 @@ go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBs go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -255,12 +255,12 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= From 9f10115af751e1501991e19167c82588c79d307f Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 17 Aug 2026 10:29:10 +0200 Subject: [PATCH 08/12] fix: update PostgreSQL version to 18.6-1.pgdg13+2 in Dockerfile (#1141) Signed-off-by: Markus Wieland --- postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 6aa9c65b1..24baed44b 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -73,7 +73,7 @@ RUN set -ex; \ ENV PG_MAJOR 18 ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin -ENV PG_VERSION 18.6-1.pgdg13+1 +ENV PG_VERSION 18.6-1.pgdg13+2 RUN set -ex; \ \ From 67daed47673964490cc0d72243eaaab1c4b53a95 Mon Sep 17 00:00:00 2001 From: Marcel <156897072+mblos@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:20:38 +0200 Subject: [PATCH 09/12] feat: CR controller checks for host overload scenario and resolves via reservation re-placements (#1125) This PR adds automated detection and remediation when a host accumulates more committed reservation blocks than its actual capacity allows: Cortex now evicts reservation slots from oversubscribed hosts so that committed capacity guarantees can be honored. A grace period is enforced before eviction to allow other controllers (e.g. failover) to self-heal first. Operators can enable detection-only mode to observe violations via metrics before turning on active remediation. - Automatically restore capacity headroom on oversubscribed hosts - Avoid unnecessary evictions during transient spikes via configurable grace period - `enableOversubscriptionUnplaceReservations: false` reports violations without evicting --------- Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com> --- cmd/manager/main.go | 14 + .../bundles/cortex-nova/templates/alerts.yaml | 70 +++ helm/bundles/cortex-nova/values.yaml | 10 + .../reservations/capacity_accounting.go | 44 ++ .../reservations/capacity_accounting_test.go | 184 +++++++ .../committed_resource_controller_test.go | 23 + .../reservations/commitments/config.go | 17 + .../reservations/commitments/field_index.go | 1 - .../host_oversubscription_controller.go | 400 +++++++++++++++ .../host_oversubscription_controller_test.go | 460 ++++++++++++++++++ .../commitments/integration_test.go | 187 ++++++- .../commitments/payg_rollback_test.go | 5 +- .../commitments/reservation_controller.go | 68 ++- .../reservation_controller_monitor.go | 58 +++ .../reservation_controller_test.go | 133 ++++- .../scheduling/reservations/field_index.go | 56 +++ 16 files changed, 1715 insertions(+), 15 deletions(-) create mode 100644 internal/scheduling/reservations/commitments/host_oversubscription_controller.go create mode 100644 internal/scheduling/reservations/commitments/host_oversubscription_controller_test.go create mode 100644 internal/scheduling/reservations/commitments/reservation_controller_monitor.go create mode 100644 internal/scheduling/reservations/field_index.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index fca0c0550..f69cc9005 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -668,6 +668,20 @@ func main() { os.Exit(1) } } + + if commitmentsConfig.ReservationController.EnableOversubscriptionCheck { + reservationControllerMonitor := commitments.NewReservationControllerMonitor() + metrics.Registry.MustRegister(&reservationControllerMonitor) + + if err := (&commitments.HostOversubscriptionController{ + Client: multiclusterClient, + Conf: commitmentsConfig.ReservationController, + Monitor: &reservationControllerMonitor, + }).SetupWithManager(mgr, multiclusterClient); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "HostOversubscription") + os.Exit(1) + } + } } if slices.Contains(mainConfig.EnabledControllers, "datasource-controllers") { setupLog.Info("enabling controller", "controller", "datasource-controllers") diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 133d8468a..6b26d749c 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -749,4 +749,74 @@ spec: resource router is mapping the same object to multiple clusters, or an object was created out-of-band on the wrong cluster. Investigate the affected resources and the routing configuration. + + {{- if .Values.kvm.enabled }} + - alert: CortexNovaHostReservationsOversubscribed + # Fires when the sum of running VM allocations + reservation blocks (committed + + # failover) exceeds the host's effective capacity for CPU or memory. + # This can happen due to: concurrent slot creation with stale informer cache, + # operator-driven VM migrations where the slot stays on the old host, or + # capacity changes (e.g. hardware replacement changing EffectiveCapacity). + # The 10m hold-off tolerates the known migration window: after a VM departs, + # the slot remains on the old host until the usage reconciler cleans it up. + # Note: `reserved` only counts Ready reservations — violations during the + # initial unready window (slot just created) are not captured by this alert. + expr: | + ( + max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="utilized"}) + + max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="reserved"}) + + max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="failover"}) + - max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_total) + ) > 0 + for: 15m + labels: + context: committed-resource-capacity + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity + annotations: + summary: "Host {{ "{{" }} $labels.compute_host {{ "}}" }} reservation blocks exceed capacity for {{ "{{" }} $labels.resource {{ "}}" }}" + description: > + The total of running VM allocations and reservation blocks (committed resource + + failover) on host {{ "{{" }} $labels.compute_host {{ "}}" }} exceeds its effective + capacity for {{ "{{" }} $labels.resource {{ "}}" }} by {{ "{{" }} $value | humanize {{ "}}" }}. + This means the host is over-subscribed and committed resource or failover guarantees may not be + satisfied. Common causes: VM migrations with slot not yet reclaimed, outdated/stale reservations, or out of sync issues. + If problem remains, inspect the reservations on this host and check the CR/failover controller logs. + - alert: CortexCommittedResourceHostOversubscribed + expr: max by (host, az, resource) (cortex_committed_resource_host_oversubscribed) > 0 + for: 15m + labels: + context: committed-resource-capacity + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity + annotations: + summary: "Host {{ "{{" }} $labels.host {{ "}}" }} CR slots exceed effective capacity for {{ "{{" }} $labels.resource {{ "}}" }}" + description: > + Committed resource reservation slots on host {{ "{{" }} $labels.host {{ "}}" }} exceed + its effective capacity for {{ "{{" }} $labels.resource {{ "}}" }} by {{ "{{" }} $value | humanize {{ "}}" }}. + The oversubscription controller was unable to resolve this by evicting unallocated slots. + Inspect the CR reservations on this host and check the oversubscription controller logs. + - alert: CortexCommittedResourceHostOversubscriptionEvictionsHigh + expr: sum by (az) (increase(cortex_committed_resource_host_oversubscribed_evicted_reservations_total{service="cortex-nova-metrics"}[1h])) > 10 + labels: + context: committed-resource-capacity + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity + annotations: + summary: "High CR slot eviction rate in AZ {{ "{{" }} $labels.az {{ "}}" }}" + description: > + More than 10 committed resource reservation slots were evicted due to oversubscription + in AZ {{ "{{" }} $labels.az {{ "}}" }} within the last hour ({{ "{{" }} $value | humanize {{ "}}" }} evictions). + This indicates recurring oversubscription events — investigate capacity changes, + concurrent slot creation races, or VM migration patterns in this AZ. + {{- end }} {{- end }} diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 6524c095e..cec441eaa 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -185,6 +185,16 @@ cortex-scheduling-controllers: # How long after a VM is allocated to a reservation before it is expected to appear # on the target host; allocations not confirmed within this window are removed allocationGracePeriod: "15m" + # How long to wait after detecting host over-subscription before evicting reservation slots. + # Gives other controllers (e.g. failover) time to self-heal. + oversubscriptionGracePeriod: "3m" + # Minimum time between consecutive over-subscription checks for the same host. + # Should be shorter than oversubscriptionGracePeriod. + oversubscriptionMinCheckInterval: "1m" + # Enable host over-subscription detection + enableOversubscriptionCheck: true + # When false, violations are detected and exposed via metrics but no slots are evicted. + enableOversubscriptionReservationEviction: true # URL of the nova external scheduler API for placement decisions schedulerURL: "http://localhost:8080/scheduler/nova/external" # Keystone credentials used to resolve domain IDs to domain names for the diff --git a/internal/scheduling/reservations/capacity_accounting.go b/internal/scheduling/reservations/capacity_accounting.go index ab305d5d5..72cd19f1f 100644 --- a/internal/scheduling/reservations/capacity_accounting.go +++ b/internal/scheduling/reservations/capacity_accounting.go @@ -25,6 +25,7 @@ import ( // res itself is excluded from step 3 to avoid subtracting its own block from free capacity. // Returns false when the hypervisor has no capacity data. func HostHasCapacityForReservation(allReservations []v1alpha1.Reservation, hv hv1.Hypervisor, res *v1alpha1.Reservation) bool { + // TODO consider refactor with HostFreeCapacity effCap := hv.Status.EffectiveCapacity if effCap == nil { effCap = hv.Status.Capacity @@ -75,6 +76,49 @@ func HostHasCapacityForReservation(allReservations []v1alpha1.Reservation, hv hv return true } +// HostFreeCapacity computes the remaining free capacity on hv after subtracting +// hv.Status.Allocation and UnusedReservationCapacity for all reservations on this host. +// Negative values indicate over-subscription for that resource. +// Returns nil when the hypervisor has no capacity data. +// Reservations not targeting this host (via Spec.TargetHost or Status.Host) are ignored. +func HostFreeCapacity(hostReservations []v1alpha1.Reservation, hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity { + // TODO consider refactor with HostHasCapacityForReservation + effCap := hv.Status.EffectiveCapacity + if effCap == nil { + effCap = hv.Status.Capacity + } + if effCap == nil { + return nil + } + + free := make(map[hv1.ResourceName]resource.Quantity, len(effCap)) + for rn, qty := range effCap { + free[rn] = qty.DeepCopy() + } + for rn, allocated := range hv.Status.Allocation { + if f, ok := free[rn]; ok { + f.Sub(allocated) + free[rn] = f + } + } + for i := range hostReservations { + res := &hostReservations[i] + if res.Spec.TargetHost == "" { + continue // evicted; status.Host may lag until reconcile + } + if res.Spec.TargetHost != hv.Name && res.Status.Host != hv.Name { + continue + } + for rn, block := range UnusedReservationCapacity(res, false) { + if f, ok := free[rn]; ok { + f.Sub(block) + free[rn] = f + } + } + } + return free +} + // UnusedReservationCapacity returns the resources a Reservation should block on its host(s). // This is the single source of truth used by both the capacity controller and // filter_has_enough_capacity to ensure consistent accounting. diff --git a/internal/scheduling/reservations/capacity_accounting_test.go b/internal/scheduling/reservations/capacity_accounting_test.go index d13a1c400..d86e1091d 100644 --- a/internal/scheduling/reservations/capacity_accounting_test.go +++ b/internal/scheduling/reservations/capacity_accounting_test.go @@ -172,6 +172,190 @@ func TestUnusedReservationCapacity(t *testing.T) { } } +func TestHostFreeCapacity(t *testing.T) { + gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) } + cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) } + + hvWithCap := func(name string, memGiB, cpuCores int64) hv1.Hypervisor { + return hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(memGiB), + hv1.ResourceCPU: cpu(cpuCores), + }, + }, + } + } + crSlot := func(name, host string, memGiB, cpuCores int64) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(memGiB), + hv1.ResourceCPU: cpu(cpuCores), + }, + }, + Status: v1alpha1.ReservationStatus{Host: host}, + } + } + freeMemGiB := func(free map[hv1.ResourceName]resource.Quantity) int64 { + q := free[hv1.ResourceMemory] + return q.Value() / (1024 * 1024 * 1024) + } + freeCPU := func(free map[hv1.ResourceName]resource.Quantity) int64 { + q := free[hv1.ResourceCPU] + return q.Value() + } + + t.Run("no capacity data returns nil", func(t *testing.T) { + hv := hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host"}} + if got := HostFreeCapacity(nil, hv); got != nil { + t.Errorf("expected nil, got %v", got) + } + }) + + t.Run("no reservations and no allocation: free = effective capacity", func(t *testing.T) { + hv := hvWithCap("host", 1024, 256) + free := HostFreeCapacity(nil, hv) + if freeMemGiB(free) != 1024 { + t.Errorf("expected 1024 GiB free, got %d", freeMemGiB(free)) + } + if freeCPU(free) != 256 { + t.Errorf("expected 256 CPU free, got %d", freeCPU(free)) + } + }) + + t.Run("allocation subtracted from capacity", func(t *testing.T) { + hv := hvWithCap("host", 1024, 256) + hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(512), + hv1.ResourceCPU: cpu(128), + } + free := HostFreeCapacity(nil, hv) + if freeMemGiB(free) != 512 { + t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free)) + } + if freeCPU(free) != 128 { + t.Errorf("expected 128 CPU free, got %d", freeCPU(free)) + } + }) + + t.Run("reservation blocks subtracted", func(t *testing.T) { + hv := hvWithCap("host", 1024, 256) + slots := []v1alpha1.Reservation{ + crSlot("slot-1", "host", 512, 128), + } + free := HostFreeCapacity(slots, hv) + if freeMemGiB(free) != 512 { + t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free)) + } + if freeCPU(free) != 128 { + t.Errorf("expected 128 CPU free, got %d", freeCPU(free)) + } + }) + + t.Run("over-subscribed: negative free values", func(t *testing.T) { + // 5 x 1TiB slots on a 4TiB host — the production scenario + hv := hvWithCap("host", 4096, 256) + slots := []v1alpha1.Reservation{ + crSlot("slot-0", "host", 1024, 128), + crSlot("slot-1", "host", 1024, 128), + crSlot("slot-2", "host", 1024, 128), + crSlot("slot-3", "host", 1024, 128), + crSlot("slot-4", "host", 1024, 128), + } + free := HostFreeCapacity(slots, hv) + if freeMemGiB(free) != -1024 { + t.Errorf("expected -1024 GiB (over-subscribed), got %d GiB", freeMemGiB(free)) + } + if freeCPU(free) != -384 { + t.Errorf("expected -384 CPU (over-subscribed), got %d", freeCPU(free)) + } + }) + + t.Run("allocation + reservations combined", func(t *testing.T) { + hv := hvWithCap("host", 1024, 256) + hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(256), + hv1.ResourceCPU: cpu(64), + } + slots := []v1alpha1.Reservation{ + crSlot("slot-1", "host", 512, 128), + } + free := HostFreeCapacity(slots, hv) + // 1024 - 256 (alloc) - 512 (slot) = 256 GiB free + if freeMemGiB(free) != 256 { + t.Errorf("expected 256 GiB free, got %d", freeMemGiB(free)) + } + // 256 - 64 (alloc) - 128 (slot) = 64 free + if freeCPU(free) != 64 { + t.Errorf("expected 64 CPU free, got %d", freeCPU(free)) + } + }) + + t.Run("confirmed VM reduces slot block (not double counted)", func(t *testing.T) { + hv := hvWithCap("host", 1024, 256) + // 256 GiB confirmed VM already counted in Allocation + hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(256), + } + // 512 GiB slot with 256 GiB confirmed VM → block = 512-256 = 256 GiB + slot := v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "slot-1"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: gib(512)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + Allocations: map[string]v1alpha1.CommittedResourceAllocation{ + "vm-1": {Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: gib(256)}}, + }, + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: "host", + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{"vm-1": "host"}, + }, + }, + } + free := HostFreeCapacity([]v1alpha1.Reservation{slot}, hv) + // 1024 - 256 (alloc/vm) - 256 (remaining slot block) = 512 + if freeMemGiB(free) != 512 { + t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free)) + } + }) + + t.Run("reservations on other hosts are ignored even if passed in", func(t *testing.T) { + hv := hvWithCap("host", 1024, 256) + slots := []v1alpha1.Reservation{ + crSlot("slot-other", "other-host", 1024, 256), // different host — must not block + } + free := HostFreeCapacity(slots, hv) + if freeMemGiB(free) != 1024 { + t.Errorf("expected 1024 GiB free (other host ignored), got %d", freeMemGiB(free)) + } + }) + + t.Run("falls back to Capacity when EffectiveCapacity nil", func(t *testing.T) { + hv := hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host"}, + Status: hv1.HypervisorStatus{ + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(512), + }, + }, + } + free := HostFreeCapacity(nil, hv) + if freeMemGiB(free) != 512 { + t.Errorf("expected 512 GiB, got %d", freeMemGiB(free)) + } + }) +} + func TestHostHasCapacityForReservation(t *testing.T) { gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) } cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) } diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go index 293f074a9..2bab56f22 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) // ============================================================================ @@ -137,9 +138,31 @@ func newCRTestClient(scheme *runtime.Scheme, objects ...client.Object) client.Cl } return uuids }). + WithIndex(&v1alpha1.Reservation{}, reservations.IdxReservationByHost, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok { + return nil + } + hosts := make(map[string]struct{}) + if res.Spec.TargetHost != "" { + hosts[res.Spec.TargetHost] = struct{}{} + } + if res.Status.Host != "" { + hosts[res.Status.Host] = struct{}{} + } + result := make([]string, 0, len(hosts)) + for h := range hosts { + result = append(result, h) + } + return result + }). Build() } +func testGiB(n int64) resource.Quantity { + return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) +} + func reconcileReq(name string) ctrl.Request { return ctrl.Request{NamespacedName: types.NamespacedName{Name: name}} } diff --git a/internal/scheduling/reservations/commitments/config.go b/internal/scheduling/reservations/commitments/config.go index 269ba3e4a..fde73e200 100644 --- a/internal/scheduling/reservations/commitments/config.go +++ b/internal/scheduling/reservations/commitments/config.go @@ -62,6 +62,23 @@ type ReservationControllerConfig struct { // reservation during which it's expected to appear on the target host. // VMs not confirmed within this period are considered stale and removed. AllocationGracePeriod metav1.Duration `json:"allocationGracePeriod"` + // OversubscriptionGracePeriod is how long to wait after detecting host over-subscription + // before evicting slots. Gives other controllers (e.g. failover) time to self-heal. + OversubscriptionGracePeriod metav1.Duration `json:"oversubscriptionGracePeriod,omitempty"` + // OversubscriptionMinCheckInterval is the minimum time between consecutive over-subscription + // checks for the same host. Independent of RequeueIntervalActive so it can be tuned + // to a value shorter than the grace period without affecting normal reconcile cadence. + // Defaults to 30s. + OversubscriptionMinCheckInterval metav1.Duration `json:"oversubscriptionMinCheckInterval,omitempty"` + // EnableOversubscriptionCheck enables the HostOversubscriptionController, which watches + // Hypervisor objects and detects committed-resource oversubscription. + // When false, the controller is not wired up and no oversubscription detection runs. + EnableOversubscriptionCheck bool `json:"enableOversubscriptionCheck,omitempty"` + // EnableOversubscriptionReservationEviction controls whether the oversubscription controller + // actively evicts reservation slots when a host is over-subscribed. + // When false, violations are detected and exposed via metrics but no eviction is performed. + // Has no effect when EnableOversubscriptionCheck is false. + EnableOversubscriptionReservationEviction bool `json:"enableOversubscriptionReservationEviction,omitempty"` // SchedulerURL is the endpoint of the nova external scheduler. SchedulerURL string `json:"schedulerURL"` // PipelineDefault is the fallback pipeline when no FlavorGroupPipelines entry matches. diff --git a/internal/scheduling/reservations/commitments/field_index.go b/internal/scheduling/reservations/commitments/field_index.go index 1237d0da5..1a4ca1a3b 100644 --- a/internal/scheduling/reservations/commitments/field_index.go +++ b/internal/scheduling/reservations/commitments/field_index.go @@ -131,7 +131,6 @@ func indexProjectQuotaByProjectID(ctx context.Context, mcl *multicluster.Client) return err } -// indexReservationByAllocationVMUUID registers an index over all VM UUIDs present in // Spec.CommittedResourceReservation.Allocations. This allows the reservation controller // to efficiently find all other Reservation CRDs carrying a specific VM UUID without // scanning every reservation in the cluster. diff --git a/internal/scheduling/reservations/commitments/host_oversubscription_controller.go b/internal/scheduling/reservations/commitments/host_oversubscription_controller.go new file mode 100644 index 000000000..325640e07 --- /dev/null +++ b/internal/scheduling/reservations/commitments/host_oversubscription_controller.go @@ -0,0 +1,400 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +import ( + "context" + "fmt" + "sort" + "sync" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" +) + +// HostOversubscriptionController reconciles Hypervisor objects to detect and remediate +// committed-resource oversubscription. +type HostOversubscriptionController struct { + client.Client + Conf ReservationControllerConfig + Monitor *ReservationControllerMonitor + + // mu protects firstSeen and lastCheck. + // firstSeen tracks when a violation was first detected per host, used to enforce + // the grace period before evicting a reservation. + // lastCheck tracks when a host was last checked, used to rate-limit checks. + // + // NOTE: intentionally kept in memory rather than persisted to an HV annotation. + mu sync.Mutex + firstSeen map[string]time.Time + lastCheck map[string]time.Time +} + +func (r *HostOversubscriptionController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := LoggerFromContext(ctx).WithValues("component", "host-oversubscription", "host", req.Name) + r.mu.Lock() + defer r.mu.Unlock() + host := req.Name + gracePeriod := r.Conf.OversubscriptionGracePeriod.Duration + if gracePeriod == 0 { + gracePeriod = 2 * time.Minute + } + minCheckInterval := r.Conf.OversubscriptionMinCheckInterval.Duration + if minCheckInterval == 0 { + minCheckInterval = 30 * time.Second + } + + if limited, remaining := r.isRateLimited(host, minCheckInterval); limited { + logger.V(1).Info("rate limited", "remaining", remaining) + return ctrl.Result{RequeueAfter: remaining}, nil + } + r.recordCheck(host) + + var hv hv1.Hypervisor + if err := r.Get(ctx, req.NamespacedName, &hv); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + var allReservations v1alpha1.ReservationList + if err := r.List(ctx, &allReservations, client.MatchingFields{reservations.IdxReservationByHost: host}); err != nil { + return ctrl.Result{}, fmt.Errorf("list reservations for host %s: %w", host, err) + } + + requeue, err := r.checkOversubscription(ctx, host, allReservations.Items, hv, gracePeriod) + if err != nil { + return ctrl.Result{}, err + } + if requeue > 0 { + return ctrl.Result{RequeueAfter: requeue}, nil + } + return ctrl.Result{}, nil +} + +// checkOversubscription detects and remediates oversubscription for a single host. +// Returns a requeue duration if the grace period is still running, 0 if resolved or no data. +func (r *HostOversubscriptionController) checkOversubscription( + ctx context.Context, + host string, + allReservations []v1alpha1.Reservation, + hv hv1.Hypervisor, + gracePeriod time.Duration, +) (time.Duration, error) { + + logger := LoggerFromContext(ctx).WithValues("component", "host-oversubscription", "host", host) + az := hv.Labels["topology.kubernetes.io/zone"] + + readyCond := meta.FindStatusCondition(hv.Status.Conditions, hv1.ConditionTypeReady) + if readyCond == nil || readyCond.Status != metav1.ConditionTrue { + logger.V(1).Info("hypervisor not ready, skipping oversubscription check") + return 0, nil + } + + violations := computeViolations(allReservations, hv) + if violations == nil { + logger.V(1).Info("no capacity data for host, skipping oversubscription check") + return 0, nil + } + if len(violations) == 0 { + if !r.firstSeenFor(host).IsZero() { + logger.Info("host oversubscription resolved") + } + r.clearFirstSeen(host) + r.Monitor.ClearHost(host, az) + return 0, nil + } + + r.updateMonitor(host, az, violations) + + firstSeen, isNew := r.getOrSetFirstSeen(host) + if isNew { + logger.Info("oversubscription first detected, grace period starting", + "gracePeriod", gracePeriod, "violations", violations) + } + if elapsed := time.Since(firstSeen); elapsed < gracePeriod { + return gracePeriod - elapsed, nil + } + + logger.Info("grace period elapsed, evicting slot", "violations", violations) + if !r.Conf.EnableOversubscriptionReservationEviction { + logger.Info("eviction disabled — violations detected but eviction skipped (enableOversubscriptionReservationEviction=false)") + r.resetFirstSeen(host) + return gracePeriod, nil + } + target := selectEvictionTarget(ctx, allReservations, violations) + if target == nil { + logger.Info("no evictable slot found — manual intervention required, will retry after grace period") + r.resetFirstSeen(host) + return gracePeriod, nil + } + + if _, err := evictReservation(ctx, r.Client, target); err != nil { + r.updateMonitor(host, az, violations) // keep monitor current on error + return 0, err + } + r.Monitor.IncSlotsEvicted(az) + + // Recompute violations without the evicted slot to get an accurate post-eviction picture. + remaining := make([]v1alpha1.Reservation, 0, len(allReservations)-1) + for i := range allReservations { + if allReservations[i].Name != target.Name { + remaining = append(remaining, allReservations[i]) + } + } + if postViolations := computeViolations(remaining, hv); len(postViolations) > 0 { + logger.Info("host still oversubscribed after eviction", "violations", postViolations) + r.updateMonitor(host, az, postViolations) + } else { + logger.Info("host oversubscription resolved after eviction") + r.Monitor.ClearHost(host, az) + } + + r.resetFirstSeen(host) + return gracePeriod, nil +} + +func (r *HostOversubscriptionController) updateMonitor(host, az string, violations map[hv1.ResourceName]resource.Quantity) { + r.Monitor.ClearHost(host, az) + for rn, excess := range violations { + r.Monitor.SetOversubscribed(host, az, string(rn), float64(excess.Value())) + } +} + +func (r *HostOversubscriptionController) isRateLimited(host string, minInterval time.Duration) (bool, time.Duration) { + last := r.lastCheck[host] // safe on nil map — returns zero time + if !last.IsZero() { + if elapsed := time.Since(last); elapsed < minInterval { + return true, minInterval - elapsed + } + } + return false, 0 +} + +func (r *HostOversubscriptionController) recordCheck(host string) { + if r.lastCheck == nil { + r.lastCheck = make(map[string]time.Time) + } + r.lastCheck[host] = time.Now() +} + +func (r *HostOversubscriptionController) firstSeenFor(host string) time.Time { + return r.firstSeen[host] // safe on nil map — returns zero time +} + +// getOrSetFirstSeen returns the first-seen time for the host, setting it to now if not yet recorded. +// The second return value is true when the entry was just created. +func (r *HostOversubscriptionController) getOrSetFirstSeen(host string) (time.Time, bool) { + if t := r.firstSeen[host]; !t.IsZero() { + return t, false + } + r.resetFirstSeen(host) + return r.firstSeen[host], true +} + +func (r *HostOversubscriptionController) clearFirstSeen(host string) { + delete(r.firstSeen, host) // safe on nil map +} + +func (r *HostOversubscriptionController) resetFirstSeen(host string) { + if r.firstSeen == nil { + r.firstSeen = make(map[string]time.Time) + } + r.firstSeen[host] = time.Now() +} + +// SetupWithManager wires the controller into the manager. +// Primary subject: Hypervisor — one reconcile per host, requeue is stable regardless of +// which reservation triggered it. +// Secondary watch: all Reservations mapped to their host via Status.Host — any reservation +// type can affect host capacity (failover slots, VM allocations, etc.), so we watch all. +func (r *HostOversubscriptionController) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { + if err := reservations.IndexReservationByHost(context.Background(), mcl); err != nil { + return fmt.Errorf("failed to set up reservation by host index: %w", err) + } + + enqueueHost := func(res *v1alpha1.Reservation, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if host := res.Status.Host; host != "" { + q.Add(reconcile.Request{NamespacedName: client.ObjectKey{Name: host}}) + } + } + + reservationToHost := handler.Funcs{ + CreateFunc: func(_ context.Context, e event.CreateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + enqueueHost(e.Object.(*v1alpha1.Reservation), q) + }, + UpdateFunc: func(_ context.Context, e event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + enqueueHost(e.ObjectOld.(*v1alpha1.Reservation), q) // old host — reservation may have moved away + enqueueHost(e.ObjectNew.(*v1alpha1.Reservation), q) + }, + DeleteFunc: func(_ context.Context, e event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + enqueueHost(e.Object.(*v1alpha1.Reservation), q) + }, + } + + bldr := multicluster.BuildController(mcl, mgr) + var err error + bldr, err = bldr.WatchesMulticluster( + &hv1.Hypervisor{}, + &handler.EnqueueRequestForObject{}, + hvCapacityChangePredicate, + ) + if err != nil { + return err + } + bldr, err = bldr.WatchesMulticluster( + &v1alpha1.Reservation{}, + reservationToHost, + ) + if err != nil { + return err + } + return bldr.Named("host-oversubscription").Complete(r) +} + +// computeViolations returns the per-resource excess (positive quantity) for any resource where +// the host is over-subscribed. Returns nil if capacity data is unavailable, empty map if no violation. +func computeViolations(allReservations []v1alpha1.Reservation, hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity { + free := reservations.HostFreeCapacity(allReservations, hv) + if free == nil { + return nil + } + zero := resource.MustParse("0") + violations := make(map[hv1.ResourceName]resource.Quantity) + for rn, f := range free { + if f.Cmp(zero) < 0 { + excess := f.DeepCopy() + excess.Neg() + violations[rn] = excess + } + } + return violations +} + +// selectEvictionTarget picks the least-disruptive slot to evict: prefers the smallest +// unallocated slot (no running VMs, easiest to re-place), falling back to the most-idle +// allocated slot when no unallocated candidates exist. +func selectEvictionTarget( + ctx context.Context, + allReservations []v1alpha1.Reservation, + violations map[hv1.ResourceName]resource.Quantity, +) *v1alpha1.Reservation { + + logger := LoggerFromContext(ctx).WithValues("component", "host-oversubscription") + var selectedRes *v1alpha1.Reservation + + var unallocated, allocated []*v1alpha1.Reservation + for i := range allReservations { + res := &allReservations[i] + if res.Spec.Type != v1alpha1.ReservationTypeCommittedResource || res.Spec.CommittedResourceReservation == nil { + continue + } + if res.Spec.TargetHost == "" { + continue // already evicted, pending status cleanup + } + if len(res.Spec.CommittedResourceReservation.Allocations) == 0 { + unallocated = append(unallocated, res) + } else { + allocated = append(allocated, res) + } + } + + // Sort unallocated by memory asc, then CPU asc — smallest slots are easiest to re-place. + sort.SliceStable(unallocated, func(i, j int) bool { + ri, rj := unallocated[i].Spec.Resources, unallocated[j].Spec.Resources + mi, mj := ri[hv1.ResourceMemory], rj[hv1.ResourceMemory] + if c := mi.Cmp(mj); c != 0 { + return c < 0 + } + ci, cj := ri[hv1.ResourceCPU], rj[hv1.ResourceCPU] + return ci.Cmp(cj) < 0 + }) + + if len(unallocated) > 0 { + selectedRes = unallocated[0] + } + + if selectedRes == nil && len(allocated) > 0 { + type allocatedEntry struct { + res *v1alpha1.Reservation + buckets map[hv1.ResourceName]int64 + } + entries := make([]allocatedEntry, len(allocated)) + for i, res := range allocated { + unusedCap := reservations.UnusedReservationCapacity(res, false) + buckets := make(map[hv1.ResourceName]int64) + for rn, total := range res.Spec.Resources { + if t := total.Value(); t != 0 { + unused := unusedCap[rn] + buckets[rn] = unused.Value() * 10 / t + } + } + entries[i] = allocatedEntry{res: res, buckets: buckets} + } + // Sort: 1st by unused mem ratio desc, 2nd by unused CPU ratio desc, 3rd by total mem asc. + sort.SliceStable(entries, func(i, j int) bool { + if d := entries[i].buckets[hv1.ResourceMemory] - entries[j].buckets[hv1.ResourceMemory]; d != 0 { + return d > 0 + } + if d := entries[i].buckets[hv1.ResourceCPU] - entries[j].buckets[hv1.ResourceCPU]; d != 0 { + return d > 0 + } + mi, mj := entries[i].res.Spec.Resources[hv1.ResourceMemory], entries[j].res.Spec.Resources[hv1.ResourceMemory] + return mi.Cmp(mj) < 0 + }) + selectedRes = entries[0].res + } + + if selectedRes == nil { + logger.Info("no eviction target found") + return nil + } + logger.Info("eviction target selected", + "reservation", selectedRes.Name, + "total unallocated slots", len(unallocated), + "total allocated slots", len(allocated), + "slot resources", selectedRes.Spec.Resources, + "violations", violations, + ) + return selectedRes +} + +// evictReservation clears Spec.TargetHost and Spec.Allocations, and returns the resources +// freed (full Spec.Resources — the slot is fully evicted). Status cleanup is left to the reconcile loop +func evictReservation( + ctx context.Context, + c client.Client, + res *v1alpha1.Reservation, +) (map[hv1.ResourceName]resource.Quantity, error) { + + logger := LoggerFromContext(ctx).WithValues("component", "host-oversubscription", "reservation", res.Name) + + freed := reservations.UnusedReservationCapacity(res, true) + + old := res.DeepCopy() + res.Spec.TargetHost = "" + if res.Spec.CommittedResourceReservation != nil { + res.Spec.CommittedResourceReservation.Allocations = nil + } + logger.Info("evicting reservation", + "freed", freed, + "previous host", old.Spec.TargetHost, + "resources", old.Spec.Resources) + if err := c.Patch(ctx, res, client.MergeFrom(old)); err != nil { + logger.Error(err, "failed to patch reservation", "reservation", res.Name) + return nil, fmt.Errorf("failed to patch reservation %s: %w", res.Name, err) + } + return freed, nil +} diff --git a/internal/scheduling/reservations/commitments/host_oversubscription_controller_test.go b/internal/scheduling/reservations/commitments/host_oversubscription_controller_test.go new file mode 100644 index 000000000..bb62fc0c6 --- /dev/null +++ b/internal/scheduling/reservations/commitments/host_oversubscription_controller_test.go @@ -0,0 +1,460 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +import ( + "context" + "testing" + "time" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" +) + +func hvReadyCondition() metav1.Condition { + return metav1.Condition{ + Type: hv1.ConditionTypeReady, + Status: metav1.ConditionTrue, + Reason: hv1.ConditionReasonReadyReady, + LastTransitionTime: metav1.Now(), + } +} + +func TestComputeViolations(t *testing.T) { + tests := []struct { + name string + hv hv1.Hypervisor + slots []v1alpha1.Reservation + wantNil bool + wantViolation bool + wantExcessGiB int64 + }{ + { + name: "nil capacity returns nil", + hv: hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host-1"}}, + wantNil: true, + }, + { + name: "slot within capacity: no violation", + hv: hv1.Hypervisor{ + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: testGiB(1024), + }, + }, + }, + slots: []v1alpha1.Reservation{{ + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + }}, + }, + { + name: "slot exceeds capacity: violation reported", + hv: hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host-1"}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: testGiB(512), + }, + }, + }, + slots: []v1alpha1.Reservation{{ + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(768)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + }}, + wantViolation: true, + wantExcessGiB: 256, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := computeViolations(tt.slots, tt.hv) + if tt.wantNil { + if v != nil { + t.Errorf("expected nil, got %v", v) + } + return + } + if tt.wantViolation && len(v) == 0 { + t.Fatal("expected violation, got none") + } + if !tt.wantViolation && len(v) != 0 { + t.Errorf("expected no violation, got %v", v) + } + if tt.wantExcessGiB != 0 { + excess := v[hv1.ResourceMemory] + if excess.Cmp(testGiB(tt.wantExcessGiB)) != 0 { + t.Errorf("expected excess %d GiB, got %s", tt.wantExcessGiB, excess.String()) + } + } + }) + } +} + +func TestSelectEvictionTarget(t *testing.T) { + unallocated := func(name string, memGiB int64) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(memGiB)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{}, + }, + } + } + allocated := func(name string, totalGiB, usedGiB int64, vmID string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(totalGiB)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + Allocations: map[string]v1alpha1.CommittedResourceAllocation{ + vmID: {Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(usedGiB)}}, + }, + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: "host-1", + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{vmID: "host-1"}, + }, + }, + } + } + + violations := map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(128)} + + tests := []struct { + name string + slots []v1alpha1.Reservation + wantName string // empty = expect nil + }{ + { + name: "prefers smallest unallocated over any allocated", + slots: []v1alpha1.Reservation{ + allocated("big-alloc", 512, 256, "vm-1"), + unallocated("large", 512), + unallocated("small", 128), + }, + wantName: "small", + }, + { + name: "3 allocated slots: selects most-idle (validates sort ordering)", + slots: []v1alpha1.Reservation{ + allocated("mostly-used", 512, 480, "vm-1"), // ~6% unused + allocated("half-used", 512, 256, "vm-2"), // ~50% unused + allocated("idle", 512, 10, "vm-3"), // ~98% unused + }, + wantName: "idle", + }, + { + name: "skips non-CR reservation types", + slots: []v1alpha1.Reservation{{ + ObjectMeta: metav1.ObjectMeta{Name: "failover"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + }, + }}, + wantName: "", + }, + { + name: "skips already-evicted slots (TargetHost cleared)", + slots: []v1alpha1.Reservation{{ + ObjectMeta: metav1.ObjectMeta{Name: "evicted"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{}, + }, + }}, + wantName: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + target := selectEvictionTarget(context.Background(), tt.slots, violations) + if tt.wantName == "" { + if target != nil { + t.Errorf("expected nil target, got %q", target.Name) + } + return + } + if target == nil { + t.Fatal("expected a target, got nil") + } + if target.Name != tt.wantName { + t.Errorf("expected %q, got %q", tt.wantName, target.Name) + } + }) + } +} + +func TestCheckOversubscription_StartsGracePeriod(t *testing.T) { + scheme := newCRTestScheme(t) + hv := hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host-1", Labels: map[string]string{"topology.kubernetes.io/zone": "az1"}}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(1024)}, + Conditions: []metav1.Condition{hvReadyCondition()}, + }, + } + makeSlot := func(name string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + } + } + // 3 × 512 GiB slots on a 1024 GiB host → over-subscribed by 512 GiB + slots := []v1alpha1.Reservation{makeSlot("slot-1"), makeSlot("slot-2"), makeSlot("slot-3")} + k8sClient := newCRTestClient(scheme, &slots[0], &slots[1], &slots[2]) + monitor := NewReservationControllerMonitor() + controller := &HostOversubscriptionController{ + Client: k8sClient, + Monitor: &monitor, + Conf: ReservationControllerConfig{EnableOversubscriptionReservationEviction: true}, + } + + requeue, err := controller.checkOversubscription(context.Background(), "host-1", slots, hv, 2*time.Minute) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if requeue == 0 { + t.Error("expected non-zero requeue on first violation (grace period)") + } + if controller.firstSeen["host-1"].IsZero() { + t.Error("expected firstSeen recorded after first violation") + } + // No eviction during grace period — all slots must still be placed. + for _, name := range []string{"slot-1", "slot-2", "slot-3"} { + var r v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: name}, &r); err != nil { + t.Fatalf("get %s: %v", name, err) + } + if r.Spec.TargetHost == "" { + t.Errorf("%s: TargetHost cleared during grace period", name) + } + } +} + +func TestCheckOversubscription_EvictsAfterGracePeriod(t *testing.T) { + scheme := newCRTestScheme(t) + hv := hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host-1", Labels: map[string]string{"topology.kubernetes.io/zone": "az1"}}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(1024)}, + Conditions: []metav1.Condition{hvReadyCondition()}, + }, + } + makeSlot := func(name string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + } + } + slots := []v1alpha1.Reservation{makeSlot("slot-1"), makeSlot("slot-2"), makeSlot("slot-3")} + k8sClient := newCRTestClient(scheme, &slots[0], &slots[1], &slots[2]) + monitor := NewReservationControllerMonitor() + controller := &HostOversubscriptionController{ + Client: k8sClient, + Monitor: &monitor, + Conf: ReservationControllerConfig{EnableOversubscriptionReservationEviction: true}, + firstSeen: map[string]time.Time{"host-1": time.Now().Add(-3 * time.Minute)}, + } + + _, err := controller.checkOversubscription(context.Background(), "host-1", slots, hv, 2*time.Minute) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var evicted []string + for _, name := range []string{"slot-1", "slot-2", "slot-3"} { + var r v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: name}, &r); err != nil { + t.Fatalf("get %s: %v", name, err) + } + if r.Spec.TargetHost == "" { + evicted = append(evicted, name) + } + } + if len(evicted) != 1 { + t.Errorf("expected exactly 1 slot evicted, got %v", evicted) + } +} + +func TestCheckOversubscription_SkipsWhenHVNotReady(t *testing.T) { + scheme := newCRTestScheme(t) + // HV has capacity data but no Ready condition. + hv := hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host-1"}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + }, + } + // Slot is oversubscribed relative to HV capacity — should NOT be evicted. + slot := v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "slot-1"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(768)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{}, + }, + } + k8sClient := newCRTestClient(scheme, &slot) + monitor := NewReservationControllerMonitor() + controller := &HostOversubscriptionController{ + Client: k8sClient, + Monitor: &monitor, + Conf: ReservationControllerConfig{EnableOversubscriptionReservationEviction: true}, + } + + requeue, err := controller.checkOversubscription(context.Background(), "host-1", []v1alpha1.Reservation{slot}, hv, 2*time.Minute) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if requeue != 0 { + t.Errorf("expected 0 requeue when HV not ready, got %v", requeue) + } + if !controller.firstSeen["host-1"].IsZero() { + t.Error("expected firstSeen not set when HV not ready") + } +} + +func TestCheckOversubscription_NoViolation(t *testing.T) { + scheme := newCRTestScheme(t) + hv := hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host-1", Labels: map[string]string{"topology.kubernetes.io/zone": "az1"}}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(1024)}, + Conditions: []metav1.Condition{hvReadyCondition()}, + }, + } + slot := v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "slot-1"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{}, + }, + } + k8sClient := newCRTestClient(scheme, &slot) + monitor := NewReservationControllerMonitor() + controller := &HostOversubscriptionController{ + Client: k8sClient, + Monitor: &monitor, + firstSeen: map[string]time.Time{"host-1": time.Now().Add(-5 * time.Minute)}, + } + + requeue, err := controller.checkOversubscription(context.Background(), "host-1", []v1alpha1.Reservation{slot}, hv, 2*time.Minute) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if requeue != 0 { + t.Errorf("expected 0 requeue when no violation, got %v", requeue) + } + if _, still := controller.firstSeen["host-1"]; still { + t.Error("expected firstSeen cleared after violation resolved") + } +} + +func TestHostOversubscriptionController_RateLimit(t *testing.T) { + controller := &HostOversubscriptionController{} + minInterval := 30 * time.Second + + if limited, _ := controller.isRateLimited("host-1", minInterval); limited { + t.Error("expected not rate-limited before first check") + } + + controller.recordCheck("host-1") + + limited, remaining := controller.isRateLimited("host-1", minInterval) + if !limited { + t.Error("expected rate-limited immediately after recordCheck") + } + if remaining <= 0 || remaining > minInterval { + t.Errorf("remaining out of expected range (0, %v]: %v", minInterval, remaining) + } + + controller.lastCheck["host-1"] = time.Time{} + if limited, _ := controller.isRateLimited("host-1", minInterval); limited { + t.Error("expected not rate-limited after zeroing lastCheck") + } +} + +func TestEvictReservation_ClearsAllocations(t *testing.T) { + scheme := newCRTestScheme(t) + slot := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "slot-1"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + Allocations: map[string]v1alpha1.CommittedResourceAllocation{ + "vm-1": {Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(256)}}, + }, + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: "host-1", + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{"vm-1": "host-1"}, + }, + }, + } + + k8sClient := newCRTestClient(scheme, slot) + freed, err := evictReservation(context.Background(), k8sClient, slot) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + freedMem := freed[hv1.ResourceMemory] + if freedMem.Cmp(testGiB(512)) != 0 { + t.Errorf("expected freed 512 GiB, got %s", freedMem.String()) + } + + var updated v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: "slot-1"}, &updated); err != nil { + t.Fatalf("get updated reservation: %v", err) + } + if updated.Spec.TargetHost != "" { + t.Errorf("expected TargetHost cleared, got %q", updated.Spec.TargetHost) + } + if len(updated.Spec.CommittedResourceReservation.Allocations) != 0 { + t.Errorf("expected Spec.Allocations cleared, got %v", updated.Spec.CommittedResourceReservation.Allocations) + } +} diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go index ce968b4f4..154d96137 100644 --- a/internal/scheduling/reservations/commitments/integration_test.go +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -20,6 +20,7 @@ package commitments import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -403,6 +404,7 @@ type intgEnv struct { k8sClient client.Client crController *CommittedResourceController resController *CommitmentReservationController + hvController *HostOversubscriptionController schedulerSrv *httptest.Server } @@ -443,6 +445,24 @@ func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.H } return uuids }). + WithIndex(&v1alpha1.Reservation{}, reservations.IdxReservationByHost, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok { + return nil + } + hosts := make(map[string]struct{}) + if res.Spec.TargetHost != "" { + hosts[res.Spec.TargetHost] = struct{}{} + } + if res.Status.Host != "" { + hosts[res.Status.Host] = struct{}{} + } + result := make([]string, 0, len(hosts)) + for h := range hosts { + result = append(result, h) + } + return result + }). Build() schedulerSrv := httptest.NewServer(schedulerFn) @@ -456,19 +476,27 @@ func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.H }, VMSource: vmSource, } + monitor := NewReservationControllerMonitor() resCtrl := &CommitmentReservationController{ Client: k8sClient, Scheme: scheme, Conf: ReservationControllerConfig{ - SchedulerURL: schedulerSrv.URL, - AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}, - RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, + SchedulerURL: schedulerSrv.URL, + AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}, + RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, + EnableOversubscriptionCheck: true, + EnableOversubscriptionReservationEviction: true, }, } if err := resCtrl.Init(context.Background(), resCtrl.Conf); err != nil { t.Fatalf("resCtrl.Init: %v", err) } - return &intgEnv{k8sClient: k8sClient, crController: crCtrl, resController: resCtrl, schedulerSrv: schedulerSrv} + hvCtrl := &HostOversubscriptionController{ + Client: k8sClient, + Monitor: &monitor, + Conf: resCtrl.Conf, + } + return &intgEnv{k8sClient: k8sClient, crController: crCtrl, resController: resCtrl, hvController: hvCtrl, schedulerSrv: schedulerSrv} } func (e *intgEnv) close() { e.schedulerSrv.Close() } @@ -491,7 +519,18 @@ func (e *intgEnv) reconcileReservation(t *testing.T, resName string) { t.Helper() req := ctrl.Request{NamespacedName: types.NamespacedName{Name: resName}} if _, err := e.resController.Reconcile(context.Background(), req); err != nil { - t.Fatalf("reservation reconcile %s: %v", resName, err) + // "no hosts found" is a retriable error surfaced deliberately for backoff — not a test failure. + if !strings.Contains(err.Error(), "no hosts found") { + t.Fatalf("reservation reconcile %s: %v", resName, err) + } + } +} + +func (e *intgEnv) reconcileHV(t *testing.T, host string) { + t.Helper() + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: host}} + if _, err := e.hvController.Reconcile(context.Background(), req); err != nil { + t.Fatalf("HV reconcile %s: %v", host, err) } } @@ -1231,3 +1270,141 @@ func TestCRScheduling_SetsReserveForCommittedResourceIntent(t *testing.T) { t.Errorf("CR slot scheduling must set _nova_check_type=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, hint) } } + +func TestCROversubscriptionRemediation(t *testing.T) { + gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) } + cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) } + + hv := &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "host-1", + Labels: map[string]string{"topology.kubernetes.io/zone": "az1"}, + }, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(200), + hv1.ResourceCPU: cpu(200), + }, + Conditions: []metav1.Condition{{ + Type: hv1.ConditionTypeReady, + Status: metav1.ConditionTrue, + Reason: hv1.ConditionReasonReadyReady, + LastTransitionTime: metav1.Now(), + }}, + }, + } + + objects := []client.Object{hv} + for i := range 20 { + memGiB := int64(10 + i) + cores := int64(10 + i) + name := fmt.Sprintf("slot-%02d", i) + + slot := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(memGiB), + hv1.ResourceCPU: cpu(cores), + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + ResourceName: "test-flavor", + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: "host-1", + Conditions: []metav1.Condition{{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReservationActive", + }}, + }, + } + if i < 10 { + // slots 0-9: allocated — have a confirmed VM + vmID := fmt.Sprintf("vm-%02d", i) + slot.Spec.CommittedResourceReservation.Allocations = map[string]v1alpha1.CommittedResourceAllocation{ + vmID: {Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(memGiB), + hv1.ResourceCPU: cpu(cores), + }}, + } + slot.Status.CommittedResourceReservation = &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{vmID: "host-1"}, + } + } + objects = append(objects, slot) + } + + var schedulerCalls atomic.Int32 + schedulerFn := func(w http.ResponseWriter, r *http.Request) { + schedulerCalls.Add(1) + intgRejectScheduler(w, r) // reject so evicted slots don't land back on host-1 + } + env := newIntgEnv(t, append(objects, newTestFlavorKnowledge()), schedulerFn, nil) + defer env.close() + + // Reconcile host-1 — triggers oversubscription detection (firstSeen set). + env.reconcileHV(t, "host-1") + if _, detected := env.hvController.firstSeen["host-1"]; !detected { + t.Fatal("expected oversubscription to be detected after first reconcile") + } + + // Drive remediation: fast-forward grace period and reset rate limit each cycle. + resolved := false + for range 25 { + env.hvController.firstSeen["host-1"] = time.Now().Add(-3 * time.Minute) + env.hvController.lastCheck["host-1"] = time.Time{} + env.reconcileHV(t, "host-1") + + if _, still := env.hvController.firstSeen["host-1"]; !still { + resolved = true + break + } + } + if !resolved { + t.Fatal("expected oversubscription to be resolved after remediation cycles") + } + + var resList v1alpha1.ReservationList + if err := env.k8sClient.List(context.Background(), &resList); err != nil { + t.Fatalf("list reservations: %v", err) + } + + var placedSlots []v1alpha1.Reservation + for _, r := range resList.Items { + if r.Spec.TargetHost == "host-1" || r.Status.Host == "host-1" { + placedSlots = append(placedSlots, r) + } + } + free := reservations.HostFreeCapacity(placedSlots, *hv) + zero := resource.MustParse("0") + for rn, f := range free { + if f.Cmp(zero) < 0 { + t.Errorf("host still over-subscribed for %s by %s after remediation", rn, f.String()) + } + } + + for _, r := range resList.Items { + if r.Spec.TargetHost == "" && + r.Spec.CommittedResourceReservation != nil && + len(r.Spec.CommittedResourceReservation.Allocations) > 0 { + t.Errorf("slot %s has allocations but was evicted", r.Name) + } + } + + // Two reconciles needed: first clears status (PlacementRevoked), second triggers placement. + callsBefore := schedulerCalls.Load() + for _, r := range resList.Items { + if r.Spec.TargetHost == "" { + env.reconcileReservation(t, r.Name) // revoke status + env.reconcileReservation(t, r.Name) // trigger placement + break + } + } + if schedulerCalls.Load() <= callsBefore { + t.Error("expected evicted slot to trigger a scheduler call on next reconcile") + } +} diff --git a/internal/scheduling/reservations/commitments/payg_rollback_test.go b/internal/scheduling/reservations/commitments/payg_rollback_test.go index c9962436a..f1f7efeb0 100644 --- a/internal/scheduling/reservations/commitments/payg_rollback_test.go +++ b/internal/scheduling/reservations/commitments/payg_rollback_test.go @@ -5,6 +5,7 @@ package commitments import ( "context" + "strings" "testing" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -82,10 +83,10 @@ func TestCRLifecycle_PAYGRollback(t *testing.T) { } for _, res := range resList.Items { resReq := ctrl.Request{NamespacedName: types.NamespacedName{Name: res.Name}} - if _, err := env.resController.Reconcile(ctx, resReq); err != nil { + if _, err := env.resController.Reconcile(ctx, resReq); err != nil && !strings.Contains(err.Error(), "no hosts found") { t.Fatalf("reservation reconcile %s (pass 1): %v", res.Name, err) } - if _, err := env.resController.Reconcile(ctx, resReq); err != nil { + if _, err := env.resController.Reconcile(ctx, resReq); err != nil && !strings.Contains(err.Error(), "no hosts found") { t.Fatalf("reservation reconcile %s (pass 2): %v", res.Name, err) } } diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index f6bf5a385..a5fd5cd2d 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -6,6 +6,8 @@ package commitments import ( "context" "fmt" + "net/http" + "reflect" "time" "k8s.io/apimachinery/pkg/api/meta" @@ -32,7 +34,6 @@ import ( hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/go-logr/logr" "github.com/gophercloud/gophercloud/v2" - "net/http" ) // CommitmentReservationController reconciles commitment Reservation objects @@ -114,6 +115,28 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr } if res.IsReady() { + // Spec.TargetHost was cleared (e.g. oversubscription eviction) — revoke status so the + // slot re-enters the placement flow on the next reconcile. + if res.Spec.TargetHost == "" { + old := res.DeepCopy() + res.Status.Host = "" + if res.Status.CommittedResourceReservation != nil { + res.Status.CommittedResourceReservation.Allocations = nil + } + meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionFalse, + Reason: "PlacementRevoked", + Message: "target host was cleared; pending re-placement", + }) + if err := r.Status().Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + logger.Info("revoked ready status after placement eviction, slot re-enters placement flow", + "component", "oversubscription-check") + return ctrl.Result{}, nil + } + logger.V(1).Info("reservation is active, verifying allocations") // Sync ObservedParentGeneration if the CR controller bumped ParentGeneration since @@ -295,7 +318,14 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr logger.Info("selected pipeline for CR reservation", "flavorName", resourceName, "flavorGroup", flavorGroupName, - "pipeline", pipelineName) + "pipeline", pipelineName, + "reason", func() string { + cond := meta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond != nil { + return cond.Reason + } + return "initial" + }()) // Use the SchedulerClient to schedule the reservation scheduleReq := reservations.ScheduleReservationRequest{ @@ -328,7 +358,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr } if len(scheduleResp.Hosts) == 0 { - logger.Info("no hosts found for reservation", "reservation", res.Name, "flavorName", resourceName) + logger.Info("no hosts found for reservation, will retry", "reservation", res.Name, "flavorName", resourceName) old := res.DeepCopy() meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ Type: v1alpha1.ReservationConditionReady, @@ -347,14 +377,14 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // Object was deleted, no need to continue return ctrl.Result{}, nil } - return ctrl.Result{}, nil // No need to requeue, we didn't find a host. + return ctrl.Result{}, fmt.Errorf("no hosts found for reservation %s (flavor %s)", res.Name, resourceName) } // Update the reservation Spec with the found host (idx 0) // Only update Spec here - the Status will be synced in the next reconcile cycle // This avoids race conditions from doing two patches in one reconcile host := scheduleResp.Hosts[0] - logger.Info("found host for reservation", "host", host) + logger.Info("found host for reservation", "host", host, "flavorName", resourceName) old := res.DeepCopy() res.Spec.TargetHost = host @@ -718,7 +748,7 @@ func (r *CommitmentReservationController) getPipelineForFlavorGroup(flavorGroupN func (r *CommitmentReservationController) hypervisorToReservations(ctx context.Context, obj client.Object) []reconcile.Request { hvName := obj.GetName() var reservationList v1alpha1.ReservationList - if err := r.List(ctx, &reservationList); err != nil { + if err := r.List(ctx, &reservationList, client.MatchingFields{reservations.IdxReservationByHost: hvName}); err != nil { logf.FromContext(ctx).Error(err, "failed to list reservations for hypervisor", "hypervisor", hvName) return nil } @@ -811,6 +841,28 @@ var commitmentReservationPredicate = predicate.Funcs{ }, } +// hvCapacityChangePredicate fires when Status.Instances, Status.Allocation, +// Status.EffectiveCapacity, or Status.Capacity changes on a Hypervisor. Instances covers +// VM presence (used by allocation verification); Allocation, EffectiveCapacity, and Capacity +// cover capacity accounting (used by the over-subscription check; Capacity is the fallback +// when EffectiveCapacity is nil). +var hvCapacityChangePredicate = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { _, ok := e.Object.(*hv1.Hypervisor); return ok }, + DeleteFunc: func(e event.DeleteEvent) bool { _, ok := e.Object.(*hv1.Hypervisor); return ok }, + GenericFunc: func(e event.GenericEvent) bool { _, ok := e.Object.(*hv1.Hypervisor); return ok }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldHV, ok1 := e.ObjectOld.(*hv1.Hypervisor) + newHV, ok2 := e.ObjectNew.(*hv1.Hypervisor) + if !ok1 || !ok2 { + return false + } + return !reflect.DeepEqual(oldHV.Status.Instances, newHV.Status.Instances) || + !reflect.DeepEqual(oldHV.Status.Allocation, newHV.Status.Allocation) || + !reflect.DeepEqual(oldHV.Status.EffectiveCapacity, newHV.Status.EffectiveCapacity) || + !reflect.DeepEqual(oldHV.Status.Capacity, newHV.Status.Capacity) + }, +} + // SetupWithManager sets up the controller with the Manager. func (r *CommitmentReservationController) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { @@ -825,6 +877,9 @@ func (r *CommitmentReservationController) SetupWithManager(mgr ctrl.Manager, mcl if err := indexReservationByAllocationVMUUID(context.Background(), mcl); err != nil { return fmt.Errorf("failed to set up reservation allocation VM UUID index: %w", err) } + if err := reservations.IndexReservationByHost(context.Background(), mcl); err != nil { + return fmt.Errorf("failed to set up reservation by host index: %w", err) + } // Use WatchesMulticluster to watch Reservations across all configured clusters // (home + remotes). This is required because Reservation CRDs may be stored @@ -848,6 +903,7 @@ func (r *CommitmentReservationController) SetupWithManager(mgr ctrl.Manager, mcl bldr, err = bldr.WatchesMulticluster( &hv1.Hypervisor{}, handler.EnqueueRequestsFromMapFunc(r.hypervisorToReservations), + hvCapacityChangePredicate, ) if err != nil { return err diff --git a/internal/scheduling/reservations/commitments/reservation_controller_monitor.go b/internal/scheduling/reservations/commitments/reservation_controller_monitor.go new file mode 100644 index 000000000..959965d13 --- /dev/null +++ b/internal/scheduling/reservations/commitments/reservation_controller_monitor.go @@ -0,0 +1,58 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// ReservationControllerMonitor reports per-host over-subscription violations +// detected by the CommitmentReservationController. +type ReservationControllerMonitor struct { + oversubscribed *prometheus.GaugeVec + slotsEvicted *prometheus.CounterVec +} + +func NewReservationControllerMonitor() ReservationControllerMonitor { + return ReservationControllerMonitor{ + oversubscribed: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_committed_resource_host_oversubscribed", + Help: "Excess resource units by which a host's reservation blocks + VM allocations exceed its effective capacity. " + + "Non-zero when the host is over-subscribed and unresolvable via unallocated slot eviction.", + }, []string{"host", "az", "resource"}), + slotsEvicted: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_committed_resource_host_oversubscribed_evicted_reservations_total", + Help: "Total number of reservation slots evicted by the oversubscription controller. " + + "A sustained increase indicates recurring oversubscription events in an AZ.", + }, []string{"az"}), + } +} + +// SetOversubscribed records the excess amount for a host+resource pair. +// Zero clears the violation. +func (m *ReservationControllerMonitor) SetOversubscribed(host, az, resource string, excessUnits float64) { + m.oversubscribed.WithLabelValues(host, az, resource).Set(excessUnits) +} + +// ClearHost resets all resource gauges for a host that is no longer over-subscribed. +func (m *ReservationControllerMonitor) ClearHost(host, az string) { + m.oversubscribed.DeletePartialMatch(prometheus.Labels{"host": host, "az": az}) +} + +// IncSlotsEvicted increments the counter for slots evicted in the given AZ. +func (m *ReservationControllerMonitor) IncSlotsEvicted(az string) { + m.slotsEvicted.WithLabelValues(az).Inc() +} + +// Describe implements prometheus.Collector. +func (m *ReservationControllerMonitor) Describe(ch chan<- *prometheus.Desc) { + m.oversubscribed.Describe(ch) + m.slotsEvicted.Describe(ch) +} + +// Collect implements prometheus.Collector. +func (m *ReservationControllerMonitor) Collect(ch chan<- prometheus.Metric) { + m.oversubscribed.Collect(ch) + m.slotsEvicted.Collect(ch) +} diff --git a/internal/scheduling/reservations/commitments/reservation_controller_test.go b/internal/scheduling/reservations/commitments/reservation_controller_test.go index a776ac0cf..98e4cb867 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller_test.go +++ b/internal/scheduling/reservations/commitments/reservation_controller_test.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -62,13 +63,15 @@ func TestCommitmentReservationController_Reconcile(t *testing.T) { Name: "test-reservation", }, Spec: v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-1", CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ ProjectID: "test-project", ResourceName: "test-flavor", }, }, Status: v1alpha1.ReservationStatus{ + Host: "host-1", Conditions: []metav1.Condition{ { Type: v1alpha1.ReservationConditionReady, @@ -1216,3 +1219,131 @@ func TestReconcileAllocations_LiveMigration(t *testing.T) { }) } } + +func TestHvCapacityChangePredicate(t *testing.T) { + gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) } + cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) } + + makeHV := func(memGiB, cpuCores int64, instances []hv1.Instance) hv1.Hypervisor { + return hv1.Hypervisor{ + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(memGiB), + hv1.ResourceCPU: cpu(cpuCores), + }, + Instances: instances, + }, + } + } + + tests := []struct { + name string + old hv1.Hypervisor + new hv1.Hypervisor + wantFire bool + }{ + { + name: "instances changed → fires", + old: makeHV(1024, 256, nil), + new: makeHV(1024, 256, []hv1.Instance{{ID: "vm-1"}}), + wantFire: true, + }, + { + name: "effective capacity changed → fires", + old: makeHV(1024, 256, nil), + new: makeHV(2048, 256, nil), + wantFire: true, + }, + { + name: "allocation changed → fires", + old: makeHV(1024, 256, nil), + new: func() hv1.Hypervisor { + h := makeHV(1024, 256, nil) + h.Status.Allocation = map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: gib(512)} + return h + }(), + wantFire: true, + }, + { + name: "nothing relevant changed → does not fire", + old: makeHV(1024, 256, nil), + new: makeHV(1024, 256, nil), + wantFire: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + oldObj := tt.old + newObj := tt.new + got := hvCapacityChangePredicate.UpdateFunc(event.UpdateEvent{ + ObjectOld: &oldObj, + ObjectNew: &newObj, + }) + if got != tt.wantFire { + t.Errorf("hvCapacityChangePredicate.UpdateFunc = %v, want %v", got, tt.wantFire) + } + }) + } +} + +func TestReconcile_RevokesReadyWhenTargetHostCleared(t *testing.T) { + scheme := newCRTestScheme(t) + + // Slot is Ready (Status.Host set, condition true) but Spec.TargetHost was cleared — + // simulates the oversubscription eviction path. + slot := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "slot-1"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "", + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + ResourceName: "test-flavor", + ProjectID: "test-project", + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: "host-1", + Conditions: []metav1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReservationActive", + LastTransitionTime: metav1.Now(), + }, + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{"vm-1": "host-1"}, + }, + }, + } + + k8sClient := newCRTestClient(scheme, slot) + controller := &CommitmentReservationController{ + Client: k8sClient, + Conf: ReservationControllerConfig{RequeueIntervalActive: metav1.Duration{Duration: 30 * time.Minute}}, + } + + _, err := controller.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKey{Name: "slot-1"}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updated v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: "slot-1"}, &updated); err != nil { + t.Fatalf("failed to get updated reservation: %v", err) + } + if updated.Status.Host != "" { + t.Errorf("expected Status.Host cleared, got %q", updated.Status.Host) + } + if updated.Status.CommittedResourceReservation != nil && len(updated.Status.CommittedResourceReservation.Allocations) != 0 { + t.Errorf("expected Status.Allocations cleared, got %v", updated.Status.CommittedResourceReservation.Allocations) + } + cond := meta.FindStatusCondition(updated.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond == nil || cond.Status != metav1.ConditionFalse { + t.Errorf("expected Ready=False, got %v", cond) + } + if cond != nil && cond.Reason != "PlacementRevoked" { + t.Errorf("expected reason PlacementRevoked, got %q", cond.Reason) + } +} diff --git a/internal/scheduling/reservations/field_index.go b/internal/scheduling/reservations/field_index.go new file mode 100644 index 000000000..57ef4c222 --- /dev/null +++ b/internal/scheduling/reservations/field_index.go @@ -0,0 +1,56 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package reservations + +import ( + "context" + "errors" + "sync" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +// IdxReservationByHost is the field index key for looking up Reservations by host. +// Both Spec.TargetHost and Status.Host are indexed so reservations in transit +// (TargetHost != Status.Host) are found via either field. +// All reservation types are included. +const IdxReservationByHost = "reservations.host" + +var onceIndexReservationByHost sync.Once + +// IndexReservationByHost registers the shared host index on the multicluster client. +// Safe to call multiple times — registration happens only once. +func IndexReservationByHost(ctx context.Context, mcl *multicluster.Client) (err error) { + onceIndexReservationByHost.Do(func() { + log := logf.FromContext(ctx) + err = mcl.IndexField(ctx, + &v1alpha1.Reservation{}, + &v1alpha1.ReservationList{}, + IdxReservationByHost, + func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok { + log.Error(errors.New("unexpected type"), "expected Reservation", "object", obj) + return nil + } + hosts := make(map[string]struct{}) + if res.Spec.TargetHost != "" { + hosts[res.Spec.TargetHost] = struct{}{} + } + if res.Status.Host != "" { + hosts[res.Status.Host] = struct{}{} + } + result := make([]string, 0, len(hosts)) + for h := range hosts { + result = append(result, h) + } + return result + }, + ) + }) + return err +} From 61200f9cc82c50dbdd4d45fc8ed8c60fa04bea46 Mon Sep 17 00:00:00 2001 From: "cortex-ai-agents[bot]" <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:39:10 +0200 Subject: [PATCH 10/12] Release cortex v0.3.7 (#1140) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Release prep for #1139: changelog entry and helm chart version bumps - Merge this before merging #1139 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 --- CHANGELOG.md | 50 +++++++++++++++++++ helm/bundles/cortex-cinder/Chart.yaml | 8 +-- helm/bundles/cortex-crds/Chart.yaml | 4 +- helm/bundles/cortex-ironcore/Chart.yaml | 4 +- helm/bundles/cortex-manila/Chart.yaml | 8 +-- helm/bundles/cortex-nova/Chart.yaml | 8 +-- helm/bundles/cortex-placement-shim/Chart.yaml | 4 +- helm/bundles/cortex-pods/Chart.yaml | 4 +- helm/library/cortex-postgres/Chart.yaml | 2 +- helm/library/cortex-shim/Chart.yaml | 2 +- helm/library/cortex/Chart.yaml | 2 +- 11 files changed, 73 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d58e91ab6..b63bb230d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,55 @@ # Changelog +## 2026-08-17 — [#1139](https://github.com/cobaltcore-dev/cortex/pull/1139) + +### cortex v0.3.7 (sha-67daed47) + +New features: +- Host oversubscription detection and remediation — adds a new controller that automatically detects when a host has more committed reservation blocks than its capacity allows and evicts excess reservation slots to restore headroom, with a configurable grace period and detection-only mode ([#1125](https://github.com/cobaltcore-dev/cortex/pull/1125)) + +Non-breaking changes: +- Remove `go-bits/easypg` — replaces `go-bits/easypg` usage with the equivalent API from `gg/pgruntime`, as part of the upstream deprecation of easypg ([#1123](https://github.com/cobaltcore-dev/cortex/pull/1123)) +- Fix: remove unused volumes and volumeMounts for perses-plugins ([#1133](https://github.com/cobaltcore-dev/cortex/pull/1133)) +- Update `github.com/sapcc/go-bits` ([#1126](https://github.com/cobaltcore-dev/cortex/pull/1126), [#1136](https://github.com/cobaltcore-dev/cortex/pull/1136)) +- Update `kube-prometheus-stack` to v88.3.0 ([#1127](https://github.com/cobaltcore-dev/cortex/pull/1127), [#1132](https://github.com/cobaltcore-dev/cortex/pull/1132)) +- Update postgres Docker tag to v18.6 ([#1137](https://github.com/cobaltcore-dev/cortex/pull/1137), [#1141](https://github.com/cobaltcore-dev/cortex/pull/1141)) + +### cortex-shim v0.1.13 (sha-67daed47) + +Includes updated image sha-67daed47 with dependency updates. + +### cortex-postgres v0.6.13 (sha-67daed47) + +Includes updated PostgreSQL base image (v18.6). + +### cortex-nova v0.0.87 + +Includes updated charts cortex v0.3.7 and cortex-postgres v0.6.13. + +### cortex-cinder v0.0.87 + +Includes updated charts cortex v0.3.7 and cortex-postgres v0.6.13. + +### cortex-manila v0.0.87 + +Includes updated charts cortex v0.3.7 and cortex-postgres v0.6.13. + +### cortex-crds v0.0.87 + +Includes updated chart cortex v0.3.7. + +### cortex-ironcore v0.0.87 + +Includes updated chart cortex v0.3.7. + +### cortex-pods v0.0.87 + +Includes updated chart cortex v0.3.7. + +### cortex-placement-shim v0.1.13 + +Includes updated chart cortex-shim v0.1.13. + ## 2026-08-10 — [#1129](https://github.com/cobaltcore-dev/cortex/pull/1129) ### cortex v0.3.6 (sha-85d9d0cf) diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index ccfea8439..259df2f77 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.86 +version: 0.0.87 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.12 + version: 0.6.13 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index a3e17a750..639351774 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.86 +version: 0.0.87 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 8d7fdcff7..e72b63e3e 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.86 +version: 0.0.87 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index b6f293369..73c6279a2 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.86 +version: 0.0.87 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.12 + version: 0.6.13 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index bba789bd2..ffd98375f 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.86 +version: 0.0.87 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.12 + version: 0.6.13 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 083001bfb..ea9c69c2a 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.12 +version: 0.1.13 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.12 + version: 0.1.13 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 1f9365c63..6ca4f7af1 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.86 +version: 0.0.87 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.6 + version: 0.3.7 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index 51d2b675f..094113b2e 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.12 +version: 0.6.13 appVersion: "sha-7343f7cd" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 889fae5e0..dea39c604 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.12 +version: 0.1.13 appVersion: "sha-1daefe16" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index baa1e4075..9c8ebed82 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.3.6 +version: 0.3.7 appVersion: "sha-013d3d31" icon: "https://example.com/icon.png" dependencies: [] From e545057ca47287c2189e121b516c7565e764cccc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:45:14 +0200 Subject: [PATCH 11/12] bump app version [skip ci] (#1131) bump app version [skip ci] ``` bumped cortex: sha-013d3d31 -> sha-67daed47 bumped cortex-shim: sha-1daefe16 -> sha-5809e5f3 bumped cortex-postgres: sha-7343f7cd -> sha-9f10115a ``` Signed-off-by: umswmayj <140147670+umswmayj@users.noreply.github.com> Co-authored-by: umswmayj <140147670+umswmayj@users.noreply.github.com> --- helm/library/cortex-postgres/Chart.yaml | 2 +- helm/library/cortex-shim/Chart.yaml | 2 +- helm/library/cortex/Chart.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index 094113b2e..3445e5a1b 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -6,4 +6,4 @@ name: cortex-postgres description: Postgres setup for Cortex. type: application version: 0.6.13 -appVersion: "sha-7343f7cd" +appVersion: "sha-9f10115a" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index dea39c604..a19fa31a0 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -3,6 +3,6 @@ name: cortex-shim description: A Helm chart to distribute cortex shims. type: application version: 0.1.13 -appVersion: "sha-1daefe16" +appVersion: "sha-5809e5f3" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 9c8ebed82..be711e880 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -3,6 +3,6 @@ name: cortex description: A Helm chart to distribute cortex. type: application version: 0.3.7 -appVersion: "sha-013d3d31" +appVersion: "sha-67daed47" icon: "https://example.com/icon.png" dependencies: [] From 36c810ce415a0662b0456ed412768806d0ec70a8 Mon Sep 17 00:00:00 2001 From: "cortex-ai-agents[bot]" <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:46:55 +0200 Subject: [PATCH 12/12] docs: fix inaccurate live migration bullet in CR reservations (#1138) The live migration bullet in the CR reservations doc claimed no special logic existed, but PR #1048 added explicit detection and handling in the reservation controller. The controller now detects when a confirmed VM disappears from its expected host, scans all hypervisors to locate it, and either follows the VM to its new host or removes it from the reservation. This corrects the documentation to match the actual behavior. Assisted-by: Claude Code:claude-sonnet-4-20250514 [Bash] [Read] Signed-off-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> --- docs/reservations/committed-resource-reservations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reservations/committed-resource-reservations.md b/docs/reservations/committed-resource-reservations.md index 4c3f448be..ae0b1f68c 100644 --- a/docs/reservations/committed-resource-reservations.md +++ b/docs/reservations/committed-resource-reservations.md @@ -178,7 +178,7 @@ The `spec_only_unblocked` term exists because an InFlightReservation on the same **Corner cases worth noting:** - Confirmed VMs exceed reservation size (e.g. after resize): clamp `remaining` to 0, never negative - Spec-only VM larger than remaining slot: block `spec_only_unblocked` — those resources will land when the VM starts -- Live migration within a reservation: handled implicitly by `hv.Status.Allocation`, which libvirt reports on both source and target during migration; no special logic needed +- Live migration within a reservation: when a confirmed VM disappears from its expected host, the controller scans all hypervisors. If the VM is the sole allocation and the new host has capacity, the reservation follows the VM; otherwise the VM is removed from the reservation. ### InFlightReservation