From dff4fea4186acd24ae0f1e1f5645bd7ef64e0f2a Mon Sep 17 00:00:00 2001 From: Luca Miccini Date: Mon, 17 Aug 2026 09:58:38 +0200 Subject: [PATCH] Add transport URL secret rotation with consumer finalizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When infra-operator rotates a RabbitMQ transport URL (creating a new secret and user), consumer operators must hold a consumer finalizer on the old secret until all their pods have rolled out with the new credentials. Without this, infra-operator cleans up the old RabbitMQ user while pods are still connected with old credentials, causing message bus outages. Design: 1. Add a consumer finalizer to the current transport URL secret early in reconcile. Set instance.Status.TransportURLSecret for first-time setup only (empty or unchanged); during rotation the status is updated solely by FinalizeSecretRotation at the end of reconcile. 2. Pass transportURL.Status.SecretName directly to sub-CR creation functions and config generation as a parameter — never read from instance.Status.TransportURLSecret for sub-CR specs. 3. Each child (sub-CR) controller records an AppliedInputSecretHash in its status, set only after statefulset.IsReadyForInput / deployment.IsReadyForInput confirms — via an uncached API read — that the workload is fully rolled out with the expected CONFIG_HASH. 4. The parent mirrors a child's Ready condition only when its Generation == ObservedGeneration and AppliedInputSecretHash matches the current input hash; otherwise it sets the condition to Unknown. 5. Guard: FinalizeSecretRotation removes the consumer finalizer from the old secret only when every child reports the expected hash and is ready. The same pattern applies to notification transport URL secrets and application credential secrets where applicable. Co-Authored-By: Claude Opus 4.8 --- ...elemetry.openstack.org_cloudkittyapis.yaml | 5 + ...lemetry.openstack.org_cloudkittyprocs.yaml | 5 + api/go.mod | 4 +- api/go.sum | 12 +- api/v1beta1/cloudkittyapi_types.go | 4 + api/v1beta1/cloudkittyproc_types.go | 4 + api/v1beta1/telemetry_consts.go | 5 + cmd/main.go | 29 ++- ...elemetry.openstack.org_cloudkittyapis.yaml | 5 + ...lemetry.openstack.org_cloudkittyprocs.yaml | 5 + go.mod | 20 +- go.sum | 44 ++-- internal/controller/aodh_controller.go | 23 +- internal/controller/autoscaling_controller.go | 153 ++++++++---- internal/controller/ceilometer_controller.go | 219 +++++++++++++----- internal/controller/cloudkitty_controller.go | 199 +++++++++++----- .../controller/cloudkittyapi_controller.go | 31 +-- .../controller/cloudkittyproc_controller.go | 30 +-- internal/utils/utils.go | 1 + 19 files changed, 557 insertions(+), 241 deletions(-) diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml index 061f75e1e..f1a84a8dc 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -425,6 +425,11 @@ spec: type: object description: API endpoints type: object + appliedInputSecretHash: + description: |- + AppliedInputSecretHash - hash of the input secrets that the child has + fully rolled out, used by the parent to track credential rotation convergence + type: string conditions: description: Conditions items: diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 3899c9a40..8b420a13b 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -246,6 +246,11 @@ spec: description: CloudKittyProcStatus defines the observed state of CloudKitty Processor properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash - hash of the input secrets that the child has + fully rolled out, used by the parent to track credential rotation convergence + type: string conditions: description: Conditions items: diff --git a/api/go.mod b/api/go.mod index 5bc90b576..12bb0833d 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,8 +4,8 @@ go 1.26.3 require ( github.com/onsi/gomega v1.42.1 - github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 + github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 github.com/rhobs/observability-operator v1.0.0 k8s.io/api v0.33.13 k8s.io/apimachinery v0.33.13 diff --git a/api/go.sum b/api/go.sum index 4b166f360..fe3e929d4 100644 --- a/api/go.sum +++ b/api/go.sum @@ -72,14 +72,14 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= -github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= +github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce h1:XLbNNe5V6mxfJF+ETXlwgOuMAwVpidXNyBU5HD35AAQ= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce/go.mod h1:XCImRHgk9LhAWyGEYihqOPubYCNwosqe/toIylOaFAc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 h1:9YpUAOFXCC94SwoWBRrhFyCVbbwnHTce6/j3CqjxaGM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:LY+Q6ms3A6E8DAKCpmvQaUxO5O2CKYm0XMcmTZ6PCcM= +github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31 h1:DdvjIeuZcmcCj0Gaey10JSYXPUJpOVsFqk5x3aHtbrM= +github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31/go.mod h1:mrcugf1qcRO26DeSoMZFtxxZWW9a4nNk15MdBDFNP6Y= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 h1:zDxe4n0GPQ+U8VFZyyCoQed+9Xauz4G22J/X6B4gq20= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:vDcTSyaI7R6tjtZOqonr0hzq8JMOfBhJCWZy0tVjQPw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/api/v1beta1/cloudkittyapi_types.go b/api/v1beta1/cloudkittyapi_types.go index 89781335a..f26c4653c 100644 --- a/api/v1beta1/cloudkittyapi_types.go +++ b/api/v1beta1/cloudkittyapi_types.go @@ -110,6 +110,10 @@ type CloudKittyAPIStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash - hash of the input secrets that the child has + // fully rolled out, used by the parent to track credential rotation convergence + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go index ed07c978a..f51db1af9 100644 --- a/api/v1beta1/cloudkittyproc_types.go +++ b/api/v1beta1/cloudkittyproc_types.go @@ -100,6 +100,10 @@ type CloudKittyProcStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash - hash of the input secrets that the child has + // fully rolled out, used by the parent to track credential rotation convergence + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1beta1/telemetry_consts.go b/api/v1beta1/telemetry_consts.go index ce633a795..795747abf 100644 --- a/api/v1beta1/telemetry_consts.go +++ b/api/v1beta1/telemetry_consts.go @@ -18,6 +18,11 @@ package v1beta1 import "time" const ( + // TelemetryTransportConsumerFinalizer is the consumer finalizer added to + // transport secrets by all telemetry controllers (CloudKitty, Autoscaling, + // Ceilometer) to prevent premature deletion during credential rotation. + TelemetryTransportConsumerFinalizer = "openstack.org/telemetry-transport-consumer" + // DefaultServiceName - DefaultServiceName = "metric-storage" // DefaultPvcStorageRequest - diff --git a/cmd/main.go b/cmd/main.go index f8c45ca4f..75bf87def 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -284,17 +284,19 @@ func main() { os.Exit(1) } if err := (&controller.CeilometerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + APIReader: mgr.GetAPIReader(), }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Ceilometer") os.Exit(1) } if err := (&controller.AutoscalingReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + APIReader: mgr.GetAPIReader(), }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Autoscaling") os.Exit(1) @@ -318,17 +320,19 @@ func main() { os.Exit(1) } if err := (&controller.CloudKittyAPIReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + APIReader: mgr.GetAPIReader(), }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CloudKittyApi") os.Exit(1) } if err := (&controller.CloudKittyProcReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + APIReader: mgr.GetAPIReader(), }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CloudKittyProc") os.Exit(1) @@ -337,6 +341,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Kclient: kclient, + APIReader: mgr.GetAPIReader(), RESTMapper: mgr.GetRESTMapper(), Cache: mgr.GetCache(), }).SetupWithManager(mgr); err != nil { diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml index 061f75e1e..f1a84a8dc 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -425,6 +425,11 @@ spec: type: object description: API endpoints type: object + appliedInputSecretHash: + description: |- + AppliedInputSecretHash - hash of the input secrets that the child has + fully rolled out, used by the parent to track credential rotation convergence + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 3899c9a40..8b420a13b 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -246,6 +246,11 @@ spec: description: CloudKittyProcStatus defines the observed state of CloudKitty Processor properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash - hash of the input secrets that the child has + fully rolled out, used by the parent to track credential rotation convergence + type: string conditions: description: Conditions items: diff --git a/go.mod b/go.mod index 302ae5030..6824a2fbb 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,14 @@ require ( github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7 github.com/onsi/gomega v1.42.1 - github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260803182736-6801ec9b166e - github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce - github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab - github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260806140515-16161b7b6b53 - github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53 - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 - github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701 - github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260806101626-af8d24ac578b + github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260815073700-3979681dc9f0 + github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31 + github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260815074929-4ac69c11afca + github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260824142420-113dc64ecfd4 + github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260824142420-113dc64ecfd4 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 + github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260815095432-ee0e4a46015e + github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260817084829-b98138447c29 github.com/openstack-k8s-operators/telemetry-operator/api v0.0.0-00010101000000-000000000000 github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring v0.77.1-rhobs1 github.com/rhobs/observability-operator v1.0.0 @@ -62,8 +62,8 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift/api v3.9.0+incompatible // indirect - github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391 // indirect - github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391 // indirect + github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4 // indirect + github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect diff --git a/go.sum b/go.sum index c10ed2d50..aca3c7c9c 100644 --- a/go.sum +++ b/go.sum @@ -101,32 +101,32 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWu github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= -github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= +github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/openshift/api v0.0.0-20260710141509-36dec0bfafe4 h1:YVY7Mb0AdJBbNgC/Zz9R/ZwhixOmZNA8P+6ZPaDW/0M= github.com/openshift/api v0.0.0-20260710141509-36dec0bfafe4/go.mod h1:SPLf21TYPipzCO67BURkCfK6dcIIxx0oNRVWaOyRcXM= -github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260803182736-6801ec9b166e h1:hgYwM+1EUBmkLJkmXEj908Zr6AgOop4u09FLr57zZok= -github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260803182736-6801ec9b166e/go.mod h1:6FqIfKGp06VUTkyZGmR4EWr4xrtKiY6eY6qvY3ThL9o= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce h1:XLbNNe5V6mxfJF+ETXlwgOuMAwVpidXNyBU5HD35AAQ= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce/go.mod h1:XCImRHgk9LhAWyGEYihqOPubYCNwosqe/toIylOaFAc= -github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab h1:r1O+mkLoivl3yqljVPJtHYsQ3SeN6ftm7deIt+PR62w= -github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab/go.mod h1:yPuK2VNA98W6kQSo2Qc1lTjGVWJz7iKJveAOlVrgXQQ= -github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260806140515-16161b7b6b53 h1:sHprU8ZFDHbsSRpdUuN0qu/xkylJPVI4V9eBgC2GfTw= -github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:TkZgZE92VUKHHNKXwuV8b7tT9toaxsZkcGffRafKCSA= -github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53 h1:nyN+1pyCXOr+b6D6r4WdoJlrel7kRS4gKY/JYdY2Wdo= -github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:FH/zwNFlfKkJaw4lpLa2gkUtxKT9lp38vO6Lg11AiTU= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 h1:9YpUAOFXCC94SwoWBRrhFyCVbbwnHTce6/j3CqjxaGM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:LY+Q6ms3A6E8DAKCpmvQaUxO5O2CKYm0XMcmTZ6PCcM= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391 h1:cRyVdm28fshWzWkE5xgRYAzlNX4SreIluOL8s2dpm2U= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391/go.mod h1:Q5wcK4PnlDJ55H+M38CFNLzL53OHYiwq81mR/gKtJXs= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391 h1:AJEo8hauBmqrT9v//s2S5LbEb5BDKa8cQNFG5lAznXI= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391/go.mod h1:fJ9CCRKpo8mQODfv4UDJ3tjUTbyCznYslXQwCkDaGg0= -github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701 h1:rYJayTvo7sE3F1NsjLqbRzd/F9RjymfpCGvqYZrj/q8= -github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701/go.mod h1:e/G0+fuofGEMmE1v6RWNVn9cImJosyQriG3j4NABZwQ= -github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260806101626-af8d24ac578b h1:F2VJiWumzHrI2+BuqbJJUo8/+5za+hJxLR14/Z7PCsU= -github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260806101626-af8d24ac578b/go.mod h1:Xly5JV+07vXpJh5O6SB3j/5Lp+JJ1NdOr5bp/5GKWao= +github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260815073700-3979681dc9f0 h1:biAk3mQLXWklzeq8HvygsMowQyjY5pQ4/baQ2hYzW/c= +github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260815073700-3979681dc9f0/go.mod h1:eLXwcrjmUiO6twgdSeOXihCJsW4mpVarup+//uRf64Q= +github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31 h1:DdvjIeuZcmcCj0Gaey10JSYXPUJpOVsFqk5x3aHtbrM= +github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31/go.mod h1:mrcugf1qcRO26DeSoMZFtxxZWW9a4nNk15MdBDFNP6Y= +github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260815074929-4ac69c11afca h1:xC0DvwZdiVFM05+7IsSrSX8MnZ4XPVkWKITWlygoHWM= +github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260815074929-4ac69c11afca/go.mod h1:+C11m+ufdqsK777ql66CbBVoiDXKa7u5dhyEaSuZqp8= +github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260824142420-113dc64ecfd4 h1:1qH+Pd3Oe1GXfFsPSqtzHdAs2fHAZCk/132ReoIvrzs= +github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:TkZgZE92VUKHHNKXwuV8b7tT9toaxsZkcGffRafKCSA= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260824142420-113dc64ecfd4 h1:67rKzegmcbpTKUClL97vZEjij+0WhdfPO3+6IFkYt3c= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:58CB6+Dq4B7NYPmVz4S60N4/pTt70ZFABrkS/1w7tDA= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 h1:zDxe4n0GPQ+U8VFZyyCoQed+9Xauz4G22J/X6B4gq20= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:vDcTSyaI7R6tjtZOqonr0hzq8JMOfBhJCWZy0tVjQPw= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4 h1:cYQFMrD6zcVsyCXExm07a6wuVLMJmXSNQNaBK23Lu3M= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:Q5wcK4PnlDJ55H+M38CFNLzL53OHYiwq81mR/gKtJXs= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4 h1:atoyW+P7wLeLpKdfWG4vtBBErRVg23+0l3ZAM+cHibo= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:fJ9CCRKpo8mQODfv4UDJ3tjUTbyCznYslXQwCkDaGg0= +github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260815095432-ee0e4a46015e h1:3ya0F++4D3LngM1KBEgeMO8Ikk/9PRgv4mpAACqn/YY= +github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260815095432-ee0e4a46015e/go.mod h1:f3KPR3QM0jPbWGhXougv1BxnW+Bnc2vLE4TvBlYTLdo= +github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260817084829-b98138447c29 h1:ywk/LA00he4oZP5gZGmbhwh5O2kmHAyl1DRc/2JW0Vg= +github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260817084829-b98138447c29/go.mod h1:WsxYq/XES1hiBp5yqMU4Mfes9Bf/q1RuMYyRBWA3FDE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/internal/controller/aodh_controller.go b/internal/controller/aodh_controller.go index fd14fe650..a61d4652f 100644 --- a/internal/controller/aodh_controller.go +++ b/internal/controller/aodh_controller.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -360,12 +361,28 @@ func (r *AutoscalingReconciler) reconcileNormalAodh( return ctrl.Result{}, err } - if sfset.GetStatefulSet().Generation == sfset.GetStatefulSet().Status.ObservedGeneration { - instance.Status.ReadyCount = sfset.GetStatefulSet().Status.ReadyReplicas - if instance.Status.ReadyCount > 0 { + ssData := sfset.GetStatefulSet() + if ssData.Generation == ssData.Status.ObservedGeneration { + instance.Status.ReadyCount = ssData.Status.ReadyReplicas + ready := false + if statefulset.IsReady(ssData) { + ready, err = statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err + } + } + if ready { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } instance.Status.Networks = instance.Spec.Aodh.NetworkAttachmentDefinitions + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } // diff --git a/internal/controller/autoscaling_controller.go b/internal/controller/autoscaling_controller.go index 446d50cfa..e1f21e8a6 100644 --- a/internal/controller/autoscaling_controller.go +++ b/internal/controller/autoscaling_controller.go @@ -52,6 +52,7 @@ import ( env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + "github.com/openstack-k8s-operators/lib-common/modules/common/object" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" service "github.com/openstack-k8s-operators/lib-common/modules/common/service" @@ -68,17 +69,15 @@ import ( autoscaling "github.com/openstack-k8s-operators/telemetry-operator/internal/autoscaling" ) -// ErrNotificationsURLSecretNotSet is returned when NotificationsURLSecret is not set in the instance status -var ErrNotificationsURLSecretNotSet = errors.New("NotificationsURLSecret is not set") - // ErrRabbitMQConfigNil is returned when rabbitmqConfig parameter is nil var ErrRabbitMQConfigNil = errors.New("rabbitmqConfig is nil - NotificationsBus must be configured") // AutoscalingReconciler reconciles a Autoscaling object type AutoscalingReconciler struct { client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Scheme *runtime.Scheme + APIReader client.Reader } // GetLogger returns a logger object with a prefix of "conroller.name" and aditional controller context fields @@ -275,11 +274,36 @@ func (r *AutoscalingReconciler) reconcileDelete( instance.Status.ApplicationCredentialSecret, instance.Spec.Aodh.Auth.ApplicationCredentialSecret, } { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, secretName, autoscaling.ACConsumerFinalizer); err != nil { return ctrl.Result{}, err } } + // Collect transport secret names from all sources to handle mid-rotation + // deletes. Status holds the old secret; the live TransportURL CR holds + // the new one. We must remove the consumer finalizer from both. + transportSecrets := map[string]struct{}{} + if instance.Status.NotificationsURLSecret != nil && *instance.Status.NotificationsURLSecret != "" { + transportSecrets[*instance.Status.NotificationsURLSecret] = struct{}{} + } + notifTransportURL := &rabbitmqv1.TransportURL{} + err = helper.GetClient().Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf("%s-notifications-transport", autoscaling.ServiceName), + Namespace: instance.Namespace, + }, notifTransportURL) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil && notifTransportURL.Status.SecretName != "" { + transportSecrets[notifTransportURL.Status.SecretName] = struct{}{} + } + for secretName := range transportSecrets { + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, + secretName, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", autoscaling.ServiceName)) @@ -333,9 +357,8 @@ func (r *AutoscalingReconciler) reconcileNormal( // secret. Old secret finalizer removal is deferred until all // sub-conditions are true (see late phase below). if instance.Spec.Aodh.Auth.ApplicationCredentialSecret != "" { - if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace, + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, instance.Spec.Aodh.Auth.ApplicationCredentialSecret, - "", autoscaling.ACConsumerFinalizer); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -371,9 +394,7 @@ func (r *AutoscalingReconciler) reconcileNormal( Log.Info(fmt.Sprintf("NotificationBusInstanceURL %s successfully reconciled - operation: %s", notificationBusInstanceURL.Name, string(op))) } - instance.Status.NotificationsURLSecret = ¬ificationBusInstanceURL.Status.SecretName - - if instance.Status.NotificationsURLSecret == nil || *instance.Status.NotificationsURLSecret == "" { + if notificationBusInstanceURL.Status.SecretName == "" { Log.Info(fmt.Sprintf("Waiting for NotificationBusInstanceURL %s secret to be created", notificationBusInstanceURL.Name)) instance.Status.Conditions.Set(condition.FalseCondition( condition.NotificationBusInstanceReadyCondition, @@ -383,6 +404,25 @@ func (r *AutoscalingReconciler) reconcileNormal( return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } + oldNotifSecret := "" + if instance.Status.NotificationsURLSecret != nil { + oldNotifSecret = *instance.Status.NotificationsURLSecret + } + currentNotifSecret := notificationBusInstanceURL.Status.SecretName + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + currentNotifSecret, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + + notifRotationFinalized := false + instance.Status.NotificationsURLSecret = ¤tNotifSecret + defer func() { + if !notifRotationFinalized { + instance.Status.NotificationsURLSecret = &oldNotifSecret + } + }() + instance.Status.Conditions.MarkTrue(condition.NotificationBusInstanceReadyCondition, condition.NotificationBusInstanceReadyMessage) // end notificationsBus @@ -502,7 +542,7 @@ func (r *AutoscalingReconciler) reconcileNormal( // check for required NotificationsBus TransportURL secret holding transport URL string // Aodh only uses NotificationsBus, not MessagingBus // - if instance.Status.NotificationsURLSecret == nil || *instance.Status.NotificationsURLSecret == "" { + if notificationBusInstanceURL.Status.SecretName == "" { Log.Info("NotificationsURLSecret not yet available") instance.Status.Conditions.Set(condition.FalseCondition( condition.NotificationBusInstanceReadyCondition, @@ -523,7 +563,7 @@ func (r *AutoscalingReconciler) reconcileNormal( ctx, types.NamespacedName{ Namespace: instance.Namespace, - Name: *instance.Status.NotificationsURLSecret, + Name: notificationBusInstanceURL.Status.SecretName, }, transportValidateFields, helper.GetClient(), @@ -604,7 +644,7 @@ func (r *AutoscalingReconciler) reconcileNormal( // - %-scripts configmap holding scripts to e.g. bootstrap the service // - %-config configmap holding minimal autoscaling config required to get the service up, user can add additional files to be added to the service // - err = r.generateServiceConfig(ctx, helper, instance, &configMapVars, memcached, db) + err = r.generateServiceConfig(ctx, helper, instance, &configMapVars, memcached, db, notificationBusInstanceURL.Status.SecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -689,7 +729,7 @@ func (r *AutoscalingReconciler) reconcileNormal( } // run check custom configs secret - end - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, instance, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( @@ -699,10 +739,6 @@ func (r *AutoscalingReconciler) reconcileNormal( condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } instance.Status.Hash[common.InputHashName] = inputHash @@ -723,20 +759,59 @@ func (r *AutoscalingReconciler) reconcileNormal( return ctrlResult, err } - // Late phase of the AC split pattern: remove the old AC secret's - // finalizer and update status only after all sub-conditions are true. - isACRotation := instance.Status.ApplicationCredentialSecret != "" && - instance.Status.ApplicationCredentialSecret != instance.Spec.Aodh.Auth.ApplicationCredentialSecret - if isACRotation { - if instance.Status.Conditions.AllSubConditionIsTrue() { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, - instance.Status.ApplicationCredentialSecret, autoscaling.ACConsumerFinalizer); err != nil { - return ctrl.Result{}, err - } - instance.Status.ApplicationCredentialSecret = instance.Spec.Aodh.Auth.ApplicationCredentialSecret - } - } else { - instance.Status.ApplicationCredentialSecret = instance.Spec.Aodh.Auth.ApplicationCredentialSecret + // Guard the rotation on the sub-conditions computed during this reconcile, + // not instance.IsReady(): the Ready condition is reset to Unknown by + // Conditions.Init() at the top of every reconcile and only recomputed in + // the deferred PatchInstance, so IsReady() would always be false here. + guardReady := instance.Status.Conditions.AllSubConditionIsTrue() + + acSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Aodh.Auth.ApplicationCredentialSecret, + autoscaling.ACConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.ApplicationCredentialSecret = acSecretName + + secretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + oldNotifSecret, + currentNotifSecret, + telemetryv1.TelemetryTransportConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.NotificationsURLSecret = &secretName + notifRotationFinalized = true + + // Self-heal consumer finalizers stranded on secrets superseded during + // rapid rotation (A -> B -> C before the workload became ready): + // FinalizeSecretRotation only ever releases the single tracked "old" + // secret, so any intermediate secret's finalizer would otherwise leak. + // keep enumerates every secret that legitimately still holds the + // finalizer; all others in the namespace are pruned. + notifKeep := "" + if instance.Status.NotificationsURLSecret != nil { + notifKeep = *instance.Status.NotificationsURLSecret + } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, telemetryv1.TelemetryTransportConsumerFinalizer, + notifKeep, currentNotifSecret, + ); err != nil { + return ctrl.Result{}, err + } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, autoscaling.ACConsumerFinalizer, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Aodh.Auth.ApplicationCredentialSecret, + ); err != nil { + return ctrl.Result{}, err } if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -778,6 +853,7 @@ func (r *AutoscalingReconciler) generateServiceConfig( envVars *map[string]env.Setter, mc *memcachedv1.Memcached, db *mariadbv1.Database, + notificationSecretName string, ) error { Log := r.GetLogger(ctx) cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(autoscaling.ServiceName), map[string]string{}) @@ -808,12 +884,7 @@ func (r *AutoscalingReconciler) generateServiceConfig( return err } - // Ensure NotificationsURLSecret is not nil before dereferencing - if instance.Status.NotificationsURLSecret == nil { - return ErrNotificationsURLSecretNotSet - } - - transportURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } @@ -897,8 +968,8 @@ func (r *AutoscalingReconciler) generateServiceConfig( // Add NotificationsURL if configured // Always get the separate notification secret since we always create separate TransportURLs - if instance.Status.NotificationsURLSecret != nil && *instance.Status.NotificationsURLSecret != "" { - notificationInstanceURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + if notificationSecretName != "" { + notificationInstanceURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } diff --git a/internal/controller/ceilometer_controller.go b/internal/controller/ceilometer_controller.go index 9bfcfb259..c08203536 100644 --- a/internal/controller/ceilometer_controller.go +++ b/internal/controller/ceilometer_controller.go @@ -50,6 +50,7 @@ import ( env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + "github.com/openstack-k8s-operators/lib-common/modules/common/object" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" statefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" @@ -76,8 +77,9 @@ const ( // CeilometerReconciler reconciles a Ceilometer object type CeilometerReconciler struct { client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Scheme *runtime.Scheme + APIReader client.Reader } // GetLogger returns a logger object with a prefix of "conroller.name" and aditional controller context fields @@ -414,12 +416,37 @@ func (r *CeilometerReconciler) reconcileDelete(ctx context.Context, instance *te instance.Status.ApplicationCredentialSecret, instance.Spec.Auth.ApplicationCredentialSecret, } { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, secretName, ceilometer.ACConsumerFinalizer); err != nil { return ctrl.Result{}, err } } + // Collect transport secret names from all sources to handle mid-rotation + // deletes. Status holds the old secret; the live TransportURL CR holds + // the new one. We must remove the consumer finalizer from both. + transportSecrets := map[string]struct{}{} + if instance.Status.NotificationsURLSecret != nil && *instance.Status.NotificationsURLSecret != "" { + transportSecrets[*instance.Status.NotificationsURLSecret] = struct{}{} + } + notifTransportURL := &rabbitmqv1.TransportURL{} + err = helper.GetClient().Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf("%s-notifications-transport", ceilometer.ServiceName), + Namespace: instance.Namespace, + }, notifTransportURL) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil && notifTransportURL.Status.SecretName != "" { + transportSecrets[notifTransportURL.Status.SecretName] = struct{}{} + } + for secretName := range transportSecrets { + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, + secretName, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", ceilometer.ServiceName)) @@ -563,9 +590,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( Log.Info(fmt.Sprintf("NotificationBusInstanceURL %s successfully reconciled - operation: %s", notificationBusInstanceURL.Name, string(op))) } - instance.Status.NotificationsURLSecret = ¬ificationBusInstanceURL.Status.SecretName - - if instance.Status.NotificationsURLSecret == nil || *instance.Status.NotificationsURLSecret == "" { + if notificationBusInstanceURL.Status.SecretName == "" { Log.Info(fmt.Sprintf("Waiting for NotificationBusInstanceURL %s secret to be created", notificationBusInstanceURL.Name)) instance.Status.Conditions.Set(condition.FalseCondition( condition.NotificationBusInstanceReadyCondition, @@ -575,6 +600,25 @@ func (r *CeilometerReconciler) reconcileCeilometer( return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } + oldNotifSecret := "" + if instance.Status.NotificationsURLSecret != nil { + oldNotifSecret = *instance.Status.NotificationsURLSecret + } + currentNotifSecret := notificationBusInstanceURL.Status.SecretName + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + currentNotifSecret, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + + notifRotationFinalized := false + instance.Status.NotificationsURLSecret = ¤tNotifSecret + defer func() { + if !notifRotationFinalized { + instance.Status.NotificationsURLSecret = &oldNotifSecret + } + }() + instance.Status.Conditions.MarkTrue(condition.NotificationBusInstanceReadyCondition, condition.NotificationBusInstanceReadyMessage) // end notificationsBus @@ -688,7 +732,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( // - %-config configmap holding minimal ceilometer config required to get the service up, user can add additional files to be added to the service // - parameters which has passwords gets added from the OpenStack secret via the init container // - err = r.generateServiceConfig(ctx, helper, instance, &configMapVars) + err = r.generateServiceConfig(ctx, helper, instance, &configMapVars, notificationBusInstanceURL.Status.SecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -705,7 +749,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( // - %-config configmap holding minimal ceilometer-compute config required to get the service up, user can add additional files to be added to the service // - parameters which has passwords gets added from the OpenStack secret via the init container // - err = r.generateComputeServiceConfig(ctx, helper, instance, &configMapVars) + err = r.generateComputeServiceConfig(ctx, helper, instance, &configMapVars, notificationBusInstanceURL.Status.SecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -745,7 +789,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, &instance.Status.Hash, configMapVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, &instance.Status.Hash, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -754,10 +798,6 @@ func (r *CeilometerReconciler) reconcileCeilometer( condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } instance.Status.Hash[common.InputHashName] = inputHash @@ -766,9 +806,8 @@ func (r *CeilometerReconciler) reconcileCeilometer( // secret. Old secret finalizer removal is deferred until all // sub-conditions are true (see late phase below). if instance.Spec.Auth.ApplicationCredentialSecret != "" { - if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace, + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, instance.Spec.Auth.ApplicationCredentialSecret, - "", ceilometer.ACConsumerFinalizer); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -842,33 +881,28 @@ func (r *CeilometerReconciler) reconcileCeilometer( // Evaluate the last part of the reconciliation only if we see the last // version of the CR - if sfset.GetStatefulSet().Generation == sfset.GetStatefulSet().Status.ObservedGeneration { - instance.Status.ReadyCount = sfset.GetStatefulSet().Status.ReadyReplicas + ssData := sfset.GetStatefulSet() + if ssData.Generation == ssData.Status.ObservedGeneration { + instance.Status.ReadyCount = ssData.Status.ReadyReplicas instance.Status.Networks = instance.Spec.NetworkAttachmentDefinitions - svc, op, err := ceilometer.Service(instance, helper, ceilometer.CeilometerPrometheusPort, serviceLabels) + svc, svcOp, err := ceilometer.Service(instance, helper, ceilometer.CeilometerPrometheusPort, serviceLabels) if err != nil { return ctrl.Result{}, err } - if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf(msgOperation, svc.Name, string(op))) - } - if instance.Status.ReadyCount > 0 { - instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + if svcOp != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf(msgOperation, svc.Name, string(svcOp))) } - // Late phase of the AC split pattern: remove the old AC secret's - // finalizer and update status only after all sub-conditions are true. - isACRotation := instance.Status.ApplicationCredentialSecret != "" && - instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret - if isACRotation { - if instance.Status.Conditions.AllSubConditionIsTrue() { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, - instance.Status.ApplicationCredentialSecret, ceilometer.ACConsumerFinalizer); err != nil { - return ctrl.Result{}, err - } - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + ready := false + if statefulset.IsReady(ssData) { + ready, err = statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err } - } else { - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + } + if ready { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -876,7 +910,69 @@ func (r *CeilometerReconciler) reconcileCeilometer( condition.ReadyCondition, condition.ReadyMessage) } Log.Info(fmt.Sprintf(msgReconcileSuccess, ceilometer.ServiceName)) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + } + + // Guard the rotation on the sub-conditions computed during this reconcile, + // not instance.IsReady(): the Ready condition is reset to Unknown by + // Conditions.Init() at the top of every reconcile and only recomputed in + // the deferred PatchInstance, so IsReady() would always be false here. + guardReady := instance.Status.Conditions.AllSubConditionIsTrue() + + acSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + ceilometer.ACConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.ApplicationCredentialSecret = acSecretName + + secretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + oldNotifSecret, + currentNotifSecret, + telemetryv1.TelemetryTransportConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.NotificationsURLSecret = &secretName + notifRotationFinalized = true + + // Self-heal consumer finalizers stranded on secrets superseded during + // rapid rotation (A -> B -> C before the workload became ready): + // FinalizeSecretRotation only ever releases the single tracked "old" + // secret, so any intermediate secret's finalizer would otherwise leak. + // keep enumerates every secret that legitimately still holds the + // finalizer; all others in the namespace are pruned. + notifKeep := "" + if instance.Status.NotificationsURLSecret != nil { + notifKeep = *instance.Status.NotificationsURLSecret + } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, telemetryv1.TelemetryTransportConsumerFinalizer, + notifKeep, currentNotifSecret, + ); err != nil { + return ctrl.Result{}, err } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, ceilometer.ACConsumerFinalizer, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil } @@ -983,7 +1079,7 @@ func (r *CeilometerReconciler) reconcileMysqldExporter( // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, &instance.Status.MysqldExporterHash, configMapVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, &instance.Status.MysqldExporterHash, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( telemetryv1.MysqldExporterServiceConfigReadyCondition, @@ -992,10 +1088,6 @@ func (r *CeilometerReconciler) reconcileMysqldExporter( condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } instance.Status.MysqldExporterHash[common.InputHashName] = inputHash @@ -1059,8 +1151,9 @@ func (r *CeilometerReconciler) reconcileMysqldExporter( // Evaluate the last part of the reconciliation only if we see the last // version of the CR - if sfset.GetStatefulSet().Generation == sfset.GetStatefulSet().Status.ObservedGeneration { - instance.Status.MysqldExporterReadyCount = sfset.GetStatefulSet().Status.ReadyReplicas + ssData := sfset.GetStatefulSet() + if ssData.Generation == ssData.Status.ObservedGeneration { + instance.Status.MysqldExporterReadyCount = ssData.Status.ReadyReplicas svc, op, err := mysqldexporter.Service(instance, helper, serviceLabels) if err != nil { return ctrl.Result{}, err @@ -1068,8 +1161,16 @@ func (r *CeilometerReconciler) reconcileMysqldExporter( if op != controllerutil.OperationResultNone { Log.Info(fmt.Sprintf(msgOperation, svc.Name, string(op))) } - if instance.Status.MysqldExporterReadyCount > 0 { - instance.Status.Conditions.MarkTrue(telemetryv1.MysqldExporterDeploymentReadyCondition, condition.DeploymentReadyMessage) + if statefulset.IsReady(ssData) { + ready, err := statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err + } + if ready { + instance.Status.Conditions.MarkTrue(telemetryv1.MysqldExporterDeploymentReadyCondition, condition.DeploymentReadyMessage) + } } Log.Info(fmt.Sprintf(msgReconcileSuccess, mysqldexporter.ServiceName)) } @@ -1170,7 +1271,7 @@ func (r *CeilometerReconciler) reconcileKSM( // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, &instance.Status.KSMHash, configMapVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, &instance.Status.KSMHash, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( telemetryv1.KSMServiceConfigReadyCondition, @@ -1179,8 +1280,6 @@ func (r *CeilometerReconciler) reconcileKSM( condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - return ctrl.Result{}, nil } instance.Status.KSMHash[common.InputHashName] = inputHash @@ -1238,8 +1337,8 @@ func (r *CeilometerReconciler) reconcileKSM( // version of the CR ssobj := ss.GetStatefulSet() if ssobj.Generation == ssobj.Status.ObservedGeneration { - instance.Status.KSMReadyCount = ss.GetStatefulSet().Status.ReadyReplicas - if instance.Status.KSMReadyCount > 0 { + instance.Status.KSMReadyCount = ssobj.Status.ReadyReplicas + if statefulset.IsReady(ssobj) { instance.Status.Conditions.MarkTrue(telemetryv1.KSMDeploymentReadyCondition, condition.DeploymentReadyMessage) } @@ -1274,6 +1373,7 @@ func (r *CeilometerReconciler) generateServiceConfig( h *helper.Helper, instance *telemetryv1.Ceilometer, envVars *map[string]env.Setter, + notificationSecretName string, ) error { cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(ceilometer.ServiceName), map[string]string{}) customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} @@ -1289,12 +1389,7 @@ func (r *CeilometerReconciler) generateServiceConfig( return err } - // Ensure NotificationsURLSecret is not nil before dereferencing - if instance.Status.NotificationsURLSecret == nil { - return ErrNotificationsURLSecretNotSet - } - - transportURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } @@ -1371,8 +1466,8 @@ func (r *CeilometerReconciler) generateServiceConfig( // Add NotificationsURL if configured // Always get the separate notification secret since we always create separate TransportURLs - if instance.Status.NotificationsURLSecret != nil && *instance.Status.NotificationsURLSecret != "" { - notificationInstanceURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + if notificationSecretName != "" { + notificationInstanceURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } @@ -1413,6 +1508,7 @@ func (r *CeilometerReconciler) generateComputeServiceConfig( h *helper.Helper, instance *telemetryv1.Ceilometer, envVars *map[string]env.Setter, + notificationSecretName string, ) error { cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(ceilometer.ComputeServiceName), map[string]string{}) ipmiLabels := labels.GetLabels(instance, labels.GetGroupLabel(ceilometer.IpmiServiceName), map[string]string{}) @@ -1429,12 +1525,7 @@ func (r *CeilometerReconciler) generateComputeServiceConfig( return err } - // Ensure NotificationsURLSecret is not nil before dereferencing - if instance.Status.NotificationsURLSecret == nil { - return ErrNotificationsURLSecretNotSet - } - - transportURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } diff --git a/internal/controller/cloudkitty_controller.go b/internal/controller/cloudkitty_controller.go index 610fb25da..d1ce8dd4b 100644 --- a/internal/controller/cloudkitty_controller.go +++ b/internal/controller/cloudkitty_controller.go @@ -59,6 +59,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/job" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/object" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" @@ -520,12 +521,37 @@ func (r *CloudKittyReconciler) reconcileDelete(ctx context.Context, instance *te instance.Status.ApplicationCredentialSecret, instance.Spec.Auth.ApplicationCredentialSecret, } { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, secretName, cloudkitty.ACConsumerFinalizer); err != nil { return ctrl.Result{}, err } } + // Collect transport secret names from all sources to handle mid-rotation + // deletes. Status holds the old secret; the live TransportURL CR holds + // the new one. We must remove the consumer finalizer from both. + transportSecrets := map[string]struct{}{} + if instance.Status.TransportURLSecret != "" { + transportSecrets[instance.Status.TransportURLSecret] = struct{}{} + } + rpcTransportURL := &rabbitmqv1.TransportURL{} + err = helper.GetClient().Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf("%s-transport", instance.Name), + Namespace: instance.Namespace, + }, rpcTransportURL) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil && rpcTransportURL.Status.SecretName != "" { + transportSecrets[rpcTransportURL.Status.SecretName] = struct{}{} + } + for secretName := range transportSecrets { + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, + secretName, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) @@ -804,6 +830,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te err.Error())) return ctrl.Result{}, err } + if op != controllerutil.OperationResultNone { Log.Info(fmt.Sprintf("LokiStack %s successfully changed - operation: %s", lokiStack.Name, string(op))) } @@ -860,7 +887,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te // create RabbitMQ transportURL CR and get the actual URL from the associated secret that is created // - transportURL, op, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels) + transportURL, transportURLOp, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.RabbitMqTransportURLReadyCondition, @@ -871,13 +898,11 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te return ctrl.Result{}, err } - if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + if transportURLOp != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(transportURLOp))) } - instance.Status.TransportURLSecret = transportURL.Status.SecretName - - if instance.Status.TransportURLSecret == "" { + if transportURL.Status.SecretName == "" { Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) instance.Status.Conditions.Set(condition.FalseCondition( condition.RabbitMqTransportURLReadyCondition, @@ -887,6 +912,21 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te return cloudkitty.ResultRequeue, nil } + currentTransportSecret := transportURL.Status.SecretName + + // Set status early for first-time setup so PatchInstance persists it + // even on early returns. During rotation (old != current), the status + // is only updated by FinalizeSecretRotation at end of reconcile. + if instance.Status.TransportURLSecret == "" || + instance.Status.TransportURLSecret == currentTransportSecret { + instance.Status.TransportURLSecret = currentTransportSecret + } + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + currentTransportSecret, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) // end transportURL - CloudKitty only uses RPC Messaging Bus, not notifications @@ -995,7 +1035,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te // // Create Secrets required as input for the Service and calculate an overall hash of hashes // - err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db) + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db, transportURL.Status.SecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1010,7 +1050,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - _, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + _, _, err = r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1019,25 +1059,14 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) - instance.Status.Conditions.MarkFalse( - condition.ServiceConfigReadyCondition, - condition.InitReason, - condition.SeverityInfo, - condition.ServiceConfigReadyInitMessage) - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } // Early phase of the AC split pattern: only add finalizer to the new // secret. Old secret finalizer removal is deferred until all // sub-conditions are true (see late phase below). if instance.Spec.Auth.ApplicationCredentialSecret != "" { - if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace, + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, instance.Spec.Auth.ApplicationCredentialSecret, - "", cloudkitty.ACConsumerFinalizer); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1106,8 +1135,19 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te // normal reconcile tasks // + // Compute expected input-secret hash from rotating secrets so children + // can track whether they have converged on the latest credential set. + rotatingSecrets := []string{currentTransportSecret} + if instance.Spec.Auth.ApplicationCredentialSecret != "" { + rotatingSecrets = append(rotatingSecrets, instance.Spec.Auth.ApplicationCredentialSecret) + } + expectedInputHash, err := util.ObjectHash(rotatingSecrets) + if err != nil { + return ctrl.Result{}, err + } + // deploy cloudkitty-api - cloudKittyAPI, op, err := r.apiDeploymentCreateOrUpdate(ctx, instance) + cloudKittyAPI, apiOp, err := r.apiDeploymentCreateOrUpdate(ctx, instance, transportURL.Status.SecretName, expectedInputHash) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( telemetryv1.CloudKittyAPIReadyCondition, @@ -1117,12 +1157,12 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te err.Error())) return ctrl.Result{}, err } - if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf("API CR for %s successfully %s", instance.Name, string(op))) + if apiOp != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("API CR for %s successfully %s", instance.Name, string(apiOp))) } - - // Mirror values when the data in the StatefulSet is for the current generation - if cloudKittyAPI.Generation == cloudKittyAPI.Status.ObservedGeneration { + apiReady := false + if cloudKittyAPI.Generation == cloudKittyAPI.Status.ObservedGeneration && + cloudKittyAPI.Status.AppliedInputSecretHash == expectedInputHash { // Mirror CloudKittyAPI status' APIEndpoints and ReadyCount to this parent CR instance.Status.APIEndpoints = cloudKittyAPI.Status.APIEndpoints instance.Status.ServiceIDs = cloudKittyAPI.Status.ServiceIDs @@ -1133,10 +1173,17 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te if c != nil { instance.Status.Conditions.Set(c) } + apiReady = cloudKittyAPI.IsReady() + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyAPIReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } // deploy CloudKitty Processor - cloudKittyProc, op, err := r.procDeploymentCreateOrUpdate(ctx, instance) + cloudKittyProc, procOp, err := r.procDeploymentCreateOrUpdate(ctx, instance, transportURL.Status.SecretName, expectedInputHash) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( telemetryv1.CloudKittyProcReadyCondition, @@ -1146,12 +1193,12 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te err.Error())) return ctrl.Result{}, err } - if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf("Scheduler CR for %s successfully %s", instance.Name, string(op))) + if procOp != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("Scheduler CR for %s successfully %s", instance.Name, string(procOp))) } - - // Mirror values when the data in the StatefulSet is for the current generation - if cloudKittyProc.Generation == cloudKittyProc.Status.ObservedGeneration { + procReady := false + if cloudKittyProc.Generation == cloudKittyProc.Status.ObservedGeneration && + cloudKittyProc.Status.AppliedInputSecretHash == expectedInputHash { // Mirror CloudKitty Processor status' ReadyCount to this parent CR instance.Status.CloudKittyProcReadyCount = cloudKittyProc.Status.ReadyCount @@ -1160,6 +1207,13 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te if c != nil { instance.Status.Conditions.Set(c) } + procReady = cloudKittyProc.IsReady() + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyProcReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, helper, cloudkitty.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) @@ -1167,20 +1221,50 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te return ctrl.Result{}, err } - // Late phase of the AC split pattern: remove the old AC secret's - // finalizer and update status only after all sub-conditions are true. - isACRotation := instance.Status.ApplicationCredentialSecret != "" && - instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret - if isACRotation { - if instance.Status.Conditions.AllSubConditionIsTrue() { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, - instance.Status.ApplicationCredentialSecret, cloudkitty.ACConsumerFinalizer); err != nil { - return ctrl.Result{}, err - } - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret - } - } else { - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + allServicesReady := apiReady && procReady + + acSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + cloudkitty.ACConsumerFinalizer, + allServicesReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.ApplicationCredentialSecret = acSecretName + + secretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.TransportURLSecret, + currentTransportSecret, + telemetryv1.TelemetryTransportConsumerFinalizer, + allServicesReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.TransportURLSecret = secretName + + // Self-heal consumer finalizers stranded on secrets superseded during + // rapid rotation (A -> B -> C before the workload became ready): + // FinalizeSecretRotation only ever releases the single tracked "old" + // secret, so any intermediate secret's finalizer would otherwise leak. + // keep enumerates every secret that legitimately still holds the + // finalizer; all others in the namespace are pruned. + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, telemetryv1.TelemetryTransportConsumerFinalizer, + instance.Status.TransportURLSecret, currentTransportSecret, + ); err != nil { + return ctrl.Result{}, err + } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, cloudkitty.ACConsumerFinalizer, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + ); err != nil { + return ctrl.Result{}, err } Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) @@ -1200,6 +1284,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( serviceLabels map[string]string, memcached *memcachedv1.Memcached, db *mariadbv1.Database, + transportURLSecretName string, ) error { Log := r.GetLogger(ctx) // @@ -1239,7 +1324,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( return err } - transportURLSecret, _, err := secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, transportURLSecretName, instance.Namespace) if err != nil { return err } @@ -1452,12 +1537,12 @@ func (r *CloudKittyReconciler) transportURLCreateOrUpdate( return transportURL, op, err } -func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyAPI, controllerutil.OperationResult, error) { +func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty, transportURLSecretName string, expectedInputHash string) (*telemetryv1.CloudKittyAPI, controllerutil.OperationResult, error) { cloudkittyAPISpec := telemetryv1.CloudKittyAPISpec{ CloudKittyTemplate: instance.Spec.CloudKittyTemplate, CloudKittyAPITemplate: instance.Spec.CloudKittyAPI, DatabaseHostname: instance.Status.DatabaseHostname, - TransportURLSecret: instance.Status.TransportURLSecret, + TransportURLSecret: transportURLSecretName, ServiceAccount: instance.RbacResourceName(), } @@ -1480,6 +1565,11 @@ func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.Spec = cloudkittyAPISpec + if deployment.Annotations == nil { + deployment.Annotations = map[string]string{} + } + deployment.Annotations["openstack.org/transport-url-secret"] = transportURLSecretName + deployment.Annotations["openstack.org/input-secret-hash"] = expectedInputHash err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) if err != nil { @@ -1492,12 +1582,12 @@ func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, return deployment, op, err } -func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyProc, controllerutil.OperationResult, error) { +func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty, transportURLSecretName string, expectedInputHash string) (*telemetryv1.CloudKittyProc, controllerutil.OperationResult, error) { cloudKittyProcSpec := telemetryv1.CloudKittyProcSpec{ CloudKittyTemplate: instance.Spec.CloudKittyTemplate, CloudKittyProcTemplate: instance.Spec.CloudKittyProc, DatabaseHostname: instance.Status.DatabaseHostname, - TransportURLSecret: instance.Status.TransportURLSecret, + TransportURLSecret: transportURLSecretName, ServiceAccount: instance.RbacResourceName(), //TLS: instance.Spec.CloudKittyProc.TLS.Ca, } @@ -1521,6 +1611,11 @@ func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.Spec = cloudKittyProcSpec + if deployment.Annotations == nil { + deployment.Annotations = map[string]string{} + } + deployment.Annotations["openstack.org/transport-url-secret"] = transportURLSecretName + deployment.Annotations["openstack.org/input-secret-hash"] = expectedInputHash err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) if err != nil { diff --git a/internal/controller/cloudkittyapi_controller.go b/internal/controller/cloudkittyapi_controller.go index 8304737a1..9919fd59f 100644 --- a/internal/controller/cloudkittyapi_controller.go +++ b/internal/controller/cloudkittyapi_controller.go @@ -77,8 +77,9 @@ func (r *CloudKittyAPIReconciler) GetScheme() *runtime.Scheme { // CloudKittyAPIReconciler reconciles a CloudKittyAPI object type CloudKittyAPIReconciler struct { client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Scheme *runtime.Scheme + APIReader client.Reader } // GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields @@ -1017,7 +1018,7 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1026,16 +1027,6 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) - instance.Status.Conditions.MarkFalse( - condition.ServiceConfigReadyCondition, - condition.InitReason, - condition.SeverityInfo, - condition.ServiceConfigReadyInitMessage) - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } // Deploy a statefulset @@ -1130,9 +1121,19 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, err } - if instance.Status.ReadyCount > 0 { + expectedHash := instance.GetAnnotations()["openstack.org/input-secret-hash"] + ready := false + if statefulset.IsReady(ss.GetStatefulSet()) { + ready, err = statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err + } + } + if ready { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) - + instance.Status.AppliedInputSecretHash = expectedHash } else if *instance.Spec.Replicas > 0 { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, diff --git a/internal/controller/cloudkittyproc_controller.go b/internal/controller/cloudkittyproc_controller.go index 4054515f1..28ead31b0 100644 --- a/internal/controller/cloudkittyproc_controller.go +++ b/internal/controller/cloudkittyproc_controller.go @@ -73,8 +73,9 @@ func (r *CloudKittyProcReconciler) GetScheme() *runtime.Scheme { // CloudKittyProcReconciler reconciles a CloudKittyProc object type CloudKittyProcReconciler struct { client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Scheme *runtime.Scheme + APIReader client.Reader } // GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields @@ -600,7 +601,7 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -609,16 +610,6 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) - instance.Status.Conditions.MarkFalse( - condition.ServiceConfigReadyCondition, - condition.InitReason, - condition.SeverityInfo, - condition.ServiceConfigReadyInitMessage) - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) @@ -776,8 +767,19 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, err } - if instance.Status.ReadyCount > 0 { + expectedHash := instance.GetAnnotations()["openstack.org/input-secret-hash"] + ready := false + if statefulset.IsReady(ss.GetStatefulSet()) { + ready, err = statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err + } + } + if ready { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + instance.Status.AppliedInputSecretHash = expectedHash } else if *instance.Spec.Replicas > 0 { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 0461d0567..8a496262d 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -42,6 +42,7 @@ type ConditionalWatchingReconciler struct { client.Client Kclient kubernetes.Interface Scheme *runtime.Scheme + APIReader client.Reader Controller controller.Controller Watching []string RESTMapper meta.RESTMapper