diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml index 061f75e1e..f1a84a8dc 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -425,6 +425,11 @@ spec: type: object description: API endpoints type: object + appliedInputSecretHash: + description: |- + AppliedInputSecretHash - hash of the input secrets that the child has + fully rolled out, used by the parent to track credential rotation convergence + type: string conditions: description: Conditions items: diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 3899c9a40..8b420a13b 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -246,6 +246,11 @@ spec: description: CloudKittyProcStatus defines the observed state of CloudKitty Processor properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash - hash of the input secrets that the child has + fully rolled out, used by the parent to track credential rotation convergence + type: string conditions: description: Conditions items: diff --git a/api/go.mod b/api/go.mod index 5bc90b576..12bb0833d 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,8 +4,8 @@ go 1.26.3 require ( github.com/onsi/gomega v1.42.1 - github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 + github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 github.com/rhobs/observability-operator v1.0.0 k8s.io/api v0.33.13 k8s.io/apimachinery v0.33.13 diff --git a/api/go.sum b/api/go.sum index 4b166f360..fe3e929d4 100644 --- a/api/go.sum +++ b/api/go.sum @@ -72,14 +72,14 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= -github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= +github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce h1:XLbNNe5V6mxfJF+ETXlwgOuMAwVpidXNyBU5HD35AAQ= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce/go.mod h1:XCImRHgk9LhAWyGEYihqOPubYCNwosqe/toIylOaFAc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 h1:9YpUAOFXCC94SwoWBRrhFyCVbbwnHTce6/j3CqjxaGM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:LY+Q6ms3A6E8DAKCpmvQaUxO5O2CKYm0XMcmTZ6PCcM= +github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31 h1:DdvjIeuZcmcCj0Gaey10JSYXPUJpOVsFqk5x3aHtbrM= +github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31/go.mod h1:mrcugf1qcRO26DeSoMZFtxxZWW9a4nNk15MdBDFNP6Y= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 h1:zDxe4n0GPQ+U8VFZyyCoQed+9Xauz4G22J/X6B4gq20= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:vDcTSyaI7R6tjtZOqonr0hzq8JMOfBhJCWZy0tVjQPw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/api/v1beta1/cloudkittyapi_types.go b/api/v1beta1/cloudkittyapi_types.go index 89781335a..f26c4653c 100644 --- a/api/v1beta1/cloudkittyapi_types.go +++ b/api/v1beta1/cloudkittyapi_types.go @@ -110,6 +110,10 @@ type CloudKittyAPIStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash - hash of the input secrets that the child has + // fully rolled out, used by the parent to track credential rotation convergence + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go index ed07c978a..f51db1af9 100644 --- a/api/v1beta1/cloudkittyproc_types.go +++ b/api/v1beta1/cloudkittyproc_types.go @@ -100,6 +100,10 @@ type CloudKittyProcStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash - hash of the input secrets that the child has + // fully rolled out, used by the parent to track credential rotation convergence + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1beta1/telemetry_consts.go b/api/v1beta1/telemetry_consts.go index ce633a795..795747abf 100644 --- a/api/v1beta1/telemetry_consts.go +++ b/api/v1beta1/telemetry_consts.go @@ -18,6 +18,11 @@ package v1beta1 import "time" const ( + // TelemetryTransportConsumerFinalizer is the consumer finalizer added to + // transport secrets by all telemetry controllers (CloudKitty, Autoscaling, + // Ceilometer) to prevent premature deletion during credential rotation. + TelemetryTransportConsumerFinalizer = "openstack.org/telemetry-transport-consumer" + // DefaultServiceName - DefaultServiceName = "metric-storage" // DefaultPvcStorageRequest - diff --git a/cmd/main.go b/cmd/main.go index f8c45ca4f..75bf87def 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -284,17 +284,19 @@ func main() { os.Exit(1) } if err := (&controller.CeilometerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + APIReader: mgr.GetAPIReader(), }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Ceilometer") os.Exit(1) } if err := (&controller.AutoscalingReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + APIReader: mgr.GetAPIReader(), }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Autoscaling") os.Exit(1) @@ -318,17 +320,19 @@ func main() { os.Exit(1) } if err := (&controller.CloudKittyAPIReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + APIReader: mgr.GetAPIReader(), }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CloudKittyApi") os.Exit(1) } if err := (&controller.CloudKittyProcReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + APIReader: mgr.GetAPIReader(), }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CloudKittyProc") os.Exit(1) @@ -337,6 +341,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Kclient: kclient, + APIReader: mgr.GetAPIReader(), RESTMapper: mgr.GetRESTMapper(), Cache: mgr.GetCache(), }).SetupWithManager(mgr); err != nil { diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml index 061f75e1e..f1a84a8dc 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -425,6 +425,11 @@ spec: type: object description: API endpoints type: object + appliedInputSecretHash: + description: |- + AppliedInputSecretHash - hash of the input secrets that the child has + fully rolled out, used by the parent to track credential rotation convergence + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 3899c9a40..8b420a13b 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -246,6 +246,11 @@ spec: description: CloudKittyProcStatus defines the observed state of CloudKitty Processor properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash - hash of the input secrets that the child has + fully rolled out, used by the parent to track credential rotation convergence + type: string conditions: description: Conditions items: diff --git a/go.mod b/go.mod index 302ae5030..6824a2fbb 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,14 @@ require ( github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7 github.com/onsi/gomega v1.42.1 - github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260803182736-6801ec9b166e - github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce - github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab - github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260806140515-16161b7b6b53 - github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53 - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 - github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701 - github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260806101626-af8d24ac578b + github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260815073700-3979681dc9f0 + github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31 + github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260815074929-4ac69c11afca + github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260824142420-113dc64ecfd4 + github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260824142420-113dc64ecfd4 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 + github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260815095432-ee0e4a46015e + github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260817084829-b98138447c29 github.com/openstack-k8s-operators/telemetry-operator/api v0.0.0-00010101000000-000000000000 github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring v0.77.1-rhobs1 github.com/rhobs/observability-operator v1.0.0 @@ -62,8 +62,8 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift/api v3.9.0+incompatible // indirect - github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391 // indirect - github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391 // indirect + github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4 // indirect + github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect diff --git a/go.sum b/go.sum index c10ed2d50..aca3c7c9c 100644 --- a/go.sum +++ b/go.sum @@ -101,32 +101,32 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWu github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= -github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= +github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/openshift/api v0.0.0-20260710141509-36dec0bfafe4 h1:YVY7Mb0AdJBbNgC/Zz9R/ZwhixOmZNA8P+6ZPaDW/0M= github.com/openshift/api v0.0.0-20260710141509-36dec0bfafe4/go.mod h1:SPLf21TYPipzCO67BURkCfK6dcIIxx0oNRVWaOyRcXM= -github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260803182736-6801ec9b166e h1:hgYwM+1EUBmkLJkmXEj908Zr6AgOop4u09FLr57zZok= -github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260803182736-6801ec9b166e/go.mod h1:6FqIfKGp06VUTkyZGmR4EWr4xrtKiY6eY6qvY3ThL9o= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce h1:XLbNNe5V6mxfJF+ETXlwgOuMAwVpidXNyBU5HD35AAQ= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce/go.mod h1:XCImRHgk9LhAWyGEYihqOPubYCNwosqe/toIylOaFAc= -github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab h1:r1O+mkLoivl3yqljVPJtHYsQ3SeN6ftm7deIt+PR62w= -github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab/go.mod h1:yPuK2VNA98W6kQSo2Qc1lTjGVWJz7iKJveAOlVrgXQQ= -github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260806140515-16161b7b6b53 h1:sHprU8ZFDHbsSRpdUuN0qu/xkylJPVI4V9eBgC2GfTw= -github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:TkZgZE92VUKHHNKXwuV8b7tT9toaxsZkcGffRafKCSA= -github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53 h1:nyN+1pyCXOr+b6D6r4WdoJlrel7kRS4gKY/JYdY2Wdo= -github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:FH/zwNFlfKkJaw4lpLa2gkUtxKT9lp38vO6Lg11AiTU= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 h1:9YpUAOFXCC94SwoWBRrhFyCVbbwnHTce6/j3CqjxaGM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:LY+Q6ms3A6E8DAKCpmvQaUxO5O2CKYm0XMcmTZ6PCcM= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391 h1:cRyVdm28fshWzWkE5xgRYAzlNX4SreIluOL8s2dpm2U= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391/go.mod h1:Q5wcK4PnlDJ55H+M38CFNLzL53OHYiwq81mR/gKtJXs= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391 h1:AJEo8hauBmqrT9v//s2S5LbEb5BDKa8cQNFG5lAznXI= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391/go.mod h1:fJ9CCRKpo8mQODfv4UDJ3tjUTbyCznYslXQwCkDaGg0= -github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701 h1:rYJayTvo7sE3F1NsjLqbRzd/F9RjymfpCGvqYZrj/q8= -github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701/go.mod h1:e/G0+fuofGEMmE1v6RWNVn9cImJosyQriG3j4NABZwQ= -github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260806101626-af8d24ac578b h1:F2VJiWumzHrI2+BuqbJJUo8/+5za+hJxLR14/Z7PCsU= -github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260806101626-af8d24ac578b/go.mod h1:Xly5JV+07vXpJh5O6SB3j/5Lp+JJ1NdOr5bp/5GKWao= +github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260815073700-3979681dc9f0 h1:biAk3mQLXWklzeq8HvygsMowQyjY5pQ4/baQ2hYzW/c= +github.com/openstack-k8s-operators/heat-operator/api v0.6.1-0.20260815073700-3979681dc9f0/go.mod h1:eLXwcrjmUiO6twgdSeOXihCJsW4mpVarup+//uRf64Q= +github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31 h1:DdvjIeuZcmcCj0Gaey10JSYXPUJpOVsFqk5x3aHtbrM= +github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260815111556-ad0006892c31/go.mod h1:mrcugf1qcRO26DeSoMZFtxxZWW9a4nNk15MdBDFNP6Y= +github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260815074929-4ac69c11afca h1:xC0DvwZdiVFM05+7IsSrSX8MnZ4XPVkWKITWlygoHWM= +github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260815074929-4ac69c11afca/go.mod h1:+C11m+ufdqsK777ql66CbBVoiDXKa7u5dhyEaSuZqp8= +github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260824142420-113dc64ecfd4 h1:1qH+Pd3Oe1GXfFsPSqtzHdAs2fHAZCk/132ReoIvrzs= +github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:TkZgZE92VUKHHNKXwuV8b7tT9toaxsZkcGffRafKCSA= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260824142420-113dc64ecfd4 h1:67rKzegmcbpTKUClL97vZEjij+0WhdfPO3+6IFkYt3c= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:58CB6+Dq4B7NYPmVz4S60N4/pTt70ZFABrkS/1w7tDA= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 h1:zDxe4n0GPQ+U8VFZyyCoQed+9Xauz4G22J/X6B4gq20= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:vDcTSyaI7R6tjtZOqonr0hzq8JMOfBhJCWZy0tVjQPw= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4 h1:cYQFMrD6zcVsyCXExm07a6wuVLMJmXSNQNaBK23Lu3M= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:Q5wcK4PnlDJ55H+M38CFNLzL53OHYiwq81mR/gKtJXs= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4 h1:atoyW+P7wLeLpKdfWG4vtBBErRVg23+0l3ZAM+cHibo= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:fJ9CCRKpo8mQODfv4UDJ3tjUTbyCznYslXQwCkDaGg0= +github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260815095432-ee0e4a46015e h1:3ya0F++4D3LngM1KBEgeMO8Ikk/9PRgv4mpAACqn/YY= +github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260815095432-ee0e4a46015e/go.mod h1:f3KPR3QM0jPbWGhXougv1BxnW+Bnc2vLE4TvBlYTLdo= +github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260817084829-b98138447c29 h1:ywk/LA00he4oZP5gZGmbhwh5O2kmHAyl1DRc/2JW0Vg= +github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260817084829-b98138447c29/go.mod h1:WsxYq/XES1hiBp5yqMU4Mfes9Bf/q1RuMYyRBWA3FDE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/internal/controller/aodh_controller.go b/internal/controller/aodh_controller.go index fd14fe650..a61d4652f 100644 --- a/internal/controller/aodh_controller.go +++ b/internal/controller/aodh_controller.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -360,12 +361,28 @@ func (r *AutoscalingReconciler) reconcileNormalAodh( return ctrl.Result{}, err } - if sfset.GetStatefulSet().Generation == sfset.GetStatefulSet().Status.ObservedGeneration { - instance.Status.ReadyCount = sfset.GetStatefulSet().Status.ReadyReplicas - if instance.Status.ReadyCount > 0 { + ssData := sfset.GetStatefulSet() + if ssData.Generation == ssData.Status.ObservedGeneration { + instance.Status.ReadyCount = ssData.Status.ReadyReplicas + ready := false + if statefulset.IsReady(ssData) { + ready, err = statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err + } + } + if ready { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } instance.Status.Networks = instance.Spec.Aodh.NetworkAttachmentDefinitions + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } // diff --git a/internal/controller/autoscaling_controller.go b/internal/controller/autoscaling_controller.go index 446d50cfa..e1f21e8a6 100644 --- a/internal/controller/autoscaling_controller.go +++ b/internal/controller/autoscaling_controller.go @@ -52,6 +52,7 @@ import ( env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + "github.com/openstack-k8s-operators/lib-common/modules/common/object" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" service "github.com/openstack-k8s-operators/lib-common/modules/common/service" @@ -68,17 +69,15 @@ import ( autoscaling "github.com/openstack-k8s-operators/telemetry-operator/internal/autoscaling" ) -// ErrNotificationsURLSecretNotSet is returned when NotificationsURLSecret is not set in the instance status -var ErrNotificationsURLSecretNotSet = errors.New("NotificationsURLSecret is not set") - // ErrRabbitMQConfigNil is returned when rabbitmqConfig parameter is nil var ErrRabbitMQConfigNil = errors.New("rabbitmqConfig is nil - NotificationsBus must be configured") // AutoscalingReconciler reconciles a Autoscaling object type AutoscalingReconciler struct { client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Scheme *runtime.Scheme + APIReader client.Reader } // GetLogger returns a logger object with a prefix of "conroller.name" and aditional controller context fields @@ -275,11 +274,36 @@ func (r *AutoscalingReconciler) reconcileDelete( instance.Status.ApplicationCredentialSecret, instance.Spec.Aodh.Auth.ApplicationCredentialSecret, } { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, secretName, autoscaling.ACConsumerFinalizer); err != nil { return ctrl.Result{}, err } } + // Collect transport secret names from all sources to handle mid-rotation + // deletes. Status holds the old secret; the live TransportURL CR holds + // the new one. We must remove the consumer finalizer from both. + transportSecrets := map[string]struct{}{} + if instance.Status.NotificationsURLSecret != nil && *instance.Status.NotificationsURLSecret != "" { + transportSecrets[*instance.Status.NotificationsURLSecret] = struct{}{} + } + notifTransportURL := &rabbitmqv1.TransportURL{} + err = helper.GetClient().Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf("%s-notifications-transport", autoscaling.ServiceName), + Namespace: instance.Namespace, + }, notifTransportURL) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil && notifTransportURL.Status.SecretName != "" { + transportSecrets[notifTransportURL.Status.SecretName] = struct{}{} + } + for secretName := range transportSecrets { + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, + secretName, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", autoscaling.ServiceName)) @@ -333,9 +357,8 @@ func (r *AutoscalingReconciler) reconcileNormal( // secret. Old secret finalizer removal is deferred until all // sub-conditions are true (see late phase below). if instance.Spec.Aodh.Auth.ApplicationCredentialSecret != "" { - if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace, + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, instance.Spec.Aodh.Auth.ApplicationCredentialSecret, - "", autoscaling.ACConsumerFinalizer); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -371,9 +394,7 @@ func (r *AutoscalingReconciler) reconcileNormal( Log.Info(fmt.Sprintf("NotificationBusInstanceURL %s successfully reconciled - operation: %s", notificationBusInstanceURL.Name, string(op))) } - instance.Status.NotificationsURLSecret = ¬ificationBusInstanceURL.Status.SecretName - - if instance.Status.NotificationsURLSecret == nil || *instance.Status.NotificationsURLSecret == "" { + if notificationBusInstanceURL.Status.SecretName == "" { Log.Info(fmt.Sprintf("Waiting for NotificationBusInstanceURL %s secret to be created", notificationBusInstanceURL.Name)) instance.Status.Conditions.Set(condition.FalseCondition( condition.NotificationBusInstanceReadyCondition, @@ -383,6 +404,25 @@ func (r *AutoscalingReconciler) reconcileNormal( return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } + oldNotifSecret := "" + if instance.Status.NotificationsURLSecret != nil { + oldNotifSecret = *instance.Status.NotificationsURLSecret + } + currentNotifSecret := notificationBusInstanceURL.Status.SecretName + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + currentNotifSecret, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + + notifRotationFinalized := false + instance.Status.NotificationsURLSecret = ¤tNotifSecret + defer func() { + if !notifRotationFinalized { + instance.Status.NotificationsURLSecret = &oldNotifSecret + } + }() + instance.Status.Conditions.MarkTrue(condition.NotificationBusInstanceReadyCondition, condition.NotificationBusInstanceReadyMessage) // end notificationsBus @@ -502,7 +542,7 @@ func (r *AutoscalingReconciler) reconcileNormal( // check for required NotificationsBus TransportURL secret holding transport URL string // Aodh only uses NotificationsBus, not MessagingBus // - if instance.Status.NotificationsURLSecret == nil || *instance.Status.NotificationsURLSecret == "" { + if notificationBusInstanceURL.Status.SecretName == "" { Log.Info("NotificationsURLSecret not yet available") instance.Status.Conditions.Set(condition.FalseCondition( condition.NotificationBusInstanceReadyCondition, @@ -523,7 +563,7 @@ func (r *AutoscalingReconciler) reconcileNormal( ctx, types.NamespacedName{ Namespace: instance.Namespace, - Name: *instance.Status.NotificationsURLSecret, + Name: notificationBusInstanceURL.Status.SecretName, }, transportValidateFields, helper.GetClient(), @@ -604,7 +644,7 @@ func (r *AutoscalingReconciler) reconcileNormal( // - %-scripts configmap holding scripts to e.g. bootstrap the service // - %-config configmap holding minimal autoscaling config required to get the service up, user can add additional files to be added to the service // - err = r.generateServiceConfig(ctx, helper, instance, &configMapVars, memcached, db) + err = r.generateServiceConfig(ctx, helper, instance, &configMapVars, memcached, db, notificationBusInstanceURL.Status.SecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -689,7 +729,7 @@ func (r *AutoscalingReconciler) reconcileNormal( } // run check custom configs secret - end - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, instance, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( @@ -699,10 +739,6 @@ func (r *AutoscalingReconciler) reconcileNormal( condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } instance.Status.Hash[common.InputHashName] = inputHash @@ -723,20 +759,59 @@ func (r *AutoscalingReconciler) reconcileNormal( return ctrlResult, err } - // Late phase of the AC split pattern: remove the old AC secret's - // finalizer and update status only after all sub-conditions are true. - isACRotation := instance.Status.ApplicationCredentialSecret != "" && - instance.Status.ApplicationCredentialSecret != instance.Spec.Aodh.Auth.ApplicationCredentialSecret - if isACRotation { - if instance.Status.Conditions.AllSubConditionIsTrue() { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, - instance.Status.ApplicationCredentialSecret, autoscaling.ACConsumerFinalizer); err != nil { - return ctrl.Result{}, err - } - instance.Status.ApplicationCredentialSecret = instance.Spec.Aodh.Auth.ApplicationCredentialSecret - } - } else { - instance.Status.ApplicationCredentialSecret = instance.Spec.Aodh.Auth.ApplicationCredentialSecret + // Guard the rotation on the sub-conditions computed during this reconcile, + // not instance.IsReady(): the Ready condition is reset to Unknown by + // Conditions.Init() at the top of every reconcile and only recomputed in + // the deferred PatchInstance, so IsReady() would always be false here. + guardReady := instance.Status.Conditions.AllSubConditionIsTrue() + + acSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Aodh.Auth.ApplicationCredentialSecret, + autoscaling.ACConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.ApplicationCredentialSecret = acSecretName + + secretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + oldNotifSecret, + currentNotifSecret, + telemetryv1.TelemetryTransportConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.NotificationsURLSecret = &secretName + notifRotationFinalized = true + + // Self-heal consumer finalizers stranded on secrets superseded during + // rapid rotation (A -> B -> C before the workload became ready): + // FinalizeSecretRotation only ever releases the single tracked "old" + // secret, so any intermediate secret's finalizer would otherwise leak. + // keep enumerates every secret that legitimately still holds the + // finalizer; all others in the namespace are pruned. + notifKeep := "" + if instance.Status.NotificationsURLSecret != nil { + notifKeep = *instance.Status.NotificationsURLSecret + } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, telemetryv1.TelemetryTransportConsumerFinalizer, + notifKeep, currentNotifSecret, + ); err != nil { + return ctrl.Result{}, err + } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, autoscaling.ACConsumerFinalizer, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Aodh.Auth.ApplicationCredentialSecret, + ); err != nil { + return ctrl.Result{}, err } if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -778,6 +853,7 @@ func (r *AutoscalingReconciler) generateServiceConfig( envVars *map[string]env.Setter, mc *memcachedv1.Memcached, db *mariadbv1.Database, + notificationSecretName string, ) error { Log := r.GetLogger(ctx) cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(autoscaling.ServiceName), map[string]string{}) @@ -808,12 +884,7 @@ func (r *AutoscalingReconciler) generateServiceConfig( return err } - // Ensure NotificationsURLSecret is not nil before dereferencing - if instance.Status.NotificationsURLSecret == nil { - return ErrNotificationsURLSecretNotSet - } - - transportURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } @@ -897,8 +968,8 @@ func (r *AutoscalingReconciler) generateServiceConfig( // Add NotificationsURL if configured // Always get the separate notification secret since we always create separate TransportURLs - if instance.Status.NotificationsURLSecret != nil && *instance.Status.NotificationsURLSecret != "" { - notificationInstanceURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + if notificationSecretName != "" { + notificationInstanceURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } diff --git a/internal/controller/ceilometer_controller.go b/internal/controller/ceilometer_controller.go index 9bfcfb259..c08203536 100644 --- a/internal/controller/ceilometer_controller.go +++ b/internal/controller/ceilometer_controller.go @@ -50,6 +50,7 @@ import ( env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + "github.com/openstack-k8s-operators/lib-common/modules/common/object" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" statefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" @@ -76,8 +77,9 @@ const ( // CeilometerReconciler reconciles a Ceilometer object type CeilometerReconciler struct { client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Scheme *runtime.Scheme + APIReader client.Reader } // GetLogger returns a logger object with a prefix of "conroller.name" and aditional controller context fields @@ -414,12 +416,37 @@ func (r *CeilometerReconciler) reconcileDelete(ctx context.Context, instance *te instance.Status.ApplicationCredentialSecret, instance.Spec.Auth.ApplicationCredentialSecret, } { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, secretName, ceilometer.ACConsumerFinalizer); err != nil { return ctrl.Result{}, err } } + // Collect transport secret names from all sources to handle mid-rotation + // deletes. Status holds the old secret; the live TransportURL CR holds + // the new one. We must remove the consumer finalizer from both. + transportSecrets := map[string]struct{}{} + if instance.Status.NotificationsURLSecret != nil && *instance.Status.NotificationsURLSecret != "" { + transportSecrets[*instance.Status.NotificationsURLSecret] = struct{}{} + } + notifTransportURL := &rabbitmqv1.TransportURL{} + err = helper.GetClient().Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf("%s-notifications-transport", ceilometer.ServiceName), + Namespace: instance.Namespace, + }, notifTransportURL) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil && notifTransportURL.Status.SecretName != "" { + transportSecrets[notifTransportURL.Status.SecretName] = struct{}{} + } + for secretName := range transportSecrets { + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, + secretName, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", ceilometer.ServiceName)) @@ -563,9 +590,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( Log.Info(fmt.Sprintf("NotificationBusInstanceURL %s successfully reconciled - operation: %s", notificationBusInstanceURL.Name, string(op))) } - instance.Status.NotificationsURLSecret = ¬ificationBusInstanceURL.Status.SecretName - - if instance.Status.NotificationsURLSecret == nil || *instance.Status.NotificationsURLSecret == "" { + if notificationBusInstanceURL.Status.SecretName == "" { Log.Info(fmt.Sprintf("Waiting for NotificationBusInstanceURL %s secret to be created", notificationBusInstanceURL.Name)) instance.Status.Conditions.Set(condition.FalseCondition( condition.NotificationBusInstanceReadyCondition, @@ -575,6 +600,25 @@ func (r *CeilometerReconciler) reconcileCeilometer( return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } + oldNotifSecret := "" + if instance.Status.NotificationsURLSecret != nil { + oldNotifSecret = *instance.Status.NotificationsURLSecret + } + currentNotifSecret := notificationBusInstanceURL.Status.SecretName + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + currentNotifSecret, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + + notifRotationFinalized := false + instance.Status.NotificationsURLSecret = ¤tNotifSecret + defer func() { + if !notifRotationFinalized { + instance.Status.NotificationsURLSecret = &oldNotifSecret + } + }() + instance.Status.Conditions.MarkTrue(condition.NotificationBusInstanceReadyCondition, condition.NotificationBusInstanceReadyMessage) // end notificationsBus @@ -688,7 +732,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( // - %-config configmap holding minimal ceilometer config required to get the service up, user can add additional files to be added to the service // - parameters which has passwords gets added from the OpenStack secret via the init container // - err = r.generateServiceConfig(ctx, helper, instance, &configMapVars) + err = r.generateServiceConfig(ctx, helper, instance, &configMapVars, notificationBusInstanceURL.Status.SecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -705,7 +749,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( // - %-config configmap holding minimal ceilometer-compute config required to get the service up, user can add additional files to be added to the service // - parameters which has passwords gets added from the OpenStack secret via the init container // - err = r.generateComputeServiceConfig(ctx, helper, instance, &configMapVars) + err = r.generateComputeServiceConfig(ctx, helper, instance, &configMapVars, notificationBusInstanceURL.Status.SecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -745,7 +789,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, &instance.Status.Hash, configMapVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, &instance.Status.Hash, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -754,10 +798,6 @@ func (r *CeilometerReconciler) reconcileCeilometer( condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } instance.Status.Hash[common.InputHashName] = inputHash @@ -766,9 +806,8 @@ func (r *CeilometerReconciler) reconcileCeilometer( // secret. Old secret finalizer removal is deferred until all // sub-conditions are true (see late phase below). if instance.Spec.Auth.ApplicationCredentialSecret != "" { - if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace, + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, instance.Spec.Auth.ApplicationCredentialSecret, - "", ceilometer.ACConsumerFinalizer); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -842,33 +881,28 @@ func (r *CeilometerReconciler) reconcileCeilometer( // Evaluate the last part of the reconciliation only if we see the last // version of the CR - if sfset.GetStatefulSet().Generation == sfset.GetStatefulSet().Status.ObservedGeneration { - instance.Status.ReadyCount = sfset.GetStatefulSet().Status.ReadyReplicas + ssData := sfset.GetStatefulSet() + if ssData.Generation == ssData.Status.ObservedGeneration { + instance.Status.ReadyCount = ssData.Status.ReadyReplicas instance.Status.Networks = instance.Spec.NetworkAttachmentDefinitions - svc, op, err := ceilometer.Service(instance, helper, ceilometer.CeilometerPrometheusPort, serviceLabels) + svc, svcOp, err := ceilometer.Service(instance, helper, ceilometer.CeilometerPrometheusPort, serviceLabels) if err != nil { return ctrl.Result{}, err } - if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf(msgOperation, svc.Name, string(op))) - } - if instance.Status.ReadyCount > 0 { - instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + if svcOp != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf(msgOperation, svc.Name, string(svcOp))) } - // Late phase of the AC split pattern: remove the old AC secret's - // finalizer and update status only after all sub-conditions are true. - isACRotation := instance.Status.ApplicationCredentialSecret != "" && - instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret - if isACRotation { - if instance.Status.Conditions.AllSubConditionIsTrue() { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, - instance.Status.ApplicationCredentialSecret, ceilometer.ACConsumerFinalizer); err != nil { - return ctrl.Result{}, err - } - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + ready := false + if statefulset.IsReady(ssData) { + ready, err = statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err } - } else { - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + } + if ready { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -876,7 +910,69 @@ func (r *CeilometerReconciler) reconcileCeilometer( condition.ReadyCondition, condition.ReadyMessage) } Log.Info(fmt.Sprintf(msgReconcileSuccess, ceilometer.ServiceName)) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + } + + // Guard the rotation on the sub-conditions computed during this reconcile, + // not instance.IsReady(): the Ready condition is reset to Unknown by + // Conditions.Init() at the top of every reconcile and only recomputed in + // the deferred PatchInstance, so IsReady() would always be false here. + guardReady := instance.Status.Conditions.AllSubConditionIsTrue() + + acSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + ceilometer.ACConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.ApplicationCredentialSecret = acSecretName + + secretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + oldNotifSecret, + currentNotifSecret, + telemetryv1.TelemetryTransportConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.NotificationsURLSecret = &secretName + notifRotationFinalized = true + + // Self-heal consumer finalizers stranded on secrets superseded during + // rapid rotation (A -> B -> C before the workload became ready): + // FinalizeSecretRotation only ever releases the single tracked "old" + // secret, so any intermediate secret's finalizer would otherwise leak. + // keep enumerates every secret that legitimately still holds the + // finalizer; all others in the namespace are pruned. + notifKeep := "" + if instance.Status.NotificationsURLSecret != nil { + notifKeep = *instance.Status.NotificationsURLSecret + } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, telemetryv1.TelemetryTransportConsumerFinalizer, + notifKeep, currentNotifSecret, + ); err != nil { + return ctrl.Result{}, err } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, ceilometer.ACConsumerFinalizer, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + ); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil } @@ -983,7 +1079,7 @@ func (r *CeilometerReconciler) reconcileMysqldExporter( // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, &instance.Status.MysqldExporterHash, configMapVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, &instance.Status.MysqldExporterHash, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( telemetryv1.MysqldExporterServiceConfigReadyCondition, @@ -992,10 +1088,6 @@ func (r *CeilometerReconciler) reconcileMysqldExporter( condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } instance.Status.MysqldExporterHash[common.InputHashName] = inputHash @@ -1059,8 +1151,9 @@ func (r *CeilometerReconciler) reconcileMysqldExporter( // Evaluate the last part of the reconciliation only if we see the last // version of the CR - if sfset.GetStatefulSet().Generation == sfset.GetStatefulSet().Status.ObservedGeneration { - instance.Status.MysqldExporterReadyCount = sfset.GetStatefulSet().Status.ReadyReplicas + ssData := sfset.GetStatefulSet() + if ssData.Generation == ssData.Status.ObservedGeneration { + instance.Status.MysqldExporterReadyCount = ssData.Status.ReadyReplicas svc, op, err := mysqldexporter.Service(instance, helper, serviceLabels) if err != nil { return ctrl.Result{}, err @@ -1068,8 +1161,16 @@ func (r *CeilometerReconciler) reconcileMysqldExporter( if op != controllerutil.OperationResultNone { Log.Info(fmt.Sprintf(msgOperation, svc.Name, string(op))) } - if instance.Status.MysqldExporterReadyCount > 0 { - instance.Status.Conditions.MarkTrue(telemetryv1.MysqldExporterDeploymentReadyCondition, condition.DeploymentReadyMessage) + if statefulset.IsReady(ssData) { + ready, err := statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err + } + if ready { + instance.Status.Conditions.MarkTrue(telemetryv1.MysqldExporterDeploymentReadyCondition, condition.DeploymentReadyMessage) + } } Log.Info(fmt.Sprintf(msgReconcileSuccess, mysqldexporter.ServiceName)) } @@ -1170,7 +1271,7 @@ func (r *CeilometerReconciler) reconcileKSM( // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, &instance.Status.KSMHash, configMapVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, &instance.Status.KSMHash, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( telemetryv1.KSMServiceConfigReadyCondition, @@ -1179,8 +1280,6 @@ func (r *CeilometerReconciler) reconcileKSM( condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - return ctrl.Result{}, nil } instance.Status.KSMHash[common.InputHashName] = inputHash @@ -1238,8 +1337,8 @@ func (r *CeilometerReconciler) reconcileKSM( // version of the CR ssobj := ss.GetStatefulSet() if ssobj.Generation == ssobj.Status.ObservedGeneration { - instance.Status.KSMReadyCount = ss.GetStatefulSet().Status.ReadyReplicas - if instance.Status.KSMReadyCount > 0 { + instance.Status.KSMReadyCount = ssobj.Status.ReadyReplicas + if statefulset.IsReady(ssobj) { instance.Status.Conditions.MarkTrue(telemetryv1.KSMDeploymentReadyCondition, condition.DeploymentReadyMessage) } @@ -1274,6 +1373,7 @@ func (r *CeilometerReconciler) generateServiceConfig( h *helper.Helper, instance *telemetryv1.Ceilometer, envVars *map[string]env.Setter, + notificationSecretName string, ) error { cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(ceilometer.ServiceName), map[string]string{}) customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} @@ -1289,12 +1389,7 @@ func (r *CeilometerReconciler) generateServiceConfig( return err } - // Ensure NotificationsURLSecret is not nil before dereferencing - if instance.Status.NotificationsURLSecret == nil { - return ErrNotificationsURLSecretNotSet - } - - transportURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } @@ -1371,8 +1466,8 @@ func (r *CeilometerReconciler) generateServiceConfig( // Add NotificationsURL if configured // Always get the separate notification secret since we always create separate TransportURLs - if instance.Status.NotificationsURLSecret != nil && *instance.Status.NotificationsURLSecret != "" { - notificationInstanceURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + if notificationSecretName != "" { + notificationInstanceURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } @@ -1413,6 +1508,7 @@ func (r *CeilometerReconciler) generateComputeServiceConfig( h *helper.Helper, instance *telemetryv1.Ceilometer, envVars *map[string]env.Setter, + notificationSecretName string, ) error { cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(ceilometer.ComputeServiceName), map[string]string{}) ipmiLabels := labels.GetLabels(instance, labels.GetGroupLabel(ceilometer.IpmiServiceName), map[string]string{}) @@ -1429,12 +1525,7 @@ func (r *CeilometerReconciler) generateComputeServiceConfig( return err } - // Ensure NotificationsURLSecret is not nil before dereferencing - if instance.Status.NotificationsURLSecret == nil { - return ErrNotificationsURLSecretNotSet - } - - transportURLSecret, _, err := secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, notificationSecretName, instance.Namespace) if err != nil { return err } diff --git a/internal/controller/cloudkitty_controller.go b/internal/controller/cloudkitty_controller.go index 610fb25da..d1ce8dd4b 100644 --- a/internal/controller/cloudkitty_controller.go +++ b/internal/controller/cloudkitty_controller.go @@ -59,6 +59,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/job" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/object" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" @@ -520,12 +521,37 @@ func (r *CloudKittyReconciler) reconcileDelete(ctx context.Context, instance *te instance.Status.ApplicationCredentialSecret, instance.Spec.Auth.ApplicationCredentialSecret, } { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, secretName, cloudkitty.ACConsumerFinalizer); err != nil { return ctrl.Result{}, err } } + // Collect transport secret names from all sources to handle mid-rotation + // deletes. Status holds the old secret; the live TransportURL CR holds + // the new one. We must remove the consumer finalizer from both. + transportSecrets := map[string]struct{}{} + if instance.Status.TransportURLSecret != "" { + transportSecrets[instance.Status.TransportURLSecret] = struct{}{} + } + rpcTransportURL := &rabbitmqv1.TransportURL{} + err = helper.GetClient().Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf("%s-transport", instance.Name), + Namespace: instance.Namespace, + }, rpcTransportURL) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil && rpcTransportURL.Status.SecretName != "" { + transportSecrets[rpcTransportURL.Status.SecretName] = struct{}{} + } + for secretName := range transportSecrets { + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, + secretName, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) @@ -804,6 +830,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te err.Error())) return ctrl.Result{}, err } + if op != controllerutil.OperationResultNone { Log.Info(fmt.Sprintf("LokiStack %s successfully changed - operation: %s", lokiStack.Name, string(op))) } @@ -860,7 +887,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te // create RabbitMQ transportURL CR and get the actual URL from the associated secret that is created // - transportURL, op, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels) + transportURL, transportURLOp, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.RabbitMqTransportURLReadyCondition, @@ -871,13 +898,11 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te return ctrl.Result{}, err } - if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + if transportURLOp != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(transportURLOp))) } - instance.Status.TransportURLSecret = transportURL.Status.SecretName - - if instance.Status.TransportURLSecret == "" { + if transportURL.Status.SecretName == "" { Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) instance.Status.Conditions.Set(condition.FalseCondition( condition.RabbitMqTransportURLReadyCondition, @@ -887,6 +912,21 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te return cloudkitty.ResultRequeue, nil } + currentTransportSecret := transportURL.Status.SecretName + + // Set status early for first-time setup so PatchInstance persists it + // even on early returns. During rotation (old != current), the status + // is only updated by FinalizeSecretRotation at end of reconcile. + if instance.Status.TransportURLSecret == "" || + instance.Status.TransportURLSecret == currentTransportSecret { + instance.Status.TransportURLSecret = currentTransportSecret + } + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + currentTransportSecret, telemetryv1.TelemetryTransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) // end transportURL - CloudKitty only uses RPC Messaging Bus, not notifications @@ -995,7 +1035,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te // // Create Secrets required as input for the Service and calculate an overall hash of hashes // - err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db) + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db, transportURL.Status.SecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1010,7 +1050,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - _, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + _, _, err = r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1019,25 +1059,14 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) - instance.Status.Conditions.MarkFalse( - condition.ServiceConfigReadyCondition, - condition.InitReason, - condition.SeverityInfo, - condition.ServiceConfigReadyInitMessage) - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } // Early phase of the AC split pattern: only add finalizer to the new // secret. Old secret finalizer removal is deferred until all // sub-conditions are true (see late phase below). if instance.Spec.Auth.ApplicationCredentialSecret != "" { - if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace, + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, instance.Spec.Auth.ApplicationCredentialSecret, - "", cloudkitty.ACConsumerFinalizer); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1106,8 +1135,19 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te // normal reconcile tasks // + // Compute expected input-secret hash from rotating secrets so children + // can track whether they have converged on the latest credential set. + rotatingSecrets := []string{currentTransportSecret} + if instance.Spec.Auth.ApplicationCredentialSecret != "" { + rotatingSecrets = append(rotatingSecrets, instance.Spec.Auth.ApplicationCredentialSecret) + } + expectedInputHash, err := util.ObjectHash(rotatingSecrets) + if err != nil { + return ctrl.Result{}, err + } + // deploy cloudkitty-api - cloudKittyAPI, op, err := r.apiDeploymentCreateOrUpdate(ctx, instance) + cloudKittyAPI, apiOp, err := r.apiDeploymentCreateOrUpdate(ctx, instance, transportURL.Status.SecretName, expectedInputHash) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( telemetryv1.CloudKittyAPIReadyCondition, @@ -1117,12 +1157,12 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te err.Error())) return ctrl.Result{}, err } - if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf("API CR for %s successfully %s", instance.Name, string(op))) + if apiOp != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("API CR for %s successfully %s", instance.Name, string(apiOp))) } - - // Mirror values when the data in the StatefulSet is for the current generation - if cloudKittyAPI.Generation == cloudKittyAPI.Status.ObservedGeneration { + apiReady := false + if cloudKittyAPI.Generation == cloudKittyAPI.Status.ObservedGeneration && + cloudKittyAPI.Status.AppliedInputSecretHash == expectedInputHash { // Mirror CloudKittyAPI status' APIEndpoints and ReadyCount to this parent CR instance.Status.APIEndpoints = cloudKittyAPI.Status.APIEndpoints instance.Status.ServiceIDs = cloudKittyAPI.Status.ServiceIDs @@ -1133,10 +1173,17 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te if c != nil { instance.Status.Conditions.Set(c) } + apiReady = cloudKittyAPI.IsReady() + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyAPIReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } // deploy CloudKitty Processor - cloudKittyProc, op, err := r.procDeploymentCreateOrUpdate(ctx, instance) + cloudKittyProc, procOp, err := r.procDeploymentCreateOrUpdate(ctx, instance, transportURL.Status.SecretName, expectedInputHash) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( telemetryv1.CloudKittyProcReadyCondition, @@ -1146,12 +1193,12 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te err.Error())) return ctrl.Result{}, err } - if op != controllerutil.OperationResultNone { - Log.Info(fmt.Sprintf("Scheduler CR for %s successfully %s", instance.Name, string(op))) + if procOp != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("Scheduler CR for %s successfully %s", instance.Name, string(procOp))) } - - // Mirror values when the data in the StatefulSet is for the current generation - if cloudKittyProc.Generation == cloudKittyProc.Status.ObservedGeneration { + procReady := false + if cloudKittyProc.Generation == cloudKittyProc.Status.ObservedGeneration && + cloudKittyProc.Status.AppliedInputSecretHash == expectedInputHash { // Mirror CloudKitty Processor status' ReadyCount to this parent CR instance.Status.CloudKittyProcReadyCount = cloudKittyProc.Status.ReadyCount @@ -1160,6 +1207,13 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te if c != nil { instance.Status.Conditions.Set(c) } + procReady = cloudKittyProc.IsReady() + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyProcReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) } err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, helper, cloudkitty.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) @@ -1167,20 +1221,50 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te return ctrl.Result{}, err } - // Late phase of the AC split pattern: remove the old AC secret's - // finalizer and update status only after all sub-conditions are true. - isACRotation := instance.Status.ApplicationCredentialSecret != "" && - instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret - if isACRotation { - if instance.Status.Conditions.AllSubConditionIsTrue() { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, - instance.Status.ApplicationCredentialSecret, cloudkitty.ACConsumerFinalizer); err != nil { - return ctrl.Result{}, err - } - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret - } - } else { - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + allServicesReady := apiReady && procReady + + acSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + cloudkitty.ACConsumerFinalizer, + allServicesReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.ApplicationCredentialSecret = acSecretName + + secretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.TransportURLSecret, + currentTransportSecret, + telemetryv1.TelemetryTransportConsumerFinalizer, + allServicesReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.TransportURLSecret = secretName + + // Self-heal consumer finalizers stranded on secrets superseded during + // rapid rotation (A -> B -> C before the workload became ready): + // FinalizeSecretRotation only ever releases the single tracked "old" + // secret, so any intermediate secret's finalizer would otherwise leak. + // keep enumerates every secret that legitimately still holds the + // finalizer; all others in the namespace are pruned. + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, telemetryv1.TelemetryTransportConsumerFinalizer, + instance.Status.TransportURLSecret, currentTransportSecret, + ); err != nil { + return ctrl.Result{}, err + } + if err := object.PruneSecretConsumerFinalizers( + ctx, helper, instance.Namespace, cloudkitty.ACConsumerFinalizer, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + ); err != nil { + return ctrl.Result{}, err } Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) @@ -1200,6 +1284,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( serviceLabels map[string]string, memcached *memcachedv1.Memcached, db *mariadbv1.Database, + transportURLSecretName string, ) error { Log := r.GetLogger(ctx) // @@ -1239,7 +1324,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( return err } - transportURLSecret, _, err := secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, transportURLSecretName, instance.Namespace) if err != nil { return err } @@ -1452,12 +1537,12 @@ func (r *CloudKittyReconciler) transportURLCreateOrUpdate( return transportURL, op, err } -func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyAPI, controllerutil.OperationResult, error) { +func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty, transportURLSecretName string, expectedInputHash string) (*telemetryv1.CloudKittyAPI, controllerutil.OperationResult, error) { cloudkittyAPISpec := telemetryv1.CloudKittyAPISpec{ CloudKittyTemplate: instance.Spec.CloudKittyTemplate, CloudKittyAPITemplate: instance.Spec.CloudKittyAPI, DatabaseHostname: instance.Status.DatabaseHostname, - TransportURLSecret: instance.Status.TransportURLSecret, + TransportURLSecret: transportURLSecretName, ServiceAccount: instance.RbacResourceName(), } @@ -1480,6 +1565,11 @@ func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.Spec = cloudkittyAPISpec + if deployment.Annotations == nil { + deployment.Annotations = map[string]string{} + } + deployment.Annotations["openstack.org/transport-url-secret"] = transportURLSecretName + deployment.Annotations["openstack.org/input-secret-hash"] = expectedInputHash err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) if err != nil { @@ -1492,12 +1582,12 @@ func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, return deployment, op, err } -func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyProc, controllerutil.OperationResult, error) { +func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty, transportURLSecretName string, expectedInputHash string) (*telemetryv1.CloudKittyProc, controllerutil.OperationResult, error) { cloudKittyProcSpec := telemetryv1.CloudKittyProcSpec{ CloudKittyTemplate: instance.Spec.CloudKittyTemplate, CloudKittyProcTemplate: instance.Spec.CloudKittyProc, DatabaseHostname: instance.Status.DatabaseHostname, - TransportURLSecret: instance.Status.TransportURLSecret, + TransportURLSecret: transportURLSecretName, ServiceAccount: instance.RbacResourceName(), //TLS: instance.Spec.CloudKittyProc.TLS.Ca, } @@ -1521,6 +1611,11 @@ func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.Spec = cloudKittyProcSpec + if deployment.Annotations == nil { + deployment.Annotations = map[string]string{} + } + deployment.Annotations["openstack.org/transport-url-secret"] = transportURLSecretName + deployment.Annotations["openstack.org/input-secret-hash"] = expectedInputHash err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) if err != nil { diff --git a/internal/controller/cloudkittyapi_controller.go b/internal/controller/cloudkittyapi_controller.go index 8304737a1..9919fd59f 100644 --- a/internal/controller/cloudkittyapi_controller.go +++ b/internal/controller/cloudkittyapi_controller.go @@ -77,8 +77,9 @@ func (r *CloudKittyAPIReconciler) GetScheme() *runtime.Scheme { // CloudKittyAPIReconciler reconciles a CloudKittyAPI object type CloudKittyAPIReconciler struct { client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Scheme *runtime.Scheme + APIReader client.Reader } // GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields @@ -1017,7 +1018,7 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1026,16 +1027,6 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) - instance.Status.Conditions.MarkFalse( - condition.ServiceConfigReadyCondition, - condition.InitReason, - condition.SeverityInfo, - condition.ServiceConfigReadyInitMessage) - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } // Deploy a statefulset @@ -1130,9 +1121,19 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, err } - if instance.Status.ReadyCount > 0 { + expectedHash := instance.GetAnnotations()["openstack.org/input-secret-hash"] + ready := false + if statefulset.IsReady(ss.GetStatefulSet()) { + ready, err = statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err + } + } + if ready { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) - + instance.Status.AppliedInputSecretHash = expectedHash } else if *instance.Spec.Replicas > 0 { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, diff --git a/internal/controller/cloudkittyproc_controller.go b/internal/controller/cloudkittyproc_controller.go index 4054515f1..28ead31b0 100644 --- a/internal/controller/cloudkittyproc_controller.go +++ b/internal/controller/cloudkittyproc_controller.go @@ -73,8 +73,9 @@ func (r *CloudKittyProcReconciler) GetScheme() *runtime.Scheme { // CloudKittyProcReconciler reconciles a CloudKittyProc object type CloudKittyProcReconciler struct { client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Scheme *runtime.Scheme + APIReader client.Reader } // GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields @@ -600,7 +601,7 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + inputHash, _, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -609,16 +610,6 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err - } else if hashChanged { - Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) - instance.Status.Conditions.MarkFalse( - condition.ServiceConfigReadyCondition, - condition.InitReason, - condition.SeverityInfo, - condition.ServiceConfigReadyInitMessage) - // Hash changed and instance status should be updated (which will be done by main defer func), - // so we need to return and reconcile again - return ctrl.Result{}, nil } instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) @@ -776,8 +767,19 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, err } - if instance.Status.ReadyCount > 0 { + expectedHash := instance.GetAnnotations()["openstack.org/input-secret-hash"] + ready := false + if statefulset.IsReady(ss.GetStatefulSet()) { + ready, err = statefulset.IsReadyForInput(ctx, r.APIReader, + types.NamespacedName{Name: ssData.Name, Namespace: ssData.Namespace}, + inputHash) + if err != nil { + return ctrl.Result{}, err + } + } + if ready { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + instance.Status.AppliedInputSecretHash = expectedHash } else if *instance.Spec.Replicas > 0 { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 0461d0567..8a496262d 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -42,6 +42,7 @@ type ConditionalWatchingReconciler struct { client.Client Kclient kubernetes.Interface Scheme *runtime.Scheme + APIReader client.Reader Controller controller.Controller Watching []string RESTMapper meta.RESTMapper