diff --git a/api/core/v1beta1/conditions.go b/api/core/v1beta1/conditions.go index a53968c7e7..f79ce90e38 100644 --- a/api/core/v1beta1/conditions.go +++ b/api/core/v1beta1/conditions.go @@ -596,4 +596,7 @@ const ( // OpenStackVersionMinorUpdateAvailableMessage OpenStackVersionMinorUpdateAvailableMessage = "update available" + + // OpenStackVersionMinorUpdateReadyGatedMessage - format string; arg is the target stage name + OpenStackVersionMinorUpdateReadyGatedMessage = "Minor update progression stopped after stage: %s . Set annotation to next stage after %s to resume OpenStack update" ) diff --git a/api/core/v1beta1/openstackversion_types.go b/api/core/v1beta1/openstackversion_types.go index 9bc685abbe..bb4dcb94e8 100644 --- a/api/core/v1beta1/openstackversion_types.go +++ b/api/core/v1beta1/openstackversion_types.go @@ -34,6 +34,28 @@ const ( MinorUpdateControlPlane string = "Minor Update Controlplane In Progress" // MinorUpdateComplete - MinorUpdateComplete string = "Complete" + + // MinorUpdateTargetStageAnnotation - specifies the update stage after which the minor update + // should pause. All stages up to and including the named stage will be completed; subsequent + // stages will be blocked until the annotation is removed or updated to a later stage. + // Valid values: "ovn-controlplane", "ovn-dataplane", "rabbitmq", "mariadb", "memcached", + // "keystone", "controlplane". Remove the annotation to let the update proceed to completion. + MinorUpdateTargetStageAnnotation string = "core.openstack.org/minor-update-target-stage" + + // MinorUpdateStageOVNControlplane - stage name for OVN controlplane update + MinorUpdateStageOVNControlplane string = "ovn-controlplane" + // MinorUpdateStageOVNDataplane - stage name for OVN dataplane update + MinorUpdateStageOVNDataplane string = "ovn-dataplane" + // MinorUpdateStageRabbitMQ - stage name for RabbitMQ update + MinorUpdateStageRabbitMQ string = "rabbitmq" + // MinorUpdateStageMariaDB - stage name for MariaDB update + MinorUpdateStageMariaDB string = "mariadb" + // MinorUpdateStageMemcached - stage name for Memcached update + MinorUpdateStageMemcached string = "memcached" + // MinorUpdateStageKeystone - stage name for Keystone update + MinorUpdateStageKeystone string = "keystone" + // MinorUpdateStageControlplane - stage name for full controlplane update + MinorUpdateStageControlplane string = "controlplane" ) // OpenStackVersionSpec - defines the desired state of OpenStackVersion diff --git a/docs/assemblies/proc_minor-update-staged-rollout.md b/docs/assemblies/proc_minor-update-staged-rollout.md new file mode 100644 index 0000000000..0b95c2b758 --- /dev/null +++ b/docs/assemblies/proc_minor-update-staged-rollout.md @@ -0,0 +1,272 @@ +# Performing a Staged Minor Update of OpenStack + +A minor update of OpenStack environment in a fixed sequence of stages. By default the +update runs all stages automatically. The `core.openstack.org/minor-update-target-stage` +annotation lets you pause the update after any stage so you can validate the environment, +coordinate maintenance windows, or simply advance one stage at a time. + +## Examples to use staged rollouts + +- You want to verify OVN networking is healthy before allowing the rest of the update to + proceed. +- Your organisation requires a sign-off after each major component is updated. +- You are performing the update in phases across a maintenance window and need to stop at a + known safe point. + +## Understanding the update pipeline + +The update always runs stages in this order. Each stage must complete before the next one +starts. + +| Stage | What gets updated | Requires manual action? | +|--------------------|-----------------------------------------|---------------------------------------------------------| +| `ovn-controlplane` | OVN control plane images | No | +| `ovn-dataplane` | OVN controller data plane images on compute nodes | **Yes** — create an OVN `OpenStackDataPlaneDeployment` | +| `rabbitmq` | RabbitMQ images | No | +| `mariadb` | MariaDB/Galera images | No | +| `memcached` | Memcached images | No | +| `keystone` | Keystone API images | No | +| `controlplane` | All remaining control-plane services | No | +| *(completion)* | Data-plane services on compute nodes | **Yes** — create a full `OpenStackDataPlaneDeployment` | + +> **Note:** Two stages require you to create an `OpenStackDataPlaneDeployment` manually. +> The `ovn-dataplane` stage and the final data-plane completion step do not self-drive — +> the controller waits for the corresponding deployment to finish before advancing. +> See [Required manual deployments](#required-manual-deployments) below. + +## Prerequisites + +- A running cluster with a deployed OpenStack environment. +- `OpenStackControlPlane` and `OpenStackVersion` are both `Ready`. +- `status.deployedVersion` is set on the `OpenStackVersion` CR. +- A newer version is available: `status.availableVersion` differs from + `status.deployedVersion`. + +The examples below use: +- Namespace: `openstack` +- `OpenStackVersion` CR name: `openstack` + +--- + +## Performing a fully staged update + +The recommended approach is to set the annotation to the first stage before bumping +`targetVersion`, then advance the annotation one stage at a time after you have validated +each step. + +### Step 1 — Confirm an update is available + +```bash +oc get openstackversion openstack -n openstack \ + -o jsonpath='Available: {.status.availableVersion} Deployed: {.status.deployedVersion}{"\n"}' +``` + +Note the `availableVersion` value — this is `` in the commands below. + +### Step 2 — Set the initial pause point + +Choose the stage after which you want the first pause. To pause after OVN control-plane: + +```bash +oc annotate openstackversion openstack \ + core.openstack.org/minor-update-target-stage=ovn-controlplane \ + -n openstack +``` + +### Step 3 — Start the update + +```bash +oc patch openstackversion openstack -n openstack \ + --type=merge -p '{"spec":{"targetVersion":""}}' +``` + +The update begins immediately. The controller runs the `ovn-controlplane` stage and then +pauses. The `MinorUpdateOVNControlplane` condition becomes `True` and the +`MinorUpdateOVNDataplane` condition shows: + +``` +Minor update progression stopped after stage: ovn-controlplane . +Set annotation to next stage after ovn-controlplane to resume OpenStack update +``` + +### Step 4 — Validate and advance stage by stage + +After each pause, check the environment is healthy, then advance to the next stage. + +#### Checking the current update status + +```bash +oc get openstackversion openstack -n openstack \ + -o jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}' \ + | grep MinorUpdate +``` + +Completed stages show `True`. The currently blocked stage shows `False` with a message +telling you which stage just finished and what to set next. + +#### Advancing to the next stage + +Update the annotation value to the stage you want to run next. For example, after +validating the OVN control-plane, advance to `ovn-dataplane`: + +> **Before advancing to `ovn-dataplane`**, create the OVN dataplane deployment first — +> see [Required manual deployments](#required-manual-deployments). + +```bash +oc annotate openstackversion openstack \ + core.openstack.org/minor-update-target-stage=ovn-dataplane \ + --overwrite -n openstack +``` + +Continue advancing through the remaining stages as needed: + +| To run through… | Set annotation to… | +|--------------------|--------------------| +| RabbitMQ | `rabbitmq` | +| MariaDB | `mariadb` | +| Memcached | `memcached` | +| Keystone | `keystone` | +| Full control-plane | `controlplane` | + +### Step 5 — Complete the update + +When you are ready to run the final data-plane update on compute nodes, first create the +full dataplane deployment (see [Required manual deployments](#required-manual-deployments)), +then remove the annotation to let the update finish: + +```bash +oc annotate openstackversion openstack \ + core.openstack.org/minor-update-target-stage- \ + -n openstack +``` + +> The trailing `-` removes the annotation entirely. + +The controller runs the remaining stages and, once complete, sets +`status.deployedVersion` to the new version. + +### Step 6 — Confirm completion + +```bash +oc get openstackversion openstack -n openstack \ + -o jsonpath='{.status.deployedVersion}' +``` + +The output should show ``. + +--- + +## Required manual deployments + +Two stages in the process do not self-start. You must create an +`OpenStackDataPlaneDeployment` before (or at the same time as) advancing past each of them. + +### OVN data-plane deployment + +Required before the `ovn-dataplane` stage can complete. This deployment updates only the +OVN-related services on compute nodes. + +```yaml +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-deployment-ovn-update + namespace: openstack +spec: + nodeSets: + - openstack-edpm-ipam + servicesOverride: + - ovn +``` + +```bash +oc apply -f edpm-deployment-ovn-update.yaml +``` + +### Full data-plane update deployment + +Required before the final completion step can finish. This deployment updates all remaining +services on compute nodes. + +```yaml +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-deployment-update + namespace: openstack +spec: + nodeSets: + - openstack-edpm-ipam + servicesOverride: + - update +``` + +```bash +oc apply -f edpm-deployment-update.yaml +``` + +--- + +## Pausing a running update + +If you need to pause an update that is already in progress, add the annotation at any time. +The controller completes whichever stage is currently running, then stops after the stage you +named. + +```bash +oc annotate openstackversion openstack \ + core.openstack.org/minor-update-target-stage= \ + -n openstack +``` + +Replace `` with the name of the last stage you want to run before pausing. + +--- + +## Running the full update without pausing + +If you do not need staged control, omit the annotation entirely and let the controller run +all stages automatically. You still need to create both dataplane deployments at the right +time: + +1. Create the OVN dataplane deployment before or immediately after starting the update. +2. Create the dataplane update deployment before the final completion step. + +```bash +oc patch openstackversion openstack -n openstack \ + --type=merge -p '{"spec":{"targetVersion":""}}' +``` + +--- + +## Troubleshooting + +### The update appears stuck + +Check whether the blocked condition message contains `"stopped after stage"`. If it does, +the update is intentionally paused — advance or remove the annotation to continue. + +```bash +oc get openstackversion openstack -n openstack -o json | \ + jq '[.status.conditions[] | select(.reason=="Requested" and .status=="False")]' +``` + +### `MinorUpdateOVNDataplane` or `MinorUpdateDataplane` stays `False` + +These stages wait for an `OpenStackDataPlaneDeployment` to complete. Check whether the +required deployment exists and is running: + +```bash +oc get openstackdataplanedeployment -n openstack +``` + +If the deployment is missing, create it as described in +[Required manual deployments](#required-manual-deployments). + +### Checking overall update progress + +```bash +watch -n 5 "oc get openstackversion openstack -n openstack \ + -o jsonpath='{range .status.conditions[*]}{.type}{\"\t\"}{.status}{\"\t\"}{.message}{\"\n\"}{end}' \ + | grep MinorUpdate" +``` diff --git a/internal/controller/core/openstackversion_controller.go b/internal/controller/core/openstackversion_controller.go index 9f0f0d40ae..f25fd5defc 100644 --- a/internal/controller/core/openstackversion_controller.go +++ b/internal/controller/core/openstackversion_controller.go @@ -262,6 +262,25 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req // minor update in progress if instance.Status.DeployedVersion != nil && instance.Spec.TargetVersion != *instance.Status.DeployedVersion { + // targetStage is the value of the target-stage annotation. When set, the update + // completes all stages up to and including the named stage, then pauses. An empty + // string (annotation absent) means run to completion without pausing. + targetStage := instance.Annotations[corev1beta1.MinorUpdateTargetStageAnnotation] + + // gateNextStage marks the next condition as blocked and returns the signal to stop. + // It is called immediately after a stage's MarkTrue when the annotation names that stage. + gateNextStage := func(completedStage string, nextCondition condition.Type) (ctrl.Result, error) { + instance.Status.Conditions.Set(condition.FalseCondition( + nextCondition, + condition.RequestedReason, + condition.SeverityInfo, + corev1beta1.OpenStackVersionMinorUpdateReadyGatedMessage, + completedStage, completedStage)) + Log.Info("Minor update paused at target stage", "stage", completedStage, + "annotation", corev1beta1.MinorUpdateTargetStageAnnotation) + return ctrl.Result{}, nil + } + // Only check OVN when enabled to avoid hanging on a removed condition if controlPlane.Spec.Ovn.Enabled { if !openstack.OVNControllerImageMatch(ctx, controlPlane, instance) || @@ -279,6 +298,11 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateOVNControlplane, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageOVNControlplane { + return gateNextStage(corev1beta1.MinorUpdateStageOVNControlplane, + corev1beta1.OpenStackVersionMinorUpdateOVNDataplane) + } + // minor update for Dataplane OVN // Only check OVN when enabled to avoid hanging on a removed condition if controlPlane.Spec.Ovn.Enabled { @@ -365,6 +389,11 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateOVNDataplane, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageOVNDataplane { + return gateNextStage(corev1beta1.MinorUpdateStageOVNDataplane, + corev1beta1.OpenStackVersionMinorUpdateRabbitMQ) + } + // minor update for RabbitMQ if !openstack.RabbitmqImageMatch(ctx, controlPlane, instance) || !controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneRabbitMQReadyCondition) { @@ -380,6 +409,11 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateRabbitMQ, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageRabbitMQ { + return gateNextStage(corev1beta1.MinorUpdateStageRabbitMQ, + corev1beta1.OpenStackVersionMinorUpdateMariaDB) + } + // minor update for MariaDB if !openstack.GaleraImageMatch(ctx, controlPlane, instance) || !controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneMariaDBReadyCondition) { @@ -395,6 +429,11 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateMariaDB, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageMariaDB { + return gateNextStage(corev1beta1.MinorUpdateStageMariaDB, + corev1beta1.OpenStackVersionMinorUpdateMemcached) + } + // minor update for Memcached if !openstack.MemcachedImageMatch(ctx, controlPlane, instance) || !controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneMemcachedReadyCondition) { @@ -410,6 +449,11 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateMemcached, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageMemcached { + return gateNextStage(corev1beta1.MinorUpdateStageMemcached, + corev1beta1.OpenStackVersionMinorUpdateKeystone) + } + // minor update for Keystone API if !openstack.KeystoneImageMatch(ctx, controlPlane, instance) || !controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneKeystoneAPIReadyCondition) { @@ -425,6 +469,11 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateKeystone, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageKeystone { + return gateNextStage(corev1beta1.MinorUpdateStageKeystone, + corev1beta1.OpenStackVersionMinorUpdateControlplane) + } + // minor update for Controlplane in progress if !controlPlane.IsReady() { instance.Status.Conditions.Set(condition.FalseCondition( @@ -456,6 +505,11 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateReadyMessage) Log.Info("Minor update for ControlPlane completed") + if targetStage == corev1beta1.MinorUpdateStageControlplane { + return gateNextStage(corev1beta1.MinorUpdateStageControlplane, + corev1beta1.OpenStackVersionMinorUpdateDataplane) + } + if !openstack.DataplaneNodesetsDeployed(instance, dataplaneNodesets) { instance.Status.Conditions.Set(condition.FalseCondition( corev1beta1.OpenStackVersionMinorUpdateDataplane, diff --git a/test/functional/ctlplane/openstackversion_controller_test.go b/test/functional/ctlplane/openstackversion_controller_test.go index f4d23f9c26..dcefd0222d 100644 --- a/test/functional/ctlplane/openstackversion_controller_test.go +++ b/test/functional/ctlplane/openstackversion_controller_test.go @@ -1585,4 +1585,301 @@ var _ = Describe("OpenStackOperator controller", func() { }) }) + // Test target-stage annotation gates minor update at the specified stage + When("Minor update with target-stage annotation", func() { + var ( + initialVersion = "old" + updatedVersion = "0.0.1" + testRabbitMQImage = "foo/rabbit:0.0.2" + testMariaDBImage = "foo/maria:0.0.2" + testMemcachedImage = "foo/memcached:0.0.2" + testKeystoneAPIImage = "foo/keystone:0.0.2" + ) + + BeforeEach(func() { + // Lightweight controlplane spec with OVN DISABLED so that the OVN stages + // auto-complete, making it straightforward to gate at "ovn-controlplane" + // without needing to simulate OVN readiness in these tests. + spec := GetDefaultOpenStackControlPlaneSpec() + + galeraTemplate := map[string]interface{}{ + names.DBName.Name: map[string]interface{}{ + "storageRequest": "500M", + }, + } + spec["galera"] = map[string]interface{}{ + "enabled": true, + "templates": galeraTemplate, + } + + spec["horizon"] = map[string]interface{}{"enabled": false} + spec["glance"] = map[string]interface{}{"enabled": false} + spec["cinder"] = map[string]interface{}{"enabled": false} + spec["neutron"] = map[string]interface{}{"enabled": false} + spec["manila"] = map[string]interface{}{"enabled": false} + spec["heat"] = map[string]interface{}{"enabled": false} + spec["telemetry"] = map[string]interface{}{"enabled": false} + spec["tls"] = GetTLSPublicSpec() + + // OVN disabled — stages auto-complete during minor update + spec["ovn"] = map[string]interface{}{ + "enabled": false, + } + + DeferCleanup( + th.DeleteInstance, + CreateOpenStackVersion(names.OpenStackVersionName, GetDefaultOpenStackVersionSpec()), + ) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCell1CertName)) + + DeferCleanup(k8sClient.Delete, ctx, CreateCertSecret(names.RootCAPublicName)) + DeferCleanup(k8sClient.Delete, ctx, CreateCertSecret(names.RootCAInternalName)) + DeferCleanup(k8sClient.Delete, ctx, CreateCertSecret(names.RootCAOvnName)) + DeferCleanup(k8sClient.Delete, ctx, CreateCertSecret(names.RootCALibvirtName)) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.DBCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.DBCell1CertName)) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.MemcachedCertName)) + + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionInitialized, + k8s_corev1.ConditionTrue, + ) + + version := GetOpenStackVersion(names.OpenStackVersionName) + g.Expect(version).Should(Not(BeNil())) + + g.Expect(*version.Status.AvailableVersion).Should(ContainSubstring("0.0.1")) + g.Expect(version.Spec.TargetVersion).Should(ContainSubstring("0.0.1")) + updatedVersion = *version.Status.AvailableVersion + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Status.ContainerImageVersionDefaults[initialVersion] = version.Status.ContainerImageVersionDefaults[updatedVersion] + version.Status.ContainerImageVersionDefaults[initialVersion].RabbitmqImage = &testRabbitMQImage + version.Status.ContainerImageVersionDefaults[initialVersion].MariadbImage = &testMariaDBImage + version.Status.ContainerImageVersionDefaults[initialVersion].InfraMemcachedImage = &testMemcachedImage + version.Status.ContainerImageVersionDefaults[initialVersion].KeystoneAPIImage = &testKeystoneAPIImage + g.Expect(th.K8sClient.Status().Update(th.Ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Spec.TargetVersion = initialVersion + g.Expect(th.K8sClient.Update(th.Ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + g.Expect(osversion).Should(Not(BeNil())) + g.Expect(osversion.Generation).Should(Equal(osversion.Status.ObservedGeneration)) + + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionInitialized, + k8s_corev1.ConditionTrue, + ) + + g.Expect(*osversion.Status.AvailableVersion).Should(Equal(updatedVersion)) + g.Expect(osversion.Spec.TargetVersion).Should(Equal(initialVersion)) + g.Expect(osversion.Status.DeployedVersion).Should(BeNil()) + }, timeout, interval).Should(Succeed()) + + DeferCleanup( + th.DeleteInstance, + CreateOpenStackControlPlane(names.OpenStackControlplaneName, spec), + ) + + DeferCleanup( + th.DeleteInstance, + CreateDataplaneNodeSet(names.OpenStackVersionName, DefaultDataPlaneNoNodeSetSpec(false)), + ) + + dataplanenodeset := GetDataplaneNodeset(names.OpenStackVersionName) + dataplanenodeset.Status.DeployedVersion = initialVersion + Expect(th.K8sClient.Status().Update(th.Ctx, dataplanenodeset)).To(Succeed()) + + th.CreateSecret(types.NamespacedName{Name: "openstack-config-secret", Namespace: namespace}, map[string][]byte{"secure.yaml": []byte("foo")}) + th.CreateConfigMap(types.NamespacedName{Name: "openstack-config", Namespace: namespace}, map[string]interface{}{"clouds.yaml": string("foo"), "OS_CLOUD": "default"}) + + OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName) + Expect(OSCtlplane.Spec.Ovn.Enabled).Should(BeFalse()) + + SimulateControlplaneReady() + + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackControlplaneName, + ConditionGetterFunc(OpenStackControlPlaneConditionGetter), + condition.ReadyCondition, + k8s_corev1.ConditionTrue, + ) + OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName) + g.Expect(OSCtlplane.Status.DeployedVersion).Should(Equal(&initialVersion)) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + g.Expect(osversion).Should(Not(BeNil())) + g.Expect(osversion.Generation).Should(Equal(osversion.Status.ObservedGeneration)) + g.Expect(osversion.Status.DeployedVersion).Should(Equal(&initialVersion)) + }, timeout, interval).Should(Succeed()) + }) + + It("should complete the named stage then block the next stage", Serial, func() { + // Set target-stage to "ovn-controlplane". With OVN disabled the OVN + // controlplane stage auto-completes (MarkTrue); the controller then + // detects the gate and blocks the OVN dataplane stage. + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + if version.Annotations == nil { + version.Annotations = make(map[string]string) + } + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNControlplane + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Trigger minor update + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Spec.TargetVersion = updatedVersion + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Wait for initialization + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + g.Expect(osversion).Should(Not(BeNil())) + g.Expect(osversion.Generation).Should(Equal(osversion.Status.ObservedGeneration)) + + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionInitialized, + k8s_corev1.ConditionTrue, + ) + }, timeout, interval).Should(Succeed()) + + // OVN controlplane stage must have completed (True) + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateOVNControlplane, + k8s_corev1.ConditionTrue, + ) + + // OVN dataplane stage must be gated (False with target-stage message) + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateOVNDataplane) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Status).Should(Equal(k8s_corev1.ConditionFalse)) + g.Expect(cond.Reason).Should(Equal(condition.RequestedReason)) + g.Expect(cond.Message).Should(ContainSubstring(corev1.MinorUpdateStageOVNControlplane)) + }, timeout, interval).Should(Succeed()) + + // DeployedVersion must not advance while gated + osversion := GetOpenStackVersion(names.OpenStackVersionName) + Expect(osversion.Status.DeployedVersion).Should(Equal(&initialVersion)) + }) + + It("should resume update when the annotation is removed", Serial, func() { + // Gate at "ovn-controlplane" + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + if version.Annotations == nil { + version.Annotations = make(map[string]string) + } + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNControlplane + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Trigger minor update + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Spec.TargetVersion = updatedVersion + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Wait for gated state: OVNDataplane blocked + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateOVNDataplane) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Message).Should(ContainSubstring(corev1.MinorUpdateStageOVNControlplane)) + }, timeout, interval).Should(Succeed()) + + // Remove the annotation to let the update proceed + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + delete(version.Annotations, corev1.MinorUpdateTargetStageAnnotation) + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // OVN dataplane should no longer be gated; with OVN disabled it + // auto-completes so the condition becomes True. + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateOVNDataplane, + k8s_corev1.ConditionTrue, + ) + }, timeout, interval).Should(Succeed()) + }) + + It("should advance gate to a later stage when annotation value is updated", Serial, func() { + // Start gated at "ovn-controlplane" + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + if version.Annotations == nil { + version.Annotations = make(map[string]string) + } + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNControlplane + version.Spec.TargetVersion = updatedVersion + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Wait for OVNDataplane to be gated + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateOVNDataplane) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Message).Should(ContainSubstring(corev1.MinorUpdateStageOVNControlplane)) + }, timeout, interval).Should(Succeed()) + + // Advance gate to "ovn-dataplane" to let OVN dataplane auto-complete + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNDataplane + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // OVN dataplane auto-completes (OVN disabled); RabbitMQ becomes gated + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateOVNDataplane, + k8s_corev1.ConditionTrue, + ) + + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateRabbitMQ) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Status).Should(Equal(k8s_corev1.ConditionFalse)) + g.Expect(cond.Reason).Should(Equal(condition.RequestedReason)) + g.Expect(cond.Message).Should(ContainSubstring(corev1.MinorUpdateStageOVNDataplane)) + }, timeout, interval).Should(Succeed()) + }) + }) + })