diff --git a/pkg/cli/admin/upgrade/recommend/alerts.go b/pkg/cli/admin/upgrade/recommend/alerts.go index f5508f4608..f6612f48eb 100644 --- a/pkg/cli/admin/upgrade/recommend/alerts.go +++ b/pkg/cli/admin/upgrade/recommend/alerts.go @@ -6,10 +6,13 @@ import ( "fmt" "strings" + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/api/features" routev1 "github.com/openshift/api/route/v1" routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" + "k8s.io/klog/v2" "github.com/openshift/oc/pkg/cli/admin/inspectalerts" "github.com/openshift/oc/pkg/cli/admin/upgrade/status" @@ -20,6 +23,12 @@ import ( // and Unknown when we do not have enough information to make a // happy-or-sad determination. func (o *options) alerts(ctx context.Context) ([]acceptableCondition, error) { + if skip, err := o.alertsEvaluatedByCVO(ctx); err != nil { + klog.Warningf("An error occured while determining if the CVO is evaluating alerts, so the client will check. %v", err) + } else if skip { + return nil, nil + } + var alertsBytes []byte if o.mockData.alertsPath != "" { if len(o.mockData.alerts) == 0 { @@ -251,3 +260,54 @@ func (o *options) alerts(ctx context.Context) ([]acceptableCondition, error) { return conditions, nil } + +// alertsEvaluatedByCVO makes API calls to determine if we need to do client-side alert checking +func (o *options) alertsEvaluatedByCVO(ctx context.Context) (bool, error) { + featureGates := o.mockData.featureGate + infrastructure := o.mockData.infrastructure + cv := o.mockData.clusterVersion + if cv == nil { + var err error + featureGates, err = o.Client.ConfigV1().FeatureGates().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return false, err + } + + infrastructure, err = o.Client.ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return false, err + } + + cv, err = o.Client.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{}) + if err != nil { + return false, err + } + } + + // if the AcceptRisks feature gate is enabled AND oc is not running against a hosted cluster, + // the CVO is handling alerts and will generate the Recommended condition if needed + return isAcceptRisksEnabled(featureGates, cv.Status.Desired.Version) && !isHostedCluster(infrastructure), nil +} + +// isAcceptRisksEnabled checks to see if the 'ClusterUpdateAcceptRisks' feature gate is enabled +// if so, return true to skip client-side alert checking +func isAcceptRisksEnabled(featureGate *configv1.FeatureGate, clusterVersion string) bool { + if featureGate == nil { + return false + } + + for _, versionedGates := range featureGate.Status.FeatureGates { + if versionedGates.Version == clusterVersion { + for _, enabledGate := range versionedGates.Enabled { + if enabledGate.Name == features.FeatureGateClusterUpdateAcceptRisks { + return true + } + } + } + } + return false +} + +func isHostedCluster(i *configv1.Infrastructure) bool { + return i != nil && i.Status.ControlPlaneTopology == configv1.ExternalTopologyMode +} diff --git a/pkg/cli/admin/upgrade/recommend/alerts_test.go b/pkg/cli/admin/upgrade/recommend/alerts_test.go new file mode 100644 index 0000000000..6566e9710d --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/alerts_test.go @@ -0,0 +1,132 @@ +package recommend + +import ( + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/api/features" +) + +func TestIsAcceptRisksEnabled(t *testing.T) { + + for _, testCase := range []struct { + name string + featureGateConfig *configv1.FeatureGate + expected bool + }{ + { + name: "no feature gates", + }, + { + name: "ClusterUpdateAcceptRisks feature gate is enabled", + featureGateConfig: &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.22.0", + Enabled: []configv1.FeatureGateAttributes{ + { + Name: features.FeatureGateClusterUpdateAcceptRisks, + }, + }, + }, + }, + }, + }, + expected: true, + }, + { + name: "ClusterUpdateAcceptRisks feature gate is explicitly disabled", + featureGateConfig: &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.22.0", + Disabled: []configv1.FeatureGateAttributes{ + { + Name: features.FeatureGateClusterUpdateAcceptRisks, + }, + }, + }, + }, + }, + }, + }, + { + name: "ClusterUpdateAcceptRisks feature gate is not explicitly enabled or disabled", + featureGateConfig: &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.22.0", + Enabled: []configv1.FeatureGateAttributes{}, + Disabled: []configv1.FeatureGateAttributes{}, + }, + }, + }, + }, + }, + { + name: "ClusterUpdateAcceptRisks feature gate is enabled for a different cluster version", + featureGateConfig: &configv1.FeatureGate{ + Status: configv1.FeatureGateStatus{ + FeatureGates: []configv1.FeatureGateDetails{ + { + Version: "4.21.0", + Enabled: []configv1.FeatureGateAttributes{ + { + Name: features.FeatureGateClusterUpdateAcceptRisks, + }, + }, + }, + }, + }, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + actual := isAcceptRisksEnabled(testCase.featureGateConfig, "4.22.0") + + if actual != testCase.expected { + t.Errorf("%v != %v", actual, testCase.expected) + } + }) + } +} + +func TestIsHypershiftEnabled(t *testing.T) { + for _, testCase := range []struct { + name string + infrastructure *configv1.Infrastructure + expected bool + }{ + { + name: "no infrastructure", + }, + { + name: "hypershift enabled", + infrastructure: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + ControlPlaneTopology: configv1.ExternalTopologyMode, + }, + }, + expected: true, + }, + { + name: "hypershift not enabled", + infrastructure: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + ControlPlaneTopology: configv1.HighlyAvailableTopologyMode, + }, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + actual := isHostedCluster(testCase.infrastructure) + + if actual != testCase.expected { + t.Errorf("%v != %v", actual, testCase.expected) + } + }) + } +} diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-alerts.json b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-alerts.json new file mode 100644 index 0000000000..a5307ab53a --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-alerts.json @@ -0,0 +1,252 @@ +{ + "status": "success", + "data": { + "alerts": [ + { + "labels": { + "alertname": "ClusterNotUpgradeable", + "condition": "Upgradeable", + "endpoint": "metrics", + "name": "version", + "namespace": "openshift-cluster-version", + "severity": "info" + }, + "annotations": { + "description": "In most cases, you will still be able to apply patch releases. Reason MultipleReasons. For more information refer to 'oc adm upgrade' or https://console-openshift-console.apps.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org/settings/cluster/.", + "summary": "One or more cluster operators have been blocking minor or major version cluster updates for at least an hour." + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:24.835299202Z", + "value": "0e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "ClusterOperatorDegraded", + "name": "monitoring", + "namespace": "openshift-cluster-version", + "reason": "UpdatingPrometheusFailed", + "severity": "warning" + }, + "annotations": { + "description": "The monitoring operator is degraded because UpdatingPrometheusFailed, and the components it manages may have reduced quality of service. Cluster upgrades may not complete. For more information refer to 'oc get -o yaml clusteroperator monitoring' or https://console-openshift-console.apps.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org/settings/cluster/.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md", + "summary": "Cluster operator has been degraded for 30 minutes." + }, + "state": "firing", + "activeAt": "2026-06-22T18:52:24.835299202Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "ClusterOperatorDegraded", + "name": "version", + "namespace": "openshift-cluster-version", + "reason": "ClusterOperatorDegraded", + "severity": "warning" + }, + "annotations": { + "description": "The version operator is degraded because ClusterOperatorDegraded, and the components it manages may have reduced quality of service. Cluster upgrades may not complete. For more information refer to 'oc adm upgrade' or https://console-openshift-console.apps.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org/settings/cluster/.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md", + "summary": "Cluster operator has been degraded for 30 minutes." + }, + "state": "firing", + "activeAt": "2026-06-22T18:53:24.835299202Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "OpenShiftUpdateRiskMightApply", + "namespace": "openshift-cluster-version", + "reason": "Alert:firing", + "risk": "PodDisruptionBudgetAtLimit", + "severity": "warning" + }, + "annotations": { + "description": "The conditional update risk PodDisruptionBudgetAtLimit might apply to the cluster because of Alert:firing, and the cluster update to a version exposed to the risk is not recommended. For more information refer to 'oc adm upgrade'.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-version-operator/OpenShiftUpdateRiskMightApply.md", + "summary": "The cluster might have been exposed to the conditional update risk for 15 minutes." + }, + "state": "pending", + "activeAt": "2026-06-22T19:30:27.711254056Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "InsightsRecommendationActive", + "container": "insights-operator", + "description": "Enabling the **TechPreviewNoUpgrade** feature set on your cluster\ncan not be undone and prevents minor version updates. Please do\nnot enable this feature set on production clusters.\n", + "endpoint": "https", + "info_link": "https://console.redhat.com/openshift/insights/advisor/clusters/ab59f554-a72f-40c3-87e7-49ebec220e4f?first=ccx_rules_ocp.external.rules.upgrade_is_blocked_due_to_tpfg%7CTECH_PREVIEW_NO_UPGRADE_FEATURE_SET_IS_ENABLED", + "instance": "10.130.0.40:8443", + "job": "metrics", + "namespace": "openshift-insights", + "pod": "insights-operator-584747df49-wqrjp", + "service": "metrics", + "severity": "info", + "total_risk": "Important" + }, + "annotations": { + "description": "Insights recommendation \"Enabling the **TechPreviewNoUpgrade** feature set on your cluster\ncan not be undone and prevents minor version updates. Please do\nnot enable this feature set on production clusters.\n\" with total risk \"Important\" was detected on the cluster. More information is available at https://console.redhat.com/openshift/insights/advisor/clusters/ab59f554-a72f-40c3-87e7-49ebec220e4f?first=ccx_rules_ocp.external.rules.upgrade_is_blocked_due_to_tpfg%7CTECH_PREVIEW_NO_UPGRADE_FEATURE_SET_IS_ENABLED.", + "summary": "An Insights recommendation is active for this cluster." + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:43.284361153Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "TechPreviewNoUpgrade", + "container": "kube-apiserver-operator", + "endpoint": "https", + "instance": "10.130.0.23:8443", + "job": "kube-apiserver-operator", + "name": "TechPreviewNoUpgrade", + "namespace": "openshift-kube-apiserver-operator", + "pod": "kube-apiserver-operator-7d998989dd-l5tww", + "service": "metrics", + "severity": "warning" + }, + "annotations": { + "description": "Cluster has enabled Technology Preview features that cannot be undone and will prevent upgrades. The TechPreviewNoUpgrade feature set is not recommended on production clusters.", + "summary": "Cluster has enabled tech preview features that will prevent upgrades." + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:30.88338992Z", + "value": "0e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "PodDisruptionBudgetAtLimit", + "namespace": "openshift-monitoring", + "poddisruptionbudget": "prometheus-k8s", + "severity": "warning" + }, + "annotations": { + "description": "The pod disruption budget is at the minimum disruptions allowed level. The number of current healthy pods is equal to the desired healthy pods.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md", + "summary": "The pod disruption budget is preventing further disruption to pods." + }, + "state": "firing", + "activeAt": "2026-06-22T18:19:01.64123308Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "Watchdog", + "namespace": "openshift-monitoring", + "severity": "none" + }, + "annotations": { + "description": "This is an alert meant to ensure that the entire alerting pipeline is functional.\nThis alert is always firing, therefore it should always be firing in Alertmanager\nand always fire against a receiver. There are integrations with various notification\nmechanisms that send a notification when this alert is not firing. For example the\n\"DeadMansSnitch\" integration in PagerDuty.\n", + "summary": "An alert that should always be firing to certify that Alertmanager is working properly." + }, + "state": "firing", + "activeAt": "2026-06-22T17:54:52.013519252Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "TargetDown", + "job": "check-endpoints", + "namespace": "openshift-apiserver", + "service": "check-endpoints", + "severity": "warning" + }, + "annotations": { + "description": "100% of the check-endpoints/check-endpoints targets in openshift-apiserver namespace have been unreachable for more than 15 minutes. This may be a symptom of network connectivity issues, down nodes, or failures within these components. Assess the health of the infrastructure and nodes running these targets and then contact support.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/TargetDown.md", + "summary": "Some targets were not reachable from the monitoring server for an extended period of time." + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:19.290981414Z", + "value": "1e+02", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "AlertmanagerReceiversNotConfigured", + "namespace": "openshift-monitoring", + "severity": "warning" + }, + "annotations": { + "description": "Alerts are not configured to be sent to a notification system, meaning that you may not be notified in a timely fashion when important failures occur. Check the OpenShift documentation to learn how to configure notifications with Alertmanager.", + "summary": "Receivers (notification integrations) are not configured on Alertmanager" + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:19.616745055Z", + "value": "0e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "ClusterMonitoringOperatorReconciliationErrors", + "container": "cluster-monitoring-operator", + "endpoint": "https", + "instance": "10.130.0.38:8443", + "job": "cluster-monitoring-operator", + "namespace": "openshift-monitoring", + "pod": "cluster-monitoring-operator-56c95fb488-cxqcp", + "service": "cluster-monitoring-operator", + "severity": "warning" + }, + "annotations": { + "description": "Errors are occurring during reconciliation cycles. Inspect the cluster-monitoring-operator log for potential root causes.", + "summary": "Cluster Monitoring Operator is experiencing unexpected reconciliation errors." + }, + "state": "pending", + "activeAt": "2026-06-22T18:41:49.616745055Z", + "value": "0e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "KubePodNotScheduled", + "container": "kube-rbac-proxy-main", + "endpoint": "https-main", + "job": "kube-state-metrics", + "namespace": "openshift-monitoring", + "pod": "prometheus-k8s-0", + "service": "kube-state-metrics", + "severity": "warning", + "uid": "5654bd3d-f61e-4c27-a56b-63eea0effc5e" + }, + "annotations": { + "description": "Pod openshift-monitoring/prometheus-k8s-0 cannot be scheduled for more than 30 minutes.\nCheck the details of the pod with the following command:\noc describe -n openshift-monitoring pod prometheus-k8s-0", + "summary": "Pod cannot be scheduled." + }, + "state": "firing", + "activeAt": "2026-06-22T18:18:49.616745055Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "KubeStatefulSetReplicasMismatch", + "container": "kube-rbac-proxy-main", + "endpoint": "https-main", + "job": "kube-state-metrics", + "namespace": "openshift-monitoring", + "service": "kube-state-metrics", + "severity": "warning", + "statefulset": "prometheus-k8s" + }, + "annotations": { + "description": "StatefulSet openshift-monitoring/prometheus-k8s has not matched the expected number of replicas for longer than 15 minutes.", + "summary": "StatefulSet has not matched the expected number of replicas." + }, + "state": "firing", + "activeAt": "2026-06-22T18:18:49.654575017Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + } + ] + } +} \ No newline at end of file diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-cv.yaml b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-cv.yaml new file mode 100644 index 0000000000..0e817ea73a --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-cv.yaml @@ -0,0 +1,171 @@ +apiVersion: config.openshift.io/v1 +kind: ClusterVersion +metadata: + creationTimestamp: "2026-06-22T17:25:19Z" + generation: 3 + name: version + resourceVersion: "63760" + uid: bc00f3f2-da92-4342-9882-ac16001b3d13 +spec: + channel: candidate-5.0 + clusterID: ab59f554-a72f-40c3-87e7-49ebec220e4f + overrides: + - group: config.openshift.io + kind: ClusterImagePolicy + name: openshift + namespace: "" + unmanaged: true +status: + availableUpdates: null + capabilities: + enabledCapabilities: + - Build + - CSISnapshot + - CloudControllerManager + - CloudCredential + - Console + - DeploymentConfig + - ImageRegistry + - Ingress + - Insights + - MachineAPI + - NodeTuning + - OperatorLifecycleManager + - OperatorLifecycleManagerV1 + - Storage + - baremetal + - marketplace + - openshift-samples + knownCapabilities: + - Build + - CSISnapshot + - CloudControllerManager + - CloudCredential + - Console + - DeploymentConfig + - ImageRegistry + - Ingress + - Insights + - MachineAPI + - NodeTuning + - OperatorLifecycleManager + - OperatorLifecycleManagerV1 + - Storage + - baremetal + - marketplace + - openshift-samples + conditionalUpdateRisks: + - conditions: + - lastTransitionTime: "2026-06-22T18:19:01Z" + message: 'warning alert PodDisruptionBudgetAtLimit firing, which might slow + node drains. Namespace=openshift-monitoring, PodDisruptionBudget=prometheus-k8s. + The pod disruption budget is preventing further disruption to pods. The alert + description is: The pod disruption budget is at the minimum disruptions allowed + level. The number of current healthy pods is equal to the desired healthy + pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md' + reason: Alert:firing + status: "True" + type: Applies + matchingRules: + - type: Always + message: The pod disruption budget is preventing further disruption to pods. + name: PodDisruptionBudgetAtLimit + url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + conditionalUpdates: + - conditions: + - lastTransitionTime: "2026-06-22T19:30:17Z" + message: The pod disruption budget is preventing further disruption to pods. + https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + reason: PodDisruptionBudgetAtLimit + status: "False" + type: Recommended + release: + channels: + - candidate-5.0 + image: quay.io/openshift-release-dev/ocp-release@sha256:b5ad6b4bb5efbd9b9d88bd42182e85388742b4441a2ff4301c4f5aa8176d5d29 + version: 5.0.0-ec.3 + riskNames: + - PodDisruptionBudgetAtLimit + risks: + - conditions: + - lastTransitionTime: "2026-06-22T18:19:01Z" + message: 'warning alert PodDisruptionBudgetAtLimit firing, which might slow + node drains. Namespace=openshift-monitoring, PodDisruptionBudget=prometheus-k8s. + The pod disruption budget is preventing further disruption to pods. The + alert description is: The pod disruption budget is at the minimum disruptions + allowed level. The number of current healthy pods is equal to the desired + healthy pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md' + reason: Alert:firing + status: "True" + type: Applies + matchingRules: + - type: Always + message: The pod disruption budget is preventing further disruption to pods. + name: PodDisruptionBudgetAtLimit + url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + conditions: + - lastTransitionTime: "2026-06-22T19:30:17Z" + status: "True" + type: RetrievedUpdates + - lastTransitionTime: "2026-06-22T17:26:06Z" + message: |- + Cluster should not be upgraded between minor or major versions for multiple reasons: ClusterVersionOverridesSet,FeatureGates_RestrictedFeatureGates_TechPreviewNoUpgrade + * Disabling ownership via cluster version overrides prevents upgrades between minor or major versions. Please remove overrides before requesting a minor or major version update. + * Cluster operator config-operator should not be upgraded between minor or major versions: FeatureGatesUpgradeable: "TechPreviewNoUpgrade" does not allow updates + reason: MultipleReasons + status: "False" + type: Upgradeable + - lastTransitionTime: "2026-06-22T17:26:06Z" + message: Capabilities match configured spec + reason: AsExpected + status: "False" + type: ImplicitlyEnabledCapabilities + - lastTransitionTime: "2026-06-22T17:26:06Z" + message: Payload loaded version="5.0.0-ec.2" image="registry.build12.ci.openshift.org/ci-ln-8np31z2/release@sha256:e8f74fa0423f416cdcd6b35a0769765118c0f8ba9d7e0e25517e41c59483c6af" + architecture="amd64" + reason: PayloadLoaded + status: "True" + type: ReleaseAccepted + - lastTransitionTime: "2026-06-22T17:58:15Z" + message: Done applying 5.0.0-ec.2 + status: "True" + type: Available + - lastTransitionTime: "2026-06-22T18:53:00Z" + message: Cluster operator monitoring is degraded + reason: ClusterOperatorDegraded + status: "True" + type: Failing + - lastTransitionTime: "2026-06-22T17:58:15Z" + message: 'Error while reconciling 5.0.0-ec.2: the cluster operator monitoring + is degraded' + reason: ClusterOperatorDegraded + status: "False" + type: Progressing + - lastTransitionTime: "2026-06-22T17:26:51Z" + message: Disabling ownership via cluster version overrides prevents upgrades between + minor or major versions. Please remove overrides before requesting a minor or + major version update. + reason: ClusterVersionOverridesSet + status: "False" + type: UpgradeableClusterVersionOverrides + - lastTransitionTime: "2026-06-22T17:31:17Z" + message: 'Cluster operator config-operator should not be upgraded between minor + or major versions: FeatureGatesUpgradeable: "TechPreviewNoUpgrade" does not + allow updates' + reason: FeatureGates_RestrictedFeatureGates_TechPreviewNoUpgrade + status: "False" + type: UpgradeableClusterOperators + desired: + channels: + - candidate-5.0 + image: registry.build12.ci.openshift.org/ci-ln-8np31z2/release@sha256:e8f74fa0423f416cdcd6b35a0769765118c0f8ba9d7e0e25517e41c59483c6af + version: 5.0.0-ec.2 + history: + - completionTime: "2026-06-22T17:58:15Z" + image: registry.build12.ci.openshift.org/ci-ln-8np31z2/release@sha256:e8f74fa0423f416cdcd6b35a0769765118c0f8ba9d7e0e25517e41c59483c6af + startedTime: "2026-06-22T17:26:06Z" + state: Completed + verified: false + version: 5.0.0-ec.2 + observedGeneration: 2 + versionHash: cUYyzI3AJLo= diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-featuregate.yaml b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-featuregate.yaml new file mode 100644 index 0000000000..561db1f92f --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-featuregate.yaml @@ -0,0 +1,130 @@ +apiVersion: config.openshift.io/v1 +kind: FeatureGate +metadata: + annotations: + include.release.openshift.io/self-managed-high-availability: "true" + creationTimestamp: "2026-06-22T17:25:39Z" + generation: 1 + name: cluster + resourceVersion: "718" + uid: e6d830fe-e07a-4276-8730-fc8d9089d10e +spec: + featureSet: TechPreviewNoUpgrade +status: + featureGates: + - disabled: + - name: ClientsAllowCBOR + - name: ClusterAPIComputeInstall + - name: ClusterAPIControlPlaneInstall + - name: ClusterAPIInstall + - name: ClusterUpdatePreflight + - name: ConfidentialCluster + - name: EventedPLEG + - name: Example2 + - name: ExternalOIDCExternalClaimsSourcing + - name: ExternalSnapshotMetadata + - name: HyperShiftOnlyDynamicResourceAllocation + - name: MachineAPIMigrationVSphere + - name: MachineAPIOperatorDisableMachineHealthCheckController + - name: MultiArchInstallAzure + - name: NetworkConnect + - name: ProvisioningRequestAvailable + - name: ShortCertRotation + - name: VSphereMultiVCenterDay2 + enabled: + - name: AWSClusterHostedDNS + - name: AWSClusterHostedDNSInstall + - name: AWSDedicatedHosts + - name: AWSDualStackInstall + - name: AWSEuropeanSovereignCloudInstall + - name: AWSServiceLBNetworkSecurityGroup + - name: AdditionalStorageConfig + - name: AutomatedEtcdBackup + - name: AzureClusterHostedDNSInstall + - name: AzureDedicatedHosts + - name: AzureDualStackInstall + - name: AzureMultiDisk + - name: AzureWorkloadIdentity + - name: BootImageSkewEnforcement + - name: BootcNodeManagement + - name: BuildCSIVolumes + - name: CBORServingAndStorage + - name: CRDCompatibilityRequirementOperator + - name: CRIOCredentialProviderConfig + - name: ClientsPreferCBOR + - name: ClusterAPIInstallIBMCloud + - name: ClusterAPIMachineManagement + - name: ClusterAPIMachineManagementAWS + - name: ClusterAPIMachineManagementAzure + - name: ClusterAPIMachineManagementBareMetal + - name: ClusterAPIMachineManagementGCP + - name: ClusterAPIMachineManagementOpenStack + - name: ClusterAPIMachineManagementPowerVS + - name: ClusterAPIMachineManagementVSphere + - name: ClusterMonitoringConfig + - name: ClusterUpdateAcceptRisks + - name: ClusterVersionOperatorConfiguration + - name: ConfigurablePKI + - name: DNSNameResolver + - name: DualReplica + - name: DyanmicServiceEndpointIBMCloud + - name: EVPN + - name: EtcdBackendQuota + - name: EventTTL + - name: Example + - name: ExternalOIDC + - name: ExternalOIDCWithUIDAndExtraClaimMappings + - name: ExternalOIDCWithUpstreamParity + - name: GCPCustomAPIEndpoints + - name: GCPCustomAPIEndpointsInstall + - name: GCPDualStackInstall + - name: GatewayAPIWithoutOLM + - name: ImageModeStatusReporting + - name: ImageStreamImportMode + - name: IngressControllerDynamicConfigurationManager + - name: InsightsConfig + - name: InsightsOnDemandDataGather + - name: IrreconcilableMachineConfig + - name: KMSEncryption + - name: KMSv1 + - name: MachineAPIMigration + - name: MachineAPIMigrationAWS + - name: MachineAPIMigrationOpenStack + - name: ManagedBootImagesCPMS + - name: MaxUnavailableStatefulSet + - name: MetricsCollectionProfiles + - name: MinimumKubeletVersion + - name: MixedCPUsAllocation + - name: MultiDiskSetup + - name: MutableCSINodeAllocatableCount + - name: MutatingAdmissionPolicy + - name: NewOLM + - name: NewOLMBoxCutterRuntime + - name: NewOLMCatalogdAPIV1Metas + - name: NewOLMConfigAPI + - name: NewOLMOwnSingleNamespace + - name: NewOLMPreflightPermissionChecks + - name: NewOLMWebhookProviderOpenshiftServiceCA + - name: NoOverlayMode + - name: NoRegistryClusterInstall + - name: NutanixMultiSubnets + - name: OLMLifecycleAndCompatibility + - name: OSStreams + - name: OVNObservability + - name: OnPremDNSRecords + - name: OpenShiftPodSecurityAdmission + - name: SELinuxMount + - name: ServiceAccountTokenNodeBinding + - name: SignatureStores + - name: SigstoreImageVerification + - name: SigstoreImageVerificationPKI + - name: StoragePerformantSecurityPolicy + - name: TLSAdherence + - name: UpgradeStatus + - name: VSphereConfigurableMaxAllowedBlockVolumesPerNode + - name: VSphereHostVMGroupZonal + - name: VSphereMixedNodeEnv + - name: VSphereMultiDisk + - name: VSphereMultiNetworks + - name: VolumeGroupSnapshot + version: 5.0.0-ec.2 diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-infrastructure.yaml b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-infrastructure.yaml new file mode 100644 index 0000000000..ab3c42fe29 --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks-infrastructure.yaml @@ -0,0 +1,30 @@ +apiVersion: config.openshift.io/v1 +kind: Infrastructure +metadata: + creationTimestamp: "2026-06-22T17:25:11Z" + generation: 1 + name: cluster + resourceVersion: "580" + uid: e7ac2dd3-5925-44fa-b93d-fc2461223d6a +spec: + cloudConfig: + key: config + name: cloud-provider-config + platformSpec: + type: GCP +status: + apiServerInternalURI: https://api-int.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org:6443 + apiServerURL: https://api.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org:6443 + controlPlaneTopology: HighlyAvailable + cpuPartitioning: None + etcdDiscoveryDomain: "" + infrastructureName: ci-ln-8np31z2-72292-nzmck + infrastructureTopology: HighlyAvailable + platform: GCP + platformStatus: + gcp: + cloudLoadBalancerConfig: + dnsType: PlatformDefault + projectID: openshift-gce-devel-ci-2 + region: us-central1 + type: GCP diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.output b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.output new file mode 100644 index 0000000000..3f7ec49344 --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.output @@ -0,0 +1,14 @@ +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator monitoring is degraded + +Upstream update service is unset, so the cluster will use an appropriate default. +Channel: candidate-5.0 (available channels: candidate-5.0) + +Updates to 5.0: + + Version: 5.0.0-ec.3 + Image: quay.io/openshift-release-dev/ocp-release@sha256:b5ad6b4bb5efbd9b9d88bd42182e85388742b4441a2ff4301c4f5aa8176d5d29 + Reason: PodDisruptionBudgetAtLimit + Message: The pod disruption budget is preventing further disruption to pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.show-outdated-releases-output b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.show-outdated-releases-output new file mode 100644 index 0000000000..027bc72c92 --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.show-outdated-releases-output @@ -0,0 +1,11 @@ +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator monitoring is degraded + +Upstream update service is unset, so the cluster will use an appropriate default. +Channel: candidate-5.0 (available channels: candidate-5.0) + +Updates to 5.0: + VERSION ISSUES + 5.0.0-ec.3 PodDisruptionBudgetAtLimit diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.version-5.0.0-ec.3-output b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.version-5.0.0-ec.3-output new file mode 100644 index 0000000000..e8003db818 --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-handling-risks.version-5.0.0-ec.3-output @@ -0,0 +1,15 @@ +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator monitoring is degraded + +Upstream update service is unset, so the cluster will use an appropriate default. +Channel: candidate-5.0 (available channels: candidate-5.0) + +Update to 5.0.0-ec.3 Recommended=False: +Image: quay.io/openshift-release-dev/ocp-release@sha256:b5ad6b4bb5efbd9b9d88bd42182e85388742b4441a2ff4301c4f5aa8176d5d29 +Release URL: +Reason: PodDisruptionBudgetAtLimit +Message: The pod disruption budget is preventing further disruption to pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + +error: issues that apply to this cluster but which were not included in --accept: ConditionalUpdateRisk,Failing diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-alerts.json b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-alerts.json new file mode 100644 index 0000000000..a5307ab53a --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-alerts.json @@ -0,0 +1,252 @@ +{ + "status": "success", + "data": { + "alerts": [ + { + "labels": { + "alertname": "ClusterNotUpgradeable", + "condition": "Upgradeable", + "endpoint": "metrics", + "name": "version", + "namespace": "openshift-cluster-version", + "severity": "info" + }, + "annotations": { + "description": "In most cases, you will still be able to apply patch releases. Reason MultipleReasons. For more information refer to 'oc adm upgrade' or https://console-openshift-console.apps.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org/settings/cluster/.", + "summary": "One or more cluster operators have been blocking minor or major version cluster updates for at least an hour." + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:24.835299202Z", + "value": "0e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "ClusterOperatorDegraded", + "name": "monitoring", + "namespace": "openshift-cluster-version", + "reason": "UpdatingPrometheusFailed", + "severity": "warning" + }, + "annotations": { + "description": "The monitoring operator is degraded because UpdatingPrometheusFailed, and the components it manages may have reduced quality of service. Cluster upgrades may not complete. For more information refer to 'oc get -o yaml clusteroperator monitoring' or https://console-openshift-console.apps.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org/settings/cluster/.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md", + "summary": "Cluster operator has been degraded for 30 minutes." + }, + "state": "firing", + "activeAt": "2026-06-22T18:52:24.835299202Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "ClusterOperatorDegraded", + "name": "version", + "namespace": "openshift-cluster-version", + "reason": "ClusterOperatorDegraded", + "severity": "warning" + }, + "annotations": { + "description": "The version operator is degraded because ClusterOperatorDegraded, and the components it manages may have reduced quality of service. Cluster upgrades may not complete. For more information refer to 'oc adm upgrade' or https://console-openshift-console.apps.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org/settings/cluster/.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md", + "summary": "Cluster operator has been degraded for 30 minutes." + }, + "state": "firing", + "activeAt": "2026-06-22T18:53:24.835299202Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "OpenShiftUpdateRiskMightApply", + "namespace": "openshift-cluster-version", + "reason": "Alert:firing", + "risk": "PodDisruptionBudgetAtLimit", + "severity": "warning" + }, + "annotations": { + "description": "The conditional update risk PodDisruptionBudgetAtLimit might apply to the cluster because of Alert:firing, and the cluster update to a version exposed to the risk is not recommended. For more information refer to 'oc adm upgrade'.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-version-operator/OpenShiftUpdateRiskMightApply.md", + "summary": "The cluster might have been exposed to the conditional update risk for 15 minutes." + }, + "state": "pending", + "activeAt": "2026-06-22T19:30:27.711254056Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "InsightsRecommendationActive", + "container": "insights-operator", + "description": "Enabling the **TechPreviewNoUpgrade** feature set on your cluster\ncan not be undone and prevents minor version updates. Please do\nnot enable this feature set on production clusters.\n", + "endpoint": "https", + "info_link": "https://console.redhat.com/openshift/insights/advisor/clusters/ab59f554-a72f-40c3-87e7-49ebec220e4f?first=ccx_rules_ocp.external.rules.upgrade_is_blocked_due_to_tpfg%7CTECH_PREVIEW_NO_UPGRADE_FEATURE_SET_IS_ENABLED", + "instance": "10.130.0.40:8443", + "job": "metrics", + "namespace": "openshift-insights", + "pod": "insights-operator-584747df49-wqrjp", + "service": "metrics", + "severity": "info", + "total_risk": "Important" + }, + "annotations": { + "description": "Insights recommendation \"Enabling the **TechPreviewNoUpgrade** feature set on your cluster\ncan not be undone and prevents minor version updates. Please do\nnot enable this feature set on production clusters.\n\" with total risk \"Important\" was detected on the cluster. More information is available at https://console.redhat.com/openshift/insights/advisor/clusters/ab59f554-a72f-40c3-87e7-49ebec220e4f?first=ccx_rules_ocp.external.rules.upgrade_is_blocked_due_to_tpfg%7CTECH_PREVIEW_NO_UPGRADE_FEATURE_SET_IS_ENABLED.", + "summary": "An Insights recommendation is active for this cluster." + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:43.284361153Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "TechPreviewNoUpgrade", + "container": "kube-apiserver-operator", + "endpoint": "https", + "instance": "10.130.0.23:8443", + "job": "kube-apiserver-operator", + "name": "TechPreviewNoUpgrade", + "namespace": "openshift-kube-apiserver-operator", + "pod": "kube-apiserver-operator-7d998989dd-l5tww", + "service": "metrics", + "severity": "warning" + }, + "annotations": { + "description": "Cluster has enabled Technology Preview features that cannot be undone and will prevent upgrades. The TechPreviewNoUpgrade feature set is not recommended on production clusters.", + "summary": "Cluster has enabled tech preview features that will prevent upgrades." + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:30.88338992Z", + "value": "0e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "PodDisruptionBudgetAtLimit", + "namespace": "openshift-monitoring", + "poddisruptionbudget": "prometheus-k8s", + "severity": "warning" + }, + "annotations": { + "description": "The pod disruption budget is at the minimum disruptions allowed level. The number of current healthy pods is equal to the desired healthy pods.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md", + "summary": "The pod disruption budget is preventing further disruption to pods." + }, + "state": "firing", + "activeAt": "2026-06-22T18:19:01.64123308Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "Watchdog", + "namespace": "openshift-monitoring", + "severity": "none" + }, + "annotations": { + "description": "This is an alert meant to ensure that the entire alerting pipeline is functional.\nThis alert is always firing, therefore it should always be firing in Alertmanager\nand always fire against a receiver. There are integrations with various notification\nmechanisms that send a notification when this alert is not firing. For example the\n\"DeadMansSnitch\" integration in PagerDuty.\n", + "summary": "An alert that should always be firing to certify that Alertmanager is working properly." + }, + "state": "firing", + "activeAt": "2026-06-22T17:54:52.013519252Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "TargetDown", + "job": "check-endpoints", + "namespace": "openshift-apiserver", + "service": "check-endpoints", + "severity": "warning" + }, + "annotations": { + "description": "100% of the check-endpoints/check-endpoints targets in openshift-apiserver namespace have been unreachable for more than 15 minutes. This may be a symptom of network connectivity issues, down nodes, or failures within these components. Assess the health of the infrastructure and nodes running these targets and then contact support.", + "runbook_url": "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/TargetDown.md", + "summary": "Some targets were not reachable from the monitoring server for an extended period of time." + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:19.290981414Z", + "value": "1e+02", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "AlertmanagerReceiversNotConfigured", + "namespace": "openshift-monitoring", + "severity": "warning" + }, + "annotations": { + "description": "Alerts are not configured to be sent to a notification system, meaning that you may not be notified in a timely fashion when important failures occur. Check the OpenShift documentation to learn how to configure notifications with Alertmanager.", + "summary": "Receivers (notification integrations) are not configured on Alertmanager" + }, + "state": "firing", + "activeAt": "2026-06-22T17:55:19.616745055Z", + "value": "0e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "ClusterMonitoringOperatorReconciliationErrors", + "container": "cluster-monitoring-operator", + "endpoint": "https", + "instance": "10.130.0.38:8443", + "job": "cluster-monitoring-operator", + "namespace": "openshift-monitoring", + "pod": "cluster-monitoring-operator-56c95fb488-cxqcp", + "service": "cluster-monitoring-operator", + "severity": "warning" + }, + "annotations": { + "description": "Errors are occurring during reconciliation cycles. Inspect the cluster-monitoring-operator log for potential root causes.", + "summary": "Cluster Monitoring Operator is experiencing unexpected reconciliation errors." + }, + "state": "pending", + "activeAt": "2026-06-22T18:41:49.616745055Z", + "value": "0e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "KubePodNotScheduled", + "container": "kube-rbac-proxy-main", + "endpoint": "https-main", + "job": "kube-state-metrics", + "namespace": "openshift-monitoring", + "pod": "prometheus-k8s-0", + "service": "kube-state-metrics", + "severity": "warning", + "uid": "5654bd3d-f61e-4c27-a56b-63eea0effc5e" + }, + "annotations": { + "description": "Pod openshift-monitoring/prometheus-k8s-0 cannot be scheduled for more than 30 minutes.\nCheck the details of the pod with the following command:\noc describe -n openshift-monitoring pod prometheus-k8s-0", + "summary": "Pod cannot be scheduled." + }, + "state": "firing", + "activeAt": "2026-06-22T18:18:49.616745055Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + }, + { + "labels": { + "alertname": "KubeStatefulSetReplicasMismatch", + "container": "kube-rbac-proxy-main", + "endpoint": "https-main", + "job": "kube-state-metrics", + "namespace": "openshift-monitoring", + "service": "kube-state-metrics", + "severity": "warning", + "statefulset": "prometheus-k8s" + }, + "annotations": { + "description": "StatefulSet openshift-monitoring/prometheus-k8s has not matched the expected number of replicas for longer than 15 minutes.", + "summary": "StatefulSet has not matched the expected number of replicas." + }, + "state": "firing", + "activeAt": "2026-06-22T18:18:49.654575017Z", + "value": "1e+00", + "partialResponseStrategy": "WARN" + } + ] + } +} \ No newline at end of file diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-cv.yaml b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-cv.yaml new file mode 100644 index 0000000000..0e817ea73a --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-cv.yaml @@ -0,0 +1,171 @@ +apiVersion: config.openshift.io/v1 +kind: ClusterVersion +metadata: + creationTimestamp: "2026-06-22T17:25:19Z" + generation: 3 + name: version + resourceVersion: "63760" + uid: bc00f3f2-da92-4342-9882-ac16001b3d13 +spec: + channel: candidate-5.0 + clusterID: ab59f554-a72f-40c3-87e7-49ebec220e4f + overrides: + - group: config.openshift.io + kind: ClusterImagePolicy + name: openshift + namespace: "" + unmanaged: true +status: + availableUpdates: null + capabilities: + enabledCapabilities: + - Build + - CSISnapshot + - CloudControllerManager + - CloudCredential + - Console + - DeploymentConfig + - ImageRegistry + - Ingress + - Insights + - MachineAPI + - NodeTuning + - OperatorLifecycleManager + - OperatorLifecycleManagerV1 + - Storage + - baremetal + - marketplace + - openshift-samples + knownCapabilities: + - Build + - CSISnapshot + - CloudControllerManager + - CloudCredential + - Console + - DeploymentConfig + - ImageRegistry + - Ingress + - Insights + - MachineAPI + - NodeTuning + - OperatorLifecycleManager + - OperatorLifecycleManagerV1 + - Storage + - baremetal + - marketplace + - openshift-samples + conditionalUpdateRisks: + - conditions: + - lastTransitionTime: "2026-06-22T18:19:01Z" + message: 'warning alert PodDisruptionBudgetAtLimit firing, which might slow + node drains. Namespace=openshift-monitoring, PodDisruptionBudget=prometheus-k8s. + The pod disruption budget is preventing further disruption to pods. The alert + description is: The pod disruption budget is at the minimum disruptions allowed + level. The number of current healthy pods is equal to the desired healthy + pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md' + reason: Alert:firing + status: "True" + type: Applies + matchingRules: + - type: Always + message: The pod disruption budget is preventing further disruption to pods. + name: PodDisruptionBudgetAtLimit + url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + conditionalUpdates: + - conditions: + - lastTransitionTime: "2026-06-22T19:30:17Z" + message: The pod disruption budget is preventing further disruption to pods. + https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + reason: PodDisruptionBudgetAtLimit + status: "False" + type: Recommended + release: + channels: + - candidate-5.0 + image: quay.io/openshift-release-dev/ocp-release@sha256:b5ad6b4bb5efbd9b9d88bd42182e85388742b4441a2ff4301c4f5aa8176d5d29 + version: 5.0.0-ec.3 + riskNames: + - PodDisruptionBudgetAtLimit + risks: + - conditions: + - lastTransitionTime: "2026-06-22T18:19:01Z" + message: 'warning alert PodDisruptionBudgetAtLimit firing, which might slow + node drains. Namespace=openshift-monitoring, PodDisruptionBudget=prometheus-k8s. + The pod disruption budget is preventing further disruption to pods. The + alert description is: The pod disruption budget is at the minimum disruptions + allowed level. The number of current healthy pods is equal to the desired + healthy pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md' + reason: Alert:firing + status: "True" + type: Applies + matchingRules: + - type: Always + message: The pod disruption budget is preventing further disruption to pods. + name: PodDisruptionBudgetAtLimit + url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + conditions: + - lastTransitionTime: "2026-06-22T19:30:17Z" + status: "True" + type: RetrievedUpdates + - lastTransitionTime: "2026-06-22T17:26:06Z" + message: |- + Cluster should not be upgraded between minor or major versions for multiple reasons: ClusterVersionOverridesSet,FeatureGates_RestrictedFeatureGates_TechPreviewNoUpgrade + * Disabling ownership via cluster version overrides prevents upgrades between minor or major versions. Please remove overrides before requesting a minor or major version update. + * Cluster operator config-operator should not be upgraded between minor or major versions: FeatureGatesUpgradeable: "TechPreviewNoUpgrade" does not allow updates + reason: MultipleReasons + status: "False" + type: Upgradeable + - lastTransitionTime: "2026-06-22T17:26:06Z" + message: Capabilities match configured spec + reason: AsExpected + status: "False" + type: ImplicitlyEnabledCapabilities + - lastTransitionTime: "2026-06-22T17:26:06Z" + message: Payload loaded version="5.0.0-ec.2" image="registry.build12.ci.openshift.org/ci-ln-8np31z2/release@sha256:e8f74fa0423f416cdcd6b35a0769765118c0f8ba9d7e0e25517e41c59483c6af" + architecture="amd64" + reason: PayloadLoaded + status: "True" + type: ReleaseAccepted + - lastTransitionTime: "2026-06-22T17:58:15Z" + message: Done applying 5.0.0-ec.2 + status: "True" + type: Available + - lastTransitionTime: "2026-06-22T18:53:00Z" + message: Cluster operator monitoring is degraded + reason: ClusterOperatorDegraded + status: "True" + type: Failing + - lastTransitionTime: "2026-06-22T17:58:15Z" + message: 'Error while reconciling 5.0.0-ec.2: the cluster operator monitoring + is degraded' + reason: ClusterOperatorDegraded + status: "False" + type: Progressing + - lastTransitionTime: "2026-06-22T17:26:51Z" + message: Disabling ownership via cluster version overrides prevents upgrades between + minor or major versions. Please remove overrides before requesting a minor or + major version update. + reason: ClusterVersionOverridesSet + status: "False" + type: UpgradeableClusterVersionOverrides + - lastTransitionTime: "2026-06-22T17:31:17Z" + message: 'Cluster operator config-operator should not be upgraded between minor + or major versions: FeatureGatesUpgradeable: "TechPreviewNoUpgrade" does not + allow updates' + reason: FeatureGates_RestrictedFeatureGates_TechPreviewNoUpgrade + status: "False" + type: UpgradeableClusterOperators + desired: + channels: + - candidate-5.0 + image: registry.build12.ci.openshift.org/ci-ln-8np31z2/release@sha256:e8f74fa0423f416cdcd6b35a0769765118c0f8ba9d7e0e25517e41c59483c6af + version: 5.0.0-ec.2 + history: + - completionTime: "2026-06-22T17:58:15Z" + image: registry.build12.ci.openshift.org/ci-ln-8np31z2/release@sha256:e8f74fa0423f416cdcd6b35a0769765118c0f8ba9d7e0e25517e41c59483c6af + startedTime: "2026-06-22T17:26:06Z" + state: Completed + verified: false + version: 5.0.0-ec.2 + observedGeneration: 2 + versionHash: cUYyzI3AJLo= diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-featuregate.yaml b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-featuregate.yaml new file mode 100644 index 0000000000..3c7f7ca074 --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-featuregate.yaml @@ -0,0 +1,130 @@ +apiVersion: config.openshift.io/v1 +kind: FeatureGate +metadata: + annotations: + include.release.openshift.io/self-managed-high-availability: "true" + creationTimestamp: "2026-06-22T17:25:39Z" + generation: 1 + name: cluster + resourceVersion: "718" + uid: e6d830fe-e07a-4276-8730-fc8d9089d10e +spec: + featureSet: TechPreviewNoUpgrade +status: + featureGates: + - disabled: + - name: ClientsAllowCBOR + - name: ClusterAPIComputeInstall + - name: ClusterAPIControlPlaneInstall + - name: ClusterAPIInstall + - name: ClusterUpdatePreflight + - name: ConfidentialCluster + - name: EventedPLEG + - name: Example2 + - name: ExternalOIDCExternalClaimsSourcing + - name: ExternalSnapshotMetadata + - name: HyperShiftOnlyDynamicResourceAllocation + - name: MachineAPIMigrationVSphere + - name: MachineAPIOperatorDisableMachineHealthCheckController + - name: MultiArchInstallAzure + - name: NetworkConnect + - name: ProvisioningRequestAvailable + - name: ShortCertRotation + - name: VSphereMultiVCenterDay2 + - name: ClusterUpdateAcceptRisks + enabled: + - name: AWSClusterHostedDNS + - name: AWSClusterHostedDNSInstall + - name: AWSDedicatedHosts + - name: AWSDualStackInstall + - name: AWSEuropeanSovereignCloudInstall + - name: AWSServiceLBNetworkSecurityGroup + - name: AdditionalStorageConfig + - name: AutomatedEtcdBackup + - name: AzureClusterHostedDNSInstall + - name: AzureDedicatedHosts + - name: AzureDualStackInstall + - name: AzureMultiDisk + - name: AzureWorkloadIdentity + - name: BootImageSkewEnforcement + - name: BootcNodeManagement + - name: BuildCSIVolumes + - name: CBORServingAndStorage + - name: CRDCompatibilityRequirementOperator + - name: CRIOCredentialProviderConfig + - name: ClientsPreferCBOR + - name: ClusterAPIInstallIBMCloud + - name: ClusterAPIMachineManagement + - name: ClusterAPIMachineManagementAWS + - name: ClusterAPIMachineManagementAzure + - name: ClusterAPIMachineManagementBareMetal + - name: ClusterAPIMachineManagementGCP + - name: ClusterAPIMachineManagementOpenStack + - name: ClusterAPIMachineManagementPowerVS + - name: ClusterAPIMachineManagementVSphere + - name: ClusterMonitoringConfig + - name: ClusterVersionOperatorConfiguration + - name: ConfigurablePKI + - name: DNSNameResolver + - name: DualReplica + - name: DyanmicServiceEndpointIBMCloud + - name: EVPN + - name: EtcdBackendQuota + - name: EventTTL + - name: Example + - name: ExternalOIDC + - name: ExternalOIDCWithUIDAndExtraClaimMappings + - name: ExternalOIDCWithUpstreamParity + - name: GCPCustomAPIEndpoints + - name: GCPCustomAPIEndpointsInstall + - name: GCPDualStackInstall + - name: GatewayAPIWithoutOLM + - name: ImageModeStatusReporting + - name: ImageStreamImportMode + - name: IngressControllerDynamicConfigurationManager + - name: InsightsConfig + - name: InsightsOnDemandDataGather + - name: IrreconcilableMachineConfig + - name: KMSEncryption + - name: KMSv1 + - name: MachineAPIMigration + - name: MachineAPIMigrationAWS + - name: MachineAPIMigrationOpenStack + - name: ManagedBootImagesCPMS + - name: MaxUnavailableStatefulSet + - name: MetricsCollectionProfiles + - name: MinimumKubeletVersion + - name: MixedCPUsAllocation + - name: MultiDiskSetup + - name: MutableCSINodeAllocatableCount + - name: MutatingAdmissionPolicy + - name: NewOLM + - name: NewOLMBoxCutterRuntime + - name: NewOLMCatalogdAPIV1Metas + - name: NewOLMConfigAPI + - name: NewOLMOwnSingleNamespace + - name: NewOLMPreflightPermissionChecks + - name: NewOLMWebhookProviderOpenshiftServiceCA + - name: NoOverlayMode + - name: NoRegistryClusterInstall + - name: NutanixMultiSubnets + - name: OLMLifecycleAndCompatibility + - name: OSStreams + - name: OVNObservability + - name: OnPremDNSRecords + - name: OpenShiftPodSecurityAdmission + - name: SELinuxMount + - name: ServiceAccountTokenNodeBinding + - name: SignatureStores + - name: SigstoreImageVerification + - name: SigstoreImageVerificationPKI + - name: StoragePerformantSecurityPolicy + - name: TLSAdherence + - name: UpgradeStatus + - name: VSphereConfigurableMaxAllowedBlockVolumesPerNode + - name: VSphereHostVMGroupZonal + - name: VSphereMixedNodeEnv + - name: VSphereMultiDisk + - name: VSphereMultiNetworks + - name: VolumeGroupSnapshot + version: 5.0.0-ec.2 diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-infrastructure.yaml b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-infrastructure.yaml new file mode 100644 index 0000000000..ab3c42fe29 --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks-infrastructure.yaml @@ -0,0 +1,30 @@ +apiVersion: config.openshift.io/v1 +kind: Infrastructure +metadata: + creationTimestamp: "2026-06-22T17:25:11Z" + generation: 1 + name: cluster + resourceVersion: "580" + uid: e7ac2dd3-5925-44fa-b93d-fc2461223d6a +spec: + cloudConfig: + key: config + name: cloud-provider-config + platformSpec: + type: GCP +status: + apiServerInternalURI: https://api-int.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org:6443 + apiServerURL: https://api.ci-ln-8np31z2-72292.gcp-2.ci.openshift.org:6443 + controlPlaneTopology: HighlyAvailable + cpuPartitioning: None + etcdDiscoveryDomain: "" + infrastructureName: ci-ln-8np31z2-72292-nzmck + infrastructureTopology: HighlyAvailable + platform: GCP + platformStatus: + gcp: + cloudLoadBalancerConfig: + dnsType: PlatformDefault + projectID: openshift-gce-devel-ci-2 + region: us-central1 + type: GCP diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.output b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.output new file mode 100644 index 0000000000..e23bc9015e --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.output @@ -0,0 +1,23 @@ +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator monitoring is degraded + +The following conditions found no cause for concern in updating this cluster to later releases: recommended/CriticalAlerts (AsExpected), recommended/NodeAlerts (AsExpected), recommended/PodImagePullAlerts (AsExpected), recommended/UpdatePrecheckAlerts (AsExpected) + +The following conditions found cause for concern in updating this cluster to later releases: recommended/PodDisruptionBudgetAlerts/PodDisruptionBudgetAtLimit/0 + +recommended/PodDisruptionBudgetAlerts/PodDisruptionBudgetAtLimit/0=False: + + Reason: Alert:firing + Message: warning alert PodDisruptionBudgetAtLimit firing, which might slow node drains. Namespace=openshift-monitoring, PodDisruptionBudget=prometheus-k8s. The pod disruption budget is preventing further disruption to pods. The alert description is: The pod disruption budget is at the minimum disruptions allowed level. The number of current healthy pods is equal to the desired healthy pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + +Upstream update service is unset, so the cluster will use an appropriate default. +Channel: candidate-5.0 (available channels: candidate-5.0) + +Updates to 5.0: + + Version: 5.0.0-ec.3 + Image: quay.io/openshift-release-dev/ocp-release@sha256:b5ad6b4bb5efbd9b9d88bd42182e85388742b4441a2ff4301c4f5aa8176d5d29 + Reason: PodDisruptionBudgetAtLimit + Message: The pod disruption budget is preventing further disruption to pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.show-outdated-releases-output b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.show-outdated-releases-output new file mode 100644 index 0000000000..7de0e8d546 --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.show-outdated-releases-output @@ -0,0 +1,20 @@ +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator monitoring is degraded + +The following conditions found no cause for concern in updating this cluster to later releases: recommended/CriticalAlerts (AsExpected), recommended/NodeAlerts (AsExpected), recommended/PodImagePullAlerts (AsExpected), recommended/UpdatePrecheckAlerts (AsExpected) + +The following conditions found cause for concern in updating this cluster to later releases: recommended/PodDisruptionBudgetAlerts/PodDisruptionBudgetAtLimit/0 + +recommended/PodDisruptionBudgetAlerts/PodDisruptionBudgetAtLimit/0=False: + + Reason: Alert:firing + Message: warning alert PodDisruptionBudgetAtLimit firing, which might slow node drains. Namespace=openshift-monitoring, PodDisruptionBudget=prometheus-k8s. The pod disruption budget is preventing further disruption to pods. The alert description is: The pod disruption budget is at the minimum disruptions allowed level. The number of current healthy pods is equal to the desired healthy pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + +Upstream update service is unset, so the cluster will use an appropriate default. +Channel: candidate-5.0 (available channels: candidate-5.0) + +Updates to 5.0: + VERSION ISSUES + 5.0.0-ec.3 PodDisruptionBudgetAtLimit diff --git a/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.version-5.0.0-ec.3-output b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.version-5.0.0-ec.3-output new file mode 100644 index 0000000000..1da8430550 --- /dev/null +++ b/pkg/cli/admin/upgrade/recommend/examples/5.0.0-cvo-not-handling-risks.version-5.0.0-ec.3-output @@ -0,0 +1,24 @@ +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator monitoring is degraded + +The following conditions found no cause for concern in updating this cluster to later releases: recommended/CriticalAlerts (AsExpected), recommended/NodeAlerts (AsExpected), recommended/PodImagePullAlerts (AsExpected), recommended/UpdatePrecheckAlerts (AsExpected) + +The following conditions found cause for concern in updating this cluster to later releases: recommended/PodDisruptionBudgetAlerts/PodDisruptionBudgetAtLimit/0 + +recommended/PodDisruptionBudgetAlerts/PodDisruptionBudgetAtLimit/0=False: + + Reason: Alert:firing + Message: warning alert PodDisruptionBudgetAtLimit firing, which might slow node drains. Namespace=openshift-monitoring, PodDisruptionBudget=prometheus-k8s. The pod disruption budget is preventing further disruption to pods. The alert description is: The pod disruption budget is at the minimum disruptions allowed level. The number of current healthy pods is equal to the desired healthy pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + +Upstream update service is unset, so the cluster will use an appropriate default. +Channel: candidate-5.0 (available channels: candidate-5.0) + +Update to 5.0.0-ec.3 Recommended=False: +Image: quay.io/openshift-release-dev/ocp-release@sha256:b5ad6b4bb5efbd9b9d88bd42182e85388742b4441a2ff4301c4f5aa8176d5d29 +Release URL: +Reason: PodDisruptionBudgetAtLimit +Message: The pod disruption budget is preventing further disruption to pods. https://github.com/openshift/runbooks/blob/master/alerts/cluster-kube-controller-manager-operator/PodDisruptionBudgetAtLimit.md + +error: issues that apply to this cluster but which were not included in --accept: ConditionalUpdateRisk,Failing,PodDisruptionBudgetAtLimit diff --git a/pkg/cli/admin/upgrade/recommend/examples_test.go b/pkg/cli/admin/upgrade/recommend/examples_test.go index 0c55bdf93e..4ebf9f8057 100644 --- a/pkg/cli/admin/upgrade/recommend/examples_test.go +++ b/pkg/cli/admin/upgrade/recommend/examples_test.go @@ -70,6 +70,8 @@ func TestExamples(t *testing.T) { "examples/4.19.0-okd-scos.16-cv.yaml": "4.19.0-okd-scos.17", "examples/4.22.0-extend-recommended-alert-cv.yaml": "1.2.3-not-important", "examples/4.22.0-extend-recommended-critical-alert-cv.yaml": "1.2.3-not-important", + "examples/5.0.0-cvo-handling-risks-cv.yaml": "5.0.0-ec.3", + "examples/5.0.0-cvo-not-handling-risks-cv.yaml": "5.0.0-ec.3", }, outputSuffixPattern: ".version-%s-output", }, diff --git a/pkg/cli/admin/upgrade/recommend/mockresources.go b/pkg/cli/admin/upgrade/recommend/mockresources.go index 76be0da53b..eadef85dbe 100644 --- a/pkg/cli/admin/upgrade/recommend/mockresources.go +++ b/pkg/cli/admin/upgrade/recommend/mockresources.go @@ -13,12 +13,16 @@ import ( type mockData struct { // inputs - cvPath string - alertsPath string + cvPath string + alertsPath string + featureGatePath string + infrastructurePath string // outputs clusterVersion *configv1.ClusterVersion alerts []byte + featureGate *configv1.FeatureGate + infrastructure *configv1.Infrastructure } func asResourceList[T any](objects *corev1.List, decoder runtime.Decoder) ([]T, error) { @@ -78,5 +82,41 @@ func (o *mockData) load() error { } } + if o.featureGatePath != "" { + fgBytes, err := os.ReadFile(o.featureGatePath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if fgBytes != nil { + fgObj, err := runtime.Decode(decoder, fgBytes) + if err != nil { + return fmt.Errorf("error while parsing file %s: %w", o.featureGatePath, err) + } + fg, ok := fgObj.(*configv1.FeatureGate) + if !ok { + return fmt.Errorf("unexpected object type %T in %s content", fgObj, o.featureGatePath) + } + o.featureGate = fg + } + } + + if o.infrastructurePath != "" { + infraBytes, err := os.ReadFile(o.infrastructurePath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if infraBytes != nil { + infraObj, err := runtime.Decode(decoder, infraBytes) + if err != nil { + return fmt.Errorf("error while parsing file %s: %w", o.infrastructurePath, err) + } + infra, ok := infraObj.(*configv1.Infrastructure) + if !ok { + return fmt.Errorf("unexpected object type %T in %s content", infraObj, o.infrastructurePath) + } + o.infrastructure = infra + } + } + return nil } diff --git a/pkg/cli/admin/upgrade/recommend/recommend.go b/pkg/cli/admin/upgrade/recommend/recommend.go index 89c54ea00f..3d8e061268 100644 --- a/pkg/cli/admin/upgrade/recommend/recommend.go +++ b/pkg/cli/admin/upgrade/recommend/recommend.go @@ -111,6 +111,8 @@ func (o *options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string } else { cvSuffix := "-cv.yaml" o.mockData.alertsPath = strings.Replace(o.mockData.cvPath, cvSuffix, "-alerts.json", 1) + o.mockData.featureGatePath = strings.Replace(o.mockData.cvPath, cvSuffix, "-featuregate.yaml", 1) + o.mockData.infrastructurePath = strings.Replace(o.mockData.cvPath, cvSuffix, "-infrastructure.yaml", 1) err := o.mockData.load() if err != nil { return err @@ -132,7 +134,6 @@ func (o *options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string o.version = &version } } - return nil } @@ -337,7 +338,14 @@ func (o *options) Run(ctx context.Context) error { } unaccepted := issues.Difference(accept) if unaccepted.Len() > 0 { - return fmt.Errorf("issues that apply to this cluster but which were not included in --accept: %s", strings.Join(sets.List(unaccepted), ",")) + if cvoChecking, err := o.alertsEvaluatedByCVO(ctx); cvoChecking { + if err != nil { + return fmt.Errorf("failed to determine if CVO is checking alerts: %v", err) + } + fmt.Fprintf(o.Out, "Cluster update risks are being handled by the Cluster Version Operator (CVO). Please use `oc adm upgrade accept ...` to accept risks.\n") + } else { + return fmt.Errorf("issues that apply to this cluster but which were not included in --accept: %s", strings.Join(sets.List(unaccepted), ",")) + } } else if issues.Len() > 0 && !o.quiet { fmt.Fprintf(o.Out, "Update to %s has no known issues relevant to this cluster other than the accepted %s.\n", update.Release.Version, strings.Join(sets.List(issues), ",")) }