From 37069e1cbc95789b5fb771b21f07a4c41032b776 Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Wed, 19 Aug 2026 14:38:15 +0530 Subject: [PATCH 1/3] OCPBUGS-111997: Add Degraded=True exception for authentication operator during upgrade The authentication operator transiently goes Degraded for ~8.5 seconds during upgrade rollout when oauth-apiserver and oauth-server pods are briefly unavailable. This is the same pattern already excepted for kube-apiserver (OCPBUGS-38661), kube-controller-manager (OCPBUGS-38662), and kube-scheduler (OCPBUGS-38663). Add a narrow exception scoped to UnavailablePod reasons only, so transient pod rollout states are classified as flakes rather than hard failures. This unblocks the OKD SCOS 5.0 promoted upgrade job which has been failing consistently since ec.4 (OKD-424). --- .../legacycvomonitortests/operators.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go index 44a48375f26e..61273fdd185c 100644 --- a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go +++ b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go @@ -383,9 +383,11 @@ func testUpgradeOperatorStateTransitions(events monitorapi.Intervals, clientConf if checkAuthenticationAvailableExceptions(condition) { return "https://issues.redhat.com/browse/OCPBUGS-20056" } - if isTwoNode && condition.Type == configv1.OperatorDegraded && condition.Status == configv1.ConditionTrue && - strings.Contains(condition.Reason, "OAuthServerDeployment_UnavailablePod") { - return "authentication may report Degraded while oauth-openshift pods roll out during DualReplica disruptive upgrades" + if condition.Type == configv1.OperatorDegraded && condition.Status == configv1.ConditionTrue && + (condition.Reason == "APIServerDeployment_UnavailablePod" || + condition.Reason == "APIServerDeployment_UnavailablePod::OAuthServerDeployment_UnavailablePod" || + condition.Reason == "OAuthServerDeployment_UnavailablePod") { + return "https://issues.redhat.com/browse/OCPBUGS-111997" } case "console": if condition.Type == configv1.OperatorDegraded && condition.Status == configv1.ConditionTrue { From 378e1e3b4db70293aded2fd8e996c36dc49ed091 Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Wed, 19 Aug 2026 15:03:50 +0530 Subject: [PATCH 2/3] OCPBUGS-111997: Add test for authentication Degraded exception during upgrade Verifies that the authentication operator's transient Degraded=True state during upgrade is correctly excepted for UnavailablePod reasons, while unrelated Degraded reasons remain hard failures. --- .../legacycvomonitortests/operators_test.go | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go index 07ea31944413..105b83489ce9 100644 --- a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go +++ b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go @@ -674,3 +674,103 @@ func TestOverlapsNoExecuteTaintManagerTest(t *testing.T) { }) } } + +func buildOperatorConditionInterval(operator, conditionType, status, reason string, from, to time.Time) monitorapi.Interval { + return monitorapi.Interval{ + Source: monitorapi.SourceClusterOperatorMonitor, + Condition: monitorapi.Condition{ + Locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorClusterOperatorKey: operator, + }, + }, + Message: monitorapi.Message{ + Reason: monitorapi.IntervalReason(reason), + HumanMessage: "test condition", + Annotations: map[monitorapi.AnnotationKey]string{ + monitorapi.AnnotationCondition: conditionType, + monitorapi.AnnotationStatus: status, + monitorapi.AnnotationReason: reason, + }, + }, + }, + From: from, + To: to, + } +} + +func Test_authenticationDegradedExceptionDuringUpgrade(t *testing.T) { + upgradeStart := time.Date(2026, 8, 17, 16, 0, 0, 0, time.UTC) + upgradeEnd := time.Date(2026, 8, 17, 17, 30, 0, 0, time.UTC) + + upgradeEvents := makeUpgradeEventList([]upgradeEvent{ + {eventTime: upgradeStart, reason: monitorapi.UpgradeStartedReason}, + {eventTime: upgradeEnd, reason: monitorapi.UpgradeCompleteReason}, + }) + + tests := []struct { + name string + reason string + wantFatal bool + }{ + { + name: "compound UnavailablePod reason should be excepted", + reason: "APIServerDeployment_UnavailablePod::OAuthServerDeployment_UnavailablePod", + wantFatal: false, + }, + { + name: "APIServerDeployment_UnavailablePod alone should be excepted", + reason: "APIServerDeployment_UnavailablePod", + wantFatal: false, + }, + { + name: "OAuthServerDeployment_UnavailablePod alone should be excepted", + reason: "OAuthServerDeployment_UnavailablePod", + wantFatal: false, + }, + { + name: "unrelated Degraded reason should NOT be excepted", + reason: "SomeOtherDegradedReason", + wantFatal: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + condFrom := upgradeStart.Add(50 * time.Minute) + condTo := condFrom.Add(10 * time.Second) + + conditionEvent := buildOperatorConditionInterval( + "authentication", "Degraded", "True", tt.reason, + condFrom, condTo, + ) + + var events monitorapi.Intervals + events = append(events, upgradeEvents...) + events = append(events, conditionEvent) + + results := testUpgradeOperatorStateTransitions(events, nil, configv1.HighlyAvailableTopologyMode) + + testName := "[bz-apiserver-auth] clusteroperator/authentication should not change condition/Degraded" + var hasFailure, hasSuccess bool + for _, tc := range results { + if tc.Name == testName { + if tc.FailureOutput != nil { + hasFailure = true + } else { + hasSuccess = true + } + } + } + + if tt.wantFatal { + assert.True(t, hasFailure, "expected a failure JUnit for reason %s", tt.reason) + assert.False(t, hasSuccess, "expected no success JUnit for reason %s (should be hard failure)", tt.reason) + } else { + if hasFailure { + assert.True(t, hasSuccess, "expected both failure and success JUnit (flake) for reason %s", tt.reason) + } + } + }) + } +} From 635ee4001555be24b83428f0672ff47604525a4b Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Fri, 21 Aug 2026 10:43:27 +0530 Subject: [PATCH 3/3] OCPBUGS-111997: Address review feedback on auth Degraded exception - Use strings.Contains for reason matching instead of exact matches, preserving the original breadth and avoiding regressions on two-node clusters - Return a descriptive message with bug ID instead of a bare URL - Update tests to match the new strings.Contains behavior --- .../legacycvomonitortests/operators.go | 6 ++---- .../legacycvomonitortests/operators_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go index 61273fdd185c..7ad2c9b56296 100644 --- a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go +++ b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go @@ -384,10 +384,8 @@ func testUpgradeOperatorStateTransitions(events monitorapi.Intervals, clientConf return "https://issues.redhat.com/browse/OCPBUGS-20056" } if condition.Type == configv1.OperatorDegraded && condition.Status == configv1.ConditionTrue && - (condition.Reason == "APIServerDeployment_UnavailablePod" || - condition.Reason == "APIServerDeployment_UnavailablePod::OAuthServerDeployment_UnavailablePod" || - condition.Reason == "OAuthServerDeployment_UnavailablePod") { - return "https://issues.redhat.com/browse/OCPBUGS-111997" + strings.Contains(condition.Reason, "OAuthServerDeployment_UnavailablePod") { + return "authentication transiently reports Degraded while oauth-apiserver/oauth-server pods roll out during upgrade (OCPBUGS-111997)" } case "console": if condition.Type == configv1.OperatorDegraded && condition.Status == configv1.ConditionTrue { diff --git a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go index 105b83489ce9..5937f9b68bfa 100644 --- a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go +++ b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go @@ -714,20 +714,20 @@ func Test_authenticationDegradedExceptionDuringUpgrade(t *testing.T) { wantFatal bool }{ { - name: "compound UnavailablePod reason should be excepted", + name: "compound reason containing OAuthServerDeployment_UnavailablePod should be excepted", reason: "APIServerDeployment_UnavailablePod::OAuthServerDeployment_UnavailablePod", wantFatal: false, }, - { - name: "APIServerDeployment_UnavailablePod alone should be excepted", - reason: "APIServerDeployment_UnavailablePod", - wantFatal: false, - }, { name: "OAuthServerDeployment_UnavailablePod alone should be excepted", reason: "OAuthServerDeployment_UnavailablePod", wantFatal: false, }, + { + name: "APIServerDeployment_UnavailablePod alone should NOT be excepted", + reason: "APIServerDeployment_UnavailablePod", + wantFatal: true, + }, { name: "unrelated Degraded reason should NOT be excepted", reason: "SomeOtherDegradedReason",