From 108a2c417e82850a9d11c27e32da68a3673af60c Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Wed, 9 Sep 2026 17:11:26 +0200 Subject: [PATCH 1/2] USHIFT-7503: land cluster-monitoring-operator rebase in the rebase PR The nightly auto-rebase never updated the cluster-monitoring-operator images (metrics-server, kube-state-metrics, node-exporter) because the monitoring rebase ran in rebase_job_entrypoint.sh *after* rebase.py. rebase.py creates the rebase branch, pushes it, and opens the PR, so anything committed afterwards was never pushed and never landed. Move the monitoring rebase into rebase.py alongside the other component rebases (ai_model_serving, sriov, cert_manager): it now runs after rebase.sh has created the branch, so its commits are carried into the pushed PR, and its result feeds rebases_succeeded so a monitoring failure is treated like any other component failure. Also commit the monitoring assets: rebase_cluster_monitoring_operator.sh previously committed only its last-rebase marker, leaving the actual manifest/image digest bumps in assets/ uncommitted. Add commit_cluster_monitoring_operator_changes() so those changes land too. Co-Authored-By: Claude Opus 4.8 (1M context) pre-commit.check-secrets: ENABLED --- scripts/auto-rebase/rebase.py | 21 ++++++++++++++++++- .../rebase_cluster_monitoring_operator.sh | 12 +++++++++++ scripts/auto-rebase/rebase_job_entrypoint.sh | 4 ---- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/scripts/auto-rebase/rebase.py b/scripts/auto-rebase/rebase.py index 2fad3a1e17..48b09bd7ac 100755 --- a/scripts/auto-rebase/rebase.py +++ b/scripts/auto-rebase/rebase.py @@ -139,6 +139,23 @@ def run_rebase_cert_manager_sh(release): return RebaseScriptResult(success=result.returncode == 0, output=result.stdout) +def run_rebase_cluster_monitoring_operator_sh(release_amd64, release_arm64): + """Run the 'rebase_cluster_monitoring_operator.sh' script with the given release versions and return the script's output.""" + script_dir = os.path.abspath(os.path.dirname(__file__)) + args = [f"{script_dir}/rebase_cluster_monitoring_operator.sh", "to", release_amd64, release_arm64] + logging.info(f"Running: '{' '.join(args)}'") + start = timer() + result = subprocess.run( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, check=False) + logging.info(f"Return code: {result.returncode}. Output:\n" + + "==================================================\n" + + f"{result.stdout}" + + "==================================================\n") + end = timer() - start + logging.info(f"Script returned code: {result.returncode}. It ran for {end/60:.0f}m{end%60:.0f}s.") + return RebaseScriptResult(success=result.returncode == 0, output=result.stdout) + + def make_sure_rebase_script_created_new_commits_or_exit(git_repo, base_branch): """Exit the script if the 'rebase.sh' script did not create any new commits.""" if git_repo.active_branch.commit == git_repo.branches[base_branch].commit: @@ -327,8 +344,10 @@ def main(): ai_rebase_result = run_rebase_ai_model_serving_sh(rhoai_release) sriov_rebase_result = run_rebase_sriov_sh(sriov_release) cert_manager_rebase_result = run_rebase_cert_manager_sh(opm_version) + monitoring_rebase_result = run_rebase_cluster_monitoring_operator_sh(release_amd, release_arm) - rebases_succeeded = all([rebase_result.success, ai_rebase_result.success, sriov_rebase_result.success, cert_manager_rebase_result.success]) + rebases_succeeded = all([rebase_result.success, ai_rebase_result.success, sriov_rebase_result.success, + cert_manager_rebase_result.success, monitoring_rebase_result.success]) if rebases_succeeded: # TODO How can we inform team that rebase job ran successfully just there was nothing new? diff --git a/scripts/auto-rebase/rebase_cluster_monitoring_operator.sh b/scripts/auto-rebase/rebase_cluster_monitoring_operator.sh index be1124f04e..c33259385c 100755 --- a/scripts/auto-rebase/rebase_cluster_monitoring_operator.sh +++ b/scripts/auto-rebase/rebase_cluster_monitoring_operator.sh @@ -329,6 +329,17 @@ EOF fi) } +commit_cluster_monitoring_operator_changes() { + title "## Committing changes to cluster-monitoring-operator assets" + (cd "${REPOROOT}" && \ + if test -n "$(git status -s ./assets)"; then \ + git add ./assets && \ + git commit -m "update cluster-monitoring-operator manifests and images"; \ + else \ + echo "No changes in cluster-monitoring-operator assets."; \ + fi) +} + rebase_cluster_monitoring_operator_to() { local release_image_amd64="$1" local release_image_arm64="$2" @@ -338,6 +349,7 @@ rebase_cluster_monitoring_operator_to() { update_kube_state_metrics_manifests update_node_exporter_manifests update_cluster_monitoring_operator_images + commit_cluster_monitoring_operator_changes update_last_rebase "${release_image_amd64}" "${release_image_arm64}" } diff --git a/scripts/auto-rebase/rebase_job_entrypoint.sh b/scripts/auto-rebase/rebase_job_entrypoint.sh index eb9960e09f..052a46de09 100755 --- a/scripts/auto-rebase/rebase_job_entrypoint.sh +++ b/scripts/auto-rebase/rebase_job_entrypoint.sh @@ -88,10 +88,6 @@ SRIOV_RELEASE=${sriov_release} \ OPM_RELEASE=${opm_release} \ ./scripts/auto-rebase/rebase.py -# Monitoring images (metrics-server, kube-state-metrics, node-exporter) are -# part of the OCP release payload and must be rebased with every nightly. -./scripts/auto-rebase/rebase_cluster_monitoring_operator.sh to "${PULLSPEC_RELEASE_AMD64}" "${PULLSPEC_RELEASE_ARM64}" - # LVMS is not tracked in the OCP release image. Instead, rely on the # latest z-stream for a given X.Y version. Only the X.Y needs manual # updating between releases. From d2347f00f2f6425a0a51d563d47d4476d28c08a9 Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Wed, 9 Sep 2026 17:11:39 +0200 Subject: [PATCH 2/2] USHIFT-7503: surface component-rebase failures in the rebase PR Make a component-rebase failure visible in the opened PR instead of being silently lost: - Extend the committed rebase.log failure artifact to include the output of every component script (rebase.sh, ai_model_serving, sriov, cert_manager, cluster_monitoring_operator), not just rebase.sh and ai_model_serving, so the failing component's log is captured. - Build the PR title and description from rebases_succeeded instead of only rebase_result.success. A component failure after a successful core rebase now flags the PR as **FAILURE** rather than appearing as a clean NO-ISSUE success, matching the job's non-zero exit and committed failure artifacts. Co-Authored-By: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-Authored-By: Claude Opus 4.8 (1M context) pre-commit.check-secrets: ENABLED --- scripts/auto-rebase/rebase.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/auto-rebase/rebase.py b/scripts/auto-rebase/rebase.py index 48b09bd7ac..cd1c7fe5fb 100755 --- a/scripts/auto-rebase/rebase.py +++ b/scripts/auto-rebase/rebase.py @@ -360,11 +360,13 @@ def main(): else: logging.warning("Rebase script failed - everything will be committed") with open('rebase.log', mode='w', encoding='utf-8') as writer: - output = ("rebase.sh:\n" + - f"{rebase_result.output}" + - "==================================================\n" + - "rebase_ai_model_serving.sh:\n" + - f"{ai_rebase_result.output}") + separator = "==================================================\n" + output = ( + "rebase.sh:\n" + f"{rebase_result.output}" + separator + + "rebase_ai_model_serving.sh:\n" + f"{ai_rebase_result.output}" + separator + + "rebase_sriov.sh:\n" + f"{sriov_rebase_result.output}" + separator + + "rebase_cert_manager.sh:\n" + f"{cert_manager_rebase_result.output}" + separator + + "rebase_cluster_monitoring_operator.sh:\n" + f"{monitoring_rebase_result.output}") writer.write(output) if g.git_repo.active_branch.name == base_branch: # rebase.sh didn't reach the step that would create a branch @@ -391,8 +393,8 @@ def main(): g.push(rebase_branch_name, base_branch, gh.gh_repo) prow_job_url = try_create_prow_job_url() - pr_title = create_pr_title(rebase_branch_name, rebase_result.success) - desc = generate_pr_description(get_release_tag(release_amd), get_release_tag(release_arm), prow_job_url, rebase_result.success) + pr_title = create_pr_title(rebase_branch_name, rebases_succeeded) + desc = generate_pr_description(get_release_tag(release_amd), get_release_tag(release_arm), prow_job_url, rebases_succeeded) comment = "" pull_req = gh.get_existing_pr_for_a_branch(adjusted_base_branch, rebase_branch_name)