From 0946489acac5fd6870893b57013a28dbc0479baf Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Tue, 25 Aug 2026 12:24:25 -0700 Subject: [PATCH] Rerun `--failed` merge queue workflows --- .github/workflows/ci.yml | 31 +++- .github/workflows/internal-tests.yml | 2 +- Cargo.lock | 22 +-- Cargo.toml | 2 +- .../Cargo.toml | 2 +- .../src/main.rs | 171 ++++++++++++++---- tools/ci/src/main.rs | 4 +- 7 files changed, 183 insertions(+), 51 deletions(-) rename tools/ci/commands/{coordinate-internal-tests => workflow-coordinator}/Cargo.toml (83%) rename tools/ci/commands/{coordinate-internal-tests => workflow-coordinator}/src/main.rs (76%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56dd41aa0c..11193e6141d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,16 +23,24 @@ permissions: jobs: merge_queue_noop: - name: Detect merge queue no-op + name: Merge queue no-op/reuse runs-on: ubuntu-latest + permissions: + actions: write + contents: read + pull-requests: read outputs: - skip: ${{ steps.compare.outputs.skip }} + skip: ${{ steps.compare.outputs.skip == 'true' || steps.reuse.outputs.reuse == 'true' }} steps: - name: Checkout merge queue commit uses: actions/checkout@v4 + # If planning cannot check out the repository, leave `skip` unset and run normal CI. + continue-on-error: true - name: Compare merge queue commit to PR head id: compare + # If the trees cannot be compared, fall back to normal CI. + continue-on-error: true env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} HEAD_REF: ${{ github.event.merge_group.head_ref }} @@ -70,6 +78,25 @@ jobs: echo "Merge queue commit ${GITHUB_SHA} differs from PR #${pr_number} head ${pr_head_sha}; running CI normally." fi + # The merge queue coordinator is part of cargo ci and needs the repository's pinned Rust toolchain. + - uses: dsherret/rust-toolchain-file@v1 + if: ${{ github.event_name == 'merge_group' && steps.compare.outputs.skip != 'true' }} + # If setup fails, let the reuse attempt fall back to normal CI. + continue-on-error: true + + - name: Reuse merge queue run + id: reuse + if: ${{ github.event_name == 'merge_group' && steps.compare.outputs.skip != 'true' }} + # Coordinator errors leave `reuse` unset and run normal CI; a reused CI failure is reported below. + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: cargo ci other-workflows coordinate merge-queue-reuse + + - name: Report merge queue workflow reuse failure on this commit + if: ${{ steps.reuse.outputs.reuse_failed == 'true' }} + run: exit 1 + upload-build-artifacts-linux: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} diff --git a/.github/workflows/internal-tests.yml b/.github/workflows/internal-tests.yml index e2301d3da96..125ba519544 100644 --- a/.github/workflows/internal-tests.yml +++ b/.github/workflows/internal-tests.yml @@ -63,7 +63,7 @@ jobs: fi cargo run --quiet \ --manifest-path tools/ci/Cargo.toml \ - -- other-workflows coordinate-internal-tests "${args[@]}" + -- other-workflows coordinate internal-tests "${args[@]}" - name: Wait for Internal Tests to complete if: steps.filter.outputs.non_docs == 'true' diff --git a/Cargo.lock b/Cargo.lock index 22cd022fb09..0f3cf3b267c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -910,17 +910,6 @@ dependencies = [ "duct", ] -[[package]] -name = "ci-coordinate-internal-tests" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.5.50", - "duct", - "serde", - "serde_json", -] - [[package]] name = "ci-docs-build" version = "0.1.0" @@ -1035,6 +1024,17 @@ dependencies = [ "duct", ] +[[package]] +name = "ci-workflow-coordinator" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "duct", + "serde", + "serde_json", +] + [[package]] name = "ci-workflow-watch" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 44858f71b9c..421e3ed9ad5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,7 @@ members = [ "tools/ci/commands/docs", "tools/ci/commands/codeowners-check", "tools/ci/commands/cla-assistant", - "tools/ci/commands/coordinate-internal-tests", + "tools/ci/commands/workflow-coordinator", "tools/ci/commands/workflow-watch", "tools/ci/common", "tools/keynote-bench-harness", diff --git a/tools/ci/commands/coordinate-internal-tests/Cargo.toml b/tools/ci/commands/workflow-coordinator/Cargo.toml similarity index 83% rename from tools/ci/commands/coordinate-internal-tests/Cargo.toml rename to tools/ci/commands/workflow-coordinator/Cargo.toml index c4b71bd705d..908bfccb9c1 100644 --- a/tools/ci/commands/coordinate-internal-tests/Cargo.toml +++ b/tools/ci/commands/workflow-coordinator/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ci-coordinate-internal-tests" +name = "ci-workflow-coordinator" version = "0.1.0" edition.workspace = true diff --git a/tools/ci/commands/coordinate-internal-tests/src/main.rs b/tools/ci/commands/workflow-coordinator/src/main.rs similarity index 76% rename from tools/ci/commands/coordinate-internal-tests/src/main.rs rename to tools/ci/commands/workflow-coordinator/src/main.rs index 3e528a15bc5..c22f08da6a2 100644 --- a/tools/ci/commands/coordinate-internal-tests/src/main.rs +++ b/tools/ci/commands/workflow-coordinator/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; -use clap::Parser; +use clap::{Parser, Subcommand}; use duct::{cmd, Expression}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -15,18 +15,31 @@ const PUBLIC_REPO: &str = "clockworklabs/SpacetimeDB"; const PRIVATE_REPO: &str = "clockworklabs/SpacetimeDBPrivate"; const PRIVATE_WORKFLOW: &str = "ci.yml"; const PRIVATE_DEFAULT_BRANCH: &str = "master"; +const REUSE_FAILURE_STEP: &str = "Report reused CI failure on this commit"; // Must match ci.yml. -/// Selects or starts the private workflow for a public Internal Tests run. +/// Coordinates CI workflow runs. #[derive(Parser)] -#[command(about = "Selects or starts the private workflow for a public Internal Tests run.")] +#[command(about = "Coordinates CI workflow runs.")] struct Cli { - /// Immutable public commit to test. - #[arg(long)] - public_sha: String, + #[command(subcommand)] + command: CliCommand, +} + +#[derive(Subcommand)] +enum CliCommand { + /// Select or start internal tests for a public commit. + InternalTests { + /// Immutable public commit to test. + #[arg(long)] + public_sha: String, - /// Public pull request number, when coordinating a pull request run. - #[arg(long)] - public_pr_number: Option, + /// Public pull request number, when coordinating a pull request run. + #[arg(long)] + public_pr_number: Option, + }, + + /// Reuse an equivalent failed merge-queue workflow run. + MergeQueueReuse, } #[derive(Debug)] @@ -141,22 +154,14 @@ fn workflow_runs(event: &str, head_sha: &str) -> Result> { Ok(pages.into_iter().flat_map(|page| page.workflow_runs).collect()) } -fn workflow_run(run_id: u64) -> Result { - get(&format!("/repos/{PRIVATE_REPO}/actions/runs/{run_id}")) +fn workflow_run(repo: &str, run_id: u64) -> Result { + get(&format!("/repos/{repo}/actions/runs/{run_id}")) } -fn rerun_failed_jobs(run_id: u64) -> Result<()> { - cmd!( - "gh", - "run", - "rerun", - run_id.to_string(), - "--failed", - "--repo", - PRIVATE_REPO - ) - .run() - .with_context(|| format!("failed to rerun unsuccessful jobs in private run {run_id}"))?; +fn rerun_failed_jobs(repo: &str, run_id: u64) -> Result<()> { + cmd!("gh", "run", "rerun", run_id.to_string(), "--failed", "--repo", repo) + .run() + .with_context(|| format!("failed to rerun unsuccessful jobs in {repo} run {run_id}"))?; Ok(()) } @@ -249,10 +254,16 @@ struct WorkflowRun { run_attempt: u64, html_url: String, created_at: String, + head_commit: Option, #[serde(default)] pull_requests: Vec, } +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +struct WorkflowRunCommit { + tree_id: String, +} + #[derive(Clone, Debug, Deserialize, PartialEq, Eq)] struct WorkflowRunStatus { id: u64, @@ -262,6 +273,24 @@ struct WorkflowRunStatus { html_url: String, } +#[derive(Deserialize)] +struct JobsPage { + jobs: Vec, +} + +#[derive(Deserialize)] +struct Job { + conclusion: Option, + #[serde(default)] + steps: Vec, +} + +#[derive(Deserialize)] +struct JobStep { + name: String, + conclusion: Option, +} + #[derive(Serialize)] struct DispatchInputs<'a> { public_ref: &'a str, @@ -394,17 +423,17 @@ fn prepare_existing_run(run: WorkflowRun) -> Result { } println!("Re-running unsuccessful jobs in the existing private run."); - rerun_failed_jobs(selected.id)?; + rerun_failed_jobs(PRIVATE_REPO, selected.id)?; Ok(CoordinatedRun { - selected: wait_for_rerun(&selected)?, + selected: wait_for_rerun(PRIVATE_REPO, &selected)?, did_start: true, }) } /// Waits for GitHub to expose the new attempt so callers do not receive the stale completed result. -fn wait_for_rerun(run: &SelectedRun) -> Result { +fn wait_for_rerun(repo: &str, run: &SelectedRun) -> Result { for _ in 0..30 { - let current = workflow_run(run.id)?; + let current = workflow_run(repo, run.id)?; if current.run_attempt > run.attempt { return Ok(SelectedRun { id: current.id, @@ -416,7 +445,74 @@ fn wait_for_rerun(run: &SelectedRun) -> Result { } std::thread::sleep(Duration::from_secs(2)); } - bail!("timed out waiting for the private run to start its rerun") + bail!("timed out waiting for {repo} run {} to start its rerun", run.id) +} + +fn wait_for_completion(repo: &str, mut run: SelectedRun) -> Result { + while run.status != "completed" { + std::thread::sleep(Duration::from_secs(30)); + let current = workflow_run(repo, run.id)?; + run.status = current.status; + run.conclusion = current.conclusion; + run.attempt = current.run_attempt; + } + Ok(run) +} + +/// Whether this workflow failed itself, rather than only reporting a failure copied from an older run. +fn has_original_failure(repo: &str, run_id: u64) -> Result { + let pages: Vec = get_paginated(&format!( + "/repos/{repo}/actions/runs/{run_id}/jobs?filter=latest&per_page=100" + ))?; + Ok(pages.into_iter().flat_map(|page| page.jobs).any(|job| { + let reports_reused_failure = job + .steps + .iter() + .any(|step| step.name == REUSE_FAILURE_STEP && step.conclusion.as_deref() == Some("failure")); + !reports_reused_failure + && job + .conclusion + .as_deref() + .is_some_and(|conclusion| !matches!(conclusion, "success" | "skipped" | "neutral")) + })) +} + +fn previous_equivalent_run() -> Result> { + let current_tree = cmd!("git", "rev-parse", "HEAD^{tree}") + .read() + .context("failed to read the current Git tree")?; + let mut runs = get::(&format!( + "/repos/{PUBLIC_REPO}/actions/workflows/ci.yml/runs?event=merge_group&per_page=100" + ))? + .workflow_runs; + runs.sort_by(|left, right| right.created_at.cmp(&left.created_at)); + for run in runs { + if run + .head_commit + .as_ref() + .is_some_and(|commit| commit.tree_id == current_tree) + && (run.conclusion.as_deref() == Some("success") || has_original_failure(PUBLIC_REPO, run.id)?) + { + return Ok(Some(run)); + } + } + Ok(None) +} + +fn coordinate_merge_queue_reuse() -> Result<()> { + let Some(run) = previous_equivalent_run()? else { + write_github_output("reuse", false)?; + return Ok(()); + }; + println!("Found equivalent merge queue run: {}", run.html_url); + let mut selected = wait_for_completion(PUBLIC_REPO, run.into())?; + if should_rerun_failed_jobs(&selected) { + rerun_failed_jobs(PUBLIC_REPO, selected.id)?; + selected = wait_for_completion(PUBLIC_REPO, wait_for_rerun(PUBLIC_REPO, &selected)?)?; + } + write_github_output("reuse_failed", selected.conclusion.as_deref() != Some("success"))?; + write_github_output("reuse", true)?; + Ok(()) } /// Reuses or reruns `pull_request` CI for a public PR with a linked private PR. @@ -453,14 +549,20 @@ fn write_github_output(name: &str, value: impl std::fmt::Display) -> Result<()> writeln!(output, "{name}={value}").context("failed to write GITHUB_OUTPUT") } -/// Coordinates the public Internal Tests run without checking out or executing private code. +/// Coordinates CI without checking out or executing private code. fn main() -> Result<()> { - let args = Cli::parse(); - let private_source = resolve_private_source(args.public_pr_number)?; + let (public_sha, public_pr_number) = match Cli::parse().command { + CliCommand::InternalTests { + public_sha, + public_pr_number, + } => (public_sha, public_pr_number), + CliCommand::MergeQueueReuse => return coordinate_merge_queue_reuse(), + }; + let private_source = resolve_private_source(public_pr_number)?; let coordinated = match private_source { - PrivateSource::LinkedPrivatePr { pull } => coordinate_linked_private_pr(&args.public_sha, &pull)?, - PrivateSource::PrivateMaster { sha } => coordinate_public_only(&args.public_sha, &sha)?, + PrivateSource::LinkedPrivatePr { pull } => coordinate_linked_private_pr(&public_sha, &pull)?, + PrivateSource::PrivateMaster { sha } => coordinate_public_only(&public_sha, &sha)?, }; println!("View run: {}", coordinated.selected.url); @@ -496,6 +598,9 @@ mod tests { run_attempt: 1, html_url: format!("https://example.test/{id}"), created_at: created_at.to_owned(), + head_commit: Some(WorkflowRunCommit { + tree_id: "tree".to_owned(), + }), pull_requests: Vec::new(), } } diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index bb0878277d9..6e23b028931 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -65,8 +65,8 @@ const COMMANDS: &[Command] = &[ package: "ci-docs-build", }, Command { - path: &["other-workflows", "coordinate-internal-tests"], - package: "ci-coordinate-internal-tests", + path: &["other-workflows", "coordinate"], + package: "ci-workflow-coordinator", }, Command { path: &["other-workflows", "codeowners-check"],