diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index dde3813ea7dd1..98d9bd70e3104 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -29,7 +29,10 @@ DATABASE_PASSWORD_VAR = "CLICKHOUSE_TEST_STAT_PASSWORD" S3_BUCKET = "altinity-build-artifacts" GITHUB_REPO = "Altinity/ClickHouse" +GITHUB_GIT_URL = f"https://github.com/{GITHUB_REPO}.git" GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN") +# Release tags are fetched here rather than refs/tags/ so local tags are never touched. +RELEASE_TAG_REF_PREFIX = "refs/workflow-report/tags/" # Resolve from this file so discovery works from repo root or the action folder. KNOWN_FAILS_FILE = ( Path(__file__).resolve().parents[3] / "tests" / "broken_tests.yaml" @@ -216,8 +219,10 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram def _git_rev_parse(ref: str, cwd: str | None) -> str | None: + # Peel to a commit so a raw SHA that is not in the object store fails. + # git rev-parse --verify <40-hex> succeeds even when the object is missing. p = subprocess.run( - ["git", "rev-parse", "--verify", ref], + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=cwd, capture_output=True, text=True, @@ -237,7 +242,7 @@ def _git_is_ancestor(ancestor: str, descendant: str, cwd: str | None) -> bool: def _git_log_merge_prs( - baseline: str, branch_ref: str, cwd: str | None, repo: str + baseline: str, commit_sha: str, cwd: str | None, repo: str ) -> pd.DataFrame: p = subprocess.run( [ @@ -245,7 +250,7 @@ def _git_log_merge_prs( "-c", "core.quotepath=false", "log", - f"{baseline}..{branch_ref}", + f"{baseline}..{commit_sha}", "--merges", "--format=%H%x09%s", ], @@ -281,15 +286,11 @@ def _git_log_merge_prs( return df -def _find_release_baseline( - branch_ref: str, repo: str, cwd: str | None -) -> tuple[str | None, str | None, str | None]: - """Return (tag_name, tag_sha, published_date YYYY-MM-DD) for the latest - non-draft release tag that is an ancestor of branch_ref. - Version-tag refs skip the matching release so the range is previous→this tag. - """ +def _get_github_releases(repo: str) -> list[dict]: + """Non-draft GitHub releases of repo, newest first.""" if not GITHUB_TOKEN: - return None, None, None + print("GITHUB_TOKEN is not set; skipping GitHub releases lookup") + return [] headers = { "Authorization": f"token {GITHUB_TOKEN}", "Accept": "application/vnd.github.v3+json", @@ -302,20 +303,31 @@ def _find_release_baseline( raise Exception( f"GitHub API request failed: {response.status_code} {response.text}" ) + return [rel for rel in response.json() if not rel.get("draft")] + + +def _find_release_baseline( + branch_ref: str, + releases: list[dict], + cwd: str | None, + commit_sha: str, +) -> tuple[str | None, str | None, str | None]: + """Return (tag_name, tag_sha, published_date YYYY-MM-DD) for the latest + release tag in releases that is an ancestor of commit_sha. + Version-tag refs skip the matching release so the range is previous→this tag. + """ parsed = _parse_release_ref(branch_ref) skip_current = parsed is not None and parsed[2] is not None - for rel in response.json(): - if rel.get("draft"): - continue + for rel in releases: tag_name = rel.get("tag_name") if not tag_name: continue - tag_sha = _git_rev_parse(tag_name, cwd) + tag_sha = _release_tag_sha(tag_name, cwd) if not tag_sha: continue - if not _git_is_ancestor(tag_sha, branch_ref, cwd): + if not _git_is_ancestor(tag_sha, commit_sha, cwd): continue - if skip_current and _git_is_ancestor(branch_ref, tag_sha, cwd): + if skip_current and _git_is_ancestor(commit_sha, tag_sha, cwd): continue published_at = rel.get("published_at") or rel.get("created_at") or "" published_date = published_at[:10] if published_at else None @@ -323,12 +335,12 @@ def _find_release_baseline( return None, None, None -def _find_rebase_baseline(branch_ref: str, cwd: str | None) -> str | None: +def _find_rebase_baseline(commit_sha: str, cwd: str | None) -> str | None: p = subprocess.run( [ "git", "log", - branch_ref, + commit_sha, "--reverse", "-i", "-E", @@ -364,11 +376,113 @@ def _git_commit_date(sha: str, cwd: str | None) -> str | None: return date or None +def _git_is_shallow(cwd: str | None) -> bool: + p = subprocess.run( + ["git", "rev-parse", "--is-shallow-repository"], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + return p.stdout.strip() == "true" + + +def _git_fetch(options: list[str], refspecs: list[str], cwd: str | None) -> None: + p = subprocess.run( + ["git", "fetch", *options, GITHUB_GIT_URL, *refspecs], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + if p.returncode != 0: + raise Exception(f"git fetch failed: {p.stderr.strip()}") + + +def _release_tag_sha(tag_name: str, cwd: str | None) -> str | None: + return _git_rev_parse(f"{RELEASE_TAG_REF_PREFIX}{tag_name}", cwd) + + +def _read_autogenerated_versions(sha: str, cwd: str | None) -> tuple[int, int]: + p = subprocess.run( + ["git", "show", f"{sha}:cmake/autogenerated_versions.txt"], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + major = re.search(r"SET\(VERSION_MAJOR (\d+)\)", p.stdout) + minor = re.search(r"SET\(VERSION_MINOR (\d+)\)", p.stdout) + if not major or not minor: + raise Exception(f"Cannot read version from {sha}") + return int(major.group(1)), int(minor.group(1)) + + +def _filter_releases_to_version_line( + releases: list[dict], + branch_ref: str, + commit_sha: str, + cwd: str | None, +) -> list[dict]: + """Keep GitHub releases that belong on commit_sha's version line.""" + parsed = _parse_release_ref(branch_ref) + if parsed is not None: + flavour, minor_version, _ = parsed + suffix = f"altinity{flavour}" + prefix = f"v{minor_version}." + return [ + rel + for rel in releases + if (tag := rel.get("tag_name")) + and tag.startswith(prefix) + and tag.endswith(f".{suffix}") + ] + major, minor = _read_autogenerated_versions(commit_sha, cwd) + prefix = f"v{major}.{minor}." + return [ + rel + for rel in releases + if (tag := rel.get("tag_name")) and tag.startswith(prefix) + ] + + +def _fetch_release_tags(releases: list[dict], cwd: str) -> None: + """Fetch release tags into RELEASE_TAG_REF_PREFIX; skip tags missing remotely.""" + for rel in releases: + tag_name = rel.get("tag_name") + if not tag_name: + continue + try: + _git_fetch( + ["--no-tags"], + [f"+refs/tags/{tag_name}:{RELEASE_TAG_REF_PREFIX}{tag_name}"], + cwd, + ) + except Exception as e: + # NOTE (strtgbb): one deleted remote tag must not abort the section, but + # if it is the newest release the baseline moves back to an older one + print(f"WARNING: skipping release tag {tag_name}: {e}") + + +def _fetch_commit(commit_sha: str, cwd: str) -> None: + """Fetch the workflow commit from GITHUB_GIT_URL. + + Local checkouts might be a different branch and might not have the run SHA yet. + --filter/--unshallow would convert a full clone into a partial one, so those + options are used only when the checkout is already shallow. + """ + if _git_is_shallow(cwd): + _git_fetch(["--filter=tree:0", "--unshallow"], [commit_sha], cwd) + else: + _git_fetch(["--no-tags"], [commit_sha], cwd) + + def get_prs_in_release_dataframe( branch_ref: str = "HEAD", *, repo: str = GITHUB_REPO, cwd: str, + commit_sha: str, ) -> tuple[pd.DataFrame, str | None]: f""" PRs merged into branch_ref that belong in the next release notes: after the latest GitHub @@ -376,62 +490,35 @@ def get_prs_in_release_dataframe( Merge commits whose subject is from the upstream repository (ClickHouse/) are omitted. Columns: pr_number, pr_name, labels. + commit_sha is the workflow commit from the GitHub Actions API. + Returns (dataframe, baseline_date YYYY-MM-DD or None). """ - branch_sha = _git_rev_parse(branch_ref, cwd) - if not branch_sha: - raise Exception(f"Cannot resolve branch ref: {branch_ref!r}") - - # CI often checks out with --depth=1 and --no-tags; fetch tags once, then deepen - # in steps until a baseline is reachable. - # Observed baseline distances are ~1k commits and grow slowly, so DEEPEN_CAP sits - # well above that; the cap exists so a failed lookup fails fast instead of fetching - # the branch's entire (100k+ commit) history. - DEEPEN_STEP = 250 - DEEPEN_CAP = 10000 - - subprocess.run( - ["git", "fetch", "--tags", "origin"], - cwd=cwd, - capture_output=True, - text=True, - check=False, + _fetch_commit(commit_sha, cwd) + resolved = _git_rev_parse(commit_sha, cwd) + if not resolved: + raise Exception(f"Cannot resolve commit: {commit_sha!r}") + commit_sha = resolved + + releases = _filter_releases_to_version_line( + _get_github_releases(repo), branch_ref, commit_sha, cwd ) + _fetch_release_tags(releases, cwd) - parsed = _parse_release_ref(branch_ref) - fetch_ref = ( - f"refs/tags/{branch_ref}" - if parsed is not None and parsed[2] is not None - else branch_ref + _, baseline_sha, baseline_date = _find_release_baseline( + branch_ref, releases, cwd, commit_sha=commit_sha ) - - baseline_sha = None - baseline_date = None - for _ in range(DEEPEN_CAP // DEEPEN_STEP): - subprocess.run( - ["git", "fetch", f"--deepen={DEEPEN_STEP}", "origin", fetch_ref], - cwd=cwd, - capture_output=True, - text=True, - check=False, - ) - _, baseline_sha, baseline_date = _find_release_baseline( - branch_ref, repo, cwd - ) - if not baseline_sha: - baseline_sha = _find_rebase_baseline(branch_ref, cwd) - if baseline_sha: - baseline_date = _git_commit_date(baseline_sha, cwd) + if not baseline_sha: + baseline_sha = _find_rebase_baseline(commit_sha, cwd) if baseline_sha: - break + baseline_date = _git_commit_date(baseline_sha, cwd) if not baseline_sha: raise Exception( - "No release tag on this branch and no rebase bootstrap merge found " - f"within {DEEPEN_CAP} commits" + "No release tag on this branch and no rebase bootstrap merge found" ) - df = _git_log_merge_prs(baseline_sha, branch_ref, cwd, repo) + df = _git_log_merge_prs(baseline_sha, commit_sha, cwd, repo) return _enrich_prs_in_release_merge_prs(df, repo), baseline_date @@ -439,18 +526,18 @@ def _parse_release_ref(ref_name: str) -> tuple[str, str, str | None] | None: """Parse a release ref into (flavour, minor_version, full_version). Handles version tags (``v25.8.28.10001.altinitystable``) and release - branches (``stable-25.8``). ``full_version`` is None for branches, which + branches (``stable-25.8``, ``antalya-25.8``, ``fips-25.3``). ``full_version`` is None for branches, which carry no patch or tweak. Returns None for refs without a version. """ # Match tag match = re.match( - r"^v((\d+\.\d+)\.\d+\.\d+)\.altinity(stable|antalya|test)$", ref_name + r"^v((\d+\.\d+)\.\d+\.\d+)\.altinity(stable|antalya|fips|test)$", ref_name ) if match: - flavour = "antalya" if match.group(3) == "antalya" else "stable" + flavour = match.group(3) if match.group(3) in ("antalya", "fips") else "stable" return flavour, match.group(2), match.group(1) # Match branch - match = re.match(r"^(stable|antalya)-(\d+\.\d+)$", ref_name) + match = re.match(r"^(stable|antalya|fips)-(\d+\.\d+)$", ref_name) if match: return match.group(1), match.group(2), None return None @@ -1300,7 +1387,7 @@ def create_workflow_report( if pr_number == 0 and not mark_preview: try: prs_df, baseline_date = get_prs_in_release_dataframe( - branch_name, cwd=os.getcwd() + branch_name, cwd=os.getcwd(), commit_sha=commit_sha ) results_dfs["prs_in_release"] = prs_df prs_in_release_search_url = _prs_in_release_search_url( diff --git a/.github/actions/create_workflow_report/test_prs_in_release.py b/.github/actions/create_workflow_report/test_prs_in_release.py new file mode 100644 index 0000000000000..a8edce3f07f9a --- /dev/null +++ b/.github/actions/create_workflow_report/test_prs_in_release.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Unit tests for PRs-in-release baseline selection and merge filtering.""" + +import subprocess +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +import pandas as pd + +from create_workflow_report import ( + RELEASE_TAG_REF_PREFIX, + _checks_latest_test_status_cte, + _enrich_prs_in_release_merge_prs, + _fetch_release_tags, + _fetch_commit, + _find_release_baseline, + _get_github_releases, + _git_log_merge_prs, + _git_rev_parse, + _parse_release_ref, + _release_tag_sha, + _filter_releases_to_version_line, + get_prs_in_release_dataframe, +) + +CURRENT_TAG = "v26.6.4.20001.altinityantalya" +PREV_TAG = "v26.6.2.20001.altinityantalya" +CURRENT_SHA = "aa" * 20 +PREV_SHA = "bb" * 20 +# Linear history: PREV_SHA is an ancestor of CURRENT_SHA (and of itself). +COMMIT_ORDER = {PREV_SHA: 0, CURRENT_SHA: 1} +REF_TO_SHA = { + CURRENT_TAG: CURRENT_SHA, + PREV_TAG: PREV_SHA, + CURRENT_SHA: CURRENT_SHA, + PREV_SHA: PREV_SHA, + "antalya-26.6": CURRENT_SHA, +} + +RELEASES = [ + { + "draft": False, + "tag_name": CURRENT_TAG, + "published_at": "2026-09-11T14:02:51Z", + }, + { + "draft": False, + "tag_name": PREV_TAG, + "published_at": "2026-08-26T23:29:39Z", + }, +] + + +def _rev_parse(ref, cwd): + return REF_TO_SHA.get(ref.removeprefix(RELEASE_TAG_REF_PREFIX)) + + +def _is_ancestor(ancestor, descendant, cwd): + a = REF_TO_SHA.get(ancestor, ancestor) + d = REF_TO_SHA.get(descendant, descendant) + if a not in COMMIT_ORDER or d not in COMMIT_ORDER: + return False + return COMMIT_ORDER[a] <= COMMIT_ORDER[d] + + +def _releases_response(): + response = MagicMock() + response.status_code = 200 + response.json.return_value = RELEASES + [ + {"draft": True, "tag_name": "v26.9.1.1.altinityantalya"} + ] + return response + + +class TestGetGithubReleases(unittest.TestCase): + def test_skips_drafts(self): + with ( + patch("create_workflow_report.GITHUB_TOKEN", "token"), + patch( + "create_workflow_report.requests.get", + return_value=_releases_response(), + ), + ): + releases = _get_github_releases("Altinity/ClickHouse") + self.assertEqual(releases, RELEASES) + + def test_no_token_returns_empty(self): + with ( + patch("create_workflow_report.GITHUB_TOKEN", None), + patch("builtins.print") as printed, + ): + self.assertEqual(_get_github_releases("Altinity/ClickHouse"), []) + printed.assert_called_once() + self.assertIn("GITHUB_TOKEN", printed.call_args.args[0]) + + +class TestFindReleaseBaseline(unittest.TestCase): + def test_tag_ref_skips_current_github_release(self): + with ( + patch( + "create_workflow_report._git_rev_parse", + side_effect=_rev_parse, + ), + patch( + "create_workflow_report._git_is_ancestor", + side_effect=_is_ancestor, + ), + ): + tag_name, tag_sha, published_date = _find_release_baseline( + CURRENT_TAG, RELEASES, cwd=None, commit_sha=CURRENT_SHA + ) + self.assertEqual(tag_name, PREV_TAG) + self.assertEqual(tag_sha, PREV_SHA) + self.assertEqual(published_date, "2026-08-26") + + def test_branch_ref_keeps_latest_github_release(self): + with ( + patch( + "create_workflow_report._git_rev_parse", + side_effect=_rev_parse, + ), + patch( + "create_workflow_report._git_is_ancestor", + side_effect=_is_ancestor, + ), + ): + tag_name, tag_sha, published_date = _find_release_baseline( + "antalya-26.6", RELEASES, cwd=None, commit_sha=CURRENT_SHA + ) + self.assertEqual(tag_name, CURRENT_TAG) + self.assertEqual(tag_sha, CURRENT_SHA) + self.assertEqual(published_date, "2026-09-11") + + def test_ancestry_uses_commit_not_branch_name(self): + # Local antalya-26.6 may sit on the latest release; the run commit is older. + with ( + patch( + "create_workflow_report._git_rev_parse", + side_effect=_rev_parse, + ), + patch( + "create_workflow_report._git_is_ancestor", + side_effect=_is_ancestor, + ), + ): + tag_name, tag_sha, published_date = _find_release_baseline( + "antalya-26.6", RELEASES, cwd=None, commit_sha=PREV_SHA + ) + self.assertEqual(tag_name, PREV_TAG) + self.assertEqual(tag_sha, PREV_SHA) + self.assertEqual(published_date, "2026-08-26") + + +def _version_file(major, minor): + return subprocess.CompletedProcess( + [], 0, stdout=f"SET(VERSION_MAJOR {major})\nSET(VERSION_MINOR {minor})\n" + ) + + +class TestFetchCommit(unittest.TestCase): + def test_full_clone_fetches_commit_without_clone_shape_flags(self): + with ( + patch("create_workflow_report._git_is_shallow", return_value=False), + patch("create_workflow_report._git_fetch") as fetch, + ): + _fetch_commit(CURRENT_SHA, ".") + fetch.assert_called_once_with(["--no-tags"], [CURRENT_SHA], ".") + + def test_shallow_clone_unshallows_commit(self): + with ( + patch("create_workflow_report._git_is_shallow", return_value=True), + patch("create_workflow_report._git_fetch") as fetch, + ): + _fetch_commit(PREV_SHA, ".") + fetch.assert_called_once_with( + ["--filter=tree:0", "--unshallow"], [PREV_SHA], "." + ) + + +class TestReleaseTagSha(unittest.TestCase): + def test_namespaced_only_ignores_local_tag(self): + with patch( + "create_workflow_report._git_rev_parse", + side_effect=lambda ref, cwd: ( + "private" if ref.startswith(RELEASE_TAG_REF_PREFIX) else "local" + ), + ): + self.assertEqual(_release_tag_sha(CURRENT_TAG, None), "private") + with patch( + "create_workflow_report._git_rev_parse", + side_effect=lambda ref, cwd: ( + None if ref.startswith(RELEASE_TAG_REF_PREFIX) else "local" + ), + ): + self.assertIsNone(_release_tag_sha(CURRENT_TAG, None)) + + +class TestReleasesForVersionLine(unittest.TestCase): + def test_keeps_flavour_and_minor_drops_others(self): + mixed = RELEASES + [ + { + "draft": False, + "tag_name": "v25.8.33.10001.altinitystable", + "published_at": "2026-09-01T00:00:00Z", + }, + { + "draft": False, + "tag_name": "v26.6.8.10001.altinitystable", + "published_at": "2026-10-01T00:00:00Z", + }, + ] + out = _filter_releases_to_version_line(mixed, "antalya-26.6", CURRENT_SHA, None) + self.assertEqual([r["tag_name"] for r in out], [CURRENT_TAG, PREV_TAG]) + + def test_odd_branch_reads_version_file(self): + mixed = RELEASES + [ + { + "draft": False, + "tag_name": "v25.8.33.10001.altinitystable", + "published_at": "2026-09-01T00:00:00Z", + }, + ] + with patch( + "create_workflow_report._read_autogenerated_versions", + return_value=(26, 6), + ): + out = _filter_releases_to_version_line(mixed, "odd-branch", CURRENT_SHA, None) + self.assertEqual([r["tag_name"] for r in out], [CURRENT_TAG, PREV_TAG]) + + +class TestFipsRefs(unittest.TestCase): + def test_parses_fips_branch_and_tag(self): + self.assertEqual(_parse_release_ref("fips-25.3"), ("fips", "25.3", None)) + self.assertEqual( + _parse_release_ref("v25.3.8.30001.altinityfips"), + ("fips", "25.3", "25.3.8.30001"), + ) + + def test_fips_branch_keeps_only_fips_tags(self): + fips_tag = "v25.3.8.30001.altinityfips" + releases = [ + {"draft": False, "tag_name": fips_tag}, + {"draft": False, "tag_name": "v25.3.8.20001.altinityantalya"}, + {"draft": False, "tag_name": "v25.3.8.10001.altinitystable"}, + ] + out = _filter_releases_to_version_line(releases, "fips-25.3", CURRENT_SHA, None) + self.assertEqual([r["tag_name"] for r in out], [fips_tag]) + + +class TestFetchReleaseTagsWarning(unittest.TestCase): + def test_missing_tag_is_reported(self): + def _fetch(options, refspecs, cwd): + if CURRENT_TAG in refspecs[0]: + raise Exception("git fetch failed: not found") + + with ( + patch("create_workflow_report._git_fetch", side_effect=_fetch), + patch("builtins.print") as printed, + ): + _fetch_release_tags(RELEASES, ".") + message = printed.call_args.args[0] + self.assertIn(CURRENT_TAG, message) + self.assertIn("not found", message) + + +class TestGitRevParse(unittest.TestCase): + def test_missing_full_sha_is_unresolved(self): + with tempfile.TemporaryDirectory() as cwd: + subprocess.run(["git", "init", "-q"], cwd=cwd, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-q", + "--allow-empty", + "-m", + "init", + ], + cwd=cwd, + check=True, + ) + head = _git_rev_parse("HEAD", cwd) + self.assertRegex(head, r"^[0-9a-f]{40}$") + self.assertEqual(_git_rev_parse(head, cwd), head) + self.assertIsNone(_git_rev_parse("deadbeef" * 5, cwd)) + + +class TestGetPrsInReleaseCommitResolve(unittest.TestCase): + def test_raises_when_commit_cannot_resolve(self): + with ( + patch("create_workflow_report._fetch_commit"), + patch("create_workflow_report._git_rev_parse", return_value=None), + ): + with self.assertRaises(Exception) as ctx: + get_prs_in_release_dataframe( + "antalya-26.6", cwd=".", commit_sha="deadbeef" * 5 + ) + self.assertIn("Cannot resolve commit", str(ctx.exception)) + self.assertIn("deadbeef", str(ctx.exception)) + + +class TestGitLogMergePrs(unittest.TestCase): + def test_includes_fork_and_altinity_omits_upstream(self): + stdout = "\n".join( + [ + f"{'e' * 40}\tMerge pull request #2226 from strtgbb/community-pr-failure-artifacts", + f"{'a' * 40}\tMerge pull request #113355 from ClickHouse/backport/26.6/112784", + f"{'b' * 40}\tMerge pull request #2252 from Altinity/ci/antalya-26.6/container-dns", + ] + ) + with patch( + "create_workflow_report.subprocess.run", + return_value=subprocess.CompletedProcess([], 0, stdout=stdout), + ): + df = _git_log_merge_prs("baseline", "branch", None, "Altinity/ClickHouse") + self.assertEqual(set(df["pr_number"]), {2226, 2252}) + + +def _pr_response(status_code, title=""): + response = MagicMock() + response.status_code = status_code + response.text = "Not Found" if status_code == 404 else "error" + response.json.return_value = {"title": title, "labels": []} + return response + + +class TestEnrichPrsInReleaseMergePrs(unittest.TestCase): + def test_skips_404_keeps_ours(self): + # Upstream PR merged from a personal fork (head_owner != ClickHouse) + # is not in Altinity/ClickHouse; 404 means skip, not fail. + df = pd.DataFrame( + { + "pr_number": [114814, 2287], + "merge_commit_sha": ["a" * 40, "b" * 40], + "merge_subject": [ + "Merge pull request #114814 from groeneai/groeneai/backport-112327-25.8", + "Merge pull request #2287 from Altinity/bump/antalya-25.8/25.8.33", + ], + } + ) + + def _get(url, headers=None): + if url.endswith("/114814"): + return _pr_response(404) + return _pr_response(200, title="bump 25.8.33") + + with ( + patch("create_workflow_report.GITHUB_TOKEN", "token"), + patch("create_workflow_report.requests.get", side_effect=_get), + ): + out = _enrich_prs_in_release_merge_prs(df, "Altinity/ClickHouse") + self.assertEqual(list(out["pr_number"]), [2287]) + self.assertEqual(list(out["pr_name"]), ["bump 25.8.33"]) + + def test_all_404s_returns_empty(self): + df = pd.DataFrame( + { + "pr_number": [114814], + "merge_commit_sha": ["a" * 40], + "merge_subject": [ + "Merge pull request #114814 from groeneai/groeneai/backport-112327-25.8", + ], + } + ) + with ( + patch("create_workflow_report.GITHUB_TOKEN", "token"), + patch( + "create_workflow_report.requests.get", + return_value=_pr_response(404), + ), + ): + out = _enrich_prs_in_release_merge_prs(df, "Altinity/ClickHouse") + self.assertEqual(len(out), 0) + self.assertEqual(list(out.columns), ["pr_number", "pr_name", "pr_labels"]) + + def test_non_404_error_still_raises(self): + df = pd.DataFrame( + { + "pr_number": [2287], + "merge_commit_sha": ["b" * 40], + "merge_subject": [ + "Merge pull request #2287 from Altinity/bump/antalya-25.8/25.8.33", + ], + } + ) + with ( + patch("create_workflow_report.GITHUB_TOKEN", "token"), + patch( + "create_workflow_report.requests.get", + return_value=_pr_response(500), + ), + ): + with self.assertRaises(Exception) as ctx: + _enrich_prs_in_release_merge_prs(df, "Altinity/ClickHouse") + self.assertIn("500", str(ctx.exception)) + + +class TestChecksLatestTestStatusCte(unittest.TestCase): + def test_anchor_excludes_hooks_and_empty_test_name(self): + sql = _checks_latest_test_status_cte("deadbeef", "antalya-25.8") + self.assertIn("test_name != ''", sql) + self.assertIn("NOT IN ('Pre Hooks', 'Post Hooks')", sql) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 9ea30c0766a8f..1930eac75ca72 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -2,7 +2,7 @@ name: Fast Builds concurrency: - group: ${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: workflow_dispatch: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6db56e583ee4a..6dbee52fbde76 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -2,7 +2,7 @@ name: MasterCI concurrency: - group: ${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: workflow_dispatch: diff --git a/ci/praktika/yaml_generator.py b/ci/praktika/yaml_generator.py index 5a7a8d5d5ab95..d5ec52d57d276 100644 --- a/ci/praktika/yaml_generator.py +++ b/ci/praktika/yaml_generator.py @@ -83,7 +83,7 @@ class Templates: workflow_dispatch:{DISPATCH_INPUTS_BLOCK} concurrency: - group: ${{{{{{{{ github.workflow }}}}}}}} + group: ${{{{{{{{ github.workflow }}}}}}}}-${{{{{{{{ github.ref }}}}}}}} env: PYTHONUNBUFFERED: 1 @@ -103,7 +103,7 @@ class Templates: name: {NAME} concurrency: - group: ${{{{{{{{ github.workflow }}}}}}}} + group: ${{{{{{{{ github.workflow }}}}}}}}-${{{{{{{{ github.ref }}}}}}}} cancel-in-progress: true on: workflow_dispatch:{DISPATCH_INPUTS_BLOCK}{WORKFLOW_CALL}{TAG_PUSH_TRIGGER} diff --git a/ci/tests/test_yaml_generator.py b/ci/tests/test_yaml_generator.py index 26feb2658f652..4b71ad39da354 100644 --- a/ci/tests/test_yaml_generator.py +++ b/ci/tests/test_yaml_generator.py @@ -53,6 +53,7 @@ def test_dispatch_with_tags_renders_boolean_input_and_tag_trigger(): assert "type: boolean" in yaml_text assert "push:" in yaml_text assert "tags: ['*']" in yaml_text + assert "group: ${{ github.workflow }}-${{ github.ref }}" in yaml_text assert "cancel-in-progress: true" in yaml_text @@ -67,6 +68,7 @@ def test_schedule_concurrency_does_not_cancel_in_progress(): ) assert "concurrency:" in yaml_text + assert "group: ${{ github.workflow }}-${{ github.ref }}" in yaml_text assert "cancel-in-progress:" not in yaml_text