Skip to content

Commit a49c090

Browse files
tode-rltode-rl
authored
Merge pull request #10 from runloopai/ci/stlc-self-heal-fork
ci(stlc): self-heal a clean staging/production trunk fork
2 parents be62cae + 357a8a0 commit a49c090

3 files changed

Lines changed: 157 additions & 9 deletions

File tree

‎.github/scripts/stlc-heal-fork.sh‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#!/usr/bin/env bash
2+
# Heals a staging/production trunk fork with a merge commit, only when the
3+
# trunks merge cleanly and staging CI passes on that merge; otherwise explains
4+
# the fork and fails.
5+
# Expects origin/main = staging main and production/main = production main,
6+
# plus GITHUB_REPOSITORY (staging), PRODUCTION_REPO, PRODUCTION_REPO_TOKEN, and
7+
# GH_TOKEN able to read staging's Actions runs.
8+
set -euo pipefail
9+
10+
HINT="$(dirname "$0")/stlc-fork-hint.sh"
11+
HEAL_BRANCH=stlc-fork-heal
12+
CI_WORKFLOW=.github/workflows/ci.yml
13+
CI_TIMEOUT=${STLC_HEAL_CI_TIMEOUT:-1800}
14+
POLL=${STLC_HEAL_POLL:-15}
15+
16+
# One trunk ahead of the other is a fast-forward, not a fork.
17+
if git merge-base --is-ancestor origin/main production/main ||
18+
git merge-base --is-ancestor production/main origin/main; then
19+
echo "::error title=Fork heal refused::staging and production main have not forked."
20+
exit 1
21+
fi
22+
23+
if [ -z "${PRODUCTION_REPO_TOKEN:-}" ]; then
24+
bash "$HINT"
25+
echo "::error title=Fork heal blocked::PRODUCTION_REPO_TOKEN is required to heal the fork."
26+
exit 1
27+
fi
28+
29+
if ! TREE=$(git merge-tree --write-tree origin/main production/main 2>/dev/null); then
30+
bash "$HINT"
31+
exit 1
32+
fi
33+
34+
STAGING_ONLY=$(git log --format='%h %an %s' production/main..origin/main)
35+
PRODUCTION_ONLY=$(git log --format='%h %an %s' origin/main..production/main)
36+
37+
# Not stlc-bot: seal-dispatch must still re-seal the production custom code
38+
# this merge brings onto staging.
39+
MERGE=$(
40+
GIT_AUTHOR_NAME="stlc-fork-heal" GIT_AUTHOR_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \
41+
GIT_COMMITTER_NAME="stlc-fork-heal" GIT_COMMITTER_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \
42+
git commit-tree "$TREE" -p production/main -p origin/main \
43+
-m "chore: merge staging main into production to heal trunk fork"
44+
)
45+
46+
git remote set-url production "https://x-access-token:${PRODUCTION_REPO_TOKEN}@github.com/${PRODUCTION_REPO}.git"
47+
git remote set-url origin "https://x-access-token:${PRODUCTION_REPO_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
48+
push() { git -c "http.https://github.com/.extraheader=" push "$@"; }
49+
50+
# Nothing reaches main until staging CI passes on this exact merge: a clean text
51+
# merge can still fail to build. CI runs on any staging branch push; the branch
52+
# only triggers it.
53+
push --force origin "$MERGE:refs/heads/$HEAL_BRANCH"
54+
trap 'push -q origin --delete "$HEAL_BRANCH" || true' EXIT
55+
56+
blocked() {
57+
{
58+
echo "### Staging and production main have forked; heal blocked by CI"
59+
echo
60+
echo "They merge cleanly as text, but $1, so nothing was pushed to main. Fix the combination on either trunk (or push a fixed merge by hand); the next back-sync poll retries the heal."
61+
echo
62+
echo "CI run: ${CI_URL:-none found}"
63+
echo
64+
echo "**Only on staging:**"
65+
echo '```'
66+
echo "$STAGING_ONLY"
67+
echo '```'
68+
echo "**Only on production:**"
69+
echo '```'
70+
echo "$PRODUCTION_ONLY"
71+
echo '```'
72+
} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
73+
echo "::error title=Fork heal blocked::$1 on clean merge commit ${MERGE}; nothing was pushed to main. CI run: ${CI_URL:-none found}"
74+
exit 1
75+
}
76+
77+
deadline=$((SECONDS + CI_TIMEOUT))
78+
STATUS="" CONCLUSION="" CI_URL=""
79+
while :; do
80+
# A failed lookup reads as "not finished yet" and is retried until the deadline.
81+
read -r STATUS CONCLUSION CI_URL < <(
82+
gh api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${MERGE}&event=push" \
83+
--jq "[.workflow_runs[] | select(.path | startswith(\"${CI_WORKFLOW}\"))][0] | \"\(.status) \(.conclusion) \(.html_url)\""
84+
) || true
85+
[ "$CI_URL" = null ] && CI_URL=""
86+
[ "$STATUS" = completed ] && break
87+
[ "$SECONDS" -ge "$deadline" ] && blocked "staging CI did not finish within ${CI_TIMEOUT}s"
88+
sleep "$POLL"
89+
done
90+
91+
[ "$CONCLUSION" = success ] || blocked "staging CI concluded ${CONCLUSION}"
92+
93+
# Both pushes are fast-forwards (MERGE descends from each tip), so a trunk that
94+
# moved since the fetch rejects the push and the next scheduled run retries.
95+
# Production first: if staging then moves, the next run sees a fresh fork and heals it.
96+
push production "$MERGE:refs/heads/main"
97+
push origin "$MERGE:refs/heads/main"
98+
99+
SHORT=$(git rev-parse --short "$MERGE")
100+
{
101+
echo "### Healed a staging/production trunk fork"
102+
echo
103+
echo "The trunks had diverged but merged cleanly, so merge commit \`${SHORT}\` was pushed to both \`${PRODUCTION_REPO}\` main and \`${GITHUB_REPOSITORY}\` main by ${GITHUB_WORKFLOW:-stlc} run ${GITHUB_RUN_ID:-local}."
104+
echo
105+
echo "**Was only on staging:**"
106+
echo '```'
107+
echo "$STAGING_ONLY"
108+
echo '```'
109+
echo "**Was only on production:**"
110+
echo '```'
111+
echo "$PRODUCTION_ONLY"
112+
echo '```'
113+
echo "Staging CI passed on this exact commit before it was pushed: ${CI_URL}"
114+
} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
115+
echo "::warning title=Trunk fork healed::pushed clean merge commit ${SHORT} to staging and production main. See the job summary for the commits it joined."

‎.github/workflows/stlc-promote.yml‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ name: Promote SDKs
55
# next stlc build). This is a git-only reconciliation, not a release: release-please
66
# owns the actual version bump/publish gate on production. Runs on every push to
77
# staging main, because any staging-only commit (stlc build or not) forks the trunks
8-
# against the next production merge and blocks both directions; the daily schedule
9-
# is a backstop and workflow_dispatch an eager promote. The fast-forward-only guard
10-
# below makes every run a safe no-op when there's nothing to promote.
8+
# against the next production merge; the daily schedule is a backstop and
9+
# workflow_dispatch an eager promote (back-sync dispatches it on a fork). A fork
10+
# that merges cleanly and passes staging CI is healed with a merge commit on both
11+
# trunks (the only non-linear history this writes); anything else fails for a person.
12+
# Every run is a safe no-op when there's nothing to promote.
1113
on:
1214
push:
1315
branches: [main]
@@ -18,6 +20,7 @@ on:
1820

1921
permissions:
2022
contents: read
23+
actions: read
2124

2225
jobs:
2326
promote:
@@ -52,7 +55,12 @@ jobs:
5255
# After a release, production has release-please commits staging lacks, so compare trees.
5356
MERGED=$(git merge-tree --write-tree production/main origin/main) || MERGED=conflict
5457
PRODUCTION_TREE=$(git rev-parse 'production/main^{tree}')
55-
if [ "$MERGED" = "$PRODUCTION_TREE" ]; then
58+
# A fork needs healing even when production already has staging's content.
59+
if ! git merge-base --is-ancestor origin/main production/main &&
60+
! git merge-base --is-ancestor production/main origin/main; then
61+
echo "Staging and production main have forked."
62+
echo "synced=false" >> "$GITHUB_OUTPUT"
63+
elif [ "$MERGED" = "$PRODUCTION_TREE" ]; then
5664
echo "Production already contains staging's content. Nothing to promote."
5765
echo "synced=true" >> "$GITHUB_OUTPUT"
5866
else
@@ -61,12 +69,16 @@ jobs:
6169
6270
- name: Promote staging to production (fast-forward)
6371
if: steps.diff.outputs.synced == 'false'
72+
env:
73+
PRODUCTION_REPO_TOKEN: ${{ secrets.PRODUCTION_REPO_TOKEN }}
74+
GH_TOKEN: ${{ github.token }}
6475
run: |
65-
# Refuse unless production is an ancestor of staging: otherwise the trunks
66-
# have forked (production advanced without a back-sync) and FF is unsafe.
76+
# Production not an ancestor of staging means the trunks have forked
77+
# (production advanced without a back-sync) and FF is unsafe: heal with a
78+
# clean merge commit, else fail.
6779
if ! git merge-base --is-ancestor production/main origin/main; then
68-
bash .github/scripts/stlc-fork-hint.sh
69-
exit 1
80+
bash .github/scripts/stlc-heal-fork.sh
81+
exit 0
7082
fi
7183
git push production origin/main:refs/heads/main
7284
echo "Fast-forwarded production/main to staging/main."

‎.github/workflows/stlc-sync.yml‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ jobs:
2828
(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch')
2929
permissions:
3030
contents: write
31+
actions: write
3132
concurrency:
3233
group: stlc-back-sync
3334
cancel-in-progress: true
@@ -68,9 +69,29 @@ jobs:
6869
else
6970
echo "behind=true" >> "$GITHUB_OUTPUT"
7071
fi
72+
if git merge-base --is-ancestor origin/main production/main ||
73+
git merge-base --is-ancestor production/main origin/main; then
74+
echo "forked=false" >> "$GITHUB_OUTPUT"
75+
else
76+
echo "forked=true" >> "$GITHUB_OUTPUT"
77+
fi
78+
79+
- name: Hand a forked trunk to promote
80+
if: steps.diff.outputs.forked == 'true'
81+
env:
82+
GH_TOKEN: ${{ github.token }}
83+
run: |
84+
# Healing writes production main, so it runs only in promote's
85+
# production-environment job; a conflicting fork needs a person.
86+
if ! git merge-tree --write-tree origin/main production/main >/dev/null 2>&1; then
87+
bash .github/scripts/stlc-fork-hint.sh
88+
exit 1
89+
fi
90+
gh workflow run stlc-promote.yml -R "$GITHUB_REPOSITORY" --ref main
91+
echo "::warning title=Trunk fork::staging and production main have forked but merge cleanly; dispatched Promote SDKs to heal it."
7192
7293
- name: Sync production to staging (fast-forward)
73-
if: steps.diff.outputs.behind == 'true'
94+
if: steps.diff.outputs.behind == 'true' && steps.diff.outputs.forked != 'true'
7495
env:
7596
PRODUCTION_REPO_TOKEN: ${{ secrets.PRODUCTION_REPO_TOKEN }}
7697
run: |

0 commit comments

Comments
 (0)