From 506a99fb2cabdd96fd55330e40753032a4a0e7d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 19:49:20 +0000 Subject: [PATCH 1/3] Test the newest git-annex build, not the newest all-green run install-git-annex-daily.sh took con/git-annex build-ubuntu.yaml runs with ?status=success. That is the whole run's conclusion, and the nfs-home test job there has been red on every run since 2026-09-23 (con/git-annex#293), so every git-annex cell kept installing 10.20260901+git47 from run 35576118792 (2026-09-21), and BeeGFS 7.4.6 on 2026-09-28 even got git42 from 34820527339. In particular no cell has yet tested the first build with OsPath (con/git-annex#295), which is what matters for the BeeGFS rename/EBUSY failures. - bin/ci/pick-git-annex-build.sh (new): newest master schedule / workflow_dispatch run that has an unexpired git-annex-debianstandalone-packages_* artifact -- uploaded only once build-package succeeded, so that is the real signal. Run once in the `matrix` job, so all cells of a workflow run test the same build. Today it picks 36399800528 (2026-09-28); the old query still says 35576118792. - install-git-annex-daily.sh: takes that run id (positional or GIT_ANNEX_RUN_ID; picks itself when empty), writes the build URL, package and `git annex version` head to the job summary, and fails if a flag in EXPECT_BUILD_FLAGS (default "OsPath") is missing, so a cell cannot silently measure a different build. - test.yaml: matrix job gets actions: read and the pick step (continue-on-error: cells fall back to picking themselves). Tested: shellcheck (26 scripts), known-issues and unit checks; picker run against the live API through a `gh api --jq` stand-in; flag check against present / missing / disabled / substring (OsPathX) cases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013n9Q2igij49kxdtiakUzYv --- .github/workflows/test.yaml | 13 +++++ bin/ci/install-git-annex-daily.sh | 96 ++++++++++++++++++++----------- bin/ci/pick-git-annex-build.sh | 88 ++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 33 deletions(-) create mode 100755 bin/ci/pick-git-annex-build.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5faca42..b4cd98b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,12 +38,24 @@ concurrency: jobs: matrix: runs-on: ubuntu-latest + permissions: + contents: read + actions: read # to list con/git-annex runs and their artifacts outputs: cells: ${{ steps.cells.outputs.matrix }} + git-annex-run: ${{ steps.git-annex-build.outputs.run_id }} steps: - uses: actions/checkout@v4 - id: cells run: echo "matrix=$(bin/ci/matrix-json.sh)" >> "$GITHUB_OUTPUT" + # Pick the git-annex build once, so every cell of this run tests + # the same one. On failure the cells fall back to picking it + # themselves (empty run id) rather than all going red. + - id: git-annex-build + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bin/ci/pick-git-annex-build.sh test: needs: matrix @@ -77,6 +89,7 @@ jobs: if: matrix.target == 'git-annex' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GIT_ANNEX_RUN_ID: ${{ needs.matrix.outputs.git-annex-run }} run: bin/ci/install-git-annex-daily.sh - name: Configure git identity diff --git a/bin/ci/install-git-annex-daily.sh b/bin/ci/install-git-annex-daily.sh index 5745d94..f3f88f4 100755 --- a/bin/ci/install-git-annex-daily.sh +++ b/bin/ci/install-git-annex-daily.sh @@ -4,49 +4,48 @@ # # Generated with Claude Code 2.1.233 / Claude Opus 4.7 # -# Download the latest successful git-annex build artifact from -# con/git-annex's "Build git-annex on Ubuntu" workflow and install it. -# Appends the git-annex-standalone bin dir to $GITHUB_PATH so subsequent -# steps see it. -# -# usage: -# bin/ci/install-git-annex-daily.sh -# -# Requires GH_TOKEN in the environment (for the actions REST API). +# Download a git-annex build artifact from con/git-annex's "Build +# git-annex on Ubuntu" workflow and install it. Appends the +# git-annex-standalone bin dir to $GITHUB_PATH so subsequent steps see it, +# and records which build was tested in $GITHUB_STEP_SUMMARY. set -euo pipefail export DEBIAN_FRONTEND=noninteractive -: "${GH_TOKEN:?GH_TOKEN must be set (secrets.GITHUB_TOKEN in a workflow)}" +usage() { + cat <<'USAGE' +usage: bin/ci/install-git-annex-daily.sh [run-id] -# `gh run list --status success --workflow "..."` has been observed -# returning stale (expired-artifact) runs on the runner's gh version. -# Use REST directly -- it orders newest-first reliably. -runs="$(gh api \ - 'repos/con/git-annex/actions/workflows/build-ubuntu.yaml/runs?status=success&per_page=10' \ - --jq '.workflow_runs[].id')" -if [ -z "$runs" ]; then - echo "no successful con/git-annex runs found" >&2 - exit 1 -fi + run-id con/git-annex build-ubuntu.yaml run to install from + (default: bin/ci/pick-git-annex-build.sh picks the newest + master build with a package) -run_id="" -for r in $runs; do - n="$(gh api "repos/con/git-annex/actions/runs/$r/artifacts" \ - --jq '[.artifacts[] | select(.expired==false and (.name | startswith("git-annex-debianstandalone-packages_")))] | length')" - if [ "$n" -gt 0 ]; then - run_id="$r" - break - fi -done +env overrides: + GIT_ANNEX_RUN_ID same as the positional run-id + GIT_ANNEX_BUILD_REPO repo the run belongs to (default: con/git-annex) + EXPECT_BUILD_FLAGS space-separated flags `git annex version` must + list, else fail (default: "OsPath"; "" disables) + GH_TOKEN required, for the GitHub REST API +USAGE +} + +case "${1:-}" in + -h|--help) usage; exit 0 ;; +esac +[ $# -le 1 ] || { usage >&2; exit 2; } + +: "${GH_TOKEN:?GH_TOKEN must be set (secrets.GITHUB_TOKEN in a workflow)}" + +here="$(cd "$(dirname "$0")" && pwd)" +repo="${GIT_ANNEX_BUILD_REPO:-con/git-annex}" +run_id="${1:-${GIT_ANNEX_RUN_ID:-}}" if [ -z "$run_id" ]; then - echo "no run with unexpired debianstandalone artifact" >&2 - exit 1 + run_id="$(GIT_ANNEX_BUILD_REPO="$repo" GITHUB_OUTPUT='' "$here/pick-git-annex-build.sh")" fi echo "downloading from run $run_id" mkdir -p /tmp/ga -gh run download --repo con/git-annex "$run_id" --dir /tmp/ga \ +gh run download --repo "$repo" "$run_id" --dir /tmp/ga \ --pattern 'git-annex-debianstandalone*' deb="$(find /tmp/ga -name '*.deb' -print -quit)" @@ -57,7 +56,38 @@ fi echo "installing $deb" sudo apt-get -o "DPkg::Lock::Timeout=60" install -y "$deb" -git-annex version | head -3 +version="$(git-annex version)" +head -3 <<< "$version" +flags="$(sed -n 's/^build flags: //p' <<< "$version")" + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### git-annex under test" + echo + echo "- build: https://github.com/$repo/actions/runs/$run_id" + echo "- package: \`$(basename "$deb")\`" + echo + echo '```' + head -3 <<< "$version" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" +fi + +# The build flags are not cosmetic: e.g. OsPath silently drops out when a +# build dependency is missing, and BeeGFS results differ with and without +# it. Fail here rather than let a cell quietly measure a different build. +missing="" +for f in ${EXPECT_BUILD_FLAGS-OsPath}; do + case " $flags " in + *" $f "*) ;; + *) missing="$missing $f" ;; + esac +done +if [ -n "$missing" ]; then + echo "E: build from run $run_id lacks expected build flag(s):$missing" >&2 + echo "E: build flags: $flags" >&2 + exit 1 +fi # Add the standalone bundle's bin dir to PATH for subsequent steps. if [ -n "${GITHUB_PATH:-}" ]; then diff --git a/bin/ci/pick-git-annex-build.sh b/bin/ci/pick-git-annex-build.sh new file mode 100755 index 0000000..6f711b6 --- /dev/null +++ b/bin/ci/pick-git-annex-build.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Pick the con/git-annex "Build git-annex on Ubuntu" run whose +# debianstandalone package the git-annex cells should install, and print +# its run id (also to $GITHUB_OUTPUT as run_id=..., when set). +# +# Why not simply `?status=success`: that is the *whole* run's +# conclusion, and one known-red test job (nfs-home, con/git-annex#293) +# marks every run failed, so filtering on it silently kept us on a +# weeks-old build. The package artifact is uploaded only once +# build-package succeeded, so "has an unexpired +# git-annex-debianstandalone-packages_* artifact" is the actual signal. +# Only master's scheduled/dispatched runs count: pull_request builds +# carry unmerged patches. +# +# Run once per workflow (the `matrix` job) and hand the id to every +# cell, so all cells of one run test the same build. + +set -euo pipefail + +usage() { + cat <<'USAGE' +usage: bin/ci/pick-git-annex-build.sh [repo] + + repo GitHub repo with the build-ubuntu.yaml workflow + (default: con/git-annex) + +env overrides: + GIT_ANNEX_BUILD_REPO same as the positional repo + GIT_ANNEX_BUILD_BRANCH branch to take runs from (default: master) + GIT_ANNEX_BUILD_EVENTS space-separated run events to accept + (default: "schedule workflow_dispatch") + GIT_ANNEX_BUILD_SCAN how many recent runs to consider (default: 30) + GH_TOKEN required, for the GitHub REST API +USAGE +} + +case "${1:-}" in + -h|--help) usage; exit 0 ;; +esac +[ $# -le 1 ] || { usage >&2; exit 2; } + +repo="${1:-${GIT_ANNEX_BUILD_REPO:-con/git-annex}}" +branch="${GIT_ANNEX_BUILD_BRANCH:-master}" +events="${GIT_ANNEX_BUILD_EVENTS:-schedule workflow_dispatch}" +scan="${GIT_ANNEX_BUILD_SCAN:-30}" + +: "${GH_TOKEN:?GH_TOKEN must be set (secrets.GITHUB_TOKEN in a workflow)}" + +# `gh run list` has been observed returning stale (expired-artifact) +# runs on the runner's gh version; REST orders newest-first reliably. +runs="$(gh api \ + "repos/$repo/actions/workflows/build-ubuntu.yaml/runs?branch=$branch&per_page=$scan" \ + --jq '.workflow_runs[] | "\(.id) \(.event) \(.conclusion // .status) \(.created_at)"')" +if [ -z "$runs" ]; then + echo "E: no $repo build-ubuntu.yaml runs on $branch" >&2 + exit 1 +fi + +run_id="" +while read -r id event state created; do + case " $events " in + *" $event "*) ;; + *) continue ;; + esac + n="$(gh api "repos/$repo/actions/runs/$id/artifacts" \ + --jq '[.artifacts[] | select(.expired==false and (.name | startswith("git-annex-debianstandalone-packages_")))] | length')" + if [ "$n" -gt 0 ]; then + run_id="$id" + # The run's own conclusion is informational only (see header). + echo "I: picked $repo run $id ($event, $created, run $state)" >&2 + break + fi +done <<< "$runs" + +if [ -z "$run_id" ]; then + echo "E: none of the last $scan $branch runs ($events) has an unexpired debianstandalone artifact" >&2 + exit 1 +fi + +echo "$run_id" +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" +fi From 5291abf0b50d1db7f8d021a8236571ff318bf771 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 19:53:33 +0000 Subject: [PATCH 2/3] pick-git-annex-build: log every candidate; keep gh off the loop's stdin In this PR's first run the matrix job picked 35319817132 (2026-09-18, no OsPath) although 11 newer master runs, each with its package artifact, precede it -- the same query picks 36399800528 locally. install-git-annex-daily.sh's new flag check caught it (the cells went red with "lacks expected build flag(s): OsPath" instead of silently testing an old build), but the picker must not skip them. The cause is not reproducible here, so: log run id, event, date, conclusion and the artifact count for every candidate considered, and iterate over an array with gh reading /dev/null instead of a `while read ... <<< "$runs"` loop whose stdin gh would share. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013n9Q2igij49kxdtiakUzYv --- bin/ci/pick-git-annex-build.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/ci/pick-git-annex-build.sh b/bin/ci/pick-git-annex-build.sh index 6f711b6..e24586a 100755 --- a/bin/ci/pick-git-annex-build.sh +++ b/bin/ci/pick-git-annex-build.sh @@ -61,21 +61,26 @@ if [ -z "$runs" ]; then exit 1 fi +# An array, not `while read ... <<< "$runs"`: gh inside such a loop +# shares its stdin and can swallow the remaining lines. &2 if [ "$n" -gt 0 ]; then run_id="$id" - # The run's own conclusion is informational only (see header). - echo "I: picked $repo run $id ($event, $created, run $state)" >&2 break fi -done <<< "$runs" +done if [ -z "$run_id" ]; then echo "E: none of the last $scan $branch runs ($events) has an unexpired debianstandalone artifact" >&2 From 0b7db7d7d7276cab4bd51678226465d10edfeb0e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 21:13:03 +0000 Subject: [PATCH 3/3] known-issues: drop beegfs-annex-export-busy -- fixed by an OsPath build With this PR's picker the git-annex cells finally ran the first OsPath build (con/git-annex run 36399800528, 10.20260901+git71), and on both BeeGFS versions every test group passes: pass 838 fail 0, no renamePath/EBUSY, no hang -- known_issues.py flagged the entry "did not reproduce -- fixed?" on both cells. The 2026-09-28 matrix on the non-OsPath git42/git47 builds (run 36417303910) failed exactly the entry's tests (7.4.6: 8 failures, 24 EBUSY; 8.1.0: 3 failures and the `conflict resolution (removed file)` hang to exit 124). Upstream git47..git71 changes nothing in the export or rename paths, so the build flag is the difference, as the upstream BeeGFS bug report said. Dropping the entry (rather than keeping it as known) makes a recurrence a red cell; install-git-annex-daily.sh already refuses non-OsPath builds. GOTCHAS.md's "BeeGFS / git-annex test" section now records the resolution, the evidence, and the one-off ~9.5 min stall seen on 7.4.6; the generated known-issues block was regenerated with `bin/ci/known_issues.py gotchas`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013n9Q2igij49kxdtiakUzYv --- GOTCHAS.md | 59 +++++++++++++++++++++++++---------------- evals/known-issues.yaml | 13 --------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/GOTCHAS.md b/GOTCHAS.md index 06bba2b..af18c25 100644 --- a/GOTCHAS.md +++ b/GOTCHAS.md @@ -187,15 +187,6 @@ Identical assertions fail on 7.4.6 and 8.1.0. See: [BeeGFS (`bin/eval-under-beegfs`)](#beegfs-bineval-under-beegfs) - -### `beegfs-annex-export-busy`: git-annex export/import fails on BeeGFS with EBUSY on rename - -**Cells:** `beegfs-7.4.6-git-annex`, `beegfs-8.1.0-git-annex` \ -**Tags:** `needs-triage` \ -**Tests:** `Tests.Repo Tests v10 *.export and import`, `Tests.Repo Tests v10 *.export and import of subdir`, `Tests.Repo Tests v10 *.git-remote-annex exporttree` - -See: , [BeeGFS / git-annex test](#beegfs--git-annex-test) - ### `loop-annex-diskreserve`: git-annex test on a loop image no larger than annex.diskreserve @@ -295,20 +286,42 @@ control row cannot catch a git-annex regression. ### BeeGFS / git-annex test -The original motivating bug, now `beegfs-annex-export-busy`: the same 8 -of the 9 (repo mode x test) combinations of `export and import`, `export -and import of subdir` and `git-remote-annex exporttree` fail on both -versions, with `renamePath:rename ... resource busy`. - -Note the useful negative result beside it: `BeeGFS * / git testsuite` -**passes** on both versions. Whatever BeeGFS does differently, it is not -breaking git's index, refs, or object plumbing -- so the cause sits in -what git-annex layers on top, in how our git-annex is built, or in the -syscalls the pjdfstest column is flagging. - -Working hypothesis for the build: our git-annex lacks the OsPath support -upstream's builds have (see the upstream bug report -linked from the issue). Unconfirmed until a build with it runs here. +The original motivating bug: 8 of the 9 (repo mode x test) combinations +of `export and import`, `export and import of subdir` and +`git-remote-annex exporttree` failed on both BeeGFS versions, with + + git-annex: renamePath:rename '.git/annex/othertmp/...' to '.git/annex/export.ex/...': resource busy (Device or resource busy) + +(plus the same EBUSY from `mv`), and on 8.1.0 the suite could also hang +in `Repo Tests v10 unlocked` at `conflict resolution (removed file)` +until the 2400s timeout. + +**Resolved by building git-annex with OsPath**, as the upstream report + +said. Our con/git-annex standalone had silently been built without it: +the flag is on by default but automatic, and its `file-io >= 0.2.0` +dependency was missing from the build image (con/git-annex#295 adds it; +con/git-annex#296 makes that CI require the flag). With the first OsPath +build (con/git-annex run 36399800528, `10.20260901+git71`), both +`BeeGFS * / git-annex test` cells pass all 26 test groups -- `pass 838 +fail 0`, no EBUSY, no hang -- where the same 2026-09-28 matrix on the +non-OsPath git42/git47 builds failed exactly those tests (eval-under runs +36417303910 vs 36475373813). git47..git71 upstream touches nothing in +the export or rename paths, so the build flag is the difference. + +So the known issue is gone, and a return of those failures is a real +regression. `bin/ci/install-git-annex-daily.sh` refuses a build lacking +`OsPath` (`EXPECT_BUILD_FLAGS`), so these cells cannot quietly go back +to measuring a build without it. + +Two things still worth knowing: + +- `BeeGFS * / git testsuite` passes on both versions too: BeeGFS does + not break git's index, refs or object plumbing. +- On 7.4.6 one run stalled ~9.5 minutes across several concurrent tests + (`storeKey`, `sync`, `add`, ... each ~560-600s) and then passed; the + suite took 22m instead of ~6m. Not a failure, but it eats into the + 2400s budget if it recurs. ## Red that is not a finding diff --git a/evals/known-issues.yaml b/evals/known-issues.yaml index ef52a2e..4a3605d 100644 --- a/evals/known-issues.yaml +++ b/evals/known-issues.yaml @@ -144,19 +144,6 @@ issues: Identical assertions fail on 7.4.6 and 8.1.0. # Seeded from run 36050917994; split by syscall once each is understood. - - id: beegfs-annex-export-busy - title: git-annex export/import fails on BeeGFS with EBUSY on rename - backends: ["beegfs-*"] - targets: [git-annex] - tests: - - "Tests.Repo Tests v10 *.export and import" - - "Tests.Repo Tests v10 *.export and import of subdir" - - "Tests.Repo Tests v10 *.git-remote-annex exporttree" - tags: [needs-triage] - links: - - https://git-annex.branchable.com/bugs/35_failed_tests_on_beegfs/ - - "GOTCHAS.md#beegfs--git-annex-test" - # --------------------------------------------------------- loop images - id: loop-annex-diskreserve title: git-annex test on a loop image no larger than annex.diskreserve