diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5faca42..b4cd98b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,12 +38,24 @@ concurrency: jobs: matrix: runs-on: ubuntu-latest + permissions: + contents: read + actions: read # to list con/git-annex runs and their artifacts outputs: cells: ${{ steps.cells.outputs.matrix }} + git-annex-run: ${{ steps.git-annex-build.outputs.run_id }} steps: - uses: actions/checkout@v4 - id: cells run: echo "matrix=$(bin/ci/matrix-json.sh)" >> "$GITHUB_OUTPUT" + # Pick the git-annex build once, so every cell of this run tests + # the same one. On failure the cells fall back to picking it + # themselves (empty run id) rather than all going red. + - id: git-annex-build + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bin/ci/pick-git-annex-build.sh test: needs: matrix @@ -77,6 +89,7 @@ jobs: if: matrix.target == 'git-annex' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GIT_ANNEX_RUN_ID: ${{ needs.matrix.outputs.git-annex-run }} run: bin/ci/install-git-annex-daily.sh - name: Configure git identity diff --git a/GOTCHAS.md b/GOTCHAS.md index 06bba2b..af18c25 100644 --- a/GOTCHAS.md +++ b/GOTCHAS.md @@ -187,15 +187,6 @@ Identical assertions fail on 7.4.6 and 8.1.0. See: [BeeGFS (`bin/eval-under-beegfs`)](#beegfs-bineval-under-beegfs) - -### `beegfs-annex-export-busy`: git-annex export/import fails on BeeGFS with EBUSY on rename - -**Cells:** `beegfs-7.4.6-git-annex`, `beegfs-8.1.0-git-annex` \ -**Tags:** `needs-triage` \ -**Tests:** `Tests.Repo Tests v10 *.export and import`, `Tests.Repo Tests v10 *.export and import of subdir`, `Tests.Repo Tests v10 *.git-remote-annex exporttree` - -See: , [BeeGFS / git-annex test](#beegfs--git-annex-test) - ### `loop-annex-diskreserve`: git-annex test on a loop image no larger than annex.diskreserve @@ -295,20 +286,42 @@ control row cannot catch a git-annex regression. ### BeeGFS / git-annex test -The original motivating bug, now `beegfs-annex-export-busy`: the same 8 -of the 9 (repo mode x test) combinations of `export and import`, `export -and import of subdir` and `git-remote-annex exporttree` fail on both -versions, with `renamePath:rename ... resource busy`. - -Note the useful negative result beside it: `BeeGFS * / git testsuite` -**passes** on both versions. Whatever BeeGFS does differently, it is not -breaking git's index, refs, or object plumbing -- so the cause sits in -what git-annex layers on top, in how our git-annex is built, or in the -syscalls the pjdfstest column is flagging. - -Working hypothesis for the build: our git-annex lacks the OsPath support -upstream's builds have (see the upstream bug report -linked from the issue). Unconfirmed until a build with it runs here. +The original motivating bug: 8 of the 9 (repo mode x test) combinations +of `export and import`, `export and import of subdir` and +`git-remote-annex exporttree` failed on both BeeGFS versions, with + + git-annex: renamePath:rename '.git/annex/othertmp/...' to '.git/annex/export.ex/...': resource busy (Device or resource busy) + +(plus the same EBUSY from `mv`), and on 8.1.0 the suite could also hang +in `Repo Tests v10 unlocked` at `conflict resolution (removed file)` +until the 2400s timeout. + +**Resolved by building git-annex with OsPath**, as the upstream report + +said. Our con/git-annex standalone had silently been built without it: +the flag is on by default but automatic, and its `file-io >= 0.2.0` +dependency was missing from the build image (con/git-annex#295 adds it; +con/git-annex#296 makes that CI require the flag). With the first OsPath +build (con/git-annex run 36399800528, `10.20260901+git71`), both +`BeeGFS * / git-annex test` cells pass all 26 test groups -- `pass 838 +fail 0`, no EBUSY, no hang -- where the same 2026-09-28 matrix on the +non-OsPath git42/git47 builds failed exactly those tests (eval-under runs +36417303910 vs 36475373813). git47..git71 upstream touches nothing in +the export or rename paths, so the build flag is the difference. + +So the known issue is gone, and a return of those failures is a real +regression. `bin/ci/install-git-annex-daily.sh` refuses a build lacking +`OsPath` (`EXPECT_BUILD_FLAGS`), so these cells cannot quietly go back +to measuring a build without it. + +Two things still worth knowing: + +- `BeeGFS * / git testsuite` passes on both versions too: BeeGFS does + not break git's index, refs or object plumbing. +- On 7.4.6 one run stalled ~9.5 minutes across several concurrent tests + (`storeKey`, `sync`, `add`, ... each ~560-600s) and then passed; the + suite took 22m instead of ~6m. Not a failure, but it eats into the + 2400s budget if it recurs. ## Red that is not a finding diff --git a/bin/ci/install-git-annex-daily.sh b/bin/ci/install-git-annex-daily.sh index 5745d94..f3f88f4 100755 --- a/bin/ci/install-git-annex-daily.sh +++ b/bin/ci/install-git-annex-daily.sh @@ -4,49 +4,48 @@ # # Generated with Claude Code 2.1.233 / Claude Opus 4.7 # -# Download the latest successful git-annex build artifact from -# con/git-annex's "Build git-annex on Ubuntu" workflow and install it. -# Appends the git-annex-standalone bin dir to $GITHUB_PATH so subsequent -# steps see it. -# -# usage: -# bin/ci/install-git-annex-daily.sh -# -# Requires GH_TOKEN in the environment (for the actions REST API). +# Download a git-annex build artifact from con/git-annex's "Build +# git-annex on Ubuntu" workflow and install it. Appends the +# git-annex-standalone bin dir to $GITHUB_PATH so subsequent steps see it, +# and records which build was tested in $GITHUB_STEP_SUMMARY. set -euo pipefail export DEBIAN_FRONTEND=noninteractive -: "${GH_TOKEN:?GH_TOKEN must be set (secrets.GITHUB_TOKEN in a workflow)}" +usage() { + cat <<'USAGE' +usage: bin/ci/install-git-annex-daily.sh [run-id] -# `gh run list --status success --workflow "..."` has been observed -# returning stale (expired-artifact) runs on the runner's gh version. -# Use REST directly -- it orders newest-first reliably. -runs="$(gh api \ - 'repos/con/git-annex/actions/workflows/build-ubuntu.yaml/runs?status=success&per_page=10' \ - --jq '.workflow_runs[].id')" -if [ -z "$runs" ]; then - echo "no successful con/git-annex runs found" >&2 - exit 1 -fi + run-id con/git-annex build-ubuntu.yaml run to install from + (default: bin/ci/pick-git-annex-build.sh picks the newest + master build with a package) -run_id="" -for r in $runs; do - n="$(gh api "repos/con/git-annex/actions/runs/$r/artifacts" \ - --jq '[.artifacts[] | select(.expired==false and (.name | startswith("git-annex-debianstandalone-packages_")))] | length')" - if [ "$n" -gt 0 ]; then - run_id="$r" - break - fi -done +env overrides: + GIT_ANNEX_RUN_ID same as the positional run-id + GIT_ANNEX_BUILD_REPO repo the run belongs to (default: con/git-annex) + EXPECT_BUILD_FLAGS space-separated flags `git annex version` must + list, else fail (default: "OsPath"; "" disables) + GH_TOKEN required, for the GitHub REST API +USAGE +} + +case "${1:-}" in + -h|--help) usage; exit 0 ;; +esac +[ $# -le 1 ] || { usage >&2; exit 2; } + +: "${GH_TOKEN:?GH_TOKEN must be set (secrets.GITHUB_TOKEN in a workflow)}" + +here="$(cd "$(dirname "$0")" && pwd)" +repo="${GIT_ANNEX_BUILD_REPO:-con/git-annex}" +run_id="${1:-${GIT_ANNEX_RUN_ID:-}}" if [ -z "$run_id" ]; then - echo "no run with unexpired debianstandalone artifact" >&2 - exit 1 + run_id="$(GIT_ANNEX_BUILD_REPO="$repo" GITHUB_OUTPUT='' "$here/pick-git-annex-build.sh")" fi echo "downloading from run $run_id" mkdir -p /tmp/ga -gh run download --repo con/git-annex "$run_id" --dir /tmp/ga \ +gh run download --repo "$repo" "$run_id" --dir /tmp/ga \ --pattern 'git-annex-debianstandalone*' deb="$(find /tmp/ga -name '*.deb' -print -quit)" @@ -57,7 +56,38 @@ fi echo "installing $deb" sudo apt-get -o "DPkg::Lock::Timeout=60" install -y "$deb" -git-annex version | head -3 +version="$(git-annex version)" +head -3 <<< "$version" +flags="$(sed -n 's/^build flags: //p' <<< "$version")" + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### git-annex under test" + echo + echo "- build: https://github.com/$repo/actions/runs/$run_id" + echo "- package: \`$(basename "$deb")\`" + echo + echo '```' + head -3 <<< "$version" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" +fi + +# The build flags are not cosmetic: e.g. OsPath silently drops out when a +# build dependency is missing, and BeeGFS results differ with and without +# it. Fail here rather than let a cell quietly measure a different build. +missing="" +for f in ${EXPECT_BUILD_FLAGS-OsPath}; do + case " $flags " in + *" $f "*) ;; + *) missing="$missing $f" ;; + esac +done +if [ -n "$missing" ]; then + echo "E: build from run $run_id lacks expected build flag(s):$missing" >&2 + echo "E: build flags: $flags" >&2 + exit 1 +fi # Add the standalone bundle's bin dir to PATH for subsequent steps. if [ -n "${GITHUB_PATH:-}" ]; then diff --git a/bin/ci/pick-git-annex-build.sh b/bin/ci/pick-git-annex-build.sh new file mode 100755 index 0000000..e24586a --- /dev/null +++ b/bin/ci/pick-git-annex-build.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +# SPDX-License-Identifier: MIT +# +# Generated with Claude Code +# +# Pick the con/git-annex "Build git-annex on Ubuntu" run whose +# debianstandalone package the git-annex cells should install, and print +# its run id (also to $GITHUB_OUTPUT as run_id=..., when set). +# +# Why not simply `?status=success`: that is the *whole* run's +# conclusion, and one known-red test job (nfs-home, con/git-annex#293) +# marks every run failed, so filtering on it silently kept us on a +# weeks-old build. The package artifact is uploaded only once +# build-package succeeded, so "has an unexpired +# git-annex-debianstandalone-packages_* artifact" is the actual signal. +# Only master's scheduled/dispatched runs count: pull_request builds +# carry unmerged patches. +# +# Run once per workflow (the `matrix` job) and hand the id to every +# cell, so all cells of one run test the same build. + +set -euo pipefail + +usage() { + cat <<'USAGE' +usage: bin/ci/pick-git-annex-build.sh [repo] + + repo GitHub repo with the build-ubuntu.yaml workflow + (default: con/git-annex) + +env overrides: + GIT_ANNEX_BUILD_REPO same as the positional repo + GIT_ANNEX_BUILD_BRANCH branch to take runs from (default: master) + GIT_ANNEX_BUILD_EVENTS space-separated run events to accept + (default: "schedule workflow_dispatch") + GIT_ANNEX_BUILD_SCAN how many recent runs to consider (default: 30) + GH_TOKEN required, for the GitHub REST API +USAGE +} + +case "${1:-}" in + -h|--help) usage; exit 0 ;; +esac +[ $# -le 1 ] || { usage >&2; exit 2; } + +repo="${1:-${GIT_ANNEX_BUILD_REPO:-con/git-annex}}" +branch="${GIT_ANNEX_BUILD_BRANCH:-master}" +events="${GIT_ANNEX_BUILD_EVENTS:-schedule workflow_dispatch}" +scan="${GIT_ANNEX_BUILD_SCAN:-30}" + +: "${GH_TOKEN:?GH_TOKEN must be set (secrets.GITHUB_TOKEN in a workflow)}" + +# `gh run list` has been observed returning stale (expired-artifact) +# runs on the runner's gh version; REST orders newest-first reliably. +runs="$(gh api \ + "repos/$repo/actions/workflows/build-ubuntu.yaml/runs?branch=$branch&per_page=$scan" \ + --jq '.workflow_runs[] | "\(.id) \(.event) \(.conclusion // .status) \(.created_at)"')" +if [ -z "$runs" ]; then + echo "E: no $repo build-ubuntu.yaml runs on $branch" >&2 + exit 1 +fi + +# An array, not `while read ... <<< "$runs"`: gh inside such a loop +# shares its stdin and can swallow the remaining lines. &2 + if [ "$n" -gt 0 ]; then + run_id="$id" + break + fi +done + +if [ -z "$run_id" ]; then + echo "E: none of the last $scan $branch runs ($events) has an unexpired debianstandalone artifact" >&2 + exit 1 +fi + +echo "$run_id" +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" +fi diff --git a/evals/known-issues.yaml b/evals/known-issues.yaml index ef52a2e..4a3605d 100644 --- a/evals/known-issues.yaml +++ b/evals/known-issues.yaml @@ -144,19 +144,6 @@ issues: Identical assertions fail on 7.4.6 and 8.1.0. # Seeded from run 36050917994; split by syscall once each is understood. - - id: beegfs-annex-export-busy - title: git-annex export/import fails on BeeGFS with EBUSY on rename - backends: ["beegfs-*"] - targets: [git-annex] - tests: - - "Tests.Repo Tests v10 *.export and import" - - "Tests.Repo Tests v10 *.export and import of subdir" - - "Tests.Repo Tests v10 *.git-remote-annex exporttree" - tags: [needs-triage] - links: - - https://git-annex.branchable.com/bugs/35_failed_tests_on_beegfs/ - - "GOTCHAS.md#beegfs--git-annex-test" - # --------------------------------------------------------- loop images - id: loop-annex-diskreserve title: git-annex test on a loop image no larger than annex.diskreserve