From 7a5ce863923978396e5b4b121f9aaf3f88cc2db2 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 22 Sep 2026 13:18:15 +0000 Subject: [PATCH 1/3] patches: CI diagnostic for the intermittent NFS LinkAnnexFailed The nightly `test-annex (nfs-home)` job fails in a different test each night, always at `git annex add` of an unlocked file (" failed to link to annex") or at `git annex unlock` ("unlock failed"). Both are linkAnnex returning LinkAnnexFailed, which is reachable by two paths that print nothing at all: * linkOrCopy returned Nothing - a swallowed IOException, or cp / preserveGitMode returning False; * the source file's inode cache (inode, size, high resolution mtime) differed before and after the copy, so checksrcunchanged deleted the destination. The transcripts in the CI logs cannot tell those apart, so add a warning to each, including the before/after inode caches. Diagnostic only, on paths that already fail; drop it once the cause is known. Not compile-tested locally (no GHC available where this was written); a PR touching patches/*.patch triggers the builds, which will verify it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01B89nUooZLfThcTA4fSMPGf --- ...22-f68b252dbd-diag-linkannex-failure.patch | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 patches/20260922-f68b252dbd-diag-linkannex-failure.patch diff --git a/patches/20260922-f68b252dbd-diag-linkannex-failure.patch b/patches/20260922-f68b252dbd-diag-linkannex-failure.patch new file mode 100644 index 0000000000..186c2da391 --- /dev/null +++ b/patches/20260922-f68b252dbd-diag-linkannex-failure.patch @@ -0,0 +1,126 @@ +Description: CI diagnostic: report why linkAnnex failed + The nightly `test-annex (nfs-home)` job intermittently fails with + " failed to link to annex" (git-annex add of an unlocked file) or + "unlock failed", in a different test each night. Both messages come + from linkAnnex returning LinkAnnexFailed, which is reachable by two + paths that say nothing at all about what went wrong: + . + * linkOrCopy returned Nothing - either an IOException was swallowed by + catchDefaultIO/catchBoolIO, or cp / the chmod in preserveGitMode + returned False; + * the source file's inode cache (inode, size, high resolution mtime) + did not compare equal before and after the copy, so checksrcunchanged + deleted the destination. + . + The CI transcripts cannot distinguish these, which makes the NFS + failures impossible to diagnose after the fact. This patch adds a + warning to each of those paths, including the before/after inode caches, + so that the next failing run says which one it is. + . + Diagnostic only - no behaviour change beyond the extra messages on + paths that already fail. Drop it once the cause is understood. + . + Drafted with Claude Code assistance. +Origin: vendor, https://github.com/con/git-annex +Author: Yaroslav Halchenko +Forwarded: not-needed +Last-Update: 2026-09-22 +SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +SPDX-License-Identifier: AGPL-3.0-or-later +--- +diff --git a/Annex/Content/LowLevel.hs b/Annex/Content/LowLevel.hs +index 49fc442..0a4daa9 100644 +--- a/Annex/Content/LowLevel.hs ++++ b/Annex/Content/LowLevel.hs +@@ -47,12 +47,16 @@ linkOrCopy :: Key -> OsPath -> OsPath -> Maybe FileMode -> Annex (Maybe LinkedOr + linkOrCopy = linkOrCopy' (annexThin <$> Annex.getGitConfig) + + linkOrCopy' :: Annex Bool -> Key -> OsPath -> OsPath -> Maybe FileMode -> Annex (Maybe LinkedOrCopied) +-linkOrCopy' canhardlink key src dest destmode = catchDefaultIO Nothing $ +- ifM canhardlink ++linkOrCopy' canhardlink key src dest destmode = go `catchIO` reportfailure ++ where ++ go = ifM canhardlink + ( hardlinkorcopy + , copy =<< getstat + ) +- where ++ reportfailure e = do ++ warning $ UnquotedString $ ++ "DIAGNOSTIC: linkOrCopy threw: " ++ show e ++ return Nothing + hardlinkorcopy = do + s <- getstat + if linkCount s > 1 +@@ -75,14 +79,29 @@ checkedCopyFile key src dest destmode = catchBoolIO $ + =<< liftIO (R.getFileStatus (fromOsPath src)) + + checkedCopyFile' :: Key -> OsPath -> OsPath -> Maybe FileMode -> FileStatus -> Annex Bool +-checkedCopyFile' key src dest destmode s = catchBoolIO $ do +- sz <- liftIO $ getFileSize' src s +- ifM (checkDiskSpace' sz (Just $ takeDirectory dest) key 0 True) +- ( liftIO $ +- copyFileExternal CopyAllMetaData src dest +- <&&> preserveGitMode dest destmode +- , return False +- ) ++checkedCopyFile' key src dest destmode s = go `catchIO` reportfailure ++ where ++ go = do ++ sz <- liftIO $ getFileSize' src s ++ ifM (checkDiskSpace' sz (Just $ takeDirectory dest) key 0 True) ++ ( ifM (liftIO $ copyFileExternal CopyAllMetaData src dest) ++ ( ifM (liftIO $ preserveGitMode dest destmode) ++ ( return True ++ , do ++ diagnostic "preserveGitMode failed" ++ return False ++ ) ++ , do ++ diagnostic "copyFileExternal failed" ++ return False ++ ) ++ , return False ++ ) ++ reportfailure e = do ++ diagnostic ("exception: " ++ show e) ++ return False ++ diagnostic m = warning $ UnquotedString $ ++ "DIAGNOSTIC: checkedCopyFile: " ++ m + + preserveGitMode :: OsPath -> Maybe FileMode -> IO Bool + preserveGitMode f (Just mode) +diff --git a/Annex/Content.hs b/Annex/Content.hs +index e3b2202..530b70b 100644 +--- a/Annex/Content.hs ++++ b/Annex/Content.hs +@@ -631,7 +631,12 @@ linkAnnex fromto key src (Just srcic) dest destmode = + else Database.Keys.addInodeCaches key [srcic] + return LinkAnnexNoop + Nothing -> linkOrCopy key src dest destmode >>= \case +- Nothing -> failed ++ Nothing -> do ++ warning $ "DIAGNOSTIC: linkAnnex: linkOrCopy failed for " ++ <> QuotedPath src ++ <> " => " ++ <> QuotedPath dest ++ failed + Just r -> do + case fromto of + From -> thawContent dest +@@ -649,7 +654,14 @@ linkAnnex fromto key src (Just srcic) dest destmode = + Database.Keys.addInodeCaches key $ + catMaybes [destic, Just srcic] + return LinkAnnexOk +- _ -> do ++ msrcic' -> do ++ warning $ "DIAGNOSTIC: linkAnnex: inode cache of " ++ <> QuotedPath src ++ <> UnquotedString (" changed while copying it; before: " ++ ++ showInodeCache srcic ++ ++ "; after: " ++ ++ maybe "(missing or not a regular file)" ++ showInodeCache msrcic') + liftIO $ removeWhenExistsWith removeFile dest + failed + From 9a9d0dbe02518f9cfe35f2ca86cd80e3650af6bf Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 22 Sep 2026 13:27:41 +0000 Subject: [PATCH 2/3] CI: branch-local tuning of build-ubuntu for the NFS diagnostic runs This branch exists only to exercise the LinkAnnexFailed diagnostic patch against NFS; it is not meant to be merged. Point its Ubuntu workflow at that single question: * test-annex runs only the nfs-home flavor, three times per invocation (one run reproduces the failure in roughly two nights out of three). * test-annex-more and test-datalad are disabled; they say nothing about this failure and cost ~40 minutes of runners per invocation. * The failure e-mails are disabled: failures here are the expected outcome, and an "Ubuntu build failed" mail per night would read as if master were broken. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01B89nUooZLfThcTA4fSMPGf --- .github/workflows/build-ubuntu.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-ubuntu.yaml b/.github/workflows/build-ubuntu.yaml index 98b8c76824..b7ca85b6b9 100644 --- a/.github/workflows/build-ubuntu.yaml +++ b/.github/workflows/build-ubuntu.yaml @@ -224,7 +224,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Send e-mail on failed run - if: failure() && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) + # Branch-local: failures are expected on this branch; do not notify. + if: false # freeze to v17 uses: dawidd6/action-send-mail@42942bc2f8fba4e611b459a018967a6a7c78c68c with: @@ -247,8 +248,13 @@ jobs: needs: build-package strategy: matrix: - flavor: ["normal", "crippled-tmp", "crippled-home", "nfs-home", "custom-config1"] + # Branch-local (diagnostic branch, never merged into master): + # only the flavor that hits the intermittent LinkAnnexFailed on NFS, + # run several times per invocation since a single run reproduces it + # in roughly two nights out of three. + flavor: ["nfs-home"] os: [ubuntu-24.04] + rep: [1, 2, 3] fail-fast: false steps: - name: Checkout this repository @@ -359,7 +365,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Send e-mail on failed run - if: failure() && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) + # Branch-local: failures are expected on this branch; do not notify. + if: false # freeze to v17 uses: dawidd6/action-send-mail@42942bc2f8fba4e611b459a018967a6a7c78c68c with: @@ -378,6 +385,8 @@ jobs: See for more information. test-annex-more: + # Branch-local: not needed for the NFS diagnostic runs. + if: false runs-on: ubuntu-24.04 needs: build-package steps: @@ -428,6 +437,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test-datalad: + # Branch-local: not needed for the NFS diagnostic runs. + if: false runs-on: ubuntu-24.04 needs: build-package strategy: From 756a55be171d6a295766637c10a3e71a5c25c989 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 23 Sep 2026 15:34:30 +0000 Subject: [PATCH 3/3] CI: attribute-cache experiment arm for the NFS diagnostic runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first harness run showed both failures are linkAnnex's checksrcunchanged, with inode and size unchanged and only the mtime moving (+12 ms in one case, +54.7 s in the other) — consistent with NFS client attribute caching rather than a real modification. Test that directly: run the nfs-home flavor three ways, three times each, in one invocation so the arms see the same runner conditions. * default — what the nightlies mount, the control; * actimeo0 — `-o actimeo=0`, attribute caching off and nothing else, the clean probe of the hypothesis; * noac — `-o noac`, that plus synchronous writes, the workaround usually recommended to NFS users. The effective mount options are printed per job so the log records what was actually negotiated. The suite timeout goes 3600 -> 7200s because noac writes synchronously and is much slower; a job dying at the timeout would make the arm unreadable. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01B89nUooZLfThcTA4fSMPGf --- .github/workflows/build-ubuntu.yaml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ubuntu.yaml b/.github/workflows/build-ubuntu.yaml index b7ca85b6b9..4e655ec58a 100644 --- a/.github/workflows/build-ubuntu.yaml +++ b/.github/workflows/build-ubuntu.yaml @@ -254,6 +254,11 @@ jobs: # in roughly two nights out of three. flavor: ["nfs-home"] os: [ubuntu-24.04] + # Attribute-cache experiment: is the LinkAnnexFailed mtime jitter + # caused by NFS client attribute caching? default = what the + # nightlies run, actimeo0 = attribute caching off only, noac = + # that plus synchronous writes (the usual recommendation). + acmode: [default, actimeo0, noac] rep: [1, 2, 3] fail-fast: false steps: @@ -304,7 +309,14 @@ jobs: echo "/tmp/nfsmount_ localhost(rw)" | sudo bash -c 'cat - > /etc/exports' sudo apt-get install -y nfs-kernel-server sudo exportfs -a - sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount + case "${{ matrix.acmode }}" in + noac) mountopts=(-o noac) ;; + actimeo0) mountopts=(-o actimeo=0) ;; + *) mountopts=() ;; + esac + sudo mount -t nfs "${mountopts[@]}" localhost:/tmp/nfsmount_ /tmp/nfsmount + echo "NFS mount options in effect (acmode=${{ matrix.acmode }}):" + findmnt -no OPTIONS /tmp/nfsmount fi case "${{ matrix.flavor }}" in @@ -351,7 +363,10 @@ jobs: cd $HOME export | grep -e crippledfs || : - timeout 3600 git annex test "${test_opts[@]:-}" + # Branch-local: noac writes synchronously and is much slower, + # so give the suite room rather than have it die at the timeout + # and leave the experiment unreadable. + timeout 7200 git annex test "${test_opts[@]:-}" - name: Set final PR status if: always() && github.event.inputs.pr != ''