From 788e7e9737ddf5c31ca0ad51621b123e6085529f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 06:49:45 +0000 Subject: [PATCH 1/2] fix(ci): remove the e2e container before removing its image `docker stop` returns as soon as the container exits, but the daemon reaps a `--rm` container asynchronously after that. The `docker rmi localai-tests` that follows teardown-e2e then loses the race against the reaper and fails with "conflict: ... is using its referenced image", so make exits 1 and the job goes red after every spec has passed. This is why the E2E Backend Tests job fails at random across pull requests. Runs 33435319093, 33435332991, 33412165884 and 33444669207 all report "SUCCESS! -- 235 Passed | 0 Failed" and then die in teardown. `docker rm -f` is synchronous, so the image reference is gone before teardown-e2e returns. It also covers the case where no container is running, which `docker stop` could not because it rejects an empty argument list. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-5 [Claude Code] --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ebedb2c98248..cf3248c3be56 100644 --- a/Makefile +++ b/Makefile @@ -393,9 +393,17 @@ test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image $(MAKE) teardown-e2e docker rmi localai-tests +# `docker stop` returns as soon as the container exits, but Docker reaps a +# `--rm` container asynchronously after that. The `docker rmi localai-tests` in +# test-e2e then loses the race against the reaper and fails on a still +# referenced image, turning a green suite red. Removing the container ourselves +# is synchronous, so the image reference is gone before we return. It also +# covers the case where nothing is running, which `docker stop` could not +# because it rejects an empty argument list. teardown-e2e: rm -rf $(TEST_DIR) || true - docker stop $$(docker ps -q --filter ancestor=localai-tests) + @CONTAINERS=$$(docker ps -aq --filter ancestor=localai-tests 2>/dev/null); \ + if [ -n "$$CONTAINERS" ]; then docker rm -f $$CONTAINERS || true; fi ######################################################## ## Integration and unit tests From 8d5cf812e2a8c61888fc791fce5cfa4ca6890e1a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 06:49:53 +0000 Subject: [PATCH 2/2] fix(ci): open a tmate session only when a PR asks for one The tmate step runs on every failure and then holds the runner until GitHub cancels the job at the 6 hour limit. A one second cleanup race in the e2e teardown therefore costs a whole ubuntu-latest slot. The recent run list is full of 6h, 7h and 12h cancelled runs for that reason. The step now needs the `ci-debug` label on the pull request, so a session opens when somebody wants to debug and never otherwise. The 30 minute step timeout caps the cost when the label is left behind. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-5 [Claude Code] --- .github/workflows/test.yml | 14 ++++++++++++-- .github/workflows/tests-aio.yml | 7 ++++++- .github/workflows/tests-e2e.yml | 7 ++++++- .github/workflows/tests-pii-ner-e2e.yml | 7 ++++++- .github/workflows/tests-ui-e2e.yml | 7 ++++++- 5 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7929f6b652a9..74d57567fdcd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,8 +80,13 @@ jobs: coverage/coverage.out coverage/coverage.html if-no-files-found: ignore + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true @@ -125,8 +130,13 @@ jobs: export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH" PATH="$PATH:$HOME/go/bin" make protogen-go PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml index f8d3d34f077c..738f96daccc9 100644 --- a/.github/workflows/tests-aio.yml +++ b/.github/workflows/tests-aio.yml @@ -77,8 +77,13 @@ jobs: - name: Test run: | PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index 3c1cb711c79b..b5c84d2a85bc 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -63,8 +63,13 @@ jobs: - name: Test Backend E2E run: | PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-pii-ner-e2e.yml b/.github/workflows/tests-pii-ner-e2e.yml index 800f67190b9f..c9ab182f260b 100644 --- a/.github/workflows/tests-pii-ner-e2e.yml +++ b/.github/workflows/tests-pii-ner-e2e.yml @@ -88,8 +88,13 @@ jobs: # CPU and runs the token_classify capability spec (byte-offset contract). - name: Run live PII NER backend E2E run: PATH="$PATH:$HOME/go/bin" make test-extra-backend-privacy-filter + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true diff --git a/.github/workflows/tests-ui-e2e.yml b/.github/workflows/tests-ui-e2e.yml index 1a170fcf365e..6d45c8e9ff9d 100644 --- a/.github/workflows/tests-ui-e2e.yml +++ b/.github/workflows/tests-ui-e2e.yml @@ -75,8 +75,13 @@ jobs: path: core/http/react-ui/coverage/ if-no-files-found: ignore retention-days: 7 + # tmate keeps the runner busy until the 6 hour job limit, so a single + # failure costs a whole runner slot. Only open a session when someone + # asked for one by labelling the pull request `ci-debug`, and cap the + # session so a forgotten label cannot idle a runner either. - name: Setup tmate session if tests fail - if: ${{ failure() }} + if: ${{ failure() && contains(github.event.pull_request.labels.*.name, 'ci-debug') }} + timeout-minutes: 30 uses: mxschmitt/action-tmate@v3.23 with: detached: true