From a4823bae3aa771fd07a068f14d0ad5b4ac799007 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 16:23:39 +0000 Subject: [PATCH] fix(ci): unbreak the e2e build and the darwin vllm-metal pin Two independent breakages on master make every open pull request red, for reasons unrelated to the changes under review. The e2e backend suite stopped compiling. Reply.message is `bytes` in backend.proto, so res.GetMessage() returns []byte, and strings.ToUpper wants a string. Every other call site in the file already converts. tests/e2e-backends sits behind a build tag, so `go build ./...` never compiled it and the breakage reached master unnoticed. The darwin vllm build stopped resolving. Upstream vllm-metal deleted its old dev tags and re-versioned to track the vLLM release it targets, so the pinned wheel 404s. The coupled vLLM release also moved out of upstream's install.sh into .github/vllm-release-tag.commit, and the wheel's platform tag moved from macosx_11_0 to macosx_15_0. Read the wheel name from the release's own asset listing rather than composing it from a hardcoded platform segment, so a platform-tag change cannot silently 404 again, and resolve the vLLM version from the new metadata file with a fallback to the legacy installer. The bump script and the extractor learn the same two-source lookup, so the next nightly run converges on the pin checked in here instead of reintroducing the break. Assisted-by: Claude:claude-opus-5 Signed-off-by: Ettore Di Giacinto --- .github/bump_vllm_metal.sh | 21 +++++++----- backend/python/vllm/install.sh | 32 +++++++++++++------ .../build/extract-vllm-metal-version_test.sh | 21 ++++++++++++ scripts/lib/extract-vllm-metal-version.sh | 10 +++++- tests/e2e-backends/backend_test.go | 2 +- 5 files changed, 66 insertions(+), 20 deletions(-) diff --git a/.github/bump_vllm_metal.sh b/.github/bump_vllm_metal.sh index d2aedf4bcb9c..459f97fd22a0 100755 --- a/.github/bump_vllm_metal.sh +++ b/.github/bump_vllm_metal.sh @@ -3,9 +3,9 @@ # darwin (Apple Silicon) install path. The macOS/Metal build # (backend/python/vllm/install.sh, Darwin branch) installs vllm-metal, which is # version-locked to a specific vLLM source release. install.sh derives that vLLM -# version at build time from vllm-metal's own installer at the pinned -# tag, so there is only ONE value to bump here -- mirroring bump_vllm_wheel.sh, -# which bumps the Linux cu130 wheel pin. +# version, and the wheel asset name, at build time from the pinned tag, so there +# is only ONE value to bump here -- mirroring bump_vllm_wheel.sh, which bumps the +# Linux cu130 wheel pin. # # This deliberately tracks vllm-project/vllm-metal, NOT vllm-project/vllm: the # darwin build can only use the exact vLLM version vllm-metal supports, so it may @@ -23,15 +23,20 @@ if [ -z "$FILE" ] || [ -z "$REPO" ] || [ -z "$VAR" ]; then exit 1 fi -# vllm-metal ships frequent dev releases, all flagged as non-prerelease, so -# /releases/latest returns the newest one (with its cp312 wheel asset). +# vllm-metal ships frequent .dev releases, flagged as prereleases, alongside the +# stable ones. /releases/latest skips the prereleases and returns the newest +# stable tag, which is what darwin should pin: upstream deletes and re-cuts .dev +# tags, and a pin to a deleted tag 404s the whole build. LATEST_TAG=$(gh_curl -H "Accept: application/vnd.github+json" \ "https://api.github.com/repos/$REPO/releases/latest" \ | python3 -c "import json,sys; print(json.load(sys.stdin)['tag_name'])") -# The coupled vLLM source version lives in vllm-metal's installer at that tag. -NEW_VLLM_VERSION=$(gh_curl \ - "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh" \ +# The coupled vLLM release lives in .github/vllm-release-tag.commit at that tag +# (since vllm-metal 0.28); releases predating that file pinned it inline in their +# own install.sh. The extractor reads both forms. +NEW_VLLM_VERSION=$( { gh_curl \ + "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/.github/vllm-release-tag.commit" \ + || gh_curl "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh"; } \ | "$(dirname "${BASH_SOURCE[0]}")/../scripts/lib/extract-vllm-metal-version.sh") if [ -z "$LATEST_TAG" ] || [ -z "$NEW_VLLM_VERSION" ]; then diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index 68d5ba257b5c..a4124977c282 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -119,14 +119,18 @@ if [ "$(uname -s)" = "Darwin" ]; then # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux # vllm pin (requirements-cublas13-after.txt, bumped independently against # vllm/vllm) until vllm-metal supports a newer vLLM. - VLLM_METAL_VERSION="v0.3.0.dev20260818075955" + VLLM_METAL_VERSION="v0.28.0" # The coupled vLLM source version is whatever this vllm-metal release builds - # against. Derive it from - # the PINNED tag rather than hardcoding a second value that could drift. The - # tag is immutable, so this stays reproducible across rebuilds. - VLLM_VERSION=$(curl -fsSL "https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}/install.sh" \ - | "$backend_dir/../../../scripts/lib/extract-vllm-metal-version.sh") + # against. Derive it from the PINNED tag rather than hardcoding a second value + # that could drift. The tag is immutable, so this stays reproducible across + # rebuilds. Since vllm-metal 0.28 the coupling is declared in + # .github/vllm-release-tag.commit; older releases pinned it inline in their + # own install.sh, so fall back to that. The extractor reads both forms. + _vllm_metal_raw="https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}" + VLLM_VERSION=$( { curl -fsSL "${_vllm_metal_raw}/.github/vllm-release-tag.commit" \ + || curl -fsSL "${_vllm_metal_raw}/install.sh"; } \ + | "$backend_dir/../../../scripts/lib/extract-vllm-metal-version.sh" || true) if [ -z "${VLLM_VERSION}" ]; then echo "ERROR: could not derive the vLLM version from vllm-metal ${VLLM_METAL_VERSION}" >&2 exit 1 @@ -153,10 +157,18 @@ if [ "$(uname -s)" = "Darwin" ]; then # 2) Install the prebuilt vllm-metal wheel for the PINNED release. It pulls # mlx / mlx-metal as deps and registers the `metal` platform plugin that # backend.py resolves to at engine-init time. Build the release-asset URL - # deterministically (tag + the cp312/arm64 wheel name) rather than querying - # api.github.com, whose unauthenticated rate limit (60/hr per IP) 403s on - # shared CI runners. The wheel version is the tag without its leading 'v'. - _metal_wheel="vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-macosx_11_0_arm64.whl" + # from the release's OWN asset listing rather than composing it from a + # hardcoded platform tag: upstream raised its macOS deployment target + # (macosx_11_0 -> macosx_15_0) and every composed URL started to 404. + # expanded_assets is the plain release page, not api.github.com, whose + # unauthenticated rate limit (60/hr per IP) 403s on shared CI runners. + # The wheel version is the tag without its leading 'v'. + _metal_wheel=$(curl -fsSL "https://github.com/vllm-project/vllm-metal/releases/expanded_assets/${VLLM_METAL_VERSION}" \ + | grep -oE "vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-[A-Za-z0-9_]+\.whl" | head -1 || true) + if [ -z "${_metal_wheel}" ]; then + echo "ERROR: no cp312 wheel asset on vllm-metal release ${VLLM_METAL_VERSION}" >&2 + exit 1 + fi _metal_wheel_url="https://github.com/vllm-project/vllm-metal/releases/download/${VLLM_METAL_VERSION}/${_metal_wheel}" echo "Installing vllm-metal wheel: ${_metal_wheel_url}" uv pip install "${_metal_wheel_url}" diff --git a/scripts/build/extract-vllm-metal-version_test.sh b/scripts/build/extract-vllm-metal-version_test.sh index f04bcfea38b9..1018b30ca351 100755 --- a/scripts/build/extract-vllm-metal-version_test.sh +++ b/scripts/build/extract-vllm-metal-version_test.sh @@ -21,6 +21,17 @@ assert_version "0.26.0" ' local vllm_v="0.26.0"' assert_version "0.26.0" 'VLLM_VERSION="0.26.0"' assert_version "0.26.1" ' VLLM_VERSION = "0.26.1" # comment' +# .github/vllm-release-tag.commit form: a lone vLLM release tag. +assert_version "0.28.0" 'v0.28.0' +assert_version "0.28.0" '0.28.0' +assert_version "0.28.0" ' v0.28.0 ' + +# A whole upstream installer must still yield the inline pin, not a version-like +# fragment of some other line. +assert_version "0.26.0" 'set -e +vllm_wheel="vllm-1.2.3-cp312.whl" +VLLM_VERSION="0.26.0"' + if printf '%s\n' 'VLLM_VERSION="not-a-version"' | "$extractor"; then echo "malformed versions must be rejected" >&2 exit 1 @@ -30,3 +41,13 @@ if printf '%s\n' 'VLLM_VERSION="0.26.0"garbage' | "$extractor"; then echo "trailing assignment content must be rejected" >&2 exit 1 fi + +if printf '%s\n' 'not-a-tag' | "$extractor"; then + echo "malformed release tags must be rejected" >&2 + exit 1 +fi + +if printf '%s\n' 'vllm-1.2.3-cp312.whl' | "$extractor"; then + echo "a version embedded in a longer line must be rejected" >&2 + exit 1 +fi diff --git a/scripts/lib/extract-vllm-metal-version.sh b/scripts/lib/extract-vllm-metal-version.sh index a5e101c63c13..6019dca3e28e 100755 --- a/scripts/lib/extract-vllm-metal-version.sh +++ b/scripts/lib/extract-vllm-metal-version.sh @@ -1,5 +1,13 @@ #!/bin/bash set -euo pipefail -grep -m1 -oE '^[[:space:]]*(local[[:space:]]+)?(vllm_v|VLLM_VERSION)[[:space:]]*=[[:space:]]*"[0-9]+\.[0-9]+\.[0-9]+"[[:space:]]*(#.*)?$' \ +# Print the bare X.Y.Z vLLM version a vllm-metal release builds against, reading +# whichever form the release declares it in on stdin: +# +# * .github/vllm-release-tag.commit -- a lone "vX.Y.Z" vLLM release tag. This is +# the source of truth since vllm-metal 0.28, which also re-versioned the +# project so its own version tracks the vLLM version it targets. +# * install.sh -- releases predating that file pinned VLLM_VERSION="X.Y.Z" +# (earlier still: vllm_v="X.Y.Z") inline in their installer. +grep -m1 -oE '^[[:space:]]*((local[[:space:]]+)?(vllm_v|VLLM_VERSION)[[:space:]]*=[[:space:]]*"[0-9]+\.[0-9]+\.[0-9]+"[[:space:]]*(#.*)?|v?[0-9]+\.[0-9]+\.[0-9]+[[:space:]]*)$' \ | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' diff --git a/tests/e2e-backends/backend_test.go b/tests/e2e-backends/backend_test.go index 73b30f9909c2..9ebd044c825a 100644 --- a/tests/e2e-backends/backend_test.go +++ b/tests/e2e-backends/backend_test.go @@ -473,7 +473,7 @@ var _ = Describe("Backend container", Ordered, func() { Expect(res.GetPromptTokens()).To(BeNumerically(">", 128), "prompt is too short to span multiple prefill batches; this spec would not prove anything") } - Expect(strings.ToUpper(res.GetMessage())).To(ContainSubstring(needle), + Expect(strings.ToUpper(string(res.GetMessage()))).To(ContainSubstring(needle), "a long prompt lost information the model repeats correctly from a short one - "+ "batched prefill is corrupting state (check the backend's device architecture flags)") GinkgoWriter.Printf("LongPrefill: prompt_tokens=%d tokens=%d msg=%q\n",