From ebea76fa414731389bfc02951d5128942e7a7bfe Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Wed, 2 Sep 2026 10:24:17 +0200 Subject: [PATCH 1/2] fix(ds4): build CUDA kernels for the target architecture The ds4 backend compiled its CUDA objects with no -arch. Upstream's Makefile leaves CUDA_ARCH empty and its `cuda` target refuses to build without one, offering `cuda-spark` (sm_121) and `cuda-generic` (native) instead. We invoke its object targets directly, which bypasses that guard, so nvcc fell back to its default architecture and the kernels ran as JIT'd PTX on the real GPU. On GB10 (sm_121) that silently corrupted inference: any prompt over roughly 128 tokens produced text unrelated to the input and never closed its thinking block, so content came back empty and the chat showed only reasoning; longer prompts failed with "cuda decode failed". It also cost close to two orders of magnitude of prefill throughput. Measured on one box, same model, same prompt, same GPU, upstream ds4 at the pinned commit, differing only in the nvcc flags: make -B ds4 (archless, as we build it) garbage output 4.21 t/s make cuda-spark (compute_121a/sm_121a) correct output 325.70 t/s Select an architecture list from CUDA_MAJOR_VERSION, which the backend matrix already declares for both ds4 cublas entries but Dockerfile.ds4 never forwarded. Upstream's CUDA_ARCH takes a single value, so it cannot express the fat binary these images need; NVCC_ARCH_FLAGS is overridden instead, since a command-line assignment wins over its `:=`. The lists are copied from vllm-cpp rather than invented so the two CUDA images cover the same GPUs, with l4t/arm64 covering Orin, Thor and GB10. An empty CUDA_MAJOR_VERSION keeps upstream's `native` behaviour for local developer builds, and no CI runner has a GPU to enumerate. DS4_CUDA_HAVE_MXF4 is deliberately left unset: upstream defines it only for single-arch sm_120/sm_121 builds and guards it with a plain #ifdef rather than __CUDA_ARCH__, so it cannot be combined with older archs. It gates an optional MXFP4 indexer fast path whose #ifndef branch returns 0 and falls back cleanly, so omitting it costs speed on GB10, not correctness. Assisted-by: Claude Code:claude-opus-5 Signed-off-by: Claudio Maradonna --- backend/Dockerfile.ds4 | 4 ++- backend/cpp/ds4/Makefile | 63 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile.ds4 b/backend/Dockerfile.ds4 index 370d1eaae3a1..5d4f02a8db9e 100644 --- a/backend/Dockerfile.ds4 +++ b/backend/Dockerfile.ds4 @@ -10,6 +10,7 @@ FROM ${BASE_IMAGE} AS builder ARG BUILD_TYPE ARG TARGETARCH ARG TARGETVARIANT +ARG CUDA_MAJOR_VERSION ENV BUILD_TYPE=${BUILD_TYPE} \ DEBIAN_FRONTEND=noninteractive \ @@ -35,7 +36,8 @@ RUN apt-get update && \ COPY . /LocalAI RUN --mount=type=cache,target=/root/.ccache,id=ds4-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ - make -C /LocalAI/backend/cpp/ds4 BUILD_TYPE=${BUILD_TYPE} NATIVE=false grpc-server package + make -C /LocalAI/backend/cpp/ds4 BUILD_TYPE=${BUILD_TYPE} \ + CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION} NATIVE=false grpc-server package FROM scratch COPY --from=builder /LocalAI/backend/cpp/ds4/package/. ./ diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index b171fa391d76..807a01a09ec0 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -18,6 +18,67 @@ UNAME_S := $(shell uname -s) CMAKE_ARGS ?= -DCMAKE_BUILD_TYPE=Release +# nvcc must be told the target architecture explicitly for a cublas build, and +# this is not a tuning knob. Upstream's Makefile leaves CUDA_ARCH empty and its +# `cuda` target REFUSES to build without one, offering `cuda-spark` +# (CUDA_ARCH=sm_121) and `cuda-generic` (CUDA_ARCH=native) instead. We drive its +# object targets directly, which bypasses that guard: nvcc then compiles with no +# -arch at all, and the kernels run as JIT'd PTX for its default architecture. +# On GB10 (sm_121) that silently produced corrupt inference output above a +# ~128-token prefill batch and ~77x slower prefill (4.21 t/s vs 325.70 t/s, +# measured on the same box with the same model). No CI runner has a GPU, so +# `native` has nothing to enumerate there. +# +# Upstream's CUDA_ARCH takes a SINGLE value (see its sm_120/sm_121 special cases +# and the `-arch=$(CUDA_ARCH)` fallback), so it cannot express the fat binary +# these images need. NVCC_ARCH_FLAGS is overridden instead: a command-line +# assignment wins over the `:=` in upstream's Makefile, and its NVCCFLAGS +# expands whatever we pass. +# +# The architecture lists are copied from backend/go/vllm-cpp/Makefile rather +# than invented, so the two CUDA images cover the same GPUs: amd64 datacenter + +# consumer, and l4t/arm64 covering Orin (87), Thor (110) and GB10 (121a). +# +# -DDS4_CUDA_HAVE_MXF4=1 is deliberately NOT set. Upstream only defines it for +# single-arch sm_120/sm_121 builds and guards the code with a plain #ifdef +# rather than __CUDA_ARCH__, so it cannot be combined with older archs in one +# fat binary. It gates an optional MXFP4 indexer fast path whose #ifndef branch +# returns 0 and falls back to the generic path, so omitting it costs some speed +# on GB10, not correctness. Revisit if upstream adds __CUDA_ARCH__ guards. +# +# An EMPTY CUDA_MAJOR_VERSION means a local developer build, not CI: fall back +# to upstream's own `native` handling, which needs a GPU present but is what a +# developer building on their own machine wants. Both variables are `?=` so an +# explicit value on the command line always wins. +UNAME_M := $(shell uname -m) +CUDA_MAJOR_VERSION ?= +ifeq ($(BUILD_TYPE),cublas) +ifeq ($(CUDA_MAJOR_VERSION),13) +ifeq ($(UNAME_M),aarch64) + DS4_NVCC_ARCH_FLAGS ?= -gencode arch=compute_87,code=sm_87 \ + -gencode arch=compute_90a,code=sm_90a \ + -gencode arch=compute_100a,code=sm_100a \ + -gencode arch=compute_110,code=sm_110 \ + -gencode arch=compute_121a,code=sm_121a +else + DS4_NVCC_ARCH_FLAGS ?= -gencode arch=compute_80,code=sm_80 \ + -gencode arch=compute_86,code=sm_86 \ + -gencode arch=compute_89,code=sm_89 \ + -gencode arch=compute_90a,code=sm_90a \ + -gencode arch=compute_100a,code=sm_100a \ + -gencode arch=compute_103a,code=sm_103a \ + -gencode arch=compute_120a,code=sm_120a \ + -gencode arch=compute_121a,code=sm_121a +endif + DS4_ARCH_MAKEVARS := NVCC_ARCH_FLAGS="$(DS4_NVCC_ARCH_FLAGS)" +else ifeq ($(CUDA_MAJOR_VERSION),) + # Local build: let upstream resolve the host GPU. + DS4_ARCH_MAKEVARS := CUDA_ARCH=native +else + $(error CUDA_MAJOR_VERSION=$(CUDA_MAJOR_VERSION) has no architecture list here (13 does). Leave it empty for a native build, or pass DS4_NVCC_ARCH_FLAGS explicitly.) +endif +endif + # Upstream splits distributed inference, tensor-parallel transport, the SSD # expert cache, and layer placement into GPU-agnostic translation units. They # are shared by every GPU mode, so append them unconditionally below. @@ -57,7 +118,7 @@ ds4: # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA). ds4/ds4.o: ds4 ifeq ($(BUILD_TYPE),cublas) - +$(MAKE) -C ds4 $(DS4_OBJ_TARGET) + +$(MAKE) -C ds4 $(DS4_ARCH_MAKEVARS) $(DS4_OBJ_TARGET) else ifeq ($(UNAME_S),Darwin) +$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o else From bdd509f1c0331f25f31415152e8c65dbeaff4730 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Wed, 2 Sep 2026 11:08:17 +0200 Subject: [PATCH 2/2] test(ds4): cover the multi-batch prefill regression The architecture fix has no automated guard: every existing e2e spec uses a short prompt, and the miscompiled backend answered short prompts correctly. The corruption only appears once a prompt spans more than one prefill batch, so the whole suite passed against a backend that produced garbage in normal use. Add an opt-in "long_prefill" capability to the backend e2e suite that sends a prompt well past one batch with a known needle and asserts the answer still reflects it, and document in the ds4 guide why the build must never omit an nvcc architecture, how to check which flags a configuration resolves to without compiling, and how to run the new spec. Assisted-by: Claude Code:claude-opus-5 Signed-off-by: Claudio Maradonna --- .agents/ds4-backend.md | 50 ++++++++++++++++++++++++++++++ tests/e2e-backends/backend_test.go | 47 ++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/.agents/ds4-backend.md b/.agents/ds4-backend.md index c1b649857c85..8bf1eb45f23c 100644 --- a/.agents/ds4-backend.md +++ b/.agents/ds4-backend.md @@ -77,6 +77,56 @@ spectrum. **Metal (Darwin) only** - it is a no-op on CUDA/CPU. Enable with budget). Gallery entries built on this: `deepseek-v4-flash-q4-ssd` (153 GB Flash on a 128 GB Mac) and `deepseek-v4-pro-q2-ssd` (433 GB Pro, experimental). +## CUDA architecture (do not build without one) + +`backend/cpp/ds4/Makefile` drives upstream's **object targets** directly +(`$(MAKE) -C ds4 ds4.o ds4_cuda.o ...`), which bypasses upstream's own guard: +its `cuda` target refuses to build unless `CUDA_ARCH` is set, and offers +`cuda-spark` (sm_121, DGX Spark / GB10) and `cuda-generic` (native) instead. +Built with no `-arch`, nvcc targets its default architecture and the kernels run +as JIT'd PTX. On GB10 that silently corrupted every prefill batch of >=128 +tokens - the model emitted text unrelated to the prompt and never closed its +thinking block, so `content` came back empty - and cost close to two orders of +magnitude of prefill throughput (4.21 t/s vs 325.70 t/s, same box, same model). +Short prompts stayed correct, which is why it went unnoticed. + +The Makefile therefore picks a gencode list from `CUDA_MAJOR_VERSION` (a build +arg the backend matrix already declares, forwarded by `Dockerfile.ds4`) and +`uname -m`, and passes it as `NVCC_ARCH_FLAGS` to the sub-make. Upstream's +`CUDA_ARCH` accepts a single value, so it cannot express the fat binary the +shipped images need; a command-line assignment beats its `:=`. An empty +`CUDA_MAJOR_VERSION` falls back to upstream's `native` for local developer +builds, and an unrecognised one is a hard error - no CI runner has a GPU, so a +silent `native` there is exactly the failure mode this guards against. + +`DS4_CUDA_HAVE_MXF4` is deliberately unset: upstream defines it only for +single-arch sm_120/sm_121 builds and guards it with a plain `#ifdef` rather than +`__CUDA_ARCH__`, so it cannot be combined with older archs. It gates an optional +MXFP4 indexer fast path whose `#ifndef` branch returns 0, so omitting it costs +speed, not correctness. + +### Verifying a build + +Check which flags a configuration resolves to, without compiling anything: + +``` +make -C backend/cpp/ds4 BUILD_TYPE=cublas CUDA_MAJOR_VERSION=13 NATIVE=false \ + --eval='show: ; @echo [$(DS4_ARCH_MAKEVARS)]' show +``` + +Do not use `make -n` for this: the recipe is `+$(MAKE) ...`, and the `+` prefix +makes it run even under `-n`. + +Then exercise the failure mode itself against a built backend. It only appears +above one prefill batch, so the ordinary `predict` spec cannot catch it: + +``` +BACKEND_BINARY=$(pwd)/backend/cpp/ds4/package/run.sh \ +BACKEND_TEST_MODEL_FILE=/path/to/ds4flash.gguf \ +BACKEND_TEST_CAPS=health,load,predict,long_prefill \ +go test -count=1 -timeout=30m -v ./tests/e2e-backends/... +``` + ## Build matrix | Build | Where | Notes | diff --git a/tests/e2e-backends/backend_test.go b/tests/e2e-backends/backend_test.go index a86a2a08896a..73b30f9909c2 100644 --- a/tests/e2e-backends/backend_test.go +++ b/tests/e2e-backends/backend_test.go @@ -62,6 +62,11 @@ import ( // model output into ChatDelta.tool_calls. // "image" exercises the GenerateImage RPC and asserts a // non-empty file is written to the requested dst path. +// "long_prefill" sends a prompt long enough to span more +// than one prefill batch and asserts the answer still +// reflects the prompt. Catches GPU backends whose kernels +// were built for the wrong architecture, which corrupt +// batched prefill while short prompts stay correct. // BACKEND_TEST_IMAGE_PROMPT Override the positive prompt for the image spec // (default: "a photograph of an astronaut riding a horse"). // BACKEND_TEST_IMAGE_STEPS Override the diffusion step count for the image spec @@ -108,6 +113,7 @@ const ( capVoiceAnalyze = "voice_analyze" capAudioTransform = "audio_transform" capLogprobs = "logprobs" + capLongPrefill = "long_prefill" capLogitBias = "logit_bias" capTokenize = "tokenize" capTokenClassify = "token_classify" @@ -433,6 +439,47 @@ var _ = Describe("Backend container", Ordered, func() { res.GetMessage(), res.GetTokens(), res.GetPromptTokens()) }) + // Regression guard for GPU backends compiled without an explicit device + // architecture. LocalAI built ds4's CUDA objects with no -arch/-gencode, so + // on a GB10 (sm_121) the kernels ran as JIT'd PTX for nvcc's default + // architecture and silently corrupted any prefill batch of 128 tokens or + // more: the model produced text unrelated to the prompt. Short prompts stayed + // correct, so every other spec here passed. Only a prompt long enough to need + // a multi-batch prefill exposes it. + It("answers a prompt long enough to span multiple prefill batches", func() { + if !caps[capLongPrefill] { + Skip("long_prefill capability not enabled") + } + const needle = "PLATYPUS" + filler := strings.Repeat("The merchants kept careful ledgers of every voyage they financed. ", 20) + longPrompt := "Read this passage.\n\n" + filler + + "\nThe secret word is " + needle + ".\n" + filler + + "\nQuestion: what is the secret word?\nAnswer: The secret word is" + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + res, err := client.Predict(ctx, &pb.PredictOptions{ + Prompt: longPrompt, + Tokens: 300, + Temperature: 0.1, + TopK: 40, + TopP: 0.9, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.GetMessage()).NotTo(BeEmpty(), "long prompt produced empty output") + // Only meaningful if the prompt really did exceed one batch. Backends that + // do not report prompt tokens still run the substring assertion below. + if res.GetPromptTokens() > 0 { + Expect(res.GetPromptTokens()).To(BeNumerically(">", 128), + "prompt is too short to span multiple prefill batches; this spec would not prove anything") + } + Expect(strings.ToUpper(res.GetMessage())).To(ContainSubstring(needle), + "a long prompt lost information the model repeats correctly from a short one - "+ + "batched prefill is corrupting state (check the backend's device architecture flags)") + GinkgoWriter.Printf("LongPrefill: prompt_tokens=%d tokens=%d msg=%q\n", + res.GetPromptTokens(), res.GetTokens(), res.GetMessage()) + }) + // Regression guard for the raw-prompt tokenize RPC. The llama.cpp handler // read the prompt from the wrong JSON key ("content" instead of "prompt"), // so any non-empty prompt threw and the RPC returned "Unexpected error in