NXP backend: remove #22179 workaround for QAT channels-last segfault

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-release/2.13
+release/2.14

.ci/docker/common/install_docs_reqs.sh

@@ -20,7 +20,9 @@ if [ -n "$BUILD_DOCS" ]; then
 
   apt-get update
   apt-get install -y --no-install-recommends yarn
-  yarn global add katex --prefix /usr/local
+  # katex 0.18.5 requires commander@15 / node >= 22.12; pin to the last
+  # release compatible with the node 16 installed above
+  yarn global add katex@0.18.4 --prefix /usr/local
 
   sudo apt-get -y install doxygen
 

.ci/docker/common/install_pytorch.sh

@@ -76,21 +76,31 @@ install_pytorch_and_domains() {
   # the image compiler cannot satisfy. The venv inherits the image's
   # site-packages, so PyTorch still builds against the same numpy.
   #
-  # Keep the list in sync with pytorch/pyproject.toml [build-system].requires.
+  # Keep in sync with pytorch/pyproject.toml [build-system].requires.
   local build_venv=/tmp/pytorch-build-venv
   rm -rf "${build_venv}"
   conda_run python -m venv --system-site-packages "${build_venv}"
+  # No pip cmake: scikit-build-core would prefer it over the image's, and it
+  # searches site-packages, where MKL and libomp are not.
   conda_run "${build_venv}/bin/pip" install build "scikit-build-core>=1.0" \
-    "setuptools>=77.0.0,<82" "cmake>=3.27,<4" ninja "packaging>=24.2" \
-    "typing-extensions>=4.10.0" pyyaml six
-  conda_run "${build_venv}/bin/python" -m build --wheel --no-isolation
+    ninja "packaging>=24.2" "typing-extensions>=4.10.0" pyyaml six numpy
+  # These images have no module scanner, and nothing here uses modules.
+  conda_run env CMAKE_CXX_SCAN_FOR_MODULES=OFF \
+    "${build_venv}/bin/python" -m build --wheel --no-isolation
   rm -rf "${build_venv}"
   pip_install "$(echo dist/*.whl)"
 
+  # A build with no BLAS succeeds silently. Run from / to import the wheel.
+  (cd / && conda_run python -c "
+import torch
+assert torch._C.has_lapack, 'built without LAPACK'
+torch.linalg.qr(torch.randn(4, 4))
+")
+
   # Grab the pinned audio and vision commits from PyTorch
   TORCHAUDIO_VERSION=release/2.11
   export TORCHAUDIO_VERSION
-  TORCHVISION_VERSION=release/0.28
+  TORCHVISION_VERSION=release/0.29
   export TORCHVISION_VERSION
 
   install_domains

.ci/scripts/build-qnn-windows-msvc.ps1

@@ -4,13 +4,35 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+param(
+    [switch]$SkipX86Windows,
+    [switch]$SkipArm64Windows
+)
+
 $ErrorActionPreference = "Stop"
 
-conda create --yes --quiet -n et python=3.12
-conda activate et
+if ($SkipX86Windows -eq $SkipArm64Windows) {
+    Write-Error "Specify exactly one of -SkipArm64Windows (to build x86_64) or -SkipX86Windows (to build arm64)."
+    exit 1
+}
 
-# Install CI requirements
-pip install -r .ci/docker/requirements-ci.txt
+if ($SkipX86Windows) {
+    $ArchLabel = "arm64"
+    py -3.12 -m venv et
+    .\et\Scripts\Activate.ps1
+    # ARM64 prebuilt wheels are not available for some Python modules.
+    # To unblock the build process, only a minimal set of dependencies
+    # is installed via pip. `PyYAML`/`torch` for ExecuTorch's codegen,
+    # `requests` for download_qnn_sdk.py.
+    pip install pyyaml requests
+    pip install torch --index-url https://download.pytorch.org/whl/cpu
+} else {
+    $ArchLabel = "x86_64"
+    conda create --yes --quiet -n et python=3.12
+    conda activate et
+    # Install CI requirements
+    pip install -r .ci/docker/requirements-ci.txt
+}
 
 # Provision the QNN SDK
 if ($env:QNN_SDK_ROOT -and (Test-Path -Path $env:QNN_SDK_ROOT)) {
@@ -44,27 +66,26 @@ if (-not (Test-Path -Path (Join-Path $env:QNN_SDK_ROOT "include\QNN"))) {
     exit 1
 }
 
-# Test x86_64 Windows host build
-.\backends\qualcomm\scripts\build.ps1 -SkipArm64Windows -Release
+if ($SkipArm64Windows) {
+    .\backends\qualcomm\scripts\build.ps1 -SkipArm64Windows -Release
+} else {
+    .\backends\qualcomm\scripts\build.ps1 -SkipX86Windows -Release
+}
 
-$x86Artifacts = @(
-    "build-x86_64-windows\backends\qualcomm\Release\PyQnnManagerAdaptor*.pyd",
-    "build-x86_64-windows\backends\qualcomm\Release\qnn_executorch_backend.dll",
-    "build-x86_64-windows\examples\qualcomm\executor_runner\Release\qnn_executor_runner.exe"
+$Artifacts = @(
+    "build-$ArchLabel-windows\backends\qualcomm\Release\qnn_executorch_backend.dll",
+    "build-$ArchLabel-windows\examples\qualcomm\executor_runner\Release\qnn_executor_runner.exe"
 )
-foreach ($artifact in $x86Artifacts) {
+if ($SkipArm64Windows) {
+    # Only run PyQnnManagerAdaptor validation for x86_64 Windows artifacts,
+    # since AOT is not fully supported on native ARM64 Windows.
+    $Artifacts += "build-x86_64-windows\backends\qualcomm\Release\PyQnnManagerAdaptor*.pyd"
+}
+foreach ($artifact in $Artifacts) {
     if (-not (Get-ChildItem -Path $artifact -ErrorAction SilentlyContinue)) {
-        Write-Error "ERROR: x86_64 artifact not found: $artifact"
+        Write-Error "ERROR: $ArchLabel artifact not found: $artifact"
         exit 1
     }
 }
 
-# The ARM64 MSVC toolchain is currently not installed in the Windows CI
-# environment. Enabling this build configuration results in build failures
-# due to the missing ARM64 platform definition.
-# `.\backends\qualcomm\scripts\build.ps1 -SkipX86Windows -Release`
-#
-# Temporarily disable this build option until ARM64 MSVC support is available
-# in CI. The configuration can be re-enabled in a future update.
-
-Write-Host "PASSED: QNN backend Windows MSVC build completed"
+Write-Host "PASSED: QNN backend Windows MSVC build ($ArchLabel) completed"

.ci/scripts/export_model_artifact.sh

@@ -294,6 +294,26 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then
   fi
 fi
 
+# Downloads and compiler caches go in scratch dirs outside OUTPUT_DIR because the CI job
+# templates upload OUTPUT_DIR even when the job fails. A failed export also empties
+# OUTPUT_DIR, but only if it started out empty, so a local run with output_dir=. cannot
+# delete the checkout. Scratch goes under RUNNER_TEMP, which the runner wipes between
+# jobs, with a fallback for containers where RUNNER_TEMP is not writable.
+LOCAL_MODEL_DIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/model_XXXXXX" 2>/dev/null || mktemp -d)
+SCRATCH_DIRS=("$LOCAL_MODEL_DIR")
+OUTPUT_DIR_WAS_EMPTY=0
+[ -n "$(ls -A -- "$OUTPUT_DIR" 2>/dev/null)" ] || OUTPUT_DIR_WAS_EMPTY=1
+cleanup() {
+  local rc=$?
+  set +e
+  rm -rf "${SCRATCH_DIRS[@]}"
+  if [ "$rc" -ne 0 ] && [ "$OUTPUT_DIR_WAS_EMPTY" = 1 ] && [ -d "$OUTPUT_DIR" ]; then
+    echo "Export failed with exit code $rc; removing partial output from ${OUTPUT_DIR}"
+    (cd -- "$OUTPUT_DIR" && find . -mindepth 1 -delete)
+  fi
+}
+trap cleanup EXIT
+
 echo "::group::Export $MODEL_NAME"
 
 if [ -n "$EXTRA_PIP" ]; then
@@ -386,8 +406,7 @@ fi
 if [ "$MODEL_NAME" = "voxtral_realtime" ]; then
   pip install safetensors huggingface_hub
 
-  # Download model weights from HuggingFace (requires HF_TOKEN for gated model)
-  LOCAL_MODEL_DIR="${OUTPUT_DIR}/model_weights"
+  # Download model weights outside OUTPUT_DIR to avoid uploading on failure (requires HF_TOKEN for gated model)
   python -c "from huggingface_hub import snapshot_download; snapshot_download('${HF_MODEL}', local_dir='${LOCAL_MODEL_DIR}')"
 
   # Per-component quantization flags
@@ -437,7 +456,6 @@ if [ "$MODEL_NAME" = "voxtral_realtime" ]; then
   fi
   # Copy tokenizer from downloaded model weights
   cp "$LOCAL_MODEL_DIR/tekken.json" "${OUTPUT_DIR}/tekken.json"
-  rm -rf "$LOCAL_MODEL_DIR"
   ls -al "${OUTPUT_DIR}"
   echo "::endgroup::"
   exit 0
@@ -448,12 +466,11 @@ if [ "$MODEL_NAME" = "qwen3_5_moe" ]; then
   pip install safetensors huggingface_hub
   pip install -r examples/models/qwen3_5_moe/requirements.txt
 
-  # Download prequantized model outside OUTPUT_DIR to avoid uploading on failure
-  LOCAL_MODEL_DIR=$(mktemp -d)
   INDUCTOR_CACHE=$(mktemp -d "${RUNNER_TEMP:-/tmp}/inductor_cache_XXXXXX")
   INDUCTOR_TMPDIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tmpdir_XXXXXX")
-  trap 'rm -rf "$LOCAL_MODEL_DIR" "$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR"' EXIT
+  SCRATCH_DIRS+=("$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR")
 
+  # Download prequantized model outside OUTPUT_DIR to avoid uploading on failure
   python -c "from huggingface_hub import snapshot_download; snapshot_download('${HF_MODEL}', local_dir='${LOCAL_MODEL_DIR}')"
 
   # Sanity check: run inference on the prequantized model
@@ -521,10 +538,9 @@ fi
 if [ "$MODEL_NAME" = "muse_glimmer" ]; then
   pip install safetensors huggingface_hub gguf
 
-  LOCAL_MODEL_DIR=$(mktemp -d)
   INDUCTOR_CACHE=$(mktemp -d "${RUNNER_TEMP:-/tmp}/inductor_cache_XXXXXX")
   INDUCTOR_TMPDIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tmpdir_XXXXXX")
-  trap 'rm -rf "$LOCAL_MODEL_DIR" "$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR"' EXIT
+  SCRATCH_DIRS+=("$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR")
 
   case "$QUANT_NAME" in
     kquant-17gb)
@@ -588,14 +604,13 @@ fi
 if [ "$MODEL_NAME" = "gemma4_31b" ]; then
   pip install safetensors huggingface_hub gguf
 
-  # Download GGUF + tokenizer outside OUTPUT_DIR to avoid uploading on failure.
-  # The unsloth GGUF repo ships the .gguf but no tokenizer.json, so the tokenizer
-  # is fetched from the (non-GGUF) unsloth/gemma-4-31B-it repo.
-  LOCAL_MODEL_DIR=$(mktemp -d)
   INDUCTOR_CACHE=$(mktemp -d "${RUNNER_TEMP:-/tmp}/inductor_cache_XXXXXX")
   INDUCTOR_TMPDIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tmpdir_XXXXXX")
-  trap 'rm -rf "$LOCAL_MODEL_DIR" "$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR"' EXIT
+  SCRATCH_DIRS+=("$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR")
 
+  # Download GGUF + tokenizer outside OUTPUT_DIR to avoid uploading on failure.
+  # The unsloth GGUF repo ships the .gguf but no tokenizer.json, so the tokenizer
+  # is fetched from the (non-GGUF) unsloth/gemma-4-31B-it repo.
   GGUF_FILE="gemma-4-31B-it-Q4_K_M.gguf"
   python -c "from huggingface_hub import hf_hub_download; hf_hub_download('unsloth/gemma-4-31B-it-GGUF', '${GGUF_FILE}', local_dir='${LOCAL_MODEL_DIR}')"
   python -c "from huggingface_hub import hf_hub_download; hf_hub_download('unsloth/gemma-4-31B-it', 'tokenizer.json', local_dir='${LOCAL_MODEL_DIR}')"

.ci/scripts/gather_test_models.py

@@ -17,23 +17,23 @@
 from examples.xnnpack import MODEL_NAME_TO_OPTIONS, QuantType
 
 DEFAULT_RUNNERS = {
-    "linux": "linux.2xlarge",
+    "linux": "mt-l-x86iavx512-8-64",
     "macos": "macos-m1-stable",
 }
 CUSTOM_RUNNERS = {
     "linux": {
         # This one runs OOM on smaller runner, the root cause is unclear (T163016365)
-        "w2l": "linux.4xlarge.memory",
-        "ic4": "linux.4xlarge.memory",
-        "resnet50": "linux.4xlarge.memory",
-        "llava": "linux.4xlarge.memory",
-        "llama3_2_vision_encoder": "linux.4xlarge.memory",
-        "llama3_2_text_decoder": "linux.4xlarge.memory",
+        "w2l": "mt-l-x86iavx512-16-128",
+        "ic4": "mt-l-x86iavx512-16-128",
+        "resnet50": "mt-l-x86iavx512-16-128",
+        "llava": "mt-l-x86iavx512-16-128",
+        "llama3_2_vision_encoder": "mt-l-x86iavx512-16-128",
+        "llama3_2_text_decoder": "mt-l-x86iavx512-16-128",
         # This one causes timeout on smaller runner, the root cause is unclear (T161064121)
-        "dl3": "linux.4xlarge.memory",
-        "emformer_join": "linux.4xlarge.memory",
-        "emformer_predict": "linux.4xlarge.memory",
-        "phi_4_mini": "linux.4xlarge.memory",
+        "dl3": "mt-l-x86iavx512-16-128",
+        "emformer_join": "mt-l-x86iavx512-16-128",
+        "emformer_predict": "mt-l-x86iavx512-16-128",
+        "phi_4_mini": "mt-l-x86iavx512-16-128",
     }
 }
 
@@ -146,7 +146,7 @@ def export_models_for_ci() -> dict[str, dict]:
                 "build-tool": "buck2",
                 "model": "mv3",
                 "backend": backend,
-                "runner": "linux.2xlarge",
+                "runner": "mt-l-x86iavx512-8-64",
                 "timeout": DEFAULT_TIMEOUT,
             }
             models["include"].append(record)
@@ -175,7 +175,7 @@ def export_models_for_ci() -> dict[str, dict]:
             "build-tool": "cmake",
             "model": name,
             "backend": backend,
-            "runner": DEFAULT_RUNNERS.get(target_os, "linux.2xlarge"),
+            "runner": DEFAULT_RUNNERS.get(target_os, "mt-l-x86iavx512-8-64"),
             "timeout": DEFAULT_TIMEOUT,
         }
 

.ci/scripts/pytest-parallelism.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Pin pytest-xdist's worker count, but only when the container is allowed less
+# CPU than the machine it landed on. `auto` asks psutil for the machine's
+# physical cores, which inside an OSDC pod is the whole node, so the workers
+# exhaust the pod's memory. nproc honours the pod's cpuset, which is what
+# pytorch relies on for OMP_NUM_THREADS on the same fleet.
+#
+# Left alone when unconstrained. `auto` already discounts hyperthreads there,
+# while nproc counts them, so overriding it would double the workers and halve
+# the memory each one gets.
+
+if [[ -z "${PYTEST_XDIST_AUTO_NUM_WORKERS:-}" ]] && command -v nproc >/dev/null 2>&1; then
+  cpus_allowed="$(nproc)"
+  cpus_installed="$(nproc --all)"
+  echo "pytest-xdist: ${cpus_allowed} of ${cpus_installed} CPUs available"
+  if [[ "${cpus_allowed}" -lt "${cpus_installed}" ]]; then
+    export PYTEST_XDIST_AUTO_NUM_WORKERS="${cpus_allowed}"
+    echo "PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS}"
+  fi
+fi

.ci/scripts/setup-samsung-linux-deps.sh

@@ -165,6 +165,7 @@ install_enn_backend() {
   echo "NDK will be installed/used at: ${ANDROID_NDK_ROOT}"
 
   bash backends/samsung/build.sh --build all
+  bash examples/samsung/build.sh
 
   export EXECUTORCH_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
   export PYTHONPATH="${PYTHONPATH:-}:${EXECUTORCH_ROOT}/.."

.ci/scripts/test-cuda-build.sh

@@ -49,10 +49,16 @@ import executorch
 print('SUCCESS: ExecuTorch imported successfully')
 "
 
+    python -m pip check
+
     # Test CUDA availability and show details
-    python -c "
+    EXPECTED_CUDA_VERSION="$cuda_version" python -c "
 try:
+    import os
     import torch
+    assert torch.version.cuda == os.environ['EXPECTED_CUDA_VERSION'], (
+        torch.version.cuda, os.environ['EXPECTED_CUDA_VERSION']
+    )
     print('INFO: PyTorch version:', torch.__version__)
     print('INFO: CUDA available:', torch.cuda.is_available())
 
@@ -68,7 +74,8 @@ try:
         x = torch.randn(10, 10).to(device)
         y = torch.randn(10, 10).to(device)
         z = torch.mm(x, y)
-        print('SUCCESS: CUDA tensor operation completed on device:', z.device)
+        torch.testing.assert_close(z.cpu(), x.cpu() @ y.cpu())
+        print('SUCCESS: CUDA tensor operation matched CPU on device:', z.device)
         print('INFO: Result tensor shape:', z.shape)
 
         print('SUCCESS: ExecuTorch CUDA integration verified')

.ci/scripts/test-rocm-aoti.sh

@@ -7,7 +7,7 @@
 
 set -euo pipefail
 
-ROCM_VERSION="${ROCM_VERSION:-7.1}"
+ROCM_VERSION="${ROCM_VERSION:-7.2}"
 ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
 PYTORCH_ROCM_INDEX="${PYTORCH_ROCM_INDEX:-https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}}"
 TORCHAO_ROCM_WHEEL_BASE="${TORCHAO_ROCM_WHEEL_BASE:-https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}}"

.ci/scripts/test-rocm-voxtral.sh

@@ -7,7 +7,7 @@
 
 set -euo pipefail
 
-ROCM_VERSION="${ROCM_VERSION:-7.1}"
+ROCM_VERSION="${ROCM_VERSION:-7.2}"
 ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
 EXPECTED_ROCM_ARCH="${EXPECTED_ROCM_ARCH:-gfx950}"
 EXPECTED_WARP_SIZE="${EXPECTED_WARP_SIZE:-64}"

.ci/scripts/test_backend.sh

@@ -7,6 +7,9 @@
 # LICENSE file in the root directory of this source tree.
 set -eux
 
+# Cap pytest-xdist's `auto` workers to the container's CPU quota.
+source .ci/scripts/pytest-parallelism.sh
+
 SUITE=$1
 FLOW=$2
 ARTIFACT_DIR=$3
@@ -151,11 +154,13 @@ if [[ "$FLOW" == *nxp* ]]; then
     export NXP_RUNNER_PATH="$(pwd)/examples/nxp/executor_runner/build/nxp_executor_runner"
 fi
 
-GOLDEN_DIR="${ARTIFACT_DIR}/golden-artifacts"
-export GOLDEN_ARTIFACTS_DIR="${GOLDEN_DIR}"
-
 EXIT_CODE=0
-PYTEST_ARGS=(-c /dev/null -n auto)
+# An Ethos-U failure captures a few hundred thousand lines of Vela operator
+# listings, and the runner agent throws System.OutOfMemoryException processing
+# a step that size, taking the whole job down before pytest can report. The
+# reason for each failure is in its exception message and traceback, which are
+# unaffected.
+PYTEST_ARGS=(-c /dev/null -n auto --show-capture=no)
 if [[ ${#PYTEST_RETRY_ARGS[@]} -gt 0 ]]; then
     PYTEST_ARGS+=("${PYTEST_RETRY_ARGS[@]}")
 fi