diff --git a/.ci/scripts/tests/test_filter_cuda_matrix.py b/.ci/scripts/tests/test_filter_cuda_matrix.py new file mode 100644 index 00000000000..02ad71e5ca4 --- /dev/null +++ b/.ci/scripts/tests/test_filter_cuda_matrix.py @@ -0,0 +1,301 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Tests for the CUDA release matrix filter. +# +# The filter decides which wheel rows a release builds and exits non-zero when its inputs disagree +# with what the project can publish. Two of its comments record past bugs it now guards against, and +# a regression in any of them would surface only as a broken release, so each gate is pinned here. + +import importlib.util +import json +import unittest +from pathlib import Path +from unittest import mock + +import yaml + +ROOT = Path(__file__).resolve().parents[3] + + +def _load_filter(): + """Load the script by path, since .github/scripts is not an importable package.""" + path = ROOT / ".github" / "scripts" / "filter_cuda_matrix.py" + spec = importlib.util.spec_from_file_location("filter_cuda_matrix", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +FILTER = _load_filter() + + +def _full_matrix(): + """Every supported python and CUDA pair. + + The filter refuses anything less: one gate rejects a matrix that would leave a CUDA train + unpublished, another rejects a missing python and CUDA combination. Built from the module's own + lists so it cannot go stale when either grows. + + That also means it shrinks when either list shrinks, and every gate keeps passing. Measured: + deleting cu132 and 3.13 from the filter left all sixteen cases here green. TestPublishedSets + below is what notices that, so this fixture does not have to. + """ + return { + "include": [ + {"python_version": python, "desired_cuda": cuda} + for python in FILTER.SUPPORTED_PYTHON_VERSIONS + for cuda in FILTER.SUPPORTED_CUDA_VERSIONS + ] + } + + +def _run(matrix, limit="false", extra=None): + argv = ["--matrix", json.dumps(matrix), "--limit-pr-builds", limit] + (extra or []) + with mock.patch("builtins.print") as printed: + FILTER.main(argv) + return printed + + +def _emitted(printed): + return json.loads(printed.call_args_list[-1].args[0]) + + +class TestRanking(unittest.TestCase): + def test_prefers_the_requested_cuda_over_a_newer_one(self): + # The ranking deliberately scores a version above the requested one NEGATIVELY, so a newer + # one never outranks the one a machine here can actually run. A fixture offering only + # versions at or below the request never executes that branch. + newer = [ + c for c in FILTER.SUPPORTED_CUDA_VERSIONS if c > FILTER.PR_CUDA_VERSION + ] + items = [ + { + "python_version": FILTER.PR_PYTHON_VERSION, + "desired_cuda": FILTER.PR_CUDA_VERSION, + } + ] + [ + {"python_version": FILTER.PR_PYTHON_VERSION, "desired_cuda": c} + for c in newer + ] + picked = FILTER.only_pull_request_row(items) + self.assertEqual(picked[0]["desired_cuda"], FILTER.PR_CUDA_VERSION) + + def test_cuda_closeness_outranks_the_python_match(self): + # Closeness is the FIRST element of the sort key, deliberately. Ranking python first is a + # recorded past bug: it picked a wheel for a CUDA version nothing on hand can execute. + other_python = next( + p for p in FILTER.SUPPORTED_PYTHON_VERSIONS if p != FILTER.PR_PYTHON_VERSION + ) + other_cuda = next( + c for c in FILTER.SUPPORTED_CUDA_VERSIONS if c != FILTER.PR_CUDA_VERSION + ) + items = [ + {"python_version": other_python, "desired_cuda": FILTER.PR_CUDA_VERSION}, + {"python_version": FILTER.PR_PYTHON_VERSION, "desired_cuda": other_cuda}, + ] + picked = FILTER.only_pull_request_row(items) + self.assertEqual(picked[0]["desired_cuda"], FILTER.PR_CUDA_VERSION) + + def test_picks_the_requested_row(self): + items = [ + {"python_version": p, "desired_cuda": c} + for p in FILTER.SUPPORTED_PYTHON_VERSIONS + for c in FILTER.SUPPORTED_CUDA_VERSIONS + ] + picked = FILTER.only_pull_request_row(items) + self.assertEqual(len(picked), 1) + self.assertEqual(picked[0]["python_version"], FILTER.PR_PYTHON_VERSION) + self.assertEqual(picked[0]["desired_cuda"], FILTER.PR_CUDA_VERSION) + + def test_empty_input_gives_empty_output(self): + # Raising here would break every pull request while releases kept working, which is one of + # the two failures this function records having had. + self.assertEqual(FILTER.only_pull_request_row([]), []) + + def test_requested_cuda_absent_from_the_offer(self): + # The other recorded past bug: the requested version falls off the supported list, and the + # function still has to return one row rather than raise or return nothing. + items = [ + {"python_version": FILTER.PR_PYTHON_VERSION, "desired_cuda": c} + for c in FILTER.SUPPORTED_CUDA_VERSIONS + if c != FILTER.PR_CUDA_VERSION + ] + picked = FILTER.only_pull_request_row(items) + self.assertEqual(len(picked), 1) + + +class TestVersionRank(unittest.TestCase): + def test_newer_cuda_ranks_higher(self): + ordered = sorted(FILTER.SUPPORTED_CUDA_VERSIONS) + self.assertGreater( + FILTER._version_rank(ordered[-1]), FILTER._version_rank(ordered[0]) + ) + + def test_unknown_value_ranks_below_every_real_one(self): + # A value ranking above the real ones would silently take over the pull request row. + self.assertEqual(FILTER._version_rank("not-a-version"), -1) + + +class TestKeep(unittest.TestCase): + def test_unsupported_python_is_dropped(self): + # The recorded bug: passing a 3.9 row returned success and emitted it. + matrix = _full_matrix() + matrix["include"].append( + {"python_version": "3.9", "desired_cuda": FILTER.SUPPORTED_CUDA_VERSIONS[0]} + ) + emitted = _emitted(_run(matrix)) + self.assertNotIn("3.9", [row["python_version"] for row in emitted["include"]]) + + def test_unsupported_cuda_is_dropped(self): + matrix = _full_matrix() + matrix["include"].append( + { + "python_version": FILTER.SUPPORTED_PYTHON_VERSIONS[0], + "desired_cuda": "cu999", + } + ) + emitted = _emitted(_run(matrix)) + self.assertNotIn("cu999", [row["desired_cuda"] for row in emitted["include"]]) + + +class TestGates(unittest.TestCase): + def _exit_message(self, matrix, limit="false", extra=None): + """The stderr text of the gate that fired, so a case can name which one it hit.""" + import contextlib + import io + + argv = ["--matrix", json.dumps(matrix), "--limit-pr-builds", limit] + ( + extra or [] + ) + captured = io.StringIO() + with contextlib.redirect_stderr(captured): + with self.assertRaises(SystemExit) as raised: + FILTER.main(argv) + self.assertNotEqual(raised.exception.code, 0) + return captured.getvalue() + + def _expect_exit(self, matrix, limit="false", extra=None): + with mock.patch("builtins.print"): + with self.assertRaises(SystemExit) as raised: + _run(matrix, limit=limit, extra=extra) + self.assertNotEqual(raised.exception.code, 0) + + def test_unparseable_matrix_exits_nonzero(self): + argv = ["--matrix", "{not json", "--limit-pr-builds", "false"] + with mock.patch("builtins.print"): + with self.assertRaises(SystemExit) as raised: + FILTER.main(argv) + self.assertNotEqual(raised.exception.code, 0) + + def test_absent_train_exits_nonzero(self): + # A supported train the generator offers nothing for would publish no wheel at all. + # + # Patching the supported list rather than deleting rows, because deleting every row for one + # train also creates missing combinations, so both gates fire and the test cannot tell which + # one it exercised. Adding an extra supported train makes it absent while every offered + # combination stays complete. + # These two gates cannot be separated by input: any matrix leaving a train absent also + # leaves every combination for that train missing, so the later gate always catches what the + # earlier one would. Measured. So each gate gets its own case, and the case asserts on the + # message rather than only on a nonzero exit, which is the only way to tell them apart. + offered = FILTER.SUPPORTED_CUDA_VERSIONS[:-1] + matrix = { + "include": [ + {"python_version": python, "desired_cuda": cuda} + for python in FILTER.SUPPORTED_PYTHON_VERSIONS + for cuda in offered + ] + } + message = self._exit_message(matrix) + self.assertIn("publish no wheel for that CUDA version", message) + + def test_missing_combination_exits_nonzero(self): + matrix = _full_matrix() + del matrix["include"][0] + message = self._exit_message(matrix) + self.assertIn("combination(s) produced no row", message) + + def test_jetpack_not_published_exits_nonzero(self): + # Refused explicitly rather than allowed to fall through to an empty result, so the reason a + # reader sees is the real one. Nothing passes this flag today, which is why it had no cover. + message = self._exit_message(_full_matrix(), extra=["--jetpack", "true"]) + self.assertIn("JetPack rows are not published yet", message) + + def test_empty_result_exits_nonzero(self): + self._expect_exit({"include": []}) + + def test_well_formed_matrix_passes_through(self): + matrix = _full_matrix() + emitted = _emitted(_run(matrix)) + self.assertEqual(emitted["include"], matrix["include"]) + + def test_pull_request_limit_reduces_to_one_row(self): + emitted = _emitted(_run(_full_matrix(), limit="true")) + self.assertEqual(len(emitted["include"]), 1) + + +class TestPublishedSets(unittest.TestCase): + """What a release publishes, pinned against something other than the filter's own lists. + + Every case above builds its fixture from those lists, so shrinking one shrinks the fixture with + it and every gate still passes. The published set is a promise to users rather than an + implementation detail, so dropping a row has to be a deliberate edit here too. + """ + + def test_published_cuda_versions(self): + self.assertEqual(FILTER.SUPPORTED_CUDA_VERSIONS, ["cu126", "cu130", "cu132"]) + + def test_published_python_versions(self): + self.assertEqual( + FILTER.SUPPORTED_PYTHON_VERSIONS, ["3.10", "3.11", "3.12", "3.13"] + ) + + def test_the_workflows_offer_exactly_the_published_pythons(self): + # The filter can only keep a row the generator produced, and these two workflows are what + # tell the generator which pythons to produce. A python published here but not offered + # there does trip the release gate, but only on a release run, well after the change + # landed. A python offered there and not published here is dropped without a word. + for name in ( + "build-wheels-cuda-linux.yml", + "build-wheels-cuda-aarch64-linux.yml", + ): + with self.subTest(workflow=name): + workflow = yaml.safe_load( + (ROOT / ".github" / "workflows" / name).read_text() + ) + offered = json.loads( + workflow["jobs"]["generate-matrix"]["with"]["python-versions"] + ) + self.assertEqual(offered, FILTER.SUPPORTED_PYTHON_VERSIONS) + + def test_the_pull_request_row_names_a_python_a_pull_request_is_offered(self): + # A limited pull request is offered one python only, because the shared generator replaces + # the list the workflow passes with its first entry. Naming any other one here matched no + # offered row, so the row a pull request built was not the row this file names. + offered = { + "include": [ + { + "python_version": FILTER.SUPPORTED_PYTHON_VERSIONS[0], + "desired_cuda": cuda, + } + for cuda in FILTER.SUPPORTED_CUDA_VERSIONS + ] + } + emitted = _emitted(_run(offered, limit="true")) + self.assertEqual( + emitted["include"], + [ + { + "python_version": FILTER.PR_PYTHON_VERSION, + "desired_cuda": FILTER.PR_CUDA_VERSION, + } + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.ci/scripts/wheel/cuda_arch_list.sh b/.ci/scripts/wheel/cuda_arch_list.sh new file mode 100644 index 00000000000..29484056633 --- /dev/null +++ b/.ci/scripts/wheel/cuda_arch_list.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# GPU architectures to compile device code for, chosen per release row rather than detected from +# the build machine. +# +# Without this nothing selects the architectures, so nvcc falls back to its own default and the +# wheel carries device code for that one architecture regardless of the builder's GPU. Measured: +# with the architecture list unset the compile line has no gencode flags at all. The wheel then +# installs on every machine the row claims and fails when a model runs on a different generation, +# with an error that looks like a model problem rather than a packaging one. Detection is the right +# default for a local build and the wrong one for a published artifact. +# +# The value is published as TORCH_CUDA_ARCH_LIST rather than CMAKE_CUDA_ARCHITECTURES, because +# PyTorch's CMake rejects the latter and overrides it, so setting only that reduces the build to a +# single detected architecture. + +# The architectures each row serves. Two rules decide the list, and they pull in opposite directions. +# +# The upper end follows the published PyTorch build for that train, read from its own library rather than +# chosen by reasoning about which GPUs matter. A delegate is only useful where torch already runs, and an +# architecture torch supports but this wheel omits produces a wheel that installs and then fails at the +# first kernel launch. Two omissions found that way were the GPU on the runner that tests these wheels, +# and a common desktop card. +# +# The lower end does NOT follow torch. It stops at 8.0 even though torch reaches further down, because one +# source here compiles an integer matrix-multiply path only at 8.0 and above. Below that a user gets a +# delegate that loads, runs most models, and fails on one needing that operator, which is worse than a row +# that never claimed the device. So these lists are narrower than torch at the bottom on purpose. +_cuda_arch_x86_64_cu130="8.0 8.6 8.9 9.0 10.0 12.0" +_cuda_arch_x86_64_cu132="${_cuda_arch_x86_64_cu130}" + +# The architectures the published aarch64 PyTorch CUDA build covers, read from its own library on an ARM +# machine, for the same reason as the x86_64 rows above. Includes the ARM module whose train matches. +_cuda_arch_aarch64_cu130="8.0 9.0 10.0 11.0 12.0" +_cuda_arch_aarch64_cu132="${_cuda_arch_aarch64_cu130}" + +# The older CUDA train. +# +# The two architectures do not carry identical lists, because each covers what the published PyTorch +# build for that architecture covers, and those differ. Matching them to each other instead would mean +# advertising a GPU on one architecture that PyTorch cannot serve there. +# +# The smaller embedded modules are deliberately absent, with one exception. An embedded-only +# architecture in a generic wheel would advertise a device the row cannot otherwise serve, since +# those devices also need the CUDA, TensorRT and PyTorch pinned by their own software release +# rather than the ones a generic wheel resolves. +# +# 8.7 is that exception. This is the only row whose CUDA major matches what that module's software +# release ships, and the wheel declares no PyTorch, so the user supplies the build that carries +# their architecture. Omitting it does not protect them from a bad pairing, it only removes the +# device code they need. +# +# The floor is 8.0 rather than the oldest architecture PyTorch still carries. One of these sources compiles +# an integer matrix-multiply path only at 8.0 and newer, so an older architecture would get a delegate that +# loads, runs most models, and fails on one that needs that operator. Claiming hardware the delegate only +# partly serves is the same problem the embedded modules have, so the row leaves it out for the same reason. +_cuda_arch_x86_64_cu126="8.0 8.6 8.9 9.0" +_cuda_arch_aarch64_cu126="8.0 8.7 9.0" + +# A CUDA train with no architecture list would leave the build detecting the builder's GPU, which is +# the failure this file exists to prevent. Adding a train to the release matrix without adding its +# architectures should fail loudly rather than silently produce a single-GPU wheel. +_executorch_unknown_train() { + echo "cuda_arch_list.sh: no GPU architecture list for CUDA train '$1' on $(uname -m)." >&2 + echo "Add one before building this row, or the wheel ships device code for one GPU only." >&2 + return 64 +} + +# The architectures for the current row, space separated in the dotted form PyTorch expects. +executorch_cuda_arch_list() { + local machine + machine="$(uname -m)" + # The wheel build exports the row's CUDA train as CU_VERSION. DESIRED_CUDA is the name of the + # matrix field rather than of the variable, so reading only that leaves every row falling back to + # detecting the builder's GPU. + local train="${CU_VERSION:-${DESIRED_CUDA:-}}" + # A CPU row names no CUDA train and needs no architectures, so it is not an error. + # + # A CUDA row always names one, so an empty value there means the row lost it. Treating that as a CPU + # row let the build fall back to detecting the builder's GPU, which produces a wheel carrying device + # code for whatever machine happened to build it while every check still reports green. + case "${train}" in + "" | cpu | CPU | none | NONE) + if [ "${EXECUTORCH_BUILD_CUDA:-}" = "1" ]; then + echo "this is a CUDA build but the row's CUDA version is '${train}', which names no CUDA" >&2 + echo "train. Refusing to detect the builder GPU instead." >&2 + return 65 + fi + return 0 + ;; + esac + # The value arrives as cu130, while some callers pass 13.0 instead. + train="${train#cu}" + train="${train//./}" + + case "${machine}" in + aarch64 | arm64) + case "${train}" in + 126) printf '%s' "${_cuda_arch_aarch64_cu126}" ;; + 130) printf '%s' "${_cuda_arch_aarch64_cu130}" ;; + 132) printf '%s' "${_cuda_arch_aarch64_cu132}" ;; + *) _executorch_unknown_train "${train}" ;; + esac + ;; + x86_64) + case "${train}" in + 126) printf '%s' "${_cuda_arch_x86_64_cu126}" ;; + 130) printf '%s' "${_cuda_arch_x86_64_cu130}" ;; + 132) printf '%s' "${_cuda_arch_x86_64_cu132}" ;; + *) _executorch_unknown_train "${train}" ;; + esac + ;; + *) _executorch_unknown_train "${train}" ;; + esac +} + +# No entry carries a "+PTX" suffix. The portable form that lets a newer GPU compile this code at +# load time is added once, for the newest architecture only, where the list is turned into +# CMAKE_CUDA_ARCHITECTURES in backends/cuda/CMakeLists.txt. Suffixing it here as well would only +# produce a duplicate for that entry to drop again. diff --git a/.ci/scripts/wheel/envvar_cuda_linux.sh b/.ci/scripts/wheel/envvar_cuda_linux.sh new file mode 100644 index 00000000000..fda1a6b88a3 --- /dev/null +++ b/.ci/scripts/wheel/envvar_cuda_linux.sh @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# This file is sourced into the environment before building a pip wheel. It +# should typically only contain shell variable assignments. Be sure to export +# any variables so that subprocesses will see them. + +source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh" + +# Ask for the CUDA delegate explicitly rather than letting the build detect a toolkit. A detected +# build is fine locally, but a release row states what it is producing, and a row that silently +# produced a CPU wheel because the toolkit was missing would publish under a CUDA name. +export EXECUTORCH_BUILD_CUDA=1 +export CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_CUDA=ON" + +# Fail the build if CUDA is not actually present. Without this the packaging step would look for +# CUDA libraries that were never built and report a confusing missing-file error several minutes +# after the real problem. +# A regular file, executable, and able to answer: a directory also passes an execute-bit test, and a +# stub or a broken wrapper passes both, which would let the row report a toolkit it cannot compile with. +_executorch_nvcc="${CUDA_HOME:-/usr/local/cuda}/bin/nvcc" +if [ ! -f "${_executorch_nvcc}" ] || [ ! -x "${_executorch_nvcc}" ] || + ! "${_executorch_nvcc}" --version >/dev/null 2>&1; then + echo "EXECUTORCH_BUILD_CUDA is set but ${_executorch_nvcc} is not a working nvcc. This row cannot build a CUDA wheel." >&2 + exit 1 +fi + +# Compile device code for the GPUs this release row claims, rather than for whichever GPU the +# builder happens to have. A wheel built by detection alone installs on every machine the row covers +# and then fails when a model runs on a different generation. +source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/cuda_arch_list.sh" +# The status is checked rather than only the output, so an unrecognised row reports why it stopped. +# A bare assignment would end the build on the lookup's own exit status with no message, since this +# file is sourced into a shell that exits on a failing command. +if ! _executorch_cuda_arch="$(executorch_cuda_arch_list)"; then + echo "could not resolve GPU architectures for CU_VERSION=${CU_VERSION:-unset}" >&2 + exit 1 +fi +if [ -n "${_executorch_cuda_arch}" ]; then + export TORCH_CUDA_ARCH_LIST="${_executorch_cuda_arch}" + echo "building device code for: ${TORCH_CUDA_ARCH_LIST}" +fi diff --git a/.ci/scripts/wheel/test_cuda_linux.py b/.ci/scripts/wheel/test_cuda_linux.py new file mode 100644 index 00000000000..5a8e20b1665 --- /dev/null +++ b/.ci/scripts/wheel/test_cuda_linux.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Smoke test for a CUDA wheel row. + +Runs the checks a GPU wheel needs, then the packaging, backend, and C++ SDK checks a CPU wheel +gets. The extra CUDA checks exist because a GPU wheel can install cleanly, import cleanly, and +still be unusable: + + the CUDA libraries can be absent while the wheel is still named as a CUDA build + the runtime dependency can be undeclared, so a user has nothing to resolve it from + the loader path can point at the build machine's toolkit, which no user has + the device code can cover no GPU the row claims, which only appears when a model runs + +A model-execution check runs where a device exists. The x86_64 rows land on a GPU runner, so +a model is exported through the CUDA partitioner and run there, and its output is compared +against eager. The aarch64 rows have no accelerator on their validation runner, so that check +skips and prints why, which keeps a green result on those rows from standing for work that did +not happen. Everything else here is inspection of the shipped artifacts, which needs no device. +""" + +import os +import pathlib +import platform +import re +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Optional, Set + +import test_base +import test_cpp_sdk +import test_shared_libraries +from examples.models import Backend, Model + + +def _package_dir() -> Path: + import executorch + + return Path(executorch.__path__[0]) + + +def test_cuda_libraries_are_shipped() -> None: + """The row is named for CUDA, so the CUDA libraries have to be in it.""" + lib_dir = _package_dir() / "lib" + shipped = {path.name for path in lib_dir.iterdir()} if lib_dir.is_dir() else set() + expected = { + "libexecutorch_backend_cuda.so", + "libexecutorch_extension_cuda.so", + } + missing = sorted(expected - shipped) + assert not missing, ( + f"this is a CUDA row but {missing} are not in the wheel, so it would install as a " + f"CUDA build with no CUDA delegate. Shipped: {sorted(shipped)}" + ) + print(f"✓ the CUDA libraries ship ({len(expected)} of them)") + + +def test_cuda_runtime_is_declared() -> None: + """The wheel links the CUDA runtime without bundling it, so it must declare it. + + Without this a user installs the wheel and has nothing to resolve libcudart from, which + surfaces as a loader error at the first import rather than as a resolution failure at + install time. + """ + import importlib.metadata as metadata + + requirements = metadata.requires("executorch") or [] + cuda = [ + requirement + for requirement in requirements + if "nvidia" in requirement.lower() or "cuda" in requirement.lower() + ] + assert cuda, ( + "this is a CUDA row but the wheel declares no CUDA runtime dependency, so nothing " + "would install the libraries its delegate links" + ) + print(f"✓ the CUDA runtime is declared ({len(cuda)} requirements)") + + +def test_cuda_libraries_resolve_relatively() -> None: + """Each CUDA library must reach its runtime through a relative path. + + An absolute toolkit path names the machine that built the wheel. It resolves there and + nowhere else, so the wheel would work only on a builder. + + Every shipped library that links the CUDA runtime is inspected, wherever it lives. Naming + only the two in lib/ skipped libaoti_cuda_shims.so, which sits under backends/cuda/, links + cudart, and carries the device code, so an absolute toolkit path on the library + that matters most shipped green. + """ + readelf = test_shared_libraries._tool("readelf") + assert readelf is not None, "readelf is required to inspect the wheel" + + package_dir = _package_dir() + libraries = sorted(test_shared_libraries._shipped_shared_objects(package_dir)) + # Without this the loop below finds nothing on a wheel that ships no CUDA library and + # reports a pass, which is the same as having no check at all. + assert libraries, f"no shared libraries found under {package_dir}" + + linked_to_cuda = [] + for library in libraries: + output = subprocess.run( + [readelf, "-d", str(library)], capture_output=True, text=True, check=True + ).stdout + if any("NEEDED" in line and "libcud" in line for line in output.splitlines()): + linked_to_cuda.append((library, output)) + + assert linked_to_cuda, ( + "no shipped library links the CUDA runtime, so this check inspected nothing. A CUDA " + "row must ship the libraries it is named for." + ) + for library, output in linked_to_cuda: + name = library.relative_to(package_dir) + entries: list[str] = [] + for line in output.splitlines(): + if "RPATH" in line or "RUNPATH" in line: + entries += line.split("[", 1)[1].rstrip("]").strip().split(":") + # Resolved against the library's own directory rather than pattern-matched. Accepting any + # entry that merely starts with $ORIGIN and mentions nvidia pins the presence of a hop, not + # that the hop lands anywhere: $ORIGIN/nvidia-does-not-exist satisfied the old form. A hop + # at the wrong depth is exactly the defect this check exists to catch. + relative = [ + entry + for entry in entries + if entry.startswith("$ORIGIN") and "nvidia" in entry + # Stat last, so the filesystem is only touched for an entry that could match. + and (library.parent / entry.replace("$ORIGIN/", "").replace("$ORIGIN", ".")) + .resolve() + .is_dir() + ] + assert relative, ( + f"{name} links the CUDA runtime but records no relative path that RESOLVES to an " + f"installed CUDA wheel directory, so it can only resolve where the builder had a " + f"toolkit: " + f"{entries}" + ) + print(f"✓ {name} resolves the CUDA runtime relatively ({relative[0]})") + + +def _row_architectures() -> list[str]: + """The architectures this row claims, from the same script the build uses. + + A refusal from that script is a fault, not an absence. It returns non-zero when a CUDA row reaches it + with no version, which is precisely the case that would otherwise build device code for whatever GPU the + builder happens to have, so swallowing it here would hide the one failure this check exists to catch. + + EXECUTORCH_BUILD_CUDA is passed through because that is how the build invokes the script, and the + refusal is conditional on it. Without it the script returned an empty list on a CUDA row that had lost + its version, this check reported nothing to do, and the assertion below could never fire. + """ + script = pathlib.Path(__file__).parent / "cuda_arch_list.sh" + assert script.is_file(), f"the architecture script is missing at {script}" + result = subprocess.run( + ["bash", "-c", f"source {script}; executorch_cuda_arch_list"], + capture_output=True, + text=True, + check=False, + env={**os.environ, "EXECUTORCH_BUILD_CUDA": "1"}, + ) + assert result.returncode == 0, ( + f"the architecture script refused this row with exit {result.returncode}, so the build had no list " + f"to compile against: {result.stderr.strip()[:300]}" + ) + # "8.0 9.0" describes sm_80 and sm_90. + return ["sm_" + value.replace(".", "") for value in result.stdout.split()] + + +def test_device_code_covers_the_row() -> None: + """Every GPU the row claims must have device code in the shipped libraries. + + A row that promises a GPU it did not compile for produces a wheel that installs and then dies + at the first kernel launch, which is the worst failure to publish. + """ + expected = _row_architectures() + if not expected: + print("- this row claims no GPU architectures, nothing to check") + return + + cuobjdump = test_shared_libraries._tool("cuobjdump") + if cuobjdump is None: + raise AssertionError( + "cuobjdump is required to check device code, and this is a CUDA row. Without it a " + "wheel missing code for a claimed GPU would ship unnoticed." + ) + + # Searched across every shipped library rather than a named one. The kernels are compiled + # into their own library, not into the delegate, and which library holds them is an internal + # detail. What the row promises is that the wheel covers those GPUs. + present: set[str] = set() + inspected = [] + with_device_code: dict = {} + for library in sorted(_package_dir().rglob("*.so")): + listed = subprocess.run( + [cuobjdump, "--list-elf", str(library)], + capture_output=True, + text=True, + check=False, + ).stdout + found = { + token + for token in listed.replace(".", " ").split() + if token.startswith("sm_") + } + if found: + inspected.append(f"{library.name} ({', '.join(sorted(found))})") + present |= found + with_device_code[library.name] = found + + assert inspected, ( + "no shipped library contains any GPU device code, so this wheel cannot run a model on any " + f"GPU, while the row claims {expected}" + ) + missing = sorted(set(expected) - present) + assert not missing, ( + f"the row claims {expected} but the wheel carries no device code for {missing}. " + f"Found: {inspected}. A user with one of those GPUs would install this wheel and fail at " + "the first kernel launch." + ) + # The other direction matters just as much. Device code for an architecture the row does not + # claim means the build did not use the row's list, so whatever selected the architectures + # ignored it. That went unnoticed once already: a selection bug substituted a single default + # architecture and this check stayed green because it only looked for what was absent. + unexpected = sorted(present - set(expected)) + assert not unexpected, ( + f"the row claims {sorted(set(expected))} but the wheel also carries device code for " + f"{unexpected}. Found: {inspected}. The build did not use the row's list, so the artifact " + "does not match what the row published." + ) + # Every library that carries device code has to cover the row on its own. Unioning + # across libraries let a library with kernels cover only part of the row while an + # unrelated object supplied the rest, so on a GPU the first one did not compile for + # there was no executable kernel even though the union looked complete. + short = sorted(set(expected)) + for library in sorted(with_device_code): + library_missing = sorted(set(expected) - with_device_code[library]) + assert not library_missing, ( + f"{library} carries GPU device code but none for {library_missing}, while the row " + f"claims {short}. Checking the union across libraries hid this: another shipped " + "object supplied those architectures, and on such a GPU this library would have no " + "executable kernel." + ) + print(f"✓ device code covers the row in every library that has any: {inspected}") + + +def test_portable_device_code_is_present() -> None: + """The newest architecture must also ship in its portable form. + + The build appends "+PTX" for the top architecture so a GPU newer than any in the row can + still run, by having the driver compile that portable form at load time. Without it such a + GPU gets no usable code at all. + + Checked with --list-ptx rather than --list-elf. --list-elf prints byte-identical output for + a library built with or without the portable form, so it cannot see this. --list-ptx prints + an entry only for the library that has it. The entry is named for the target architecture, + "sm_90.ptx" rather than "compute_90.ptx", which is what the real tool prints. + """ + expected = _row_architectures() + if not expected: + print("- this row claims no GPU architectures, nothing to check") + return + + cuobjdump = test_shared_libraries._tool("cuobjdump") + assert ( + cuobjdump is not None + ), "cuobjdump is required to check the portable device code, and this is a CUDA row." + + # The newest architecture in the row, which is the one the build makes portable. + newest = max(expected, key=lambda name: int(name.removeprefix("sm_"))) + + found_in = [] + for library in sorted(_package_dir().rglob("*.so")): + listed = subprocess.run( + [cuobjdump, "--list-ptx", str(library)], + capture_output=True, + text=True, + check=False, + ).stdout + if newest in listed.replace(".", " ").split(): + found_in.append(library.name) + + assert found_in, ( + f"no shipped library carries portable device code for {newest}, the newest architecture " + f"in this row ({sorted(expected)}). A GPU newer than {newest} would install this wheel and " + "find no code it can run. The build appends the portable form for exactly this case, so " + "either it was dropped or the spelling in the architecture list is wrong." + ) + print(f"✓ portable device code for {newest} ships in {', '.join(found_in)}") + + +def test_the_delegate_registers() -> None: + """The delegate has to appear in the runtime's backend list, not merely be present as a file. + + Registration happens in a static initializer, which a normal link discards because nothing in the + program references it. Keeping it alive needs a linker option, and a wheel whose delegate ships but + does not register would load a delegated program and fail with an unregistered backend. That is the + failure this whole layout is most able to introduce, so it is worth asserting rather than assuming. + + Needs no GPU: registration is a link-time property, checked by importing. + """ + from executorch.extension.pybindings.portable_lib import ( + _get_registered_backend_names, + ) + + registered = _get_registered_backend_names() + assert "CudaBackend" in registered, ( + f"the wheel ships the CUDA delegate but CudaBackend is not registered: {registered}. " + "The library is present and its static initializer did not run, which means the option " + "that keeps it on the link line stopped working." + ) + print(f"✓ the delegate registers: CudaBackend among {len(registered)} backend(s)") + + +# Run in a child interpreter by test_a_model_runs_through_the_delegate, so the corrected library +# search path is in place before glibc caches it. Kept as source rather than a separate file +# because the smoke test is invoked directly and ships as one module. +_EXECUTION_CHILD = """ +import torch + +from executorch.backends.cuda.cuda_partitioner import CudaPartitioner +from executorch.exir import to_edge_transform_and_lower +from executorch.extension.pybindings.portable_lib import ( + _load_for_executorch_from_buffer, +) + + +class Add(torch.nn.Module): + def forward(self, x, y): + return x + y + + +example = (torch.randn(4, 8), torch.randn(4, 8)) +eager = Add()(*example) + +exported = torch.export.export(Add().eval(), example) +lowered = to_edge_transform_and_lower( + exported, partitioner=[CudaPartitioner([])] +).to_executorch() + +# The program has to actually carry the delegate, or this passes while proving nothing about it. +assert b"CudaBackend" in lowered.buffer, ( + "the exported program contains no CUDA delegate, so running it would not exercise the " + "shipped delegate at all" +) + +# Loaded from the buffer rather than a file, and the output is brought back to host, because the +# delegate returns device memory and comparing it against an eager result on the host otherwise +# fails with an invalid argument rather than a mismatch. +module = _load_for_executorch_from_buffer(lowered.buffer) +actual = module.forward(list(example))[0] + +torch.testing.assert_close(actual.cpu(), eager, rtol=1e-3, atol=1e-3) +capability = torch.cuda.get_device_capability(0) +print(f"PASS: a CUDA-delegated model ran on sm_{capability[0]}{capability[1]} and matched eager") +""" + + +def _cxx_runtime_dir() -> Optional[Path]: + """The directory holding the C++ runtime this environment's compiler links against. + + The kernel library the delegate loads is compiled here, so it records this runtime's symbol + versions. Returning the directory lets the caller put it where the loader will look. + """ + prefix = os.environ.get("CONDA_PREFIX") or os.environ.get("CONDA_ENV") + if not prefix: + return None + lib = Path(prefix) / "lib" + return lib if (lib / "libstdc++.so.6").exists() else None + + +def _version_key(version: str) -> tuple: + return tuple(int(part) for part in version.split("_")[1].split(".")) + + +def _glibcxx_versions(library: Path) -> Set[str]: + """The GLIBCXX version nodes a shared object defines or requires. + + Read with readelf rather than by parsing a filename, so a runtime that satisfies the kernel + starts being used the moment it is present instead of when a list is updated by hand. + """ + readelf = test_shared_libraries._tool("readelf") + if readelf is None: + return set() + completed = subprocess.run( + [readelf, "--version-info", "--wide", str(library)], + capture_output=True, + text=True, + check=False, + ) + return set(re.findall(r"GLIBCXX_[0-9.]+", completed.stdout)) + + +def test_a_model_runs_through_the_delegate() -> None: + """Export a model to the CUDA delegate and run it, comparing against eager. + + Every other check in this file reads a shipped artifact. This one executes, because a wheel whose + libraries are all present and correctly linked can still fail to compute, and nothing above would + notice. The x86_64 rows land on a GPU runner, so this is the row where that can be proven. + + Only the aarch64 rows may skip, and they say why, so a green result never stands for work that + did not happen. On x86_64 the absent device or the mismatched torch build is itself the failure: + that row is the one place execution is proven, and letting it report success for a check it did + not run is how a delegate that cannot compute reaches a release. + + The body runs in a child interpreter. Export compiles the kernel library with this environment's + compiler, so the library records that compiler's C++ runtime versions, while dlopen resolves + libstdc++.so.6 from the loader's search path. Where the system copy is older than the compiler's + the load fails on a missing version node. glibc caches the search list at startup, so the + corrected path has to be in place before an interpreter begins, which is what the child gives. + Only this check runs with it, because the checks above prove the shipped libraries resolve + without a widened search path and would stop measuring that if it were widened for them too. + """ + import torch + + execution_required = platform.machine() in ("x86_64", "amd64") + + if not torch.cuda.is_available(): + assert not execution_required, ( + "no CUDA device is visible to this x86_64 row, which is the row that proves the " + "delegate computes. The runner lost its GPU or the installed torch cannot reach it; " + "either way nothing here executed." + ) + print( + "SKIP: no CUDA device on this runner, so the delegate cannot execute here. " + "This row still verifies the shipped libraries, their declared dependencies, " + "their loader paths and the device code they carry." + ) + return + + capability = torch.cuda.get_device_capability(0) + device = f"sm_{capability[0]}{capability[1]}" + if device not in torch.cuda.get_arch_list(): + assert not execution_required, ( + f"the torch resolved for this x86_64 row carries no code for {device}, the runner's " + f"own GPU, so the execution check cannot run: {torch.cuda.get_arch_list()}" + ) + print( + f"SKIP: the installed torch carries no code for {device}, so nothing can run on this " + f"device regardless of what the wheel ships." + ) + return + + environment = dict(os.environ) + runtime_dir = _cxx_runtime_dir() + if runtime_dir is not None: + search_path = environment.get("LD_LIBRARY_PATH") + environment["LD_LIBRARY_PATH"] = ( + f"{runtime_dir}:{search_path}" if search_path else str(runtime_dir) + ) + + completed = subprocess.run( + [sys.executable, "-c", _EXECUTION_CHILD], + capture_output=True, + text=True, + env=environment, + check=False, + ) + print(completed.stdout, end="") + if completed.returncode != 0: + # The compiled kernel's own requirement against what the loader offered, so a version + # mismatch reports the two numbers rather than only the load error the child saw. + detail = "" + if runtime_dir is not None: + offered = _glibcxx_versions(runtime_dir / "libstdc++.so.6") + if offered: + detail = ( + f" The runtime at {runtime_dir} offers up to " + f"{max(offered, key=_version_key)}." + ) + raise AssertionError( + f"a CUDA-delegated model did not run on {device}.{detail}\n" + f"{completed.stdout}\n{completed.stderr}" + ) + + +if __name__ == "__main__": + assert platform.system() == "Linux", "the CUDA rows are Linux only" + + test_cuda_libraries_are_shipped() + test_cuda_runtime_is_declared() + test_cuda_libraries_resolve_relatively() + test_device_code_covers_the_row() + test_portable_device_code_is_present() + test_the_delegate_registers() + test_a_model_runs_through_the_delegate() + + # The backend registrations a CPU Linux row asserts also apply here: the CUDA build enables + # OpenVINO on every Linux architecture and downloads the QNN SDK on x86_64, so a CUDA wheel + # carries both backends and needs both to register. + from executorch.extension.pybindings.portable_lib import ( + _get_registered_backend_names, + ) + + registered = _get_registered_backend_names() + if platform.machine() in ("x86_64", "amd64"): + assert ( + "QnnBackend" in registered + ), f"QnnBackend not found in registered backends: {registered}" + print("✓ QnnBackend is registered") + assert ( + "OpenvinoBackend" in registered + ), f"OpenvinoBackend not found in registered backends: {registered}" + print("✓ OpenvinoBackend is registered") + + test_base.test_cmsis_nn_install() + + # The packaging and linking checks a CPU wheel is held to still apply: one owner per + # component, no build-tree paths, and a C++ application able to link what the wheel + # ships. + with tempfile.TemporaryDirectory() as work_dir: + test_shared_libraries.run_tests(Path(work_dir)) + with tempfile.TemporaryDirectory() as work_dir: + test_cpp_sdk.run_tests(Path(work_dir)) + + test_base.run_tests( + model_tests=[ + test_base.ModelTest( + model=Model.Mv3, + backend=Backend.XnnpackQuantizationDelegation, + ), + ] + ) diff --git a/.ci/scripts/wheel/test_shared_libraries.py b/.ci/scripts/wheel/test_shared_libraries.py index e8e15f7596a..a21480318ab 100644 --- a/.ci/scripts/wheel/test_shared_libraries.py +++ b/.ci/scripts/wheel/test_shared_libraries.py @@ -465,6 +465,16 @@ def _wheel_cuda_train() -> str: _REQUIRED_ON_A_CUDA_WHEEL = "cuda-wheel-only" +# The exact dependency names packaging declares per CUDA train, mirroring +# _CUDA_RUNTIME_PACKAGES in setup.py. Listed here rather than imported because setup.py +# runs a build when imported, and duplicated deliberately so a rename on the packaging +# side has to be made here too rather than silently agreeing with itself. +_EXPECTED_CUDA_PACKAGES = { + "12": ("nvidia-cuda-runtime-cu12",), + "13": ("nvidia-cuda-runtime",), +} + + # Each component the wheel ships as its own library, the symbols that identify it, # and the library that must own them. `required` says whether the owner has to be # present: the optimized kernels are optional, because a wheel built without them @@ -1777,17 +1787,34 @@ def test_model_matches_eager_pytorch(work_dir: Path) -> None: def test_declared_dependencies_match_the_wheel_tag() -> None: - """A CPU wheel must not declare the CUDA runtime, and a CUDA wheel must declare it. + """A CPU wheel must not declare the CUDA runtime, and a CUDA wheel must declare its own train. The tag is what a user resolves against, so a mismatch is a promise the wheel cannot keep in either direction: a CPU wheel that pulls the CUDA packages costs a user hundreds of megabytes it never loads, and a CUDA wheel that declares nothing leaves the runtime unresolvable. + Declaring the wrong train is the quiet case, and the reason this checks the names rather than + only their presence. The CUDA 12 packages are published with a "-cu12" suffix and the CUDA 13 + ones without, so a cu130 wheel that asked for the cu12 packages would install a runtime its + libraries cannot load, while looking correctly specified. + This is metadata only, so no library check can see it. A CPU wheel that wrongly declared the CUDA runtime passed every other check in this file. """ requirements = importlib.metadata.requires("executorch") or [] - cuda = sorted(r.split()[0] for r in requirements if r.lower().startswith("nvidia")) + + # Split off any environment marker AND any version specifier. The name, the specifier + # and the marker can arrive as one token, so taking the first whitespace-separated + # word left "nvidia-cuda-runtime-cu12==12.6.77" as the name and made a correctly + # specified wheel fail the moment any CUDA dependency gained a pin. + def distribution_name(requirement: str) -> str: + return re.split(r"[\s;\[<>=!~(]", requirement.strip(), maxsplit=1)[0] + + cuda = sorted( + name + for name in (distribution_name(r) for r in requirements) + if name.lower().startswith("nvidia") + ) # The local version segment of the installed version states what the wheel was built for. version = importlib.metadata.version("executorch") @@ -1799,7 +1826,35 @@ def test_declared_dependencies_match_the_wheel_tag() -> None: f"version {version} says this is a CUDA wheel, but it declares no CUDA runtime " "packages, so nothing resolves the runtime it links" ) - print(f"✓ this CUDA wheel declares the runtime ({len(cuda)} packages)") + # Compared as sets in both directions rather than as a name suffix: for CUDA 13 the + # expected suffix is the empty string and every name ends with that, so a suffix test + # accepted a name from any train whose spelling happened not to be one of the two + # literals it also excluded. Measured: a cu130 wheel declaring nvidia-cuda-runtime-cu11 + # passed. The reverse check catches the other side of the same defect: a wheel that + # declares one package and omits the others still cannot load, and one-direction only + # would accept it. + train = local[len("cu") : len("cu") + 2] + expected = set(_EXPECTED_CUDA_PACKAGES.get(train, ())) + assert expected, ( + f"version {version} names CUDA train {train}, which this check has no expected " + f"package list for. Add it beside the packaging list it mirrors." + ) + actual = set(cuda) + wrong = sorted(actual - expected) + missing = sorted(expected - actual) + assert not wrong, ( + f"version {version} is a CUDA {train} wheel, but it declares {wrong}, which belong to " + f"another CUDA train. Expected only {sorted(expected)}. A user would install a runtime " + "this wheel's libraries cannot load." + ) + assert not missing, ( + f"version {version} is a CUDA {train} wheel, but it does not declare {missing} " + f"(expected {sorted(expected)}). A user installing this wheel would end up without part " + "of the CUDA runtime the wheel's libraries need." + ) + print( + f"✓ this CUDA {train} wheel declares its own runtime ({len(cuda)} packages)" + ) else: assert not cuda, ( f"version {version} is not a CUDA wheel, yet it declares {cuda}. A user installing it " diff --git a/.github/scripts/filter_cuda_matrix.py b/.github/scripts/filter_cuda_matrix.py new file mode 100644 index 00000000000..35d4938f67d --- /dev/null +++ b/.github/scripts/filter_cuda_matrix.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Narrow the generated build matrix to the rows a GPU wheel can honestly support. + +The shared matrix generator emits every CUDA version and Python version it knows about. +Building all of them would publish wheels for combinations nothing can verify, and a GPU +wheel that installs and then cannot run is worse than one that does not exist: the failure +appears when a model runs, and it looks like a model problem rather than a packaging one. + +A row is kept only when both of these hold: + + a GPU exists that the row's device code covers + a PyTorch build is published for that CUDA version and architecture + +The x86_64 rows run a model as part of their smoke test, because their runner has a GPU. +The aarch64 rows have no accelerator, so that check skips there and prints why. This filter +decides only which rows exist, not what each one checks. + +The values below are the current answers to those questions. They are written out rather +than derived because each one is an external fact that can change independently. +""" + +import argparse +import json +import sys +from typing import Any, Dict, List + +# Python versions that are deliberately NOT published, with the reason, so a row naming one +# is rejected for a stated cause rather than for merely being absent from the supported list. +# 3.14 is excluded because the current CPU wheel rows already fail on it for an unrelated +# reason in the example requirements, so a GPU row would inherit a known-broken build. The +# free-threaded builds are excluded because the CUDA dependencies are not published for them. +# +# This is documentation, not the gate. The gate is SUPPORTED_PYTHON_VERSIONS below: anything +# not on that list is rejected whether or not it appears here. +DISABLED_PYTHON_VERSIONS: List[str] = ["3.13t", "3.14", "3.14t", "3.15", "3.15t"] + +# CUDA versions to publish. +# +# Chosen so that every consumer row can find a matching wheel rather than by what is +# convenient to verify. A delegate built against one of these has to be able to depend on an +# ExecuTorch wheel for the same CUDA version, and a missing version means that consumer has +# nothing to depend on: +# +# cu126 the floor, and what Jetson devices are limited to +# cu130 the generator's stable choice, and the default for accelerator consumers +# cu132 the newest, which consumers building against a current TensorRT need +# +# cu132 is included because omitting it would leave a published consumer row with no +# ExecuTorch wheel to pair with. It is executable on a device one minor behind, since CUDA +# minor versions are compatible, so a cu132 wheel has been run end to end on a CUDA 13.0 +# device. The packaging properties are checked on every row regardless. +SUPPORTED_CUDA_VERSIONS: List[str] = ["cu126", "cu130", "cu132"] + +# Python versions to publish, stated rather than derived for the same reason the CUDA +# versions are. Deriving them from the rows that survived the filter made the release +# guard below unable to notice a python that disappeared from every supported train: with +# nothing left to compare, a release quietly published nine wheels instead of twelve. +# Keep in step with the python-versions list in the CUDA wheel workflows. +SUPPORTED_PYTHON_VERSIONS: List[str] = ["3.10", "3.11", "3.12", "3.13"] + +# The single row built for a pull request. A full matrix on every push would cost hours for +# little signal, and cu130 is the version with a machine on hand that can run a model on it. +# +# The python is not a free choice. When a pull request is limited, the shared generator replaces +# the offered python list with its first entry, so that entry is the only python any row can +# carry. Naming a different one here matched no offered row: the tiebreaker below never fired and +# the pull request silently built whichever python the generator had left, so the constant +# described a row that was never built. +PR_PYTHON_VERSION: str = SUPPORTED_PYTHON_VERSIONS[0] +PR_CUDA_VERSION: str = "cu130" + +# Jetson devices are their own row: a JetPack image, one Python version, and one CUDA +# version. Kept empty on purpose today, so no Jetson row is emitted. +# +# The generic aarch64 CUDA 12.6 wheel does compile sm_87 device code for one embedded +# module, so the wheel itself is not the blocker. What is: published PyTorch stopped +# shipping sm_87 device code after 2.8.0, so a Jetson row today would produce a wheel +# whose PyTorch dependency cannot execute on the device. Populate this when that +# changes. +# +# Because both lists are empty, asking for the JetPack rows can only produce an empty result. +# No workflow asks, and the request is rejected up front with that reason rather than left to +# surface as the generic "the filter produced no rows" message, which reads as a broken +# matrix rather than as a row that is deliberately not built yet. +JETPACK_PYTHON_VERSIONS: List[str] = [] +JETPACK_CUDA_VERSIONS: List[str] = [] +JETPACK_CONTAINER_IMAGE: str = "nvcr.io/nvidia/l4t-jetpack:r36.4.0" + + +def keep(item: Dict[str, Any], is_jetpack: bool) -> bool: + """Whether this row should be built, adjusting its container image where needed.""" + # An allowlist, the same shape as the CUDA test below. Testing only the disabled list + # let any python not on it through: passing a 3.9 row returned success and emitted it, + # and the only thing preventing that today is both workflows happening to pin the list + # they pass in. + if item["python_version"] not in SUPPORTED_PYTHON_VERSIONS: + return False + + if is_jetpack: + if ( + item["python_version"] in JETPACK_PYTHON_VERSIONS + and item["desired_cuda"] in JETPACK_CUDA_VERSIONS + ): + item["container_image"] = JETPACK_CONTAINER_IMAGE + return True + return False + + if item["desired_cuda"] not in SUPPORTED_CUDA_VERSIONS: + return False + + return True + + +def _version_rank(cuda: str) -> int: + """Where a CUDA version sits in the supported list, or -1 when it is not supported at all.""" + try: + return SUPPORTED_CUDA_VERSIONS.index(cuda) + except ValueError: + return -1 + + +def only_pull_request_row(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """One representative row, so a pull request does not build the whole matrix. + + Chosen by preference rather than exact match, so a request that does not appear in the + generated matrix degrades to the closest supported combination instead of falling off the + end. + """ + if not items: + return [] + + # Looked up once, and tolerantly: a PR_CUDA_VERSION that falls off SUPPORTED_CUDA_VERSIONS used to + # raise here and break every pull request while releases kept working, which is the wrong way round + # for a constant that only chooses which single row to build. + wanted = _version_rank(PR_CUDA_VERSION) + + def rank(item: Dict[str, Any]) -> tuple: + # Closeness peaks at the requested version, then falls off, and it outranks the python match. + # Ranking python first picked a wheel for a CUDA version nothing on hand can execute whenever the + # generator skewed the two axes, and the point of building one row is to get signal from it. + offered = _version_rank(item["desired_cuda"]) + # Negative above the requested version, so a newer one never outranks an older one a machine here + # can actually run. + closeness = offered if offered <= wanted else wanted - offered + return (closeness, item["python_version"] == PR_PYTHON_VERSION) + + return [max(items, key=rank)] + + +def main(argv: List[str]) -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", required=True, help="the generated matrix, as JSON") + parser.add_argument( + "--jetpack", default="false", help="build the Jetson row instead" + ) + parser.add_argument("--limit-pr-builds", default="false", help="build one row only") + args = parser.parse_args(argv) + + try: + matrix = json.loads(args.matrix) + except json.JSONDecodeError as error: + print(f"could not parse the matrix: {error}", file=sys.stderr) + sys.exit(1) + + is_jetpack = args.jetpack.lower() == "true" + if is_jetpack and not (JETPACK_PYTHON_VERSIONS and JETPACK_CUDA_VERSIONS): + # Rejected here rather than allowed to fall through to an empty result, so the reason + # is the actual one. Nothing passes this flag today. + print( + "the JetPack rows are not published yet: JETPACK_PYTHON_VERSIONS and " + "JETPACK_CUDA_VERSIONS are empty because published PyTorch carries no device code " + "for that GPU architecture, so any wheel built here could not run on the device. " + "Populate both lists to enable this row.", + file=sys.stderr, + ) + sys.exit(1) + items = [item for item in matrix.get("include", []) if keep(item, is_jetpack)] + + if args.limit_pr_builds.lower() == "true" and items: + items = only_pull_request_row(items) + elif items and not is_jetpack: + # A release has to publish every combination this policy advertises. Comparing the result against + # what the generator offered cannot catch anything, because both sides apply the same conditions, so + # the difference is empty by construction and the check never fires. The policy's own list is the + # thing to compare against: a CUDA version the generator stopped offering otherwise disappears from + # the release silently, and a missing job is a green check for a wheel that was never built. + # + # The generic rows only. A JetPack release advertises the single pair its own lists name rather than + # every supported CUDA version, so checking it against this list would fail a correct release. + # + # Both axes come from this policy's own lists, not from the matrix. Reading the generator's python + # axis pulled in rows this policy never builds, and deriving it from the rows that survived went + # blind to a python that disappeared from every supported train. The generator lives in another + # repository and its axes move independently of what this policy promises to publish. + built = {(item["python_version"], item["desired_cuda"]) for item in items} + # A train that produced no row at all is missing for every python, so reporting it per python + # would read as a python problem. Named on its own instead, and first, because the per-pair + # report below would otherwise bury it. + absent_trains = sorted( + set(SUPPORTED_CUDA_VERSIONS) - {cuda for _, cuda in built} + ) + if absent_trains: + print( + f"this policy publishes {SUPPORTED_CUDA_VERSIONS}, but the generator offered no row " + f"this filter could keep for {absent_trains}, so a release would publish no wheel for " + "that CUDA version at all", + file=sys.stderr, + ) + sys.exit(1) + missing = sorted( + f"{python}/{cuda}" + for python in SUPPORTED_PYTHON_VERSIONS + for cuda in SUPPORTED_CUDA_VERSIONS + if (python, cuda) not in built + ) + if missing: + print( + f"this policy publishes {SUPPORTED_CUDA_VERSIONS} for each of " + f"{SUPPORTED_PYTHON_VERSIONS}, but {len(missing)} combination(s) produced no row, so a " + f"release would publish no wheel for them: {missing}", + file=sys.stderr, + ) + sys.exit(1) + + # Fail loudly on an empty result. A silently empty matrix produces a workflow with no + # build job, which shows up as a green check for a build that never happened. + if not items: + print( + "the filter produced no rows to build, so nothing would be verified. " + f"jetpack={is_jetpack}, supported CUDA={SUPPORTED_CUDA_VERSIONS}", + file=sys.stderr, + ) + sys.exit(1) + + print(json.dumps({"include": items})) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/.github/workflows/build-wheels-cuda-aarch64-linux.yml b/.github/workflows/build-wheels-cuda-aarch64-linux.yml new file mode 100644 index 00000000000..b5c982b3b92 --- /dev/null +++ b/.github/workflows/build-wheels-cuda-aarch64-linux.yml @@ -0,0 +1,114 @@ +# From https://github.com/pytorch/test-infra/wiki/Using-Nova-Reusable-Build-Workflows +name: Build Aarch64 Linux CUDA Wheels + +on: + pull_request: + paths: + - .ci/**/* + - .github/scripts/filter_cuda_matrix.py + - .github/workflows/build-wheels-cuda-aarch64-linux.yml + - '**/CMakeLists.txt' + - backends/cuda/**/* + - examples/**/* + - extension/cuda/**/* + - install_requirements.py + - install_utils.py + - pyproject.toml + - setup.py + - docs/source/using-executorch-cpp.md + - tools/cmake/**/* + # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories + # setup.py copies from, so adding one there cannot silently drop it from this list. + - devtools/etdump/**.h + - extension/**.h + - runtime/**.h + push: + branches: + - nightly + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + - ciflow/binaries/* + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux-aarch64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cuda: enable + with-cpu: disable + with-rocm: disable + python-versions: '["3.10", "3.11", "3.12", "3.13"]' + + # The generator emits every CUDA version it knows about. Publishing all of them would ship + # wheels for combinations nothing can verify, so this keeps only the rows a device exists for + # and a matching PyTorch is published for. These runners carry no accelerator, unlike the + # x86_64 ones, so the smoke test here checks packaging rather than execution. The script fails + # rather than emitting an empty matrix, because a workflow with no build job reads as a pass. + filter-matrix: + needs: generate-matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.filter.outputs.matrix }} + steps: + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - uses: actions/checkout@v4 + - name: Filter the matrix + id: filter + run: | + set -eou pipefail + MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} + # Mirrors the shared generator, which drops its own row limit when this label is + # present. Clamping here regardless meant the label was accepted as a trigger and + # then ignored, so the full matrix could never be exercised before a release. + LIMIT_PR=${{ (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ciflow/binaries/all')) && 'true' || 'false' }} + MATRIX_BLOB="$(python3 .github/scripts/filter_cuda_matrix.py \ + --matrix "${MATRIX_BLOB}" --limit-pr-builds "${LIMIT_PR}")" + echo "${MATRIX_BLOB}" + echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" + + build: + needs: filter-matrix + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/executorch + pre-script: .ci/scripts/wheel/pre_build_script.sh + post-script: .ci/scripts/wheel/post_build_script.sh + smoke-test-script: .ci/scripts/wheel/test_cuda_linux.py + package-name: executorch + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + submodules: recursive + env-var-script: .ci/scripts/wheel/envvar_cuda_linux.sh + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + # Required for aarch64. Without it the shared build workflow prepares an x86_64 job + # and skips the aarch64 conda install, so the first build step fails on a missing + # conda. + architecture: aarch64 diff --git a/.github/workflows/build-wheels-cuda-linux.yml b/.github/workflows/build-wheels-cuda-linux.yml new file mode 100644 index 00000000000..8609e5addc5 --- /dev/null +++ b/.github/workflows/build-wheels-cuda-linux.yml @@ -0,0 +1,109 @@ +# From https://github.com/pytorch/test-infra/wiki/Using-Nova-Reusable-Build-Workflows +name: Build Linux CUDA Wheels + +on: + pull_request: + paths: + - .ci/**/* + - .github/scripts/filter_cuda_matrix.py + - .github/workflows/build-wheels-cuda-linux.yml + - '**/CMakeLists.txt' + - backends/cuda/**/* + - examples/**/* + - extension/cuda/**/* + - install_requirements.py + - install_utils.py + - pyproject.toml + - setup.py + - docs/source/using-executorch-cpp.md + - tools/cmake/**/* + # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories + # setup.py copies from, so adding one there cannot silently drop it from this list. + - devtools/etdump/**.h + - extension/**.h + - runtime/**.h + push: + branches: + - nightly + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + - ciflow/binaries/* + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cuda: enable + with-cpu: disable + with-rocm: disable + python-versions: '["3.10", "3.11", "3.12", "3.13"]' + + # The generator emits every CUDA version it knows about. Publishing all of them would ship + # wheels for combinations nothing can verify, so this keeps only the rows with a GPU to run + # them on. The script fails rather than emitting an empty matrix, because a workflow with no + # build job reads as a pass. + filter-matrix: + needs: generate-matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.filter.outputs.matrix }} + steps: + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - uses: actions/checkout@v4 + - name: Filter the matrix + id: filter + run: | + set -eou pipefail + MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} + # Mirrors the shared generator, which drops its own row limit when this label is + # present. Clamping here regardless meant the label was accepted as a trigger and + # then ignored, so the full matrix could never be exercised before a release. + LIMIT_PR=${{ (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ciflow/binaries/all')) && 'true' || 'false' }} + MATRIX_BLOB="$(python3 .github/scripts/filter_cuda_matrix.py \ + --matrix "${MATRIX_BLOB}" --limit-pr-builds "${LIMIT_PR}")" + echo "${MATRIX_BLOB}" + echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" + + build: + needs: filter-matrix + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/executorch + pre-script: .ci/scripts/wheel/pre_build_script.sh + post-script: .ci/scripts/wheel/post_build_script.sh + smoke-test-script: .ci/scripts/wheel/test_cuda_linux.py + package-name: executorch + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + submodules: recursive + env-var-script: .ci/scripts/wheel/envvar_cuda_linux.sh + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} diff --git a/backends/cuda/CMakeLists.txt b/backends/cuda/CMakeLists.txt index 2c752dda618..21f8f5e2914 100644 --- a/backends/cuda/CMakeLists.txt +++ b/backends/cuda/CMakeLists.txt @@ -50,6 +50,116 @@ if(NOT EXECUTORCH_BUILD_ROCM AND NOT CMAKE_CUDA_COMPILER) check_language(CUDA) endif() +# Take the architectures from the release row when it names them, before the +# language is enabled, since CMake fixes them at that point. Without this the +# build uses CMake's default, which on some devices is older than the intrinsics +# these sources use, and the compile fails with an undefined identifier that +# looks like a source problem. +# +# TORCH_CUDA_ARCH_LIST is the variable the surrounding build environment already +# sets, in PyTorch's dotted form. CMake wants bare integers, so "9.0" becomes +# 90. A "+PTX" suffix asks for the portable form in addition to the compiled +# one, which is what PyTorch means by it, so it adds the -virtual kind rather +# than replacing the -real one. Torch sets this to OFF at root scope when it is +# defined, warning that it ignores the value, so a defined value does not mean a +# caller chose it. OFF is treated as absent here, otherwise asking for an +# architecture through the preset silently compiles for whatever torch's own +# gencode flags select instead. CMAKE_CUDA_ARCHITECTURES is deliberately not +# read here. torch's CMake rejects and overrides it, which is why the release +# rows publish TORCH_CUDA_ARCH_LIST instead, and CMake fills the cache entry +# with a default of its own once the CUDA language is enabled. torch enables +# that language before this directory is added, so the cache always holds a +# value and cannot be read as a caller's intent: doing so replaced torch's +# autodetected architecture with CMake's default and broke the build on a device +# the default does not cover. +if(EXECUTORCH_BUILD_ROCM) + # ROCm never enables the CUDA language below, so there is no architecture list + # to choose and nothing here would be read. +elseif(DEFINED ENV{TORCH_CUDA_ARCH_LIST}) + string(REPLACE "." "" _executorch_cuda_arch_list "$ENV{TORCH_CUDA_ARCH_LIST}") + string(REPLACE " " ";" _executorch_cuda_arch_list + "${_executorch_cuda_arch_list}" + ) + set(_executorch_cuda_arch_resolved "") + set(_executorch_cuda_arch_newest "") + set(_executorch_cuda_arch_newest_rank 0) + # Mapped to CMake's explicit kinds rather than left bare. A bare number asks + # nvcc for both the binary and the portable form, which measured at about 2.5x + # the device code for a four architecture row. Only the newest needs the + # portable form, since that is what lets a device newer than every entry here + # run this code by compiling it on load. torch's own "+PTX" spelling of that + # request is dropped on the way, because nvcc rejects the suffix literally and + # the -virtual entry below already covers what it asks for. + foreach(_arch IN LISTS _executorch_cuda_arch_list) + string(REGEX REPLACE "\\+PTX$" "" _arch "${_arch}") + list(APPEND _executorch_cuda_arch_resolved "${_arch}-real") + # Compared numerically rather than taken as the last entry written. Nothing + # orders TORCH_CUDA_ARCH_LIST, so a row spelled "9.0 8.0" put the portable + # form on 8.0 and left a device newer than 9.0 with nothing to compile on + # load. The trailing letters some architectures carry are dropped for the + # comparison only, and a family name reduces to nothing and is skipped, + # which is the same answer the numeric filter below reaches. + string(REGEX MATCH "^[0-9]+" _executorch_cuda_arch_rank "${_arch}") + if(_executorch_cuda_arch_rank AND _executorch_cuda_arch_rank GREATER + _executorch_cuda_arch_newest_rank + ) + set(_executorch_cuda_arch_newest_rank "${_executorch_cuda_arch_rank}") + set(_executorch_cuda_arch_newest "${_arch}") + endif() + endforeach() + if(_executorch_cuda_arch_newest) + list(APPEND _executorch_cuda_arch_resolved + "${_executorch_cuda_arch_newest}-virtual" + ) + endif() + # A row that names both "12.0" and "12.0+PTX" collapses to the same entry + # twice, which would duplicate a gencode on the compile line. + list(REMOVE_DUPLICATES _executorch_cuda_arch_resolved) + # torch also accepts family names such as Ampere or Hopper, which its own + # select_compute_arch.cmake documents and has already turned into gencode + # flags by the time this directory is added. Dropped rather than rejected: + # failing here would break a configuration that builds today, and passing a + # name through to CMAKE_CUDA_ARCHITECTURES would reach nvcc, which only + # accepts numbers. + set(_executorch_cuda_arch_numeric) + foreach(_arch IN LISTS _executorch_cuda_arch_resolved) + if(_arch MATCHES "^[0-9]+[a-z]*(-real|-virtual)?$") + list(APPEND _executorch_cuda_arch_numeric "${_arch}") + else() + message( + STATUS + "executorch: leaving TORCH_CUDA_ARCH_LIST entry \"${_arch}\" to torch, since it names a " + "GPU family rather than a compute capability" + ) + endif() + endforeach() + # Left unset when nothing numeric remains, so torch's resolution is what + # decides. Setting it to an empty string would instead ask CMake to compile + # for no architecture at all. + if(_executorch_cuda_arch_numeric) + set(CMAKE_CUDA_ARCHITECTURES "${_executorch_cuda_arch_numeric}") + endif() + message( + STATUS + "CUDA architectures from TORCH_CUDA_ARCH_LIST: ${CMAKE_CUDA_ARCHITECTURES}" + ) +else() + # torch sets this to OFF on purpose and drives nvcc with its own gencode + # flags, so there is nothing to choose here and overriding it would replace a + # working set of architectures with one value. + # + # An empty value is a different case: nothing selected the architectures, and + # an empty list makes nvcc fall back to its built-in compute_50, which current + # CUDA rejects outright. Ask CMake to detect the builder's GPU instead, which + # is what a source build wants. + if("${CMAKE_CUDA_ARCHITECTURES}" STREQUAL "") + set(CMAKE_CUDA_ARCHITECTURES native) + endif() + message( + STATUS "CUDA architectures left to torch: ${CMAKE_CUDA_ARCHITECTURES}" + ) +endif() + if(NOT EXECUTORCH_BUILD_ROCM AND CMAKE_CUDA_COMPILER) enable_language(CUDA) endif() diff --git a/docs/source/using-executorch-cpp.md b/docs/source/using-executorch-cpp.md index 4f165b7bbf8..4b8191afa9f 100644 --- a/docs/source/using-executorch-cpp.md +++ b/docs/source/using-executorch-cpp.md @@ -259,6 +259,97 @@ stops the runtime. You should not need `LD_LIBRARY_PATH`. The shipped libraries record where their neighbours live, so they find each other once the program links against the installed package. +### Running on a GPU with the CUDA package + +The CUDA build is a separate package. Releases cover CUDA 12.6, 13.0 and 13.2, so pick the index +matching the CUDA version you have (`cu126`, `cu130` or `cu132`). For CUDA 12.6: + +``` +pip install executorch torch \ + --index-url https://download.pytorch.org/whl/cu126 \ + --extra-index-url https://pypi.org/simple +``` + +CUDA wheels are built for Python 3.10 through 3.13. On a newer Python there is no CUDA wheel to +install, so pip falls back to the CPU one. + +The second index is required: a bare `--index-url` replaces PyPI rather than adding to it, and some +dependencies are only on PyPI. The torch you install has to come from the same CUDA index, because +exporting a model for CUDA runs through torch. + +Everything above stays the same. Add the CUDA backend to both CMake lines: + +```cmake +find_package(executorch REQUIRED COMPONENTS kernels_optimized backend_cuda) + +target_link_libraries(app PRIVATE executorch::runtime + executorch::kernels_optimized + executorch::backend_cuda) +``` + +The model has to be exported for CUDA as well, on a machine with a GPU. That step also needs the +CUDA compiler (`nvcc`) on your `PATH`, because the backend compiles the model into GPU code ahead +of time. `pip install` does not provide it, so install the CUDA Toolkit for this step and check it +with `nvcc --version`. + +```python +# export_cuda.py, the same model as before with one line added +import torch +from executorch.exir import to_edge_transform_and_lower +from executorch.backends.cuda.cuda_partitioner import CudaPartitioner +from executorch.extension.export_util.utils import save_pte_program + +class Add(torch.nn.Module): + def forward(self, x, y): + return x + y + +example = (torch.ones(2, 2), torch.ones(2, 2)) +program = to_edge_transform_and_lower( + torch.export.export(Add(), example), partitioner=[CudaPartitioner([])] +).to_executorch() +save_pte_program(program, "model", ".") +``` + +`save_pte_program` is used instead of writing the buffer by hand because the CUDA backend puts its +model weights in a **separate data file** next to `model.pte`. The compiled GPU code stays +inside `model.pte`. Writing only the program file loses the weights, so the model then fails when it +runs. + +The backend chooses that file's name, so check what was written: + +``` +$ ls +aoti_cuda_blob.ptd model.pte +``` + +Load both from C++, passing the data file as the second argument: + +```cpp +Module module("model.pte", "aoti_cuda_blob.ptd"); +``` + +The CUDA backend is still experimental, so exporting prints a warning saying so. + +By default the runtime copies inputs to the GPU and results back, so your program keeps passing +ordinary CPU tensors and nothing else changes. + +One thing to check first, and the numbers matter. If a model fails with a message about no kernel +image being available for the device, your GPU is not one these packages were built for. The floor +is **compute capability 8.0**, which means an NVIDIA Ampere generation card or newer. + +Check your own GPU: + +``` +python -c 'import torch; print(torch.cuda.get_device_capability())' +``` + +A result below `(8, 0)` is not covered. Above the floor the answer depends on the package: each one +covers what the PyTorch build for the same platform and CUDA version covers, and the ARM packages +reach fewer cards in that range than the x86_64 ones, so a card at or above `(8, 0)` can still be +outside an ARM package. Note that `torch.cuda.get_arch_list()` is not the right check either: +PyTorch builds for a wider set at the bottom than these packages do, so a GPU can appear in that +list and still not be supported. + ### Building from source diff --git a/install_requirements.py b/install_requirements.py index 1aedcf6f0f8..648da1df243 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -7,6 +7,7 @@ import argparse import os +import platform import subprocess import sys @@ -45,7 +46,19 @@ def install_requirements(use_pytorch_nightly): # Determine the appropriate PyTorch URL based on CUDA delegate status torch_url = determine_torch_url(TORCH_URL_BASE) - torchao_url = determine_torch_url(TORCHAO_URL_BASE) + # torchao's CUDA channel publishes x86_64 only, so asking for a CUDA build makes the pin + # unsatisfiable on aarch64. Only that case is special-cased: falling back everywhere would + # change which torchao a CPU x86_64 install resolves, and the CUDA build is genuinely wanted + # where it exists. Nothing in the wheel links or bundles torchao; it is a quantization + # workflow dependency of the examples and tests. + if platform.machine().lower() in ("aarch64", "arm64"): + # The cpu channel specifically, not the index root. The root carries every variant, and a + # pin without a local segment admits all of them while ordering a local segment highest, + # so the xpu channel's pure python wheel would win on version before pip compares wheel + # tags, silently replacing the compiled aarch64 build. + torchao_url = f"{TORCHAO_URL_BASE}/cpu" + else: + torchao_url = determine_torch_url(TORCHAO_URL_BASE) # pip packages needed by exir. TORCH_PACKAGE = [