diff --git a/.github/workflows/ci-pixi-source-test.yml b/.github/workflows/ci-pixi-source-test.yml index cdae1b0cf26..c821454def8 100644 --- a/.github/workflows/ci-pixi-source-test.yml +++ b/.github/workflows/ci-pixi-source-test.yml @@ -15,6 +15,8 @@ # - build-smoke (PRs): CPU-only. Source-builds bindings + core, imports them, # builds the cython test extensions and checks placement. Catches the # compile / ABI / .so-placement regressions WITHOUT a GPU. +# - build-identity-roundtrip (PRs): CPU-only. cu13 -> cu12 -> cu13 in one +# checkout, so build artifacts cannot be reused across CUDA majors. # - full-test (nightly + manual): GPU runner, full `pixi run test`. name: "CI: pixi run test (source build)" @@ -108,12 +110,69 @@ jobs: done echo "cython test extensions placed correctly" + # ── PR guard: build artifacts must be CUDA-major aware ── + # + # Neither Cython nor setuptools tracks the CUDA major in its own up-to-date + # check: Cython does not hash `compile_time_env`, and an editable install's + # .so is named by the Python ABI tag alone. Before build_hooks keyed them, + # a cu13 build followed by a cu12 build in the same checkout failed while + # compiling cu13-generated C++ against CUDA 12 headers. + # + # The round trip (not just cu13 -> cu12) is what catches the second half: + # coming back to cu13 must not silently reuse the cu12 extension. + # + # cuda_core only: cuda_bindings cannot be source-built in its cu12 + # environment at all, for reasons unrelated to stale artifacts. + build-identity-roundtrip: + name: "cu13 -> cu12 -> cu13 round trip (linux-64, CPU)" + if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout ${{ github.event.repository.name }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + # Full history + tags so setuptools-scm derives the real (13.x) + # package version; a shallow checkout yields 0.1.dev1, which trips + # cuda.core's "cuda.bindings 12.x or 13.x must be installed" guard. + fetch-depth: 0 + + - name: Setup pixi + # Pinned to a commit SHA; install logic lives in the action and is + # auditable/pinned (vs. a curl|bash of an unverified installer). + uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 + with: + pixi-version: ${{ env.PIXI_VERSION }} + run-install: false + + - name: Build cu13, then cu12, then cu13 again in one checkout + run: | + for cuda_env in cu13 cu12 cu13; do + echo "::group::${cuda_env}" + pixi run --manifest-path cuda_core -e "${cuda_env}" \ + python -c "import cuda.core; print('core import OK')" + echo "::endgroup::" + done + # The last build was cu13, and each major must have kept its own + # generated sources rather than overwriting the other's. + stamp=$(cat cuda_core/build/.build-cuda-major) + if [ "${stamp}" != "13" ]; then + echo "::error::build stamp is '${stamp}', expected 13" + exit 1 + fi + for major in cu12 cu13; do + if [ ! -d "cuda_core/build/cython/${major}" ]; then + echo "::error::no ${major} generated-source directory" + exit 1 + fi + done + # ── Nightly: full `pixi run test` on a GPU runner ── full-test: name: "pixi run test (${{ inputs.cuda-env || 'cu13' }}, linux-64, GPU)" if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.repository_owner == 'nvidia' }} runs-on: "linux-amd64-gpu-l4-latest-1" # same label scheme as test-wheel-linux.yml - timeout-minutes: 90 + timeout-minutes: 60 container: options: -u root --security-opt seccomp=unconfined --shm-size 16g image: ubuntu:24.04 diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index 4aec4981c53..b9663d586c0 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -119,6 +119,51 @@ def _determine_cuda_major_version() -> str: # used later by setup() _extensions = None +# Records the CUDA major of the last completed build, so setup.py can force +# build_ext when it changes. Written by record_build_major(). +_BUILD_MAJOR_STAMP = os.path.join("build", ".build-cuda-major") + +force_build_ext = False + + +def _check_build_major() -> str: + """Return the CUDA major to key build artifacts by, and set force_build_ext. + + Cython's up-to-date check does not hash ``compile_time_env``, so generated + sources for one CUDA major would otherwise be reused for another. Keying + the generated-source directory fixes that, but not the compiled extension: + in an editable install it lands in the source tree under a name keyed by + the Python ABI tag alone, with nowhere to record the CUDA major. On a + cu12 -> cu13 -> cu12 round trip build_ext would find the older cu12 + generated source next to the newer cu13 .so and skip the rebuild, so the + major is also stamped and build_ext forced whenever it changes. + """ + global force_build_ext + + cuda_major = _determine_cuda_major_version() + try: + with open(_BUILD_MAJOR_STAMP, encoding="utf-8") as f: + previous = f.read().strip() + except FileNotFoundError: + previous = None + + if previous is not None and previous != cuda_major: + print(f"CUDA major changed ({previous} -> {cuda_major}); forcing a full rebuild") + force_build_ext = True + + return cuda_major + + +def record_build_major() -> None: + """Stamp the CUDA major of the build that just completed. + + setup.py calls this after build_ext succeeds, so that a build which failed + partway through does not claim outputs it never produced. + """ + os.makedirs(os.path.dirname(_BUILD_MAJOR_STAMP), exist_ok=True) + with open(_BUILD_MAJOR_STAMP, "w", encoding="utf-8") as f: + f.write(_determine_cuda_major_version() + "\n") + def _build_cuda_core(debug=False): # Customizing the build hooks is needed because we must defer cythonization until cuda-bindings, @@ -128,6 +173,8 @@ def _build_cuda_core(debug=False): # This function populates "_extensions". global _extensions + cuda_major = _check_build_major() + # Add cuda-bindings to sys.path so Cython can find .pxd files # This is needed for editable installs where meta path finders don't work for Cython # We need to add the directory containing the 'cuda' package so Cython can resolve @@ -211,7 +258,7 @@ def get_sources(mod_name): ) nthreads = int(os.environ.get("CUDA_PYTHON_PARALLEL_LEVEL", os.cpu_count() // 2)) - compile_time_env = {"CUDA_CORE_BUILD_MAJOR": int(_determine_cuda_major_version())} + compile_time_env = {"CUDA_CORE_BUILD_MAJOR": int(cuda_major)} compiler_directives = {"embedsignature": True, "warn.deprecated.IF": False, "freethreading_compatible": True} _CythonOptions.warning_errors = True if COMPILE_FOR_COVERAGE: @@ -220,7 +267,9 @@ def get_sources(mod_name): ext_modules, verbose=True, language_level=3, - build_dir="." if COMPILE_FOR_COVERAGE else "build/cython", + # CUDA_PYTHON_COVERAGE deliberately generates in-tree so the sources can + # be packaged; every other build gets its own per-configuration cache. + build_dir="." if COMPILE_FOR_COVERAGE else f"build/cython/cu{cuda_major}", nthreads=nthreads, compiler_directives=compiler_directives, compile_time_env=compile_time_env, diff --git a/cuda_core/setup.py b/cuda_core/setup.py index e0d745b1f21..d72e708f522 100644 --- a/cuda_core/setup.py +++ b/cuda_core/setup.py @@ -51,6 +51,13 @@ def _build_aoti_shim_lib(compiler, plat_name): class build_ext(_build_ext): # noqa: N801 + def finalize_options(self): + super().finalize_options() + # A cu13 .so in the source tree looks perfectly fresh to a cu12 build; + # see build_hooks._check_build_major(). + if build_hooks.force_build_ext: + self.force = True + def _configure_windows_tensor_bridge(self): if os.name != "nt" or getattr(self.compiler, "compiler_type", None) != "msvc": return @@ -74,6 +81,7 @@ def build_extensions(self): self.parallel = nthreads self._configure_windows_tensor_bridge() super().build_extensions() + build_hooks.record_build_major() class build_py(_build_py): # noqa: N801 diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index 121ed1be053..24e0ff4fafc 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -135,3 +135,47 @@ def test_missing_cuda_path_raises_error(self): pytest.raises(RuntimeError, match="CUDA_PATH or CUDA_HOME"), ): build_hooks._determine_cuda_major_version() + + +@pytest.fixture +def build_tree(tmp_path, monkeypatch): + """Run the stamp helpers against a scratch source tree. + + The stamp path is relative because PEP 517 hooks always run with the + package directory as the working directory. + """ + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(build_hooks, "force_build_ext", False) + build_hooks._get_cuda_path.cache_clear() + build_hooks._determine_cuda_major_version.cache_clear() + get_cuda_path_or_home.cache_clear() + monkeypatch.setenv("CUDA_CORE_BUILD_MAJOR", "13") + return tmp_path + + +def _write_stamp(build_tree, cuda_major): + stamp = build_tree / build_hooks._BUILD_MAJOR_STAMP + stamp.parent.mkdir(parents=True, exist_ok=True) + stamp.write_text(cuda_major + "\n") + + +class TestBuildMajorStamp: + """Tests for _check_build_major() and record_build_major().""" + + def test_first_build_does_not_force(self, build_tree): + assert build_hooks._check_build_major() == "13" + assert build_hooks.force_build_ext is False + + def test_same_major_does_not_force(self, build_tree): + _write_stamp(build_tree, "13") + assert build_hooks._check_build_major() == "13" + assert build_hooks.force_build_ext is False + + def test_changed_major_forces_rebuild(self, build_tree): + _write_stamp(build_tree, "12") + assert build_hooks._check_build_major() == "13" + assert build_hooks.force_build_ext is True + + def test_record_writes_stamp(self, build_tree): + build_hooks.record_build_major() + assert (build_tree / build_hooks._BUILD_MAJOR_STAMP).read_text().strip() == "13"