build: make Cython/extension artifacts CUDA-major aware

.github/workflows/ci-pixi-source-test.yml

@@ -15,6 +15,8 @@
 #   - build-smoke (PRs): CPU-only. Source-builds bindings + core, imports them,
 #     builds the cython test extensions and checks placement. Catches the
 #     compile / ABI / .so-placement regressions WITHOUT a GPU.
+#   - build-identity-roundtrip (PRs): CPU-only. cu13 -> cu12 -> cu13 in one
+#     checkout, so build artifacts cannot be reused across CUDA majors.
 #   - full-test (nightly + manual): GPU runner, full `pixi run test`.
 
 name: "CI: pixi run test (source build)"
@@ -108,12 +110,69 @@ jobs:
           done
           echo "cython test extensions placed correctly"
 
+  # ── PR guard: build artifacts must be CUDA-major aware ──
+  #
+  # Neither Cython nor setuptools tracks the CUDA major in its own up-to-date
+  # check: Cython does not hash `compile_time_env`, and an editable install's
+  # .so is named by the Python ABI tag alone. Before build_hooks keyed them,
+  # a cu13 build followed by a cu12 build in the same checkout failed while
+  # compiling cu13-generated C++ against CUDA 12 headers.
+  #
+  # The round trip (not just cu13 -> cu12) is what catches the second half:
+  # coming back to cu13 must not silently reuse the cu12 extension.
+  #
+  # cuda_core only: cuda_bindings cannot be source-built in its cu12
+  # environment at all, for reasons unrelated to stale artifacts.
+  build-identity-roundtrip:
+    name: "cu13 -> cu12 -> cu13 round trip (linux-64, CPU)"
+    if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          # Full history + tags so setuptools-scm derives the real (13.x)
+          # package version; a shallow checkout yields 0.1.dev1, which trips
+          # cuda.core's "cuda.bindings 12.x or 13.x must be installed" guard.
+          fetch-depth: 0
+
+      - name: Setup pixi
+        # Pinned to a commit SHA; install logic lives in the action and is
+        # auditable/pinned (vs. a curl|bash of an unverified installer).
+        uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
+        with:
+          pixi-version: ${{ env.PIXI_VERSION }}
+          run-install: false
+
+      - name: Build cu13, then cu12, then cu13 again in one checkout
+        run: |
+          for cuda_env in cu13 cu12 cu13; do
+            echo "::group::${cuda_env}"
+            pixi run --manifest-path cuda_core -e "${cuda_env}" \
+              python -c "import cuda.core; print('core import OK')"
+            echo "::endgroup::"
+          done
+          # The last build was cu13, and each major must have kept its own
+          # generated sources rather than overwriting the other's.
+          stamp=$(cat cuda_core/build/.build-cuda-major)
+          if [ "${stamp}" != "13" ]; then
+            echo "::error::build stamp is '${stamp}', expected 13"
+            exit 1
+          fi
+          for major in cu12 cu13; do
+            if [ ! -d "cuda_core/build/cython/${major}" ]; then
+              echo "::error::no ${major} generated-source directory"
+              exit 1
+            fi
+          done
+
   # ── Nightly: full `pixi run test` on a GPU runner ──
   full-test:
     name: "pixi run test (${{ inputs.cuda-env || 'cu13' }}, linux-64, GPU)"
     if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.repository_owner == 'nvidia' }}
     runs-on: "linux-amd64-gpu-l4-latest-1" # same label scheme as test-wheel-linux.yml
-    timeout-minutes: 90
+    timeout-minutes: 60
     container:
       options: -u root --security-opt seccomp=unconfined --shm-size 16g
       image: ubuntu:24.04

cuda_core/build_hooks.py

@@ -119,6 +119,51 @@ def _determine_cuda_major_version() -> str:
 # used later by setup()
 _extensions = None
 
+# Records the CUDA major of the last completed build, so setup.py can force
+# build_ext when it changes. Written by record_build_major().
+_BUILD_MAJOR_STAMP = os.path.join("build", ".build-cuda-major")
+
+force_build_ext = False
+
+
+def _check_build_major() -> str:
+    """Return the CUDA major to key build artifacts by, and set force_build_ext.
+
+    Cython's up-to-date check does not hash ``compile_time_env``, so generated
+    sources for one CUDA major would otherwise be reused for another. Keying
+    the generated-source directory fixes that, but not the compiled extension:
+    in an editable install it lands in the source tree under a name keyed by
+    the Python ABI tag alone, with nowhere to record the CUDA major. On a
+    cu12 -> cu13 -> cu12 round trip build_ext would find the older cu12
+    generated source next to the newer cu13 .so and skip the rebuild, so the
+    major is also stamped and build_ext forced whenever it changes.
+    """
+    global force_build_ext
+
+    cuda_major = _determine_cuda_major_version()
+    try:
+        with open(_BUILD_MAJOR_STAMP, encoding="utf-8") as f:
+            previous = f.read().strip()
+    except FileNotFoundError:
+        previous = None
+
+    if previous is not None and previous != cuda_major:
+        print(f"CUDA major changed ({previous} -> {cuda_major}); forcing a full rebuild")
+        force_build_ext = True
+
+    return cuda_major
+
+
+def record_build_major() -> None:
+    """Stamp the CUDA major of the build that just completed.
+
+    setup.py calls this after build_ext succeeds, so that a build which failed
+    partway through does not claim outputs it never produced.
+    """
+    os.makedirs(os.path.dirname(_BUILD_MAJOR_STAMP), exist_ok=True)
+    with open(_BUILD_MAJOR_STAMP, "w", encoding="utf-8") as f:
+        f.write(_determine_cuda_major_version() + "\n")
+
 
 def _build_cuda_core(debug=False):
     # Customizing the build hooks is needed because we must defer cythonization until cuda-bindings,
@@ -128,6 +173,8 @@ def _build_cuda_core(debug=False):
     # This function populates "_extensions".
     global _extensions
 
+    cuda_major = _check_build_major()
+
     # Add cuda-bindings to sys.path so Cython can find .pxd files
     # This is needed for editable installs where meta path finders don't work for Cython
     # We need to add the directory containing the 'cuda' package so Cython can resolve
@@ -211,7 +258,7 @@ def get_sources(mod_name):
     )
 
     nthreads = int(os.environ.get("CUDA_PYTHON_PARALLEL_LEVEL", os.cpu_count() // 2))
-    compile_time_env = {"CUDA_CORE_BUILD_MAJOR": int(_determine_cuda_major_version())}
+    compile_time_env = {"CUDA_CORE_BUILD_MAJOR": int(cuda_major)}
     compiler_directives = {"embedsignature": True, "warn.deprecated.IF": False, "freethreading_compatible": True}
     _CythonOptions.warning_errors = True
     if COMPILE_FOR_COVERAGE:
@@ -220,7 +267,9 @@ def get_sources(mod_name):
         ext_modules,
         verbose=True,
         language_level=3,
-        build_dir="." if COMPILE_FOR_COVERAGE else "build/cython",
+        # CUDA_PYTHON_COVERAGE deliberately generates in-tree so the sources can
+        # be packaged; every other build gets its own per-configuration cache.
+        build_dir="." if COMPILE_FOR_COVERAGE else f"build/cython/cu{cuda_major}",
         nthreads=nthreads,
         compiler_directives=compiler_directives,
         compile_time_env=compile_time_env,

cuda_core/setup.py

@@ -51,6 +51,13 @@ def _build_aoti_shim_lib(compiler, plat_name):
 
 
 class build_ext(_build_ext):  # noqa: N801
+    def finalize_options(self):
+        super().finalize_options()
+        # A cu13 .so in the source tree looks perfectly fresh to a cu12 build;
+        # see build_hooks._check_build_major().
+        if build_hooks.force_build_ext:
+            self.force = True
+
     def _configure_windows_tensor_bridge(self):
         if os.name != "nt" or getattr(self.compiler, "compiler_type", None) != "msvc":
             return
@@ -74,6 +81,7 @@ def build_extensions(self):
         self.parallel = nthreads
         self._configure_windows_tensor_bridge()
         super().build_extensions()
+        build_hooks.record_build_major()
 
 
 class build_py(_build_py):  # noqa: N801

cuda_core/tests/test_build_hooks.py

@@ -135,3 +135,47 @@ def test_missing_cuda_path_raises_error(self):
             pytest.raises(RuntimeError, match="CUDA_PATH or CUDA_HOME"),
         ):
             build_hooks._determine_cuda_major_version()
+
+
+@pytest.fixture
+def build_tree(tmp_path, monkeypatch):
+    """Run the stamp helpers against a scratch source tree.
+
+    The stamp path is relative because PEP 517 hooks always run with the
+    package directory as the working directory.
+    """
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(build_hooks, "force_build_ext", False)
+    build_hooks._get_cuda_path.cache_clear()
+    build_hooks._determine_cuda_major_version.cache_clear()
+    get_cuda_path_or_home.cache_clear()
+    monkeypatch.setenv("CUDA_CORE_BUILD_MAJOR", "13")
+    return tmp_path
+
+
+def _write_stamp(build_tree, cuda_major):
+    stamp = build_tree / build_hooks._BUILD_MAJOR_STAMP
+    stamp.parent.mkdir(parents=True, exist_ok=True)
+    stamp.write_text(cuda_major + "\n")
+
+
+class TestBuildMajorStamp:
+    """Tests for _check_build_major() and record_build_major()."""
+
+    def test_first_build_does_not_force(self, build_tree):
+        assert build_hooks._check_build_major() == "13"
+        assert build_hooks.force_build_ext is False
+
+    def test_same_major_does_not_force(self, build_tree):
+        _write_stamp(build_tree, "13")
+        assert build_hooks._check_build_major() == "13"
+        assert build_hooks.force_build_ext is False
+
+    def test_changed_major_forces_rebuild(self, build_tree):
+        _write_stamp(build_tree, "12")
+        assert build_hooks._check_build_major() == "13"
+        assert build_hooks.force_build_ext is True
+
+    def test_record_writes_stamp(self, build_tree):
+        build_hooks.record_build_major()
+        assert (build_tree / build_hooks._BUILD_MAJOR_STAMP).read_text().strip() == "13"