diff --git a/cuda_bindings/cuda/bindings/_lib/param_packer.h b/cuda_bindings/cuda/bindings/_lib/param_packer.h index 8d4833bb200..5242535685b 100644 --- a/cuda_bindings/cuda/bindings/_lib/param_packer.h +++ b/cuda_bindings/cuda/bindings/_lib/param_packer.h @@ -30,6 +30,31 @@ PyLong_AsInt(PyObject *obj) } #endif +// PyLong_AsInt entered the public/stable CPython API in 3.13. cuda.bindings +// supports Python 3.10+, so provide a file-local backport for older builds. +// This is a copy of the CPython implementation; it is `static` (unlike the +// original) because this header is compiled into every extension module that +// includes it, mirroring the other helpers below. +#if PY_VERSION_HEX < 0x030D0000 +static int +PyLong_AsInt(PyObject *obj) +{ + int overflow; + long result = PyLong_AsLongAndOverflow(obj, &overflow); + if (overflow || result > INT_MAX || result < INT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)result; +} +#endif + +// Strong reference to the ctypes module, acquired once at import in +// init_param_packer() and intentionally never released. This ref is +// load-bearing: the ctypes_c_* pointers below are *borrowed* references into +// the ctypes module dict, and stay valid only while this strong ref keeps the +// module (and therefore its type objects) alive. Do not Py_DECREF it. static PyObject* ctypes_module = nullptr; static PyTypeObject* ctypes_c_char = nullptr; @@ -50,6 +75,10 @@ static PyTypeObject* ctypes_c_float = nullptr; static PyTypeObject* ctypes_c_double = nullptr; static PyTypeObject* ctypes_c_void_p = nullptr; +// Import ctypes and cache pointers to its scalar type objects. May throw +// std::runtime_error (translated to a Python exception via the `except +` +// declaration on init_param_packer in param_packer.pxd). Called exactly once, +// from init_param_packer() at module import while single-threaded. static void fetch_ctypes() { ctypes_module = PyImport_ImportModule("ctypes"); @@ -171,17 +200,45 @@ static void populate_feeders(PyTypeObject* target_t, PyTypeObject* source_t) } } +// Initialize all shared state once, at module import, while single-threaded +// (called as a bare module-level statement from utils.pxi -- mirroring the +// _resource_handles.pyx pattern of calling initialize_deferred_cleanup() at +// import). This fetches the ctypes type pointers and pre-builds the *entire* +// feeder table, so the hot feed() path below is afterwards a pure read of +// never-mutated global state. That is what makes feed() safe to call +// concurrently from multiple threads under free-threading (Py_MOD_GIL_NOT_USED) +// without any lock: doing the work lazily inside feed() would race the map and +// the ctypes lazy-init across threads launching distinct kernels. +// +// May throw (e.g. if `import ctypes` fails); declared `except +` in the pxd so +// the failure surfaces as a Python exception during module import rather than +// std::terminate. +static void init_param_packer() +{ + if (ctypes_module != nullptr) + return; // defensive: module import already runs exactly once + fetch_ctypes(); + // Pre-build every feeder the old lazy path could ever have created. This is + // a fixed, finite set of (target, source) type pairs, so a table built here + // is identical to the one feed() used to build on demand; any unmatched + // pair still falls through to feed() returning 0 -> the ctype() fallback in + // utils.pxi. After this, m_feeders is never mutated again. + populate_feeders(ctypes_c_int, &PyLong_Type); + populate_feeders(ctypes_c_bool, &PyBool_Type); + populate_feeders(ctypes_c_byte, &PyLong_Type); + populate_feeders(ctypes_c_double, &PyFloat_Type); + populate_feeders(ctypes_c_float, &PyFloat_Type); + populate_feeders(ctypes_c_longlong, &PyLong_Type); +} + +// Hot path. Read-only lookup on the import-time-populated feeder table; no +// mutation, no allocation, no throwing C-API translation -> effectively +// noexcept and safe under concurrent, GIL-free calls. Returns 0 for any +// unhandled (target, source) pair so the caller applies its ctype() fallback. static int feed(void* ptr, PyObject* value, PyObject* type) { PyTypeObject* pto = (PyTypeObject*)type; - if (ctypes_c_int == nullptr) - fetch_ctypes(); auto found = m_feeders.find({pto,value->ob_type}); - if (found == m_feeders.end()) - { - populate_feeders(pto, value->ob_type); - found = m_feeders.find({pto,value->ob_type}); - } if (found != m_feeders.end()) { return found->second(ptr, value); diff --git a/cuda_bindings/cuda/bindings/_lib/utils.pxi b/cuda_bindings/cuda/bindings/_lib/utils.pxi index 2796f910798..bbb5e4a1b9e 100644 --- a/cuda_bindings/cuda/bindings/_lib/utils.pxi +++ b/cuda_bindings/cuda/bindings/_lib/utils.pxi @@ -11,6 +11,11 @@ import ctypes as _ctypes cimport cuda.bindings.cydriver as cydriver cimport cuda.bindings._lib.param_packer as param_packer +# Initialize the param_packer feeder table + ctypes pointers once, at import, +# while single-threaded, so param_packer.feed() is a lock-free read-only lookup +# on the kernel-launch hot path under free-threading. See param_packer.h. +param_packer.init_param_packer() + cdef void* _callocWrapper(length, size): cdef void* out = calloc(length, size) if out is NULL: diff --git a/cuda_core/cuda/core/_cpp/resource_handles.cpp b/cuda_core/cuda/core/_cpp/resource_handles.cpp index 2102d36f22f..a1d78cce9b8 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.cpp +++ b/cuda_core/cuda/core/_cpp/resource_handles.cpp @@ -1060,10 +1060,16 @@ DevicePtrHandle deviceptr_create_mapped_graphics( // MemoryResource-owned Device Pointer Handles // ============================================================================ -static MRDeallocCallback mr_dealloc_cb = nullptr; +// Registered exactly once at import from _buffer.pyx (module-level statement, +// runs single-threaded before any MR-backed device pointer exists), so a plain +// pointer would already be safe. Made atomic as free-threading hardening: it is +// read/called from shared_ptr deleters on arbitrary threads, and the +// acquire/release pair gives a well-formed happens-before between the import +// registration and every later deleter read. +static std::atomic mr_dealloc_cb{nullptr}; void register_mr_dealloc_callback(MRDeallocCallback cb) { - mr_dealloc_cb = cb; + mr_dealloc_cb.store(cb, std::memory_order_release); } DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr) { @@ -1081,8 +1087,9 @@ DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* [mr, size](DevicePtrBox* b) { GILAcquireGuard gil; if (gil.acquired()) { - if (mr_dealloc_cb) { - mr_dealloc_cb(mr, b->resource, size, b->h_stream); + if (MRDeallocCallback cb = + mr_dealloc_cb.load(std::memory_order_acquire)) { + cb(mr, b->resource, size, b->h_stream); } Py_DECREF(mr); } diff --git a/cuda_core/cuda/core/graph/_graph_node.pyx b/cuda_core/cuda/core/graph/_graph_node.pyx index 38bc6f955e6..11e93e0b062 100644 --- a/cuda_core/cuda/core/graph/_graph_node.pyx +++ b/cuda_core/cuda/core/graph/_graph_node.pyx @@ -755,6 +755,15 @@ cdef inline KernelNode GN_launch(GraphNode self, LaunchConfig conf, Kernel ker, HANDLE_RETURN(graph_prepare_attachment( h_graph, kernel_owner, args_owner, &prepared)) + # Keep the kernel and argument objects alive because CUDA copies argument + # values but does not retain the resources they reference. + kernel_owner = ker._h_kernel + kernel_args = ker_args.kernel_args + if kernel_args is not None: + args_owner = make_opaque_py(kernel_args) + HANDLE_RETURN(graph_prepare_attachment( + h_graph, kernel_owner, args_owner, &prepared)) + with nogil: HANDLE_RETURN(cydriver.cuGraphAddKernelNode( &new_node, as_cu(h_graph), deps, num_deps, &node_params)) @@ -961,6 +970,10 @@ cdef inline MemsetNode GN_memset( height=height, ) + if dst_owner: + HANDLE_RETURN(graph_prepare_attachment( + h_graph, dst_owner, OpaqueHandle(), &prepared)) + if dst_owner: HANDLE_RETURN(graph_prepare_attachment( h_graph, dst_owner, OpaqueHandle(), &prepared))