Name the Python extension after what it contains

.ci/scripts/setup-windows.ps1

@@ -21,8 +21,8 @@ $env:CMAKE_ARGS = "$env:CMAKE_ARGS -DCMAKE_CXX_STANDARD=20"
 # The Windows CI image ships CUDA toolkits on PATH, so install_executorch
 # (setup.py) auto-enables EXECUTORCH_BUILD_CUDA whenever the detected nvcc
 # version is in SUPPORTED_CUDA_VERSIONS. CPU-only jobs install CPU torch, so a
-# CUDA build of _portable_lib then fails to load its CUDA DLLs at import time
-# ("DLL load failed while importing _portable_lib"). Force a CPU-only build
+# CUDA build of _C then fails to load its CUDA DLLs at import time
+# ("DLL load failed while importing _C"). Force a CPU-only build
 # when the caller asks for it.
 if ($cpuOnly -eq 'true') {
     $env:CMAKE_ARGS = "$env:CMAKE_ARGS -DEXECUTORCH_BUILD_CUDA=OFF"

.ci/scripts/wheel/pre_build_script.sh

@@ -128,8 +128,8 @@ if [[ $UNAME_S == *"MINGW"* || $UNAME_S == *"MSYS"* ]]; then
     # Windows wheels are CPU-only (build-wheels-windows.yml sets
     # with-cuda: disabled), but the Windows CI image ships a CUDA toolkit on
     # PATH, which makes setup.py auto-enable EXECUTORCH_BUILD_CUDA. That bakes a
-    # CUDA _portable_lib into the CPU wheel, which then fails its DLL load in the
-    # smoke test ("DLL load failed while importing _portable_lib"). Force a
+    # CUDA _C into the CPU wheel, which then fails its DLL load in the
+    # smoke test ("DLL load failed while importing _C"). Force a
     # CPU-only build.
     export CMAKE_ARGS="${CMAKE_ARGS:-} -DEXECUTORCH_BUILD_CUDA=OFF"
     echo "CMAKE_ARGS=${CMAKE_ARGS}" >> "${GITHUB_ENV}"

.ci/scripts/wheel/test_base.py

@@ -59,16 +59,16 @@ def test_cmsis_nn_install():
 
 
 def run_tests(model_tests: List[ModelTest]) -> None:
-    # Test that we can import the portable_lib module - verifies RPATH is correct
-    print("Testing portable_lib import...")
+    # Test that we can import the _C module - verifies RPATH is correct
+    print("Testing _C import...")
     try:
-        from executorch.extension.pybindings._portable_lib import (  # noqa: F401
+        from executorch.extension.pybindings._C import (  # noqa: F401
             _load_for_executorch,
         )
 
-        print("✓ Successfully imported _load_for_executorch from portable_lib")
+        print("✓ Successfully imported _load_for_executorch from _C")
     except ImportError as e:
-        print(f"✗ Failed to import portable_lib: {e}")
+        print(f"✗ Failed to import _C: {e}")
         raise
 
     # Why are we doing this envvar shenanigans? Since we build the testers, which

.ci/scripts/wheel/test_shared_libraries.py

@@ -234,6 +234,20 @@ def _declared_requirements() -> set:
     return names
 
 
+def _cmake_version() -> tuple[int, ...] | None:
+    """The running cmake's version, or None when it cannot be determined."""
+    cmake = _tool("cmake")
+    if cmake is None:
+        return None
+    probe = subprocess.run([cmake, "--version"], capture_output=True, text=True)
+    if probe.returncode != 0:
+        return None
+    match = re.search(r"cmake version (\d+)\.(\d+)", probe.stdout)
+    # Unparseable output reads as unknown rather than as old, so a future format change
+    # runs the real check instead of silently skipping it.
+    return tuple(int(part) for part in match.groups()) if match else None
+
+
 def _installed_package_dir() -> Path:
     """The installed executorch package, never the source checkout.
 
@@ -675,7 +689,20 @@ def _defines_symbol(library: Path, symbol: str) -> bool:
         # "defines nothing" would let a duplicate pass. The ELF magic bytes tell them apart
         # without depending on the reader's wording.
         with library.open("rb") as handle:
-            is_object_file = handle.read(4) == b"\x7fELF"
+            magic = handle.read(4)
+        # Mach-O too, not only ELF. Testing for the ELF magic alone made every macOS
+        # library read as "not an object file", so the assert below never fired there and
+        # an unreadable dylib was scored as defining nothing, which is the case it exists
+        # to catch. 64 and 32 bit, thin and fat, both byte orders.
+        is_object_file = magic in (
+            b"\x7fELF",
+            b"\xcf\xfa\xed\xfe",
+            b"\xce\xfa\xed\xfe",
+            b"\xfe\xed\xfa\xcf",
+            b"\xfe\xed\xfa\xce",
+            b"\xca\xfe\xba\xbe",
+            b"\xbe\xba\xfe\xca",
+        )
         assert not is_object_file, (
             f"nm could not read {library.name}, which is a shipped object file, so the symbol "
             f"checks cannot be trusted: {result.stderr.strip()[:200]}"
@@ -1116,15 +1143,26 @@ def test_python_extensions_import() -> None:
 
 
 _CUSTOM_OP_CMAKE = """\
-cmake_minimum_required(VERSION 3.24)
+# 3.28, not the 3.24 floor this package supports, because the imported targets this
+# project links are deliberately not created before that version.
+cmake_minimum_required(VERSION 3.28)
 project(custom_op_check CXX)
 
 find_package(executorch REQUIRED)
 
+# An unknown name here would be passed to the linker as a plain library rather
+# than reported, so the include directories would silently not arrive and the
+# build would fail later on a missing header.
+if(NOT TARGET executorch::runtime)
+  message(FATAL_ERROR "cannot link a target that does not exist: executorch::runtime")
+endif()
+
 add_library(custom_op_check SHARED custom_op.cpp)
-# The legacy contract: a custom-op library links the shipped Python extension,
-# which owns the operator registry it registers into.
-target_link_libraries(custom_op_check PRIVATE _portable_lib)
+# The runtime owns the operator registry. It used to live in the Python extension, so a
+# custom-op library linked that; since the split it is in libexecutorch, and linking the
+# extension from C++ cannot work because its CPython symbols only resolve inside an
+# interpreter. Linking the runtime reaches the same registry singleton.
+target_link_libraries(custom_op_check PRIVATE executorch::runtime)
 # The runtime headers include c10 headers, which belong to torch rather than to
 # this wheel, so an out-of-tree operator project supplies them the same way it
 # supplies torch itself. The package config does not and should not ship them.
@@ -1191,7 +1229,13 @@ def _dyld_load_failure(library: Path, *, with_torch: bool) -> str:
     )
     if result.returncode == 0:
         return ""
-    return (result.stdout + result.stderr).strip()
+    message = (result.stdout + result.stderr).strip()
+    # A signal leaves the message empty, and an empty return here reads as a clean load.
+    # The kernel killing the process during loading is the case an invalid signature
+    # produces, so it has to be named rather than dropped.
+    if not message and result.returncode < 0:
+        return f"the loader was killed by signal {-result.returncode}"
+    return message
 
 
 def _dyld_missing_names(message: str) -> list[str]:
@@ -1582,9 +1626,19 @@ def test_custom_op_compiles(work_dir: Path) -> None:
     if _tool("cmake") is None:
         print("- cmake unavailable, skipping the custom op check")
         return
+    # The project below requires 3.28, where the imported targets it links start existing.
+    # An older cmake refuses to configure at all, which would read as the wheel being at
+    # fault, so it is reported as the version skip it is.
+    cmake_version = _cmake_version()
+    if cmake_version is not None and cmake_version < (3, 28):
+        print(
+            f"- cmake {'.'.join(str(p) for p in cmake_version)} is older than the 3.28 "
+            "these targets need, skipping the custom op check"
+        )
+        return
 
     package_dir = _installed_package_dir()
-    if not list(package_dir.glob("extension/pybindings/_portable_lib*")):
+    if not list(package_dir.glob("extension/pybindings/_C*")):
         print("- the wheel ships no Python extension, skipping the custom op check")
         return
 
@@ -2046,10 +2100,8 @@ def test_extension_contains_no_component() -> None:
     assert _tool("nm") is not None, "nm is required to inspect the wheel"
 
     package_dir = _installed_package_dir()
-    extensions = sorted(
-        (package_dir / "extension" / "pybindings").glob("_portable_lib.*.so")
-    )
-    assert len(extensions) == 1, f"expected one _portable_lib, found {extensions}"
+    extensions = sorted((package_dir / "extension" / "pybindings").glob("_C.*.so"))
+    assert len(extensions) == 1, f"expected one _C, found {extensions}"
     extension = extensions[0]
 
     lib_dir = package_dir / "lib"

.github/workflows/lint.yml

@@ -54,7 +54,7 @@ jobs:
 
       - name: Generate mypy stubs for C++ bindings
         run: |
-          cp extension/pybindings/pybindings.pyi extension/pybindings/_portable_lib.pyi
+          cp extension/pybindings/pybindings.pyi extension/pybindings/_C.pyi
 
       - name: Run mypy
         run: |

CMakeLists.txt

@@ -486,8 +486,8 @@ if(EXECUTORCH_BUILD_PTHREADPOOL)
   if(APPLE)
     # Use hidden visibility for pthreadpool on Apple platforms to avoid issues
     # with pthreadpool symbols from libtorch_cpu taking precedence over the ones
-    # from the pthreadpool library statically linked in _portable_lib. The
-    # pthreadpool public APIs are marked as weak by default on some Apple
+    # from the pthreadpool library statically linked in the Python extension.
+    # The pthreadpool public APIs are marked as weak by default on some Apple
     # platforms, so setting to hidden visibility works around this by not
     # putting the symbol in the indirection table. See
     # https://github.com/pytorch/executorch/issues/14321 for more details.
@@ -1257,24 +1257,21 @@ if(EXECUTORCH_BUILD_PYBIND)
     list(APPEND _dep_libs aoti_common)
   endif()
 
-  # RPATH for _portable_lib.so. It sits in
+  # RPATH for the Python extension. It sits in
   # <site-packages>/executorch/extension/pybindings, so torch is three levels up
-  # and the wheel's own lib/ directory is two. Mach-O spells the loader relative
-  # token differently and takes a list rather than a colon joined string, so
-  # both differ here while the layout reasoning does not.
-  if(APPLE)
-    set(_portable_lib_origin "@loader_path")
-    set(_portable_lib_rpath_separator ";")
-  else()
-    set(_portable_lib_origin "$ORIGIN")
-    set(_portable_lib_rpath_separator ":")
-  endif()
-  set(_portable_lib_rpath "${_portable_lib_origin}/../../../torch/lib")
+  # and the wheel's own lib/ directory is two.
+  #
+  # ELF spelling. The Mach-O branch below takes only the Torch hop from this
+  # list, because nothing else records it there. The other entries reach a
+  # Mach-O build another way: executorch_target_shared_runtime_path emits lib
+  # and src/executorch/lib, and setup.py's _SIBLING_LIBRARY_DIRECTORIES adds
+  # backends/cuda in the platform's own token at packaging time.
+  set(_python_extension_rpath "$ORIGIN/../../../torch/lib")
   # An editable install copies this extension into a package directory whose
   # subdirectories are symlinks to the checkout, and the loader resolves the
   # origin token against the real path, so the runtime sits two levels up and
   # then inside src/executorch/lib rather than beside the copy.
-  string(APPEND _portable_lib_rpath ":$ORIGIN/../../src/executorch/lib")
+  string(APPEND _python_extension_rpath ":$ORIGIN/../../src/executorch/lib")
 
   if(EXECUTORCH_BUILD_EXTENSION_MODULE)
     # extension_module_static is already bundled into libexecutorch.so; linking
@@ -1324,20 +1321,12 @@ if(EXECUTORCH_BUILD_PYBIND)
   endif()
 
   if(EXECUTORCH_BUILD_CUDA OR EXECUTORCH_BUILD_ROCM)
-    string(
-      APPEND
-      _portable_lib_rpath
-      "${_portable_lib_rpath_separator}${_portable_lib_origin}/../../backends/cuda"
-    )
+    string(APPEND _python_extension_rpath ":$ORIGIN/../../backends/cuda")
   endif()
 
   if(EXECUTORCH_BUILD_QNN)
     list(APPEND _dep_libs qnn_executorch_backend)
-    string(
-      APPEND
-      _portable_lib_rpath
-      "${_portable_lib_rpath_separator}${_portable_lib_origin}/../../backends/qualcomm"
-    )
+    string(APPEND _python_extension_rpath ":$ORIGIN/../../backends/qualcomm")
   endif()
 
   if(EXECUTORCH_BUILD_ENN)
@@ -1399,10 +1388,10 @@ if(EXECUTORCH_BUILD_PYBIND)
   # portable_lib.py in the same python package. PyTorch requires C++20, so
   # pybindings must be compiled with C++20.
   set_target_properties(
-    portable_lib PROPERTIES OUTPUT_NAME "_portable_lib" CXX_STANDARD 20
+    portable_lib PROPERTIES OUTPUT_NAME "_C" CXX_STANDARD 20
   )
   target_compile_definitions(
-    portable_lib PUBLIC EXECUTORCH_PYTHON_MODULE_NAME=_portable_lib
+    portable_lib PUBLIC EXECUTORCH_PYTHON_MODULE_NAME=_C
   )
   target_include_directories(portable_lib PRIVATE ${TORCH_INCLUDE_DIRS})
   target_compile_options(portable_lib PUBLIC ${_pybind_compile_options})
@@ -1457,13 +1446,13 @@ if(EXECUTORCH_BUILD_PYBIND)
   # A token with a subpath is a distinct entry, and this target already ships
   # several.
   if(APPLE)
-    set(_portable_lib_torch_path "@loader_path/../../../torch/lib")
+    set(_python_extension_torch_path "@loader_path/../../../torch/lib")
   else()
-    set(_portable_lib_torch_path "${_portable_lib_rpath}")
+    set(_python_extension_torch_path "${_python_extension_rpath}")
   endif()
   set_target_properties(
-    portable_lib PROPERTIES BUILD_RPATH "${_portable_lib_torch_path}"
-                            INSTALL_RPATH "${_portable_lib_torch_path}"
+    portable_lib PROPERTIES BUILD_RPATH "${_python_extension_torch_path}"
+                            INSTALL_RPATH "${_python_extension_torch_path}"
   )
   executorch_target_shared_runtime_path(
     portable_lib "extension/pybindings" "executorch/extension/pybindings"
@@ -1499,10 +1488,10 @@ if(EXECUTORCH_BUILD_PYBIND)
           LIBRARY DESTINATION executorch/extension/pybindings
   )
 
-  # Copy MLX metallib next to _portable_lib.so for editable installs. MLX uses
-  # dladdr() to find the directory containing the library with MLX code, then
-  # looks for mlx.metallib in that directory. When MLX is statically linked into
-  # _portable_lib.so, we need the metallib colocated with it.
+  # Copy the MLX metallib next to the Python extension for editable installs.
+  # MLX uses dladdr() to find the directory containing the library with MLX
+  # code, then looks for mlx.metallib in that directory. When MLX is statically
+  # linked into the Python extension, we need the metallib colocated with it.
   executorch_target_copy_mlx_metallib(portable_lib)
 endif()
 

backends/mlx/CMakeLists.txt

@@ -339,16 +339,16 @@ install(
 # exist)
 #
 # where {binary_dir} is determined at runtime via dladdr() on the library
-# containing MLX code. When MLX is statically linked into _portable_lib.so, this
-# is the directory containing _portable_lib.so.
+# containing MLX code. When MLX is statically linked into _C.so, this is the
+# directory containing _C.so.
 #
 # For the installed library, we put metallib in lib/ alongside libmlx.a. The
 # metallib is produced in the mlx_external build tree (MLX_METAL_JIT=ON does not
 # install it); _mlx_metallib points there.
 install(FILES ${_mlx_metallib} DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
-# Cache the metallib path for pybindings to copy it next to _portable_lib.so
-# This enables editable installs to work correctly
+# Cache the metallib path for pybindings to copy it next to _C.so. This enables
+# editable installs to work correctly
 set(MLX_METALLIB_PATH
     "${_mlx_metallib}"
     CACHE INTERNAL "Path to mlx.metallib for pybindings"

extension/pybindings/BUCK

@@ -34,9 +34,9 @@ fbcode_target(_kind = runtime.genrule,
     outs = {
         "aten_lib.pyi": ["aten_lib.pyi"],
         "core.pyi": ["core.pyi"],
-        "_portable_lib.pyi": ["_portable_lib.pyi"],
+        "_C.pyi": ["_C.pyi"],
     },
-    cmd = "cp $(location :pybinding_types)/* $OUT/_portable_lib.pyi && cp $(location :pybinding_types)/* $OUT/aten_lib.pyi && cp $(location :pybinding_types)/* $OUT/core.pyi",
+    cmd = "cp $(location :pybinding_types)/* $OUT/_C.pyi && cp $(location :pybinding_types)/* $OUT/aten_lib.pyi && cp $(location :pybinding_types)/* $OUT/core.pyi",
     visibility = ["//executorch/extension/pybindings/..."],
 )
 
@@ -50,8 +50,8 @@ fbcode_target(_kind = executorch_pybindings,
 fbcode_target(_kind = executorch_pybindings,
     cppdeps = PORTABLE_MODULE_DEPS + MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB,
     # Give this an underscore prefix because it has a pure python wrapper.
-    python_module_name = "_portable_lib",
-    types = ["//executorch/extension/pybindings:pybindings_types_gen[_portable_lib.pyi]"],
+    python_module_name = "_C",
+    types = ["//executorch/extension/pybindings:pybindings_types_gen[_C.pyi]"],
     visibility = ["PUBLIC"],
 )
 
@@ -67,7 +67,7 @@ fbcode_target(_kind = runtime.python_library,
     srcs = ["portable_lib.py"],
     visibility = ["PUBLIC"],
     deps = [
-        ":_portable_lib",
+        ":_C",
         "//executorch/exir:_warnings",
     ],
 )

extension/pybindings/portable_lib.py

@@ -72,12 +72,12 @@
             e,
         )
 
-# Let users import everything from the C++ _portable_lib extension as if this
+# Let users import everything from the C++ _C extension as if this
 # python file defined them. Although we could import these dynamically, it
 # wouldn't preserve the static type annotations.
 #
 # Note that all of these are experimental, and subject to change without notice.
-from executorch.extension.pybindings._portable_lib import (  # noqa: F401
+from executorch.extension.pybindings._C import (  # noqa: F401
     # Disable "imported but unused" (F401) checks.
     _create_profile_block,  # noqa: F401
     _dump_profile_results,  # noqa: F401
@@ -101,7 +101,7 @@
     Verification,  # noqa: F401
 )
 
-# Clean up so that `dir(portable_lib)` is the same as `dir(_portable_lib)`
+# Clean up so that `dir(portable_lib)` is the same as `dir(_C)`
 # (apart from some __dunder__ names).
 del _torch
 del _exir_warnings