Ship a C++ SDK in the wheel

.ci/scripts/wheel/test_linux.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 
 import test_base
+import test_cpp_sdk
 import test_shared_libraries
 from examples.models import Backend, Model
 
@@ -51,6 +52,13 @@
         with tempfile.TemporaryDirectory() as work_dir:
             test_shared_libraries.run_tests(Path(work_dir))
 
+        # And that a C++ application outside the wheel can actually use them.
+        # Nothing above covers this: the Python extension links those libraries
+        # itself, so it passes whether or not the package config names them or the
+        # shipped headers are complete.
+        with tempfile.TemporaryDirectory() as work_dir:
+            test_cpp_sdk.run_tests(Path(work_dir))
+
     model_tests = [
         test_base.ModelTest(
             model=Model.Mv3,

.ci/scripts/wheel/test_linux_aarch64.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 
 import test_base
+import test_cpp_sdk
 import test_shared_libraries
 from examples.models import Backend, Model
 
@@ -36,6 +37,11 @@
     with tempfile.TemporaryDirectory() as work_dir:
         test_shared_libraries.run_tests(Path(work_dir))
 
+    # And that a C++ application outside the wheel can actually use those
+    # libraries, which nothing above covers.
+    with tempfile.TemporaryDirectory() as work_dir:
+        test_cpp_sdk.run_tests(Path(work_dir))
+
     test_base.run_tests(
         model_tests=[
             test_base.ModelTest(

.github/workflows/build-wheels-aarch64-linux.yml

@@ -13,6 +13,11 @@ on:
       - docs/source/using-executorch-cpp.md
       - install_utils.py
       - tools/cmake/**/*
+      # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories
+      # setup.py copies from, so adding one there cannot silently drop it from this list.
+      - devtools/etdump/**.h
+      - extension/**.h
+      - runtime/**.h
   push:
     branches:
       - nightly

.github/workflows/build-wheels-linux.yml

@@ -13,6 +13,11 @@ on:
       - docs/source/using-executorch-cpp.md
       - install_utils.py
       - tools/cmake/**/*
+      # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories
+      # setup.py copies from, so adding one there cannot silently drop it from this list.
+      - devtools/etdump/**.h
+      - extension/**.h
+      - runtime/**.h
   push:
     branches:
       - nightly

.github/workflows/build-wheels-macos.yml

@@ -13,6 +13,11 @@ on:
       - docs/source/using-executorch-cpp.md
       - install_utils.py
       - tools/cmake/**/*
+      # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories
+      # setup.py copies from, so adding one there cannot silently drop it from this list.
+      - devtools/etdump/**.h
+      - extension/**.h
+      - runtime/**.h
   push:
     branches:
       - nightly

.github/workflows/build-wheels-windows.yml

@@ -12,6 +12,11 @@ on:
       - docs/source/using-executorch-cpp.md
       - install_utils.py
       - tools/cmake/**/*
+      # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories
+      # setup.py copies from, so adding one there cannot silently drop it from this list.
+      - devtools/etdump/**.h
+      - extension/**.h
+      - runtime/**.h
   push:
     branches:
       - nightly

docs/source/getting-started.md

@@ -171,6 +171,23 @@ For more information on iOS integration, including an API reference, logging set
 ExecuTorch provides C++ APIs, which can be used to target embedded or mobile devices. The C++ APIs provide a greater level of control compared to other language bindings, allowing for advanced memory management, data loading, and platform integration.
 
 #### Installation
+On Linux and macOS the quickest route is the pip package, which ships the runtime as prebuilt
+libraries with headers and a CMake package, so there is nothing to build:
+
+```
+pip install executorch
+```
+
+```cmake
+find_package(executorch REQUIRED COMPONENTS kernels_optimized)
+target_link_libraries(my_target PRIVATE executorch::runtime executorch::kernels_optimized)
+```
+
+See [Using the prebuilt libraries from the pip package](using-executorch-cpp.md#using-the-prebuilt-libraries-from-the-pip-package)
+for a complete walkthrough, and [Running on a GPU with the CUDA package](using-executorch-cpp.md#running-on-a-gpu-with-the-cuda-package)
+for GPU support.
+
+For a platform the package does not cover, or to change build options, build from source instead.
 CMake is the preferred build system for the ExecuTorch C++ runtime. To use with CMake, clone the ExecuTorch repository as a subdirectory of your project, and use CMake's `add_subdirectory("executorch")` to include the dependency. The `executorch` target, as well as kernel and backend targets will be made available to link against. The runtime can also be built standalone to support diverse toolchains. See [Using ExecuTorch with C++](using-executorch-cpp.md) and [Building from Source](using-executorch-building-from-source.md) for a detailed description of build integration, targets, and cross compilation.
 
 ```

docs/source/using-executorch-building-from-source.md

@@ -1,5 +1,12 @@
 # Building from Source
 
+On Linux and macOS you may not need to build at all. `pip install executorch` ships the
+runtime as prebuilt libraries with headers and a CMake package, so a C++ program can link
+it directly. See [Using the prebuilt libraries from the pip package](using-executorch-cpp.md#using-the-prebuilt-libraries-from-the-pip-package),
+including the CUDA packages for running on a GPU. Build from source when you need a
+platform the package does not cover, a build option it does not enable, or your own
+changes to the runtime.
+
 ExecuTorch uses [CMake](https://cmake.org/) as the primary build system.
 Even if you don't use CMake directly, CMake can emit scripts for other format
 like Make, Ninja or Xcode. For information, see [cmake-generators(7)](https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html).

docs/source/using-executorch-cpp.md

@@ -40,6 +40,213 @@ Running a model using the low-level runtime APIs allows for a high-degree of con
 
 ## Building with CMake
 
+There are two ways to get the C++ runtime. Linking the prebuilt libraries from the pip
+package needs no source checkout and is the quicker option. Building from source gives
+you every option the project has, and is what you need for a platform the wheel does not
+cover.
+
+### Using the prebuilt libraries from the pip package
+
+On Linux, `pip install executorch` ships the runtime as prebuilt shared libraries together with the
+headers and a CMake package. So a C++ program can use ExecuTorch without building it from source,
+and without knowing much CMake.
+
+#### Run your first model in four steps
+
+Copy these three files into an empty folder and follow along. No prior CMake knowledge needed.
+
+**1. Install, and make a model file.**
+
+```
+pip install executorch torch --extra-index-url https://download.pytorch.org/whl/cpu
+```
+
+`torch` is named explicitly because the wheel does not depend on it, so you bring your own torch and
+keep the version under your control. You need it only to create a model file in step 1, not to run
+the C++ program.
+
+A C++ program loads a `.pte` file, which is a model that has already been exported. C++ cannot
+create one, so make it in Python first:
+
+```python
+# export.py
+import torch
+from executorch.exir import to_edge_transform_and_lower
+
+class Add(torch.nn.Module):
+    def forward(self, x, y):
+        return x + y
+
+example = (torch.ones(2, 2), torch.ones(2, 2))
+program = to_edge_transform_and_lower(
+    torch.export.export(Add(), example)
+).to_executorch()
+open("model.pte", "wb").write(program.buffer)
+```
+
+```
+python export.py
+```
+
+**2. Write the program.**
+
+```cpp
+// main.cpp
+#include <executorch/extension/module/module.h>
+#include <executorch/extension/tensor/tensor.h>
+#include <cstdio>
+
+using namespace executorch::extension;
+
+int main() {
+  Module module("model.pte");
+
+  std::array<float, 4> a{1, 2, 3, 4};
+  std::array<float, 4> b{10, 20, 30, 40};
+
+  const auto result = module.forward({make_tensor_ptr({2, 2}, a.data()),
+                                      make_tensor_ptr({2, 2}, b.data())});
+  if (!result.ok()) {
+    std::printf("forward failed: 0x%x\n", (unsigned)result.error());
+    return 1;
+  }
+
+  const auto out = result->at(0).toTensor();
+  for (int i = 0; i < out.numel(); ++i) {
+    std::printf("%g ", out.const_data_ptr<float>()[i]);
+  }
+  std::printf("\n");
+  return 0;
+}
+```
+
+**3. Write six lines of CMake.**
+
+```cmake
+# CMakeLists.txt
+cmake_minimum_required(VERSION 3.28)
+project(app CXX)
+
+find_package(executorch REQUIRED COMPONENTS kernels_optimized)
+
+add_executable(app main.cpp)
+target_link_libraries(app PRIVATE executorch::runtime
+                                  executorch::kernels_optimized)
+```
+
+Two lines matter. `find_package` finds the installed ExecuTorch, and `target_link_libraries` says
+which parts you want. Every model needs at least these two: `runtime` is the engine that executes a
+program, and a kernel component such as `kernels_optimized` provides the maths the model computes
+with. With only the engine, a model loads and then fails with a missing operator.
+
+**4. Build and run.**
+
+```
+cmake -S . -B build \
+  -DCMAKE_PREFIX_PATH="$(python -c 'import executorch, pathlib; print(pathlib.Path(executorch.__path__[0]) / "share" / "cmake")')"
+cmake --build build
+./build/app
+```
+
+```
+11 22 33 44
+```
+
+That is the two input arrays added together. The long `python -c` part just prints where pip put the
+CMake package, so CMake can find it. Run `./build/app` from the folder holding `model.pte`, because
+the path in `main.cpp` is relative.
+
+#### Adding kernels and backends
+
+Add a component to both lines to get more. Nothing else in the program changes.
+
+```cmake
+find_package(executorch REQUIRED COMPONENTS kernels_optimized backend_xnnpack)
+
+target_link_libraries(app PRIVATE executorch::runtime
+                                  executorch::kernels_optimized
+                                  executorch::backend_xnnpack)
+```
+
+These are the components the Linux package provides:
+
+| Component | What it gives you | Where |
+| --- | --- | --- |
+| `runtime` | The engine. Always needed. | Linux |
+| `kernels_optimized` | Fast CPU operators. The usual choice. | Linux |
+| `backend_xnnpack` | The XNNPACK backend, for models exported with it. | Linux |
+| `threadpool` | Multi-threaded execution. | Linux |
+| `etdump` | Profiling, to record what ran and how long it took. | Linux |
+
+To see what your own install offers, ask CMake:
+
+```cmake
+find_package(executorch REQUIRED)
+foreach(_component runtime kernels_optimized backend_xnnpack threadpool etdump)
+  if(TARGET executorch::${_component})
+    message(STATUS "have ${_component}")
+  endif()
+endforeach()
+```
+
+The macOS wheel keeps everything inside the Python extension rather than shipping separate C++
+libraries, so there is nothing for a C++ application to link there; use it from Python, or build
+from source if you need C++ on macOS.
+
+A backend is only needed if the model was exported for it. Linking XNNPACK does not make a plain
+model faster, and a model exported for XNNPACK will fail to load without it. If you are not sure
+what a model needs, start with `runtime` and `kernels_optimized` and add what the error asks for.
+
+If you would rather not choose, one variable links the common set:
+
+```cmake
+find_package(executorch REQUIRED)
+target_link_libraries(app PRIVATE ${EXECUTORCH_LIBRARIES})
+```
+
+#### When something does not work
+
+- `find_package` could not find executorch: the `-DCMAKE_PREFIX_PATH=...` argument is missing or
+  points somewhere else. Run the `python -c` line on its own and check the folder exists.
+- The program builds but fails to load the model: the path is relative, so run it from the folder
+  containing the `.pte` file.
+- A missing operator at run time: add a kernel component, usually
+  `executorch::kernels_optimized`.
+- The model fails to load complaining about a backend: link the backend it was exported for.
+- `executorch::runtime` is not a target: imported targets need CMake 3.28 or newer. On an older
+  CMake the package still works, but you name variables instead of targets, and you have to pass on
+  the definitions and the C++ standard requirement yourself:
+
+  ```cmake
+  cmake_minimum_required(VERSION 3.19)
+  project(app CXX)
+
+  find_package(executorch REQUIRED)
+
+  add_executable(app main.cpp)
+  target_include_directories(app PRIVATE ${EXECUTORCH_INCLUDE_DIRS})
+  target_compile_definitions(app PRIVATE ${EXECUTORCH_COMPILE_DEFINITIONS})
+  target_compile_features(app PRIVATE cxx_std_${EXECUTORCH_CXX_STANDARD})
+  target_link_libraries(app PRIVATE ${EXECUTORCH_LIBRARIES})
+  set_target_properties(
+    app PROPERTIES INSTALL_RPATH "${EXECUTORCH_RUNTIME_LIBRARY_DIR}"
+  )
+  ```
+
+  Leaving out `EXECUTORCH_COMPILE_DEFINITIONS` fails with a missing
+  `torch/headeronly/macros/cmake_macros.h`, because the vendored headers look for a file that only
+  exists inside a PyTorch build.
+
+  `INSTALL_RPATH` matters once you run `cmake --install`. CMake gives your program a search path while
+  it sits in the build directory and removes that path when installing, so an installed program cannot
+  find the libraries unless you record where they live.
+
+You should not need `LD_LIBRARY_PATH`. The shipped libraries record where their neighbours live, so
+they find each other once the program links against the installed package.
+
+### Building from source
+
+
 ExecuTorch uses CMake as the primary build system. Inclusion of the module and tensor APIs are controlled by the `EXECUTORCH_BUILD_EXTENSION_MODULE` and `EXECUTORCH_BUILD_EXTENSION_TENSOR` CMake options. As these APIs may not be supported on embedded systems, they are disabled by default when building from source. The low-level API surface is always included. To link, add the `executorch` target as a CMake dependency, along with `executorch_backends`, `executorch_extensions`, and `extension_kernels`, to link all configured backends, extensions, and kernels.
 
 ```

docs/source/using-executorch-faqs.md

@@ -47,6 +47,9 @@ Thread count can be set with the following function. Ensure this is done prior t
 ```
 ::executorch::extension::threadpool::get_threadpool()->_unsafe_reset_threadpool(num_threads);
 ```
+This needs `extension/threadpool/threadpool.h`, which a source build provides. The pip
+package does not publish it, because the header holds a `pthreadpool` member by value and
+the wheel vendors no pthreadpool headers to compile it against.
 
 For a deeper investigation into model performance, ExecuTorch supports operator-level performance profiling. See [Using the ExecuTorch Developer Tools to Profile a Model](tutorials/devtools-integration-tutorial) <!-- @lint-ignore --> for more information.
 

extension/memory_allocator/memory_allocator_utils.h

@@ -16,12 +16,10 @@
 #include <executorch/runtime/core/result.h>
 #include <executorch/runtime/platform/compiler.h>
 
-using executorch::runtime::Error;
-using executorch::runtime::Result;
 namespace executorch::extension::utils {
 
 // Util to get alighment adjusted allocation size
-inline Result<size_t> get_aligned_size(size_t size, size_t alignment) {
+inline runtime::Result<size_t> get_aligned_size(size_t size, size_t alignment) {
   // The minimum alignment that malloc() is guaranteed to provide.
   static constexpr size_t kMallocAlignment = alignof(std::max_align_t);
   if (alignment > kMallocAlignment) {
@@ -31,7 +29,7 @@ inline Result<size_t> get_aligned_size(size_t size, size_t alignment) {
     const size_t extra = alignment - 1;
     if ET_UNLIKELY (extra >= SIZE_MAX - size) {
       ET_LOG(Error, "Malloc size overflow: size=%zu + extra=%zu", size, extra);
-      return Result<size_t>(Error::InvalidArgument);
+      return runtime::Result<size_t>(runtime::Error::InvalidArgument);
     }
     size += extra;
   }

runtime/executor/platform_memory_allocator.h

@@ -13,6 +13,7 @@
 #include <cstdint>
 
 #include <c10/util/safe_numerics.h>
+#include <executorch/runtime/core/exec_aten/exec_aten.h>
 #include <executorch/runtime/core/memory_allocator.h>
 #include <executorch/runtime/platform/log.h>
 #include <executorch/runtime/platform/platform.h>