diff --git a/.ci/scripts/wheel/test_cpp_sdk.py b/.ci/scripts/wheel/test_cpp_sdk.py index 22266ed4ff5..81e68106826 100644 --- a/.ci/scripts/wheel/test_cpp_sdk.py +++ b/.ci/scripts/wheel/test_cpp_sdk.py @@ -57,6 +57,25 @@ def forward(self, x, image): with torch.no_grad(): expected = model(*example) +if mode == "quantized": + # Quantize with the same flow the documentation shows, so the exported program + # references the quantized operator set rather than the plain one. + # Importing this loads the ahead-of-time library, which is what registers the out + # variants of the quantized operators with torch. Without it the export fails with + # "Missing out variants: quantized_decomposed::quantize_per_tensor", because the + # lowering step has no out variant to select. + import executorch.kernels.quantized # noqa: F401 + from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( + get_symmetric_quantization_config, + XNNPACKQuantizer, + ) + from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + quantizer = XNNPACKQuantizer().set_global(get_symmetric_quantization_config()) + prepared = prepare_pt2e(torch.export.export(model, example).module(), quantizer) + prepared(*example) + model = convert_pt2e(prepared) + partitioners = [] if mode == "delegate": from executorch.backends.xnnpack.partition.xnnpack_partitioner import ( @@ -85,6 +104,11 @@ def forward(self, x, image): "expected": expected.flatten().tolist(), "delegated": mode == "delegate", "has_xnnpack": b"XnnpackBackend" in bytes(buffer), + # Whether the program actually carries quantized operators. The numeric comparison alone + # cannot tell: an unquantized export of the same model produces a closer match than the + # tolerance a quantized one needs, so it would pass while proving nothing about the + # quantized kernels. + "has_quantized": b"quantized_decomposed" in bytes(buffer), } ) ) @@ -102,6 +126,7 @@ def forward(self, x, image): #include #include +#include #include #include #include @@ -196,8 +221,14 @@ def forward(self, x, image): } worst = std::fmax(worst, diff); } - if (worst > 1e-4) { - std::printf("output differs from eager PyTorch by %g\n", worst); + // Passed in rather than fixed, because the acceptable difference depends on the + // model. A float32 model should match to within rounding, while an int8 quantized one + // legitimately differs by about one quantization step, and using the looser number + // for both would stop the float path catching a real regression. + const double tolerance = argc > 7 ? std::atof(argv[7]) : 1e-4; + if (worst > tolerance) { + std::printf( + "output differs from eager PyTorch by %g, tolerance %g\n", worst, tolerance); return 1; } @@ -335,8 +366,16 @@ def _build_consumer(work_dir: Path, name: str, components) -> Path: return consumer -def _run_consumer(consumer: Path, model: Path, reference, work_dir: Path) -> str: - """Run the application and require it to match eager PyTorch.""" +def _run_consumer( + consumer: Path, model: Path, reference, work_dir: Path, tolerance: float = 1e-4 +) -> str: + """Run the application and require it to match eager PyTorch within `tolerance`. + + The tolerance is a parameter because the acceptable difference depends on the model. + A float32 model should match to within rounding, while an int8 quantized one + legitimately differs by about one quantization step, and using the looser number for + both would stop the float path catching a real regression. + """ inputs = reference["inputs"] shape_a, data_a = _write_tensor(work_dir, "a", inputs[0]) shape_b, data_b = _write_tensor(work_dir, "b", inputs[1]) @@ -358,6 +397,7 @@ def _run_consumer(consumer: Path, model: Path, reference, work_dir: Path) -> str str(shape_b), str(data_b), str(expected), + str(tolerance), ], capture_output=True, text=True, @@ -1236,6 +1276,125 @@ def test_pre_3_28_route_builds_a_consumer_through_variables(work_dir: Path) -> N ) +def test_quantized_kernels_component_runs_a_model(work_dir: Path) -> None: + """A C++ application can run a quantized model using the shipped quantized kernels. + + Before the quantized kernels became their own library they existed only inside the + ahead-of-time extension beside the Python bindings, so a C++ application loading a + quantized program had nothing to link and failed at run time with the operators + reported missing. + + A missing library is a failure rather than a skip. The preset that builds the wheel + always enables the quantized kernels, so their absence is a regression in packaging + or in the build, not a configuration this suite has to tolerate. Skipping there + reported the whole check as coverage while running none of it. + """ + package_dir = _installed_package_dir() + # Globbed for the same reason the profiler check is: the library carries a version suffix outside a + # wheel build, and an exact name would skip this silently there rather than running it. + shipped = sorted((package_dir / "lib").glob("libexecutorch_kernels_quantized.so*")) + assert shipped, ( + "the wheel ships no quantized kernels library. The preset that builds it enables " + "them unconditionally, so this is a packaging or build regression rather than an " + "unsupported configuration." + ) + + model, reference = _export(work_dir, "quantized") + # The export has to have produced a quantized program, or the rest of this proves nothing about the + # quantized kernels. The numeric comparison cannot tell the difference: an unquantized export of the + # same model lands well inside the tolerance a quantized one needs, so it would pass while linking a + # library it never exercised. + assert reference["has_quantized"], ( + "the quantized export produced a program with no quantized operators, so this check would " + "prove nothing about the quantized kernels" + ) + consumer = _build_consumer( + work_dir, + "with-quantized", + ["runtime", "kernels_optimized", "kernels_quantized"], + ) + # One int8 quantization step over this model's output range is about 5e-3, so a + # float32 tolerance cannot be met by a correct quantized run. + output = _run_consumer(consumer, model, reference, work_dir, tolerance=2e-2) + print( + f"✓ a C++ app linking executorch::kernels_quantized runs a quantized model " + f"({output})" + ) + + +def test_aggregate_variable_excludes_the_quantized_kernels(work_dir: Path) -> None: + """`${EXECUTORCH_LIBRARIES}` must not drag in the quantized kernels. + + The export-time plugin that `executorch.kernels.quantized` loads carries its own + copy of those kernels rather than depending on the shipped library, so a process + holding both registers the same operators twice and the runtime stops on the + second one. An application that links whatever the package offers by default + would inherit that, so the component is defined but held out of the aggregate and + a consumer that wants it names it. + + Checked by reading the link line rather than by running, because the failure is a + process-wide abort that needs a Python interpreter in the same process to trigger. + What this owns is the packaging decision: is the library on the link line at all. + """ + package_dir = _installed_package_dir() + # Fatal for the same reason the check above is: the preset that builds the wheel + # always enables these kernels, so their absence is a regression rather than a + # configuration to tolerate, and skipping would report this as coverage. + assert sorted( + (package_dir / "lib").glob("libexecutorch_kernels_quantized.so*") + ), "the wheel ships no quantized kernels library, so this check cannot run" + + source_dir = work_dir / "aggregate-only" + source_dir.mkdir(parents=True, exist_ok=True) + (source_dir / "consumer.cpp").write_text(_CONSUMER_SOURCE) + # No COMPONENTS and no named target, which is the shape the older-CMake route + # forces and the documentation offers as the general case. + (source_dir / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.28)\n" + "project(consumer CXX)\n" + "find_package(executorch REQUIRED)\n" + "add_executable(consumer consumer.cpp)\n" + "target_link_libraries(consumer PRIVATE ${EXECUTORCH_LIBRARIES})\n" + ) + build_dir = work_dir / "aggregate-only-build" + config = package_dir / "share" / "cmake" / "executorch-config.cmake" + for command in ( + [ + _tool("cmake"), + "-S", + str(source_dir), + "-B", + str(build_dir), + f"-DCMAKE_PREFIX_PATH={config.parent}", + ], + [_tool("cmake"), "--build", str(build_dir)], + ): + result = subprocess.run(command, capture_output=True, text=True, check=False) + assert result.returncode == 0, ( + "an application linking only ${EXECUTORCH_LIBRARIES} could not be built:\n" + f"{result.stdout[-2000:]}\n{result.stderr[-2000:]}" + ) + + consumer = build_dir / "consumer" + dependencies = subprocess.run( + ["readelf", "-d", str(consumer)], capture_output=True, text=True, check=False + ).stdout + assert "libexecutorch_kernels_quantized" not in dependencies, ( + "an application that linked only ${EXECUTORCH_LIBRARIES} depends on the " + "quantized kernels. That library collides with the export-time plugin, so it " + "has to be opted into by name rather than handed to every consumer." + ) + # The rest of the aggregate still has to be there, or this would pass by shipping + # nothing at all. + assert "libexecutorch_kernels_optimized" in dependencies, ( + "the aggregate no longer carries the CPU kernels, so an application linking it " + "would fail at run time with the operators reported missing" + ) + print( + "✓ ${EXECUTORCH_LIBRARIES} carries the CPU kernels and not the quantized ones" + ) + + def run_tests(work_dir: Path) -> None: test_find_package_honours_a_version_request(work_dir) test_profiler_component_is_usable(work_dir) @@ -1245,6 +1404,8 @@ def run_tests(work_dir: Path) -> None: test_runtime_alone_links_but_cannot_compute(work_dir) test_kernels_component_runs_a_model(work_dir) test_pre_3_28_route_builds_a_consumer_through_variables(work_dir) + test_quantized_kernels_component_runs_a_model(work_dir) + test_aggregate_variable_excludes_the_quantized_kernels(work_dir) test_delegated_model_needs_the_delegate_component(work_dir) test_consumer_is_relocatable(work_dir) test_one_registry_in_the_cpp_process(work_dir) diff --git a/.ci/scripts/wheel/test_shared_libraries.py b/.ci/scripts/wheel/test_shared_libraries.py index 02c580d2c06..8aa7d79d0d8 100644 --- a/.ci/scripts/wheel/test_shared_libraries.py +++ b/.ci/scripts/wheel/test_shared_libraries.py @@ -63,6 +63,14 @@ # the operators are registered twice, which aborts at startup. _KERNEL_SYMBOLS = ("torch::executor::native::abs_out",) +# The quantized kernels, whose own library the wheel ships when they are built. +# A separate group because they have a separate owner, and because a wheel built +# without them ships neither the library nor these symbols. +_QUANTIZED_KERNEL_SYMBOLS = ( + "torch::executor::native::quantize_per_tensor_out", + "torch::executor::native::dequantize_per_tensor_out", +) + # The registry entry points, kept separate from the kernel implementations above. # A library that carries its own copy of these has its own registration code, which # is what this split is meant to prevent: one owner of the operator table. Checking @@ -295,7 +303,37 @@ def _defines_symbol(library: Path, symbol: str) -> bool: return False -def _assert_single_definer(symbols, what: str, owner: str | None = None) -> None: +def _is_export_only(library: Path) -> bool: + """Whether a library exists to export a model rather than to run one. + + The ahead-of-time operator libraries register kernels into torch so a model can be + exported, and they link torch to do it. They deliberately carry their own copy of + the kernels, because the copy a C++ application links is registered into a table + those libraries never read. + + Named by the caller per component rather than excluded everywhere. Counting them for + the component they duplicate would report a duplicate that is not one, and excusing + them for every component would stop this catching a second registry hiding inside + one of them. + + Matched on both the torch dependency and the name marker, because either alone + misfires: several shipped libraries link torch without being export-side, and a + name check alone would accept a runtime library that adopted the suffix. + """ + if _tool("readelf") is None: + return library.name.endswith("_aot_lib.so") + dynamic = subprocess.run( + [_tool("readelf"), "-d", str(library)], + capture_output=True, + text=True, + check=False, + ).stdout + return "libtorch.so" in dynamic and "_aot_lib" in library.name + + +def _assert_single_definer( + symbols, what: str, owner: str | None = None, allow_export_copy: bool = False +) -> None: """At most one shipped library may define each of `symbols`. The owner is named where one is expected, because counting definers alone does @@ -306,11 +344,25 @@ def _assert_single_definer(symbols, what: str, owner: str | None = None) -> None A component the wheel does not ship at all is a valid configuration, not a fault. Delegates and kernel sets are build options, so a wheel built without one has zero definers and is reported as such. What must never happen is two. + + `allow_export_copy` excuses the export-side libraries for one component only. The + quantized kernels genuinely exist twice, once in the runtime library and once in the + library torch loads at export time, because each side registers into a table the + other never reads. Loading both into one process does abort on the second + registration, so what this check enforces for that component is one owner among the + runtime libraries, not the absence of the export copy. Excusing every component + would disarm the check where duplication is a real fault: two of these libraries + defined the backend registry symbols in one released wheel and not in the release + before it, so the duplication this catches does happen. """ assert _tool("nm") is not None, "nm is required to inspect the wheel" package_dir = _installed_package_dir() - libraries = _shipped_shared_objects(package_dir) + libraries = [ + library + for library in _shipped_shared_objects(package_dir) + if not (allow_export_copy and _is_export_only(library)) + ] assert libraries, f"no shared libraries found under {package_dir}" # Every symbol is resolved before anything is reported, so a component that is only @@ -399,6 +451,12 @@ def _assert_single_definer(symbols, what: str, owner: str | None = None) -> None "libexecutorch_kernels_optimized.so", False, ), + ( + "set of quantized kernels", + _QUANTIZED_KERNEL_SYMBOLS, + "libexecutorch_kernels_quantized.so", + True, + ), # The third-party code these libraries bundle, checked separately from the # wrappers above. A wrapper can have a single owner while the implementation # underneath it is bundled into two of these, which is two real thread pools or @@ -425,6 +483,15 @@ def _assert_single_definer(symbols, what: str, owner: str | None = None) -> None ) +# The one component that legitimately exists twice. The quantized kernels are compiled into the runtime +# library and again into the library torch loads at export time, because each side registers into a +# table the other never reads, so a second definer there is expected rather than a fault. A process +# that loads both does abort on the second registration, which is why this is named per component and +# the check stays armed for every other component, where a second definer means two registries or two thread +# pools in one process. +_COMPONENTS_WITH_AN_EXPORT_COPY = frozenset({"set of quantized kernels"}) + + def test_each_component_has_one_owner() -> None: """No component may be defined by more than one library the wheel ships. @@ -442,7 +509,12 @@ def test_each_component_has_one_owner() -> None: f"the wheel ships no {owner}, which owns the {what}. Either packaging " "dropped it or the build did not produce it." ) - _assert_single_definer(symbols, what, owner if present else None) + _assert_single_definer( + symbols, + what, + owner if present else None, + allow_export_copy=what in _COMPONENTS_WITH_AN_EXPORT_COPY, + ) def test_python_extensions_import() -> None: @@ -1333,17 +1405,43 @@ def test_extension_contains_no_component() -> None: ).stdout.splitlines() if "NEEDED" in line } + # Only the libraries whose code the extension used to contain. Those are the ones + # this split moved out of it, so the extension must now resolve them from outside or + # a retention option silently failed. + # + # Not every shipped library serves Python. The quantized kernels and the CUDA + # delegate exist for a C++ application: Python registers quantized operators through + # the torch-linked ahead-of-time library at export time, and never loads the CUDA + # delegate from this extension at all. Requiring a dependency on those would demand + # the extension link code it has no use for. shipped = {path.name for path in _shipped_runtime_libraries(package_dir)} assert shipped, ( f"the wheel installed no runtime libraries under {package_dir / 'lib'}, so this check would " "compare the extension against nothing and pass" ) - unused = sorted(shipped - needed) + expected = { + name + for name in shipped + if not any( + marker in name + for marker in ("kernels_quantized", "backend_cuda", "extension_cuda") + ) + } + unused = sorted(expected - needed) assert not unused, ( f"the wheel ships {unused} but {extension.name} does not depend on them, so " "either they are dead weight or a retention option did not hold" ) + # Two shipped libraries register the same quantized operators, one for export and one for a C++ + # application, and the runtime treats a repeat registration as fatal. Reaching both from one process + # aborts it, and the only thing preventing that is this extension not depending on the run-time one. + assert not any("kernels_quantized" in name for name in needed), ( + f"{extension.name} depends on the run-time quantized library, which registers the same operators " + "as the export-time one it already loads. The runtime aborts on a repeat registration, so " + "importing this extension would kill the process." + ) + # Positive proof that the extension resolves these from elsewhere, rather than # only the absence of a visible definition. A hidden or local copy would not # appear in the dynamic symbol table at all, so "defines nothing" on its own is @@ -1386,7 +1484,7 @@ def test_extension_contains_no_component() -> None: print( f"✓ {extension.name} ({extension.stat().st_size // 1024} KiB) contains no " f"component, imports the runtime symbols it uses, and depends on all " - f"{len(shipped)} shipped libraries" + f"{len(expected)} shipped libraries it used to contain" ) @@ -1430,6 +1528,7 @@ def test_shipped_library_names_are_expected() -> None: known = ( "libexecutorch", "libexecutorch_kernels_optimized", + "libexecutorch_kernels_quantized", "libexecutorch_backend_xnnpack", "libexecutorch_threadpool", "libexecutorch_etdump", diff --git a/docs/source/using-executorch-cpp.md b/docs/source/using-executorch-cpp.md index 3431e8e826f..0a982763148 100644 --- a/docs/source/using-executorch-cpp.md +++ b/docs/source/using-executorch-cpp.md @@ -177,12 +177,13 @@ These are the components the Linux package provides: | `backend_xnnpack` | The XNNPACK backend, for models exported with it. | Linux | | `threadpool` | Multi-threaded execution. | Linux | | `etdump` | Profiling, to record what ran and how long it took. | Linux | +| `kernels_quantized` | The quantized operator kernels | Linux | To see what your own install offers, ask CMake: ```cmake find_package(executorch REQUIRED) -foreach(_component runtime kernels_optimized backend_xnnpack threadpool etdump) +foreach(_component runtime kernels_optimized kernels_quantized backend_xnnpack threadpool etdump) if(TARGET executorch::${_component}) message(STATUS "have ${_component}") endif() @@ -204,6 +205,10 @@ find_package(executorch REQUIRED) target_link_libraries(app PRIVATE ${EXECUTORCH_LIBRARIES}) ``` +The quantized kernels are deliberately left out of that variable, because loading +`executorch.kernels.quantized` in Python registers the same operators and a duplicate registration +stops the runtime. + #### When something does not work - `find_package` could not find executorch: the `-DCMAKE_PREFIX_PATH=...` argument is missing or @@ -241,6 +246,12 @@ target_link_libraries(app PRIVATE ${EXECUTORCH_LIBRARIES}) it sits in the build directory and removes that path when installing, so an installed program cannot find the libraries unless you record where they live. + Quantized kernels are not part of `EXECUTORCH_LIBRARIES`, so add them when your model needs them: + + ```cmake + target_link_libraries(app PRIVATE ${EXECUTORCH_QUANTIZED_KERNELS_LIBRARY}) + ``` + You should not need `LD_LIBRARY_PATH`. The shipped libraries record where their neighbours live, so they find each other once the program links against the installed package. diff --git a/kernels/quantized/CMakeLists.txt b/kernels/quantized/CMakeLists.txt index 760d9344054..ca24ccca035 100644 --- a/kernels/quantized/CMakeLists.txt +++ b/kernels/quantized/CMakeLists.txt @@ -97,7 +97,16 @@ if(NOT CMAKE_GENERATOR STREQUAL "Xcode" # Register quantized ops to portable_lib, so that they're available via # pybindings. - if(TARGET portable_lib) + # + # Only when this build has no shared quantized library to link instead. That + # library and this block compile the same sources against the same yaml, so + # a build producing both put two identical registrars in one process: the + # operator sets are identical by construction, and measured on the shipped + # wheel neither library carried one the other lacked. Registration is a + # static initializer, so both fire on load and the second aborts the + # process. Where the shared library exists the AOT plugin links it below + # rather than carrying its own copy. + if(TARGET portable_lib AND NOT EXECUTORCH_BUILD_SHARED) add_library(quantized_pybind_kernels_lib ${_quantized_kernels__srcs}) target_link_libraries( quantized_pybind_kernels_lib PRIVATE portable_lib executorch_core @@ -125,6 +134,15 @@ if(NOT CMAKE_GENERATOR STREQUAL "Xcode" quantized_ops_aot_lib PUBLIC quantized_ops_pybind_lib ) + endif() + + # Outside the gate above. That gate only decides whether this build compiles + # its own copy of the kernels; the runtime search path below is needed + # either way, because the plugin links torch and the Python extension + # directly through Codegen.cmake regardless. Removing it left an Apple build + # with no route to torch/lib, where the loader cannot fall back on a + # non-absolute name the way Linux can. + if(TARGET portable_lib) # pip wheels will need to be able to find the dependent libraries. On # Linux, the .so has non-absolute dependencies on libs like # "_portable_lib.so" without paths; as long as we `import torch` first, @@ -164,14 +182,14 @@ if(NOT CMAKE_GENERATOR STREQUAL "Xcode" endif() add_library(quantized_kernels ${_quantized_kernels__srcs}) -# The thread pool carries the define that switches parallel_for from a serial -# fallback to the real threaded implementation, so without it quantize, -# dequantize and choose_qparams run on one core. Guarded because a bare metal -# target builds these kernels without a thread pool at all, where the serial -# fallback is the only correct choice. target_link_libraries( quantized_kernels PRIVATE executorch_core kernels_util_all_deps ) +# The thread pool carries the define that switches parallel_for from a serial +# fallback to the real threaded implementation. Without it choose_qparams runs +# on one core, as does the ARM path in quantize. Guarded because a bare metal +# target builds these kernels without a thread pool at all, where the serial +# fallback is the only correct choice. if(TARGET extension_threadpool) target_link_libraries(quantized_kernels PRIVATE extension_threadpool) endif() @@ -200,8 +218,29 @@ if(EXECUTORCH_BUILD_SHARED) # This library holds no symbol a consumer names, only registration # constructors, so a link with --as-needed would drop it and register nothing. executorch_target_link_options_shared_lib(executorch_quantized_ops) - # Ships beside libexecutorch.so, so it resolves the runtime from there rather - # than from wherever it was built. Without this the installed library has an - # empty search path and a consumer that links it cannot start. + # The export plugin gets its kernels from this one library rather than + # compiling its own copy, so the process holds a single registrar. Attached + # here rather than beside the plugin's other properties because that block + # runs before this target exists. The plugin already routes kernels/quantized/ + # to lib/ for its runtime search path, which is where this library ships, and + # the --as-needed retention above is an interface property so it reaches the + # plugin too. + if(TARGET quantized_ops_aot_lib) + target_link_libraries(quantized_ops_aot_lib PUBLIC executorch_quantized_ops) + endif() + # Named after what the library provides rather than after the target that + # produces it, matching the optimized kernels next to it, so the shipped file + # reads as libexecutorch_kernels_quantized.so. The target name stays as it is + # because a source build already refers to it. Only for the wheel, for the + # same reason as the optimized kernels: a source install has been shipping the + # old file name with a versioned soname. + if(EXECUTORCH_BUILD_WHEEL_DO_NOT_USE) + set_target_properties( + executorch_quantized_ops PROPERTIES OUTPUT_NAME + executorch_kernels_quantized + ) + endif() + # Ships beside libexecutorch.so in the wheel's lib/ directory, so it resolves + # the runtime from there rather than from wherever it was built. executorch_target_shipped_runtime_path(executorch_quantized_ops) endif() diff --git a/setup.py b/setup.py index 61bd25677b6..b3194eb7a5d 100644 --- a/setup.py +++ b/setup.py @@ -1468,8 +1468,8 @@ def run(self): # noqa C901 if cmake_cache.is_enabled("EXECUTORCH_BUILD_MLX"): cmake_build_args += ["--target", "mlxdelegate"] - # The libraries a C++ consumer links out of the wheel. Until now they were - # reached as dependencies of the Python extension, which meant a shared + # The libraries a C++ consumer links out of the wheel. Most of them used to + # be reached as dependencies of the Python extension, which meant a shared # build with the bindings off asked for none of them and packaging then # looked for files no target had been told to produce. Each condition here # is the one its packaging entry carries, so the two lists cannot drift. @@ -1482,6 +1482,8 @@ def run(self): # noqa C901 cmake_build_args += ["--target", "extension_threadpool"] if cmake_cache.is_enabled("EXECUTORCH_BUILD_KERNELS_OPTIMIZED"): cmake_build_args += ["--target", "optimized_native_cpu_ops_lib"] + if cmake_cache.is_enabled("EXECUTORCH_BUILD_KERNELS_QUANTIZED"): + cmake_build_args += ["--target", "executorch_quantized_ops"] if cmake_cache.is_enabled("EXECUTORCH_BUILD_XNNPACK"): cmake_build_args += ["--target", "xnnpack_backend"] @@ -1602,6 +1604,19 @@ def run(self): # noqa C901 "EXECUTORCH_BUILD_KERNELS_OPTIMIZED", ], ), + # The quantized kernels, as their own library rather than code + # fused into the AOT-only extension beside the Python bindings. + # A C++ application running a quantized model could not link + # them before. + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/kernels/quantized/", + src_name="libexecutorch_kernels_quantized.so", + dst="executorch/lib/libexecutorch_kernels_quantized.so", + dependent_cmake_flags=[ + "EXECUTORCH_BUILD_SHARED", + "EXECUTORCH_BUILD_KERNELS_QUANTIZED", + ], + ), # Install the XNNPACK delegate beside them, so a process has one # copy of it instead of one per component that uses it. BuiltFile( diff --git a/tools/cmake/executorch-wheel-config.cmake b/tools/cmake/executorch-wheel-config.cmake index 6b98d04eab5..8d82cfe803c 100644 --- a/tools/cmake/executorch-wheel-config.cmake +++ b/tools/cmake/executorch-wheel-config.cmake @@ -64,10 +64,21 @@ # dependency and, for a registration-only library, the link options that keep it # from being dropped. The names, when present, are: # -# executorch::kernels_optimized -- The CPU operator kernels. Needed to run a -# model. executorch::backend_xnnpack -- The XNNPACK delegate. -# executorch::threadpool -- The shared thread pool. executorch::etdump -- -# The profiler. +# ~~~ +# executorch::kernels_optimized The CPU operator kernels. Needed to run a model. +# executorch::kernels_quantized The quantized operator kernels, for a quantized +# model. Not part of EXECUTORCH_LIBRARIES, see +# below. +# executorch::backend_xnnpack The XNNPACK delegate. +# executorch::threadpool The shared thread pool. +# executorch::etdump The profiler. +# ~~~ +# +# EXECUTORCH_LIBRARIES carries every component except the quantized kernels, +# which a consumer names explicitly instead. The export-time plugin that +# executorch.kernels.quantized loads carries its own copy of those kernels, so a +# process holding both stops on a repeated operator registration, and a consumer +# linking the aggregate would inherit that without asking for it. # # Check with if(TARGET executorch::) rather than assuming one exists. A # namespaced name that was never defined is a configure-time error that names @@ -289,6 +300,13 @@ if(_executorch_runtime_library AND NOT _executorch_targets_supported) # then a load failure saying the backend is not registered, which reads as a # model problem. Anything the wheel did not ship is simply not found and # skipped. + # + # The quantized kernels are deliberately absent, for the reason given at their + # component definition below: they collide with the export-time plugin that + # executorch.kernels.quantized loads, and a process holding both dies. This + # route has no per-component target to opt into, so they are offered through + # EXECUTORCH_QUANTIZED_KERNELS_LIBRARY instead and a consumer that wants them + # links that as well. foreach(_executorch_component IN ITEMS libexecutorch_kernels_optimized libexecutorch_backend_xnnpack libexecutorch_threadpool libexecutorch_etdump @@ -323,6 +341,21 @@ if(_executorch_runtime_library AND NOT _executorch_targets_supported) endif() endforeach() unset(_executorch_component_library) + # Held out of the aggregate above, so name it separately. A consumer that + # wants quantized operators and does not load the Python plugin in the same + # process links this too. Empty when the wheel shipped no such library. + _executorch_find_library( + EXECUTORCH_QUANTIZED_KERNELS_LIBRARY libexecutorch_kernels_quantized + ) + if(EXECUTORCH_QUANTIZED_KERNELS_LIBRARY AND CMAKE_SYSTEM_NAME STREQUAL + "Linux" + ) + # The same scoped retention the aggregate entries get, for the same reason: + # a registration-only library exports nothing the application references. + set(EXECUTORCH_QUANTIZED_KERNELS_LIBRARY + "-Wl,--push-state,--no-as-needed,${EXECUTORCH_QUANTIZED_KERNELS_LIBRARY},--pop-state" + ) + endif() message( STATUS "executorch: the prebuilt runtime is present but its imported targets need CMake 3.28 or " @@ -400,8 +433,13 @@ endif() # most: a registration-only library has no symbol the application references, so # a normal link drops it and its registration never runs. # -# Call as: _executorch_define_component( ) +# Call as: _executorch_define_component( +# [OPT_IN]) +# +# OPT_IN defines the target but keeps it out of EXECUTORCH_LIBRARIES, for a +# library a consumer has to choose deliberately rather than receive by default. function(_executorch_define_component _suffix _library_name) + cmake_parse_arguments(PARSE_ARGV 2 _component "OPT_IN" "" "") # Same reason the runtime target is skipped on older CMake: a component target # exports an $ORIGIN-relative search path, and a version that writes it wrong # produces a target that works in place and fails once deployed. @@ -424,10 +462,12 @@ function(_executorch_define_component _suffix _library_name) # returning here would hand the second caller a list with the runtime but # none of the components. A consumer linking that variable would then be # missing its kernels and fail at load with an unregistered operator. - set(EXECUTORCH_LIBRARIES - ${EXECUTORCH_LIBRARIES} ${_target} - PARENT_SCOPE - ) + if(NOT _component_OPT_IN) + set(EXECUTORCH_LIBRARIES + ${EXECUTORCH_LIBRARIES} ${_target} + PARENT_SCOPE + ) + endif() return() endif() add_library(${_target} SHARED IMPORTED) @@ -484,10 +524,12 @@ function(_executorch_define_component _suffix _library_name) "LINKER:--push-state,--no-as-needed,${_library},--pop-state" ) endif() - set(EXECUTORCH_LIBRARIES - ${EXECUTORCH_LIBRARIES} ${_target} - PARENT_SCOPE - ) + if(NOT _component_OPT_IN) + set(EXECUTORCH_LIBRARIES + ${EXECUTORCH_LIBRARIES} ${_target} + PARENT_SCOPE + ) + endif() endfunction() _executorch_define_component(threadpool executorch_threadpool) @@ -496,6 +538,25 @@ _executorch_define_component(threadpool executorch_threadpool) # checks, so it has to be defined here or a consumer following the documentation # gets a bare name that CMake hands to the linker as a literal flag. _executorch_define_component(kernels_optimized executorch_kernels_optimized) +# The quantized kernels, optional in the same way: a wheel built without them +# simply has no such library and the component is not defined. +# +# Opt in rather than part of the aggregate. The export-time plugin that +# executorch.kernels.quantized loads registers the same operator names, and the +# runtime stops on a repeat registration rather than choosing one, so a process +# holding both dies. Measured: linking this library and importing that module in +# either order aborts with "Re-registering quantized_decomposed::add.out". None +# of the other shipped components collide this way, so only this one is held +# back, and a consumer that wants it names it. +_executorch_define_component( + kernels_quantized executorch_kernels_quantized OPT_IN +) +# The same library exposed through a variable, so a consumer that follows the +# pre-3.28 recipe and later upgrades past 3.28 keeps working. Left empty when +# the wheel shipped no such library, matching the pre-3.28 branch above. +if(TARGET executorch::kernels_quantized) + set(EXECUTORCH_QUANTIZED_KERNELS_LIBRARY executorch::kernels_quantized) +endif() # The profiler. A C++ application could not record timing data from an installed # package before, because the implementation shipped only inside the Python # extension. @@ -763,13 +824,28 @@ foreach(_component ${executorch_FIND_COMPONENTS}) # One string rather than several arguments. Several make a list, and # message() joins a list with semicolons, which lands separators mid # sentence. - string( - CONCAT - executorch_NOT_FOUND_MESSAGE - "the required component '${_component}' needs CMake 3.28 or newer, because older " - "versions write the \$ORIGIN token in a runtime search path incorrectly; this " - "package is otherwise usable through EXECUTORCH_LIBRARIES" - ) + # + # The quantized kernels are held out of EXECUTORCH_LIBRARIES on purpose, + # so a consumer who wants them names + # EXECUTORCH_QUANTIZED_KERNELS_LIBRARY instead. See the OPT_IN comment + # at the component definition above. + if(_component STREQUAL "kernels_quantized") + string( + CONCAT + executorch_NOT_FOUND_MESSAGE + "the required component '${_component}' needs CMake 3.28 or newer, because " + "older versions write the \$ORIGIN token in a runtime search path incorrectly; " + "this package is otherwise usable through EXECUTORCH_QUANTIZED_KERNELS_LIBRARY" + ) + else() + string( + CONCAT + executorch_NOT_FOUND_MESSAGE + "the required component '${_component}' needs CMake 3.28 or newer, because older " + "versions write the \$ORIGIN token in a runtime search path incorrectly; this " + "package is otherwise usable through EXECUTORCH_LIBRARIES" + ) + endif() else() # One string rather than several arguments. Several make a list, and # message() joins a list with semicolons, which lands separators mid