Ship a pkg-config file for the runtime in the Linux and macOS wheels

.ci/scripts/wheel/test_cpp_sdk.py

@@ -1494,6 +1494,120 @@ def test_aggregate_variable_excludes_the_quantized_kernels(work_dir: Path) -> No
     )
 
 
+def _cmake_compile_definitions(work_dir: Path) -> list:
+    """The compile definitions the installed CMake package hands a consumer."""
+    config = _installed_package_dir() / "share" / "cmake" / "executorch-config.cmake"
+    source_dir = work_dir / "definitions-probe"
+    source_dir.mkdir(parents=True, exist_ok=True)
+    (source_dir / "CMakeLists.txt").write_text(
+        "cmake_minimum_required(VERSION 3.19)\n"
+        "project(probe NONE)\n"
+        "find_package(executorch REQUIRED)\n"
+        'message(STATUS "DEFINITIONS=${EXECUTORCH_COMPILE_DEFINITIONS}")\n'
+    )
+    result = subprocess.run(
+        [
+            _tool("cmake"),
+            "-S",
+            str(source_dir),
+            "-B",
+            str(work_dir / "definitions-probe-build"),
+            f"-DCMAKE_PREFIX_PATH={config.parent}",
+        ],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, (
+        "could not read the compile definitions from the installed CMake package:\n"
+        f"{result.stdout[-2000:]}{result.stderr[-2000:]}"
+    )
+    line = next(
+        entry for entry in result.stdout.splitlines() if "DEFINITIONS=" in entry
+    )
+    return [d for d in line.split("DEFINITIONS=", 1)[1].split(";") if d]
+
+
+def test_pkg_config_builds_a_consumer(work_dir: Path) -> None:
+    """A program built from the flags pkg-config prints finds, links and runs the runtime.
+
+    Meson, Autotools and plain Makefiles read pkg-config files rather than CMake packages. The
+    kernels are named on the command line because the file describes only the runtime, the same
+    split as the CMake components. pkg-config itself comes from the pkgconf package on PyPI, so
+    the check does not depend on one the build machine happens to have.
+    """
+    pc_dir = _installed_package_dir() / "lib" / "pkgconfig"
+    assert (pc_dir / "executorch.pc").is_file(), (
+        f"the wheel ships no pkg-config file in {pc_dir}, so a build system that reads "
+        "pkg-config cannot find the runtime"
+    )
+    venv_dir = work_dir / "pkgconf-venv"
+    subprocess.run([sys.executable, "-m", "venv", str(venv_dir)], check=True)
+    subprocess.run(
+        [str(venv_dir / "bin" / "pip"), "install", "--quiet", "pkgconf"], check=True
+    )
+    pkg_config = str(venv_dir / "bin" / "pkg-config")
+
+    # PKG_CONFIG_LIBDIR alone, so a system executorch.pc cannot be found instead of this one.
+    environment = dict(os.environ, PKG_CONFIG_LIBDIR=str(pc_dir))
+    environment.pop("PKG_CONFIG_PATH", None)
+    result = subprocess.run(
+        [pkg_config, "--cflags", "--libs", "executorch"],
+        capture_output=True,
+        text=True,
+        check=False,
+        env=environment,
+    )
+    assert result.returncode == 0, (
+        "pkg-config --cflags --libs executorch failed against the installed file:\n"
+        f"{result.stdout[-2000:]}{result.stderr[-2000:]}"
+    )
+    flags = result.stdout.split()
+
+    # Both routes describe the same libraries, so they must agree on the definitions. A
+    # missing one fails silently for the consumer: ET_EVENT_TRACER_ENABLED changes the
+    # layout of the tracer scope classes, and without ET_USE_THREADPOOL parallel_for is serial.
+    pc_definitions = sorted(f[2:] for f in flags if f.startswith("-D"))
+    cmake_definitions = sorted(_cmake_compile_definitions(work_dir))
+    assert pc_definitions == cmake_definitions, (
+        f"the pkg-config file defines {pc_definitions}, but the CMake package in the same "
+        f"wheel defines {cmake_definitions}"
+    )
+
+    source = work_dir / "pkg-config-consumer.cpp"
+    source.write_text(_CONSUMER_SOURCE)
+    consumer = work_dir / "pkg-config-consumer"
+    if sys.platform == "darwin":
+        kernels = ["-lexecutorch_kernels_optimized"]
+    else:
+        kernels = [
+            "-Wl,--push-state,--no-as-needed",
+            "-lexecutorch_kernels_optimized",
+            "-Wl,--pop-state",
+        ]
+    built = subprocess.run(
+        [
+            _tool("c++"),
+            "-std=c++17",
+            str(source),
+            "-o",
+            str(consumer),
+            *flags,
+            *kernels,
+        ],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert built.returncode == 0, (
+        f"a program built with the pkg-config flags did not compile and link:\n"
+        f"{built.stdout[-3000:]}{built.stderr[-3000:]}"
+    )
+    model, reference = _export(work_dir, "plain")
+    output = _run_consumer(consumer, model, reference, work_dir)
+    print(f"✓ a C++ app built from pkg-config flags runs a model ({output})")
+
+
 def run_tests(work_dir: Path) -> None:
     test_find_package_honours_a_version_request(work_dir)
     test_profiler_component_is_usable(work_dir)
@@ -1502,6 +1616,7 @@ def run_tests(work_dir: Path) -> None:
     test_documented_example_compiles(work_dir)
     test_runtime_alone_links_but_cannot_compute(work_dir)
     test_kernels_component_runs_a_model(work_dir)
+    test_pkg_config_builds_a_consumer(work_dir)
     test_pre_3_28_route_builds_a_consumer_through_variables(work_dir)
     test_quantized_kernels_component_runs_a_model(work_dir)
     test_aggregate_variable_excludes_the_quantized_kernels(work_dir)

CMakeLists.txt

@@ -1090,6 +1090,32 @@ if(EXECUTORCH_BUILD_SHARED)
   install(FILES ${CMAKE_CURRENT_BINARY_DIR}/executorch.pc
           DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
   )
+  # pip decides where the wheel lands, so the wheel's copy of this file locates
+  # everything relative to its own location.
+  if(EXECUTORCH_BUILD_WHEEL_DO_NOT_USE)
+    set(_executorch_pc_definitions "-DC10_USING_CUSTOM_GENERATED_MACROS")
+    if(EXECUTORCH_ENABLE_EVENT_TRACER)
+      string(APPEND _executorch_pc_definitions " -DET_EVENT_TRACER_ENABLED")
+    endif()
+    # Matches the CMake package, which adds both to the runtime whenever the
+    # thread pool ships. Without the definition, parallel_for runs serially.
+    set(_executorch_pc_libs "-lexecutorch")
+    if(TARGET extension_threadpool)
+      string(APPEND _executorch_pc_definitions " -DET_USE_THREADPOOL")
+      string(APPEND _executorch_pc_libs " -lexecutorch_threadpool")
+    endif()
+    # Separate arguments, because Meson only keeps a dependency's rpath at
+    # install when the argument starts with -Wl,-rpath.
+    if(APPLE)
+      set(_executorch_pc_rpath "-Wl,-rpath,\${libdir}")
+    else()
+      set(_executorch_pc_rpath "-Wl,--enable-new-dtags -Wl,-rpath,\${libdir}")
+    endif()
+    configure_file(
+      tools/cmake/executorch-wheel.pc.in
+      ${CMAKE_CURRENT_BINARY_DIR}/executorch-wheel.pc @ONLY
+    )
+  endif()
 endif()
 
 # The macro changes the layout of the tracer scope objects, so a consumer

docs/source/using-executorch-cpp.md

@@ -261,6 +261,49 @@ find_package(executorch REQUIRED COMPONENTS backend_mlx)
 message(STATUS "Metal kernels: ${MLX_METALLIB_PATH}")
 ```
 
+#### Using pkg-config
+
+Build systems such as Meson and Autotools read pkg-config files. The wheel ships one for the
+runtime. It covers the engine and the thread pool. Name the kernel libraries yourself, the same
+way you add CMake components. Point pkg-config at the file with an absolute path, because the
+library search path it gives the linker is built from that path:
+
+```
+export PKG_CONFIG_PATH="$(python -c 'import executorch, pathlib; print(pathlib.Path(executorch.__path__[0]) / "lib" / "pkgconfig")')${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}"
+```
+
+The kernels register themselves when they load, and nothing in your code names them, so the
+linker must be told to keep them. On Linux:
+
+```
+c++ -std=c++17 main.cpp $(pkg-config --cflags --libs executorch) \
+  -Wl,--push-state,--no-as-needed -lexecutorch_kernels_optimized -Wl,--pop-state -o app
+```
+
+On macOS:
+
+```
+c++ -std=c++17 main.cpp $(pkg-config --cflags --libs executorch) -lexecutorch_kernels_optimized -o app
+```
+
+In Meson, take the library directory from the dependency. On macOS, replace the three Linux flags
+with `'-Wl,-needed-lexecutorch_kernels_optimized'`:
+
+```
+executorch = dependency('executorch')
+libdir = executorch.get_variable(pkgconfig : 'libdir')
+executable('app', 'main.cpp',
+  dependencies : executorch,
+  link_args : ['-L' + libdir,
+               '-Wl,--push-state,--no-as-needed',
+               '-lexecutorch_kernels_optimized',
+               '-Wl,--pop-state'],
+  install : true)
+```
+
+Add `-lexecutorch_kernels_quantized` or a backend such as `-lexecutorch_backend_xnnpack` the same
+way, because they sit next to the runtime library.
+
 #### When something does not work
 
 - `find_package` could not find executorch: the `-DCMAKE_PREFIX_PATH=...` argument is missing or

setup.py

@@ -2944,6 +2944,13 @@ def iter_distribution_names(self):
                         "EXECUTORCH_BUILD_KERNELS_OPTIMIZED",
                     ],
                 ),
+                # For build systems that read pkg-config rather than CMake packages.
+                BuiltFile(
+                    src_dir="%CMAKE_CACHE_DIR%/",
+                    src_name="executorch-wheel.pc",
+                    dst="executorch/lib/pkgconfig/executorch.pc",
+                    dependent_cmake_flags=["EXECUTORCH_BUILD_SHARED"],
+                ),
                 # The CUDA delegate and the process-wide CUDA stream helper, for a
                 # wheel built from a CUDA index. Only present when the build asks for
                 # CUDA, so packaging requires that rather than looking for files a

tools/cmake/executorch-wheel.pc.in

@@ -0,0 +1,9 @@
+prefix=${pcfiledir}/../..
+libdir=${prefix}/lib
+includedir=${prefix}/include
+
+Name: ExecuTorch
+Description: On-device AI framework for PyTorch models
+Version: @PROJECT_VERSION@
+Cflags: -I${includedir} -I${includedir}/executorch/runtime/core/portable_type/c10 @_executorch_pc_definitions@
+Libs: -L${libdir} @_executorch_pc_rpath@ @_executorch_pc_libs@