[executorch][cuda] Add CudaExecutor

.github/workflows/cuda.yml

@@ -515,12 +515,22 @@ jobs:
           -v -o "addopts="
 
         cmake --preset llm-release-cuda -DEXECUTORCH_BUILD_TESTS=ON
-        cmake --build cmake-out --target test_cuda_allocator test_cuda_mutable_state test_cuda_weight_cache test_cuda_kv_cache test_cuda_guard test_cuda_stream_guard test_step_plan -j$(nproc)
+        cmake --build cmake-out --target test_cuda_allocator test_cuda_mutable_state test_cuda_weight_cache test_cuda_kv_cache test_cuda_guard test_cuda_stream_guard test_step_plan test_cuda_executor -j$(nproc)
         ctest --test-dir cmake-out -R test_cuda_allocator --output-on-failure -V
         ctest --test-dir cmake-out -R test_cuda_mutable_state --output-on-failure -V
         ctest --test-dir cmake-out -R test_cuda_weight_cache --output-on-failure -V
         ctest --test-dir cmake-out -R test_cuda_kv_cache --output-on-failure -V
         ctest --test-dir cmake-out -R test_step_plan --output-on-failure -V
+        # CudaExecutor end to end on a toy decoder, against its eager greedy
+        # output. The test skips without the model, so fail if it skipped.
+        export ET_CUDA_BATCHING_TOY_DIR="${RUNNER_TEMP:-/tmp}/cuda_batching_toy"
+        python -m executorch.backends.cuda.batching.test.export_toy_decoder --output-dir "${ET_CUDA_BATCHING_TOY_DIR}"
+        ctest --test-dir cmake-out -R test_cuda_executor --output-on-failure -V > /tmp/test_cuda_executor.log 2>&1 || { cat /tmp/test_cuda_executor.log; exit 1; }
+        cat /tmp/test_cuda_executor.log
+        if grep -q "SKIPPED" /tmp/test_cuda_executor.log; then
+          echo "::error::test_cuda_executor skipped"
+          exit 1
+        fi
         ctest --test-dir cmake-out -R test_cuda_guard --output-on-failure -V
         ctest --test-dir cmake-out -R test_cuda_stream_guard --output-on-failure -V
 

backends/cuda/batching/CMakeLists.txt

@@ -1,23 +1,50 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-# Step slicing for the CUDA batching executor: cuts a batch of packed tokens
-# into forwards the exported decode/prefill methods can run. Added from the
-# root after extension/llm/batching.
+# The batching executor for programs the CUDA backend compiled: the CUDA
+# counterpart of extension_llm_batching_module. Added from the root after
+# extension/llm/batching, whose executor seam it implements.
 
 if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
 endif()
 
-install(FILES step_plan.h
+add_library(cuda_batching cuda_executor.cpp)
+target_include_directories(cuda_batching PUBLIC ${_common_include_directories})
+target_compile_options(cuda_batching PUBLIC ${_common_compile_options})
+target_compile_definitions(cuda_batching PRIVATE CUDA_AVAILABLE=1)
+target_link_libraries(
+  cuda_batching
+  PUBLIC aoti_cuda_backend extension_llm_batching extension_llm_cache
+         extension_module extension_tensor
+  PRIVATE extension_llm_sampler
+)
+
+install(
+  TARGETS cuda_batching
+  EXPORT ExecuTorchTargets
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  INCLUDES
+  DESTINATION ${_common_include_directories}
+)
+install(FILES cuda_executor.h step_plan.h
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/backends/cuda/batching
 )
 
 if(BUILD_TESTING)
   include(${EXECUTORCH_ROOT}/tools/cmake/Test.cmake)
 
   et_cxx_test(test_step_plan SOURCES test/test_step_plan.cpp)
+
+  # Skips unless ET_CUDA_BATCHING_TOY_DIR names the output of
+  # test/export_toy_decoder.py. The program's host/device copies are portable
+  # et_copy kernels, registered by the CPU ops library.
+  et_cxx_test(
+    test_cuda_executor SOURCES test/test_cuda_executor.cpp EXTRA_LIBS
+    cuda_batching optimized_native_cpu_ops_lib
+  )
+  executorch_target_link_options_shared_lib(optimized_native_cpu_ops_lib)
 endif()