[TIRx] Post-bringup follow-ups: op-dispatch, namespaces, launch bounds, gemm-async, backend reorg

CMakeLists.txt

@@ -339,13 +339,19 @@ tvm_file_glob(GLOB CODEGEN_SRCS
   src/target/source/*.cc
   src/target/canonicalizer/*.cc
   src/target/canonicalizer/llvm/*.cc
-  src/target/cuda/*.cc
-  src/target/rocm/*.cc
-  src/target/hexagon/*.cc
-  src/target/metal/*.cc
-  src/target/opencl/*.cc
-  src/target/vulkan/vulkan_fallback_module.cc
-  src/target/webgpu/*.cc
+  src/backend/cuda/codegen/*.cc
+  src/backend/cuda/op/*.cc
+  src/backend/hexagon/codegen/*.cc
+  src/backend/metal/codegen/*.cc
+  src/backend/metal/op/*.cc
+  src/backend/opencl/codegen/*.cc
+  src/backend/rocm/codegen/*.cc
+  src/backend/trn/codegen/*.cc
+  src/backend/trn/op/*.cc
+  src/backend/trn/transform/*.cc
+  src/backend/vulkan/codegen/register.cc
+  src/backend/vulkan/codegen/vulkan_fallback_module.cc
+  src/backend/webgpu/codegen/*.cc
     )
 
 list(APPEND COMPILER_SRCS ${CODEGEN_SRCS})
@@ -361,7 +367,7 @@ tvm_file_glob(GLOB RUNTIME_SRCS
   src/runtime/rpc/minrpc/*.cc
 )
 # Note: src/runtime/extra/disco/** moves to libtvm_runtime_extra.
-# Note: src/runtime/{cuda,vulkan,opencl,metal,rocm,hexagon}/* move to per-backend DSOs.
+# Note: src/backend/*/runtime sources move to per-backend DSOs.
 set(TVM_RUNTIME_EXT_OBJS "")
 
 if(BUILD_FOR_HEXAGON)

cmake/modules/CUDA.cmake

@@ -62,8 +62,8 @@ endif(USE_CUDA)
 if(USE_CUDA)
   message(STATUS "Build cuda device runtime")
 
-  tvm_file_glob(GLOB RUNTIME_CUDA_SRCS src/runtime/cuda/*.cc)
-  tvm_file_glob(GLOB VM_CUDA_BUILTIN_SRC_CC src/runtime/vm/cuda/*.cc)
+  tvm_file_glob(GLOB RUNTIME_CUDA_SRCS src/backend/cuda/runtime/*.cc)
+  tvm_file_glob(GLOB VM_CUDA_BUILTIN_SRC_CC src/backend/cuda/runtime/vm/*.cc)
 
   add_library(tvm_runtime_cuda_objs OBJECT ${RUNTIME_CUDA_SRCS} ${VM_CUDA_BUILTIN_SRC_CC})
   target_link_libraries(tvm_runtime_cuda_objs PUBLIC tvm_ffi_header)

cmake/modules/Hexagon.cmake

@@ -50,7 +50,8 @@ macro(file_glob_append _output_list)
   set(${_output_list} ${_tmp1})
 endmacro()
 
-set(TVMRT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime")
+set(TVMRT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/backend/hexagon/runtime")
+set(TVM_CORE_RUNTIME_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/runtime")
 
 if(DEFINED USE_HEXAGON_DEVICE)
   message(WARNING "USE_HEXAGON_DEVICE is deprecated, use USE_HEXAGON instead")
@@ -79,7 +80,7 @@ endif()
 
 if(NOT USE_HEXAGON)
   # USE_HEXAGON=OFF: codegen still works through the per-backend fallback
-  # module (src/target/hexagon/hexagon_fallback_module.cc), which is always
+  # module (src/backend/hexagon/codegen/hexagon_fallback_module.cc), which is always
   # compiled into libtvm via CODEGEN_SRCS.  No opt-stub registration is
   # needed.
   return()
@@ -122,22 +123,22 @@ if(BUILD_FOR_HEXAGON)
   # When building FOR Hexagon (the DSP itself), all runtime sources go into
   # the single libtvm_runtime (static or shared). No per-backend DSO split.
   file_glob_append(RUNTIME_HEXAGON_SRCS
-    "${TVMRT_SOURCE_DIR}/hexagon/*.cc"
+    "${TVMRT_SOURCE_DIR}/*.cc"
   )
   # Add builtins to RelaxVM
   tvm_file_glob(GLOB VM_BUILTIN_SRC_CC src/runtime/vm/hexagon/*.cc)
   list(APPEND RUNTIME_SRCS ${VM_BUILTIN_SRC_CC})
 else()
   file_glob_append(RUNTIME_HEXAGON_SRCS
-    "${TVMRT_SOURCE_DIR}/hexagon/hexagon_module.cc"
+    "${TVMRT_SOURCE_DIR}/hexagon_module.cc"
   )
 endif()
 
 set(htp_supported_archs "v68" "v69" "v73" "v75")
 list(FIND htp_supported_archs "${USE_HEXAGON_ARCH}" supported_arch_index)
 if(${supported_arch_index} EQUAL -1)
   # Exclude User DMA files when building for archs below v68
-  list(REMOVE_ITEM RUNTIME_HEXAGON_SRCS "${TVMRT_SOURCE_DIR}/hexagon/hexagon_user_dma.cc")
+  list(REMOVE_ITEM RUNTIME_HEXAGON_SRCS "${TVMRT_SOURCE_DIR}/hexagon_user_dma.cc")
 endif()
 
 if(BUILD_FOR_HEXAGON)
@@ -160,7 +161,7 @@ if(BUILD_FOR_HEXAGON)
   # QHL support.
   if(USE_HEXAGON_QHL)
     file_glob_append(TVM_QHL_WRAPPER_SRCS
-      "${TVMRT_SOURCE_DIR}/hexagon/qhl/*.cc"
+      "${TVMRT_SOURCE_DIR}/qhl/*.cc"
     )
 
     include_directories(
@@ -181,20 +182,20 @@ if(BUILD_FOR_HEXAGON)
   if(${supported_arch_index} GREATER -1)
     # Hand-written ops
     file_glob_append(RUNTIME_HEXAGON_SRCS
-      "${TVMRT_SOURCE_DIR}/hexagon/ops/*.cc"
+      "${TVMRT_SOURCE_DIR}/ops/*.cc"
     )
 
     include_directories(
-      "${TVMRT_SOURCE_DIR}/hexagon/ops"
+      "${TVMRT_SOURCE_DIR}/ops"
     )
 
     set_source_files_properties(
-      "${TVMRT_SOURCE_DIR}/hexagon/ops/conv2d_quant_hvx.cc"
+      "${TVMRT_SOURCE_DIR}/ops/conv2d_quant_hvx.cc"
       PROPERTIES COMPILE_FLAGS "-mhvx"
     )
 
     set_source_files_properties(
-      "${TVMRT_SOURCE_DIR}/hexagon/ops/conv2d_fp16_hvx.cc"
+      "${TVMRT_SOURCE_DIR}/ops/conv2d_fp16_hvx.cc"
       PROPERTIES COMPILE_FLAGS "-mhvx"
     )
   endif()
@@ -249,21 +250,21 @@ if(USE_HEXAGON_RPC)
 
     add_custom_command(
       OUTPUT
-        "${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon_rpc.h"
-        "${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon_rpc_skel.c"
-        "${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon_rpc_stub.c"
+        "${TVMRT_SOURCE_DIR}/rpc/hexagon_rpc.h"
+        "${TVMRT_SOURCE_DIR}/rpc/hexagon_rpc_skel.c"
+        "${TVMRT_SOURCE_DIR}/rpc/hexagon_rpc_stub.c"
       COMMAND
         ${QAIC_EXE_PATH} ${QAIC_FLAGS}
-          "${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon_rpc.idl"
-          -o "${TVMRT_SOURCE_DIR}/hexagon/rpc"
-      MAIN_DEPENDENCY "${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon_rpc.idl"
+          "${TVMRT_SOURCE_DIR}/rpc/hexagon_rpc.idl"
+          -o "${TVMRT_SOURCE_DIR}/rpc"
+      MAIN_DEPENDENCY "${TVMRT_SOURCE_DIR}/rpc/hexagon_rpc.idl"
     )
 
     if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
         # We can't easily fix this at the source-code level, because the .c file is generated
         # by the qaic program.  But it should be safe to ignore the warning:
         # https://stackoverflow.com/questions/13905200/is-it-wise-to-ignore-gcc-clangs-wmissing-braces-warning
-        set_source_files_properties("${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon_rpc_stub.c"
+        set_source_files_properties("${TVMRT_SOURCE_DIR}/rpc/hexagon_rpc_stub.c"
             PROPERTY COMPILE_FLAGS "-Wno-missing-braces")
     endif()
   endfunction()
@@ -273,12 +274,12 @@ if(USE_HEXAGON_RPC)
     add_android_paths()
     build_rpc_idl()
     file_glob_append(RUNTIME_HEXAGON_SRCS
-      "${TVMRT_SOURCE_DIR}/hexagon/rpc/android/*.cc"
+      "${TVMRT_SOURCE_DIR}/rpc/android/*.cc"
     )
     # Add this file separately, because it's auto-generated, and glob won't
     # find it during cmake-time.
     list(APPEND RUNTIME_HEXAGON_SRCS
-      "${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon_rpc_stub.c"
+      "${TVMRT_SOURCE_DIR}/rpc/hexagon_rpc_stub.c"
     )
     list(APPEND TVM_RUNTIME_LINKER_LIBS cdsprpc)
 
@@ -289,30 +290,30 @@ if(USE_HEXAGON_RPC)
 
     # Include the generic RPC code into the TVM runtime.
     list(APPEND RUNTIME_HEXAGON_SRCS
-      "${TVMRT_SOURCE_DIR}/rpc/minrpc/minrpc_server.h"
-      "${TVMRT_SOURCE_DIR}/rpc/minrpc/rpc_reference.h"
-      "${TVMRT_SOURCE_DIR}/rpc/rpc_module.cc"
-      "${TVMRT_SOURCE_DIR}/rpc/rpc_endpoint.cc"
-      "${TVMRT_SOURCE_DIR}/rpc/rpc_session.cc"
+      "${TVM_CORE_RUNTIME_SOURCE_DIR}/rpc/minrpc/minrpc_server.h"
+      "${TVM_CORE_RUNTIME_SOURCE_DIR}/rpc/minrpc/rpc_reference.h"
+      "${TVM_CORE_RUNTIME_SOURCE_DIR}/rpc/rpc_module.cc"
+      "${TVM_CORE_RUNTIME_SOURCE_DIR}/rpc/rpc_endpoint.cc"
+      "${TVM_CORE_RUNTIME_SOURCE_DIR}/rpc/rpc_session.cc"
       # TODO(masahi): Remove rpc_local_session.cc after verifying that things work without it
-      "${TVMRT_SOURCE_DIR}/rpc/rpc_local_session.cc"
+      "${TVM_CORE_RUNTIME_SOURCE_DIR}/rpc/rpc_local_session.cc"
     )
-    set(HEXAGON_PROFILER_DIR "${TVMRT_SOURCE_DIR}/hexagon/profiler")
+    set(HEXAGON_PROFILER_DIR "${TVMRT_SOURCE_DIR}/profiler")
     # Add the hardware-specific RPC code into the skel library.
     set_property(SOURCE ${HEXAGON_PROFILER_DIR}/lwp_handler.S PROPERTY LANGUAGE C)
     add_library(hexagon_rpc_skel SHARED
-      "${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon/rpc_server.cc"
-      "${TVMRT_SOURCE_DIR}/hexagon/rpc/hexagon_rpc_skel.c"
+      "${TVMRT_SOURCE_DIR}/rpc/hexagon/rpc_server.cc"
+      "${TVMRT_SOURCE_DIR}/rpc/hexagon_rpc_skel.c"
       "${HEXAGON_PROFILER_DIR}/prof_utils.cc"
       "${HEXAGON_PROFILER_DIR}/lwp_handler.S"
     )
     target_include_directories(hexagon_rpc_skel
-      SYSTEM PRIVATE "${TVMRT_SOURCE_DIR}/hexagon/rpc"
+      SYSTEM PRIVATE "${TVMRT_SOURCE_DIR}/rpc"
     )
     # Add the simulator-specific RPC code into a shared library to be
     # executed via run_main_on_sim.
     add_library(hexagon_rpc_sim SHARED
-      "${TVMRT_SOURCE_DIR}/hexagon/rpc/simulator/rpc_server.cc"
+      "${TVMRT_SOURCE_DIR}/rpc/simulator/rpc_server.cc"
       "${HEXAGON_PROFILER_DIR}/prof_utils.cc"
       "${HEXAGON_PROFILER_DIR}/lwp_handler.S"
     )
@@ -324,7 +325,7 @@ if(USE_HEXAGON_RPC)
     find_hexagon_toolchain()
     add_hexagon_wrapper_paths()
     file_glob_append(RUNTIME_HEXAGON_SRCS
-      "${TVMRT_SOURCE_DIR}/hexagon/rpc/simulator/session.cc"
+      "${TVMRT_SOURCE_DIR}/rpc/simulator/session.cc"
     )
     list(APPEND TVM_RUNTIME_LINKER_LIBS "-lwrapper")
   endif()

cmake/modules/LLVM.cmake

@@ -44,9 +44,9 @@ if(NOT ${USE_LLVM} MATCHES ${IS_FALSE_PATTERN})
   add_definitions(-DTVM_LLVM_HAS_AARCH64_TARGET=${TVM_LLVM_HAS_AARCH64_TARGET})
   tvm_file_glob(GLOB COMPILER_LLVM_SRCS
     src/target/llvm/*.cc
-    src/target/cuda/llvm/*.cc
-    src/target/rocm/llvm/*.cc
-    src/target/hexagon/llvm/*.cc
+    src/backend/cuda/codegen/llvm/*.cc
+    src/backend/rocm/codegen/llvm/*.cc
+    src/backend/hexagon/codegen/llvm/*.cc
   )
   list(APPEND TVM_LINKER_LIBS ${LLVM_LIBS})
   list(APPEND COMPILER_SRCS ${COMPILER_LLVM_SRCS})

cmake/modules/Metal.cmake

@@ -19,7 +19,7 @@ if(USE_METAL)
   message(STATUS "Build metal device runtime")
   find_library(METAL_LIB Metal)
   find_library(FOUNDATION_LIB Foundation)
-  tvm_file_glob(GLOB RUNTIME_METAL_SRCS src/runtime/metal/*.mm)
+  tvm_file_glob(GLOB RUNTIME_METAL_SRCS src/backend/metal/runtime/*.mm)
 
   add_library(tvm_runtime_metal_objs OBJECT ${RUNTIME_METAL_SRCS})
   target_link_libraries(tvm_runtime_metal_objs PUBLIC tvm_ffi_header)
@@ -33,5 +33,5 @@ if(USE_METAL)
   tvm_configure_target_library(tvm_runtime_metal RUNTIME_MODULE)
 endif(USE_METAL)
 # When USE_METAL=OFF the codegen-side fallback in
-# src/target/metal/metal_fallback_module.cc handles construction; no opt
+# src/backend/metal/codegen/metal_fallback_module.cc handles construction; no opt
 # stub is needed (it is always compiled via CODEGEN_SRCS in CMakeLists.txt).

cmake/modules/OpenCL.cmake

@@ -16,13 +16,13 @@
 # under the License.
 
 if(USE_OPENCL)
-  tvm_file_glob(GLOB RUNTIME_OPENCL_SRCS src/runtime/opencl/*.cc)
+  tvm_file_glob(GLOB RUNTIME_OPENCL_SRCS src/backend/opencl/runtime/*.cc)
 
   set(_opencl_libs "")
   if(${USE_OPENCL} MATCHES ${IS_TRUE_PATTERN})
     message(STATUS "Enabled runtime search for OpenCL library location")
     file_glob_append(RUNTIME_OPENCL_SRCS
-      "src/runtime/opencl/opencl_wrapper/opencl_wrapper.cc"
+      "src/backend/opencl/runtime/opencl_wrapper/opencl_wrapper.cc"
     )
     include_directories(SYSTEM "3rdparty/OpenCL-Headers")
   else()

cmake/modules/ROCM.cmake

@@ -32,7 +32,7 @@ if(USE_ROCM)
   endif()
   message(STATUS "Build rocm device runtime")
 
-  tvm_file_glob(GLOB RUNTIME_ROCM_SRCS src/runtime/rocm/*.cc)
+  tvm_file_glob(GLOB RUNTIME_ROCM_SRCS src/backend/rocm/runtime/*.cc)
 
   set(_rocm_libs ${ROCM_HIPHCC_LIBRARY})
   if(ROCM_HSA_LIBRARY)

cmake/modules/Vulkan.cmake

@@ -29,12 +29,12 @@ if(USE_VULKAN)
   include_directories(SYSTEM ${Vulkan_INCLUDE_DIRS})
   message(STATUS "Build with Vulkan support")
   tvm_file_glob(GLOB COMPILER_VULKAN_SRCS
-    src/target/vulkan/build_vulkan.cc
-    src/target/vulkan/codegen_spirv.cc
-    src/target/vulkan/intrin_rule_spirv.cc
-    src/target/vulkan/ir_builder.cc
-    src/target/vulkan/spirv_support.cc
-    src/target/vulkan/spirv_utils.cc
+    src/backend/vulkan/codegen/build_vulkan.cc
+    src/backend/vulkan/codegen/codegen_spirv.cc
+    src/backend/vulkan/codegen/intrin_rule_spirv.cc
+    src/backend/vulkan/codegen/ir_builder.cc
+    src/backend/vulkan/codegen/spirv_support.cc
+    src/backend/vulkan/codegen/spirv_utils.cc
   )
   list(APPEND COMPILER_SRCS ${COMPILER_VULKAN_SRCS})
   list(APPEND TVM_LINKER_LIBS ${Vulkan_SPIRV_TOOLS_LIBRARY})
@@ -44,7 +44,7 @@ endif(USE_VULKAN)
 if(USE_VULKAN)
   message(STATUS "Build vulkan device runtime")
 
-  tvm_file_glob(GLOB RUNTIME_VULKAN_SRCS src/runtime/vulkan/*.cc)
+  tvm_file_glob(GLOB RUNTIME_VULKAN_SRCS src/backend/vulkan/runtime/*.cc)
 
   add_library(tvm_runtime_vulkan_objs OBJECT ${RUNTIME_VULKAN_SRCS})
   target_link_libraries(tvm_runtime_vulkan_objs PUBLIC tvm_ffi_header)

docs/reference/api/python/tirx/backend.rst

@@ -23,9 +23,3 @@ tvm.tirx.backend
 .. automodule:: tvm.tirx.backend
    :members:
    :imported-members:
-
-tvm.tirx.backend.adreno
-***********************
-.. automodule:: tvm.tirx.backend.adreno
-   :members:
-   :imported-members: