GH-51245: [C++][Compute][Gandiva] Add support for LLVM 23.1

ci/conda_env_gandiva.txt

@@ -16,4 +16,4 @@
 # under the License.
 
 clang
-llvmdev<23
+llvmdev<24

cpp/CMakeLists.txt

@@ -179,6 +179,7 @@ set(ARROW_DOC_DIR "${CMAKE_INSTALL_DOCDIR}")
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 
 set(ARROW_LLVM_VERSIONS
+    "23.1"
     "22.1"
     "21.1"
     "20.1"

cpp/src/arrow/compute/exec.cc

@@ -1185,16 +1185,6 @@ class ScalarAggExecutor : public KernelExecutorImpl<ScalarAggregateKernel> {
   const FunctionOptions* options_;
 };
 
-template <typename ExecutorType,
-          typename FunctionType = typename ExecutorType::FunctionType>
-Result<std::unique_ptr<KernelExecutor>> MakeExecutor(ExecContext* ctx,
-                                                     const Function* func,
-                                                     const FunctionOptions* options) {
-  DCHECK_EQ(ExecutorType::function_kind, func->kind());
-  auto typed_func = checked_cast<const FunctionType*>(func);
-  return std::make_unique<ExecutorType>(ctx, typed_func, options);
-}
-
 }  // namespace
 
 Status PropagateNulls(KernelContext* ctx, const ExecSpan& batch, ArrayData* output) {

cpp/src/arrow/compute/function_internal.h

@@ -535,8 +535,8 @@ static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
 }
 
 template <typename T>
-static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
-GenericFromScalar(const std::shared_ptr<Scalar>& value) {
+enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>> GenericFromScalar(
+    const std::shared_ptr<Scalar>& value) {
   using ValueType = typename T::value_type;
   if (value->type->id() != Type::LIST) {
     return Status::Invalid("Expected type LIST but got ", value->type->ToString());

cpp/src/arrow/compute/kernels/codegen_internal.h

@@ -481,8 +481,8 @@ struct UnboxScalar<Decimal256Type> {
 
 template <typename T, typename VisitFunc, typename NullFunc>
   requires std::is_void_v<std::invoke_result_t<VisitFunc, typename GetViewType<T>::T>>
-static void VisitArrayValuesInline(const ArraySpan& arr, VisitFunc&& valid_func,
-                                   NullFunc&& null_func) {
+void VisitArrayValuesInline(const ArraySpan& arr, VisitFunc&& valid_func,
+                            NullFunc&& null_func) {
   VisitArraySpanInline<T>(
       arr,
       [&](typename GetViewType<T>::PhysicalType v) {
@@ -494,8 +494,8 @@ static void VisitArrayValuesInline(const ArraySpan& arr, VisitFunc&& valid_func,
 template <typename T, typename VisitFunc, typename NullFunc>
   requires std::is_same_v<std::invoke_result_t<VisitFunc, typename GetViewType<T>::T>,
                           Status>
-static Status VisitArrayValuesInline(const ArraySpan& arr, VisitFunc&& valid_func,
-                                     NullFunc&& null_func) {
+Status VisitArrayValuesInline(const ArraySpan& arr, VisitFunc&& valid_func,
+                              NullFunc&& null_func) {
   return VisitArraySpanInline<T>(
       arr,
       [&](typename GetViewType<T>::PhysicalType v) {
@@ -507,8 +507,8 @@ static Status VisitArrayValuesInline(const ArraySpan& arr, VisitFunc&& valid_fun
 // Like VisitArrayValuesInline, but for binary functions.
 
 template <typename Arg0Type, typename Arg1Type, typename VisitFunc, typename NullFunc>
-static void VisitTwoArrayValuesInline(const ArraySpan& arr0, const ArraySpan& arr1,
-                                      VisitFunc&& valid_func, NullFunc&& null_func) {
+void VisitTwoArrayValuesInline(const ArraySpan& arr0, const ArraySpan& arr1,
+                               VisitFunc&& valid_func, NullFunc&& null_func) {
   ArrayIterator<Arg0Type> arr0_it(arr0);
   ArrayIterator<Arg1Type> arr1_it(arr1);
 
@@ -584,7 +584,7 @@ namespace applicator {
 // static Status Call(KernelContext*, const Scalar& arg0, const ArraySpan& arg1,
 //                    ExecResult* out)
 template <typename Operator>
-static Status SimpleBinary(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+Status SimpleBinary(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
   if (batch.length == 0) return Status::OK();
 
   if (batch[0].is_array()) {

cpp/src/arrow/compute/kernels/vector_hash.cc

@@ -136,7 +136,7 @@ class ValueCountsAction final : ActionBase {
   }
 
   template <class Index>
-  void ObserveNullNotFound(Index index) {
+  [[maybe_unused]] void ObserveNullNotFound(Index index) {
     ARROW_LOG(FATAL) << "ObserveNullNotFound without err_status should not be called";
   }
 

cpp/src/arrow/scalar.cc

@@ -1191,21 +1191,21 @@ constexpr int64_t kMillisecondsInDay = 86400000;
 
 // date to date
 template <typename To>
-enable_if_t<std::is_same<To, Date64Scalar>::value, Result<std::shared_ptr<Scalar>>>
+enable_if_t<std::is_same<To, Date64Type>::value, Result<std::shared_ptr<Scalar>>>
 CastImpl(const Date32Scalar& from, std::shared_ptr<DataType> to_type) {
   return std::make_shared<Date64Scalar>(from.value * kMillisecondsInDay,
                                         std::move(to_type));
 }
 template <typename To>
-enable_if_t<std::is_same<To, Date32Scalar>::value, Result<std::shared_ptr<Scalar>>>
+enable_if_t<std::is_same<To, Date32Type>::value, Result<std::shared_ptr<Scalar>>>
 CastImpl(const Date64Scalar& from, std::shared_ptr<DataType> to_type) {
   return std::make_shared<Date32Scalar>(
       static_cast<int32_t>(from.value / kMillisecondsInDay), std::move(to_type));
 }
 
 // timestamp to date
 template <typename To>
-enable_if_t<std::is_same<To, Date64Scalar>::value, Result<std::shared_ptr<Scalar>>>
+enable_if_t<std::is_same<To, Date64Type>::value, Result<std::shared_ptr<Scalar>>>
 CastImpl(const TimestampScalar& from, std::shared_ptr<DataType> to_type) {
   ARROW_ASSIGN_OR_RAISE(
       auto millis,
@@ -1214,7 +1214,7 @@ CastImpl(const TimestampScalar& from, std::shared_ptr<DataType> to_type) {
                                         std::move(to_type));
 }
 template <typename To>
-enable_if_t<std::is_same<To, Date32Scalar>::value, Result<std::shared_ptr<Scalar>>>
+enable_if_t<std::is_same<To, Date32Type>::value, Result<std::shared_ptr<Scalar>>>
 CastImpl(const TimestampScalar& from, std::shared_ptr<DataType> to_type) {
   ARROW_ASSIGN_OR_RAISE(
       auto millis,
@@ -1225,7 +1225,7 @@ CastImpl(const TimestampScalar& from, std::shared_ptr<DataType> to_type) {
 
 // date to timestamp
 template <typename To, typename From>
-enable_if_timestamp<Result<std::shared_ptr<To>>> CastImpl(
+enable_if_timestamp<To, Result<std::shared_ptr<Scalar>>> CastImpl(
     const DateScalar<From>& from, std::shared_ptr<DataType> to_type) {
   using ToScalar = typename TypeTraits<To>::ScalarType;
   int64_t millis = from.value;

cpp/src/arrow/scalar_test.cc

@@ -973,6 +973,39 @@ TEST(TestDateScalars, MakeScalar) {
                     Date64Scalar(-188171LL * 24 * 60 * 60 * 1000));
 }
 
+TEST(TestDateScalars, CastTo) {
+  constexpr int64_t kMillisecondsInDay = 86400000;
+
+  ASSERT_OK_AND_ASSIGN(auto casted_date64, Date32Scalar(2).CastTo(date64()));
+  EXPECT_EQ(*casted_date64, Date64Scalar(2 * kMillisecondsInDay));
+
+  ASSERT_OK_AND_ASSIGN(auto casted_date32,
+                       Date64Scalar(2 * kMillisecondsInDay).CastTo(date32()));
+  EXPECT_EQ(*casted_date32, Date32Scalar(2));
+
+  const auto timestamp_type = timestamp(TimeUnit::SECOND);
+
+  ASSERT_OK_AND_ASSIGN(auto timestamp_from_date32,
+                       Date32Scalar(2).CastTo(timestamp_type));
+  EXPECT_EQ(*timestamp_from_date32, TimestampScalar(2 * 24 * 60 * 60, timestamp_type));
+
+  ASSERT_OK_AND_ASSIGN(auto timestamp_from_date64,
+                       Date64Scalar(2 * kMillisecondsInDay).CastTo(timestamp_type));
+  EXPECT_EQ(*timestamp_from_date64, TimestampScalar(2 * 24 * 60 * 60, timestamp_type));
+
+  ASSERT_OK_AND_ASSIGN(
+      auto date64_from_timestamp,
+      TimestampScalar(2 * kMillisecondsInDay + 3, timestamp(TimeUnit::MILLI))
+          .CastTo(date64()));
+  EXPECT_EQ(*date64_from_timestamp, Date64Scalar(2 * kMillisecondsInDay));
+
+  ASSERT_OK_AND_ASSIGN(
+      auto date32_from_timestamp,
+      TimestampScalar(2 * kMillisecondsInDay + 3, timestamp(TimeUnit::MILLI))
+          .CastTo(date32()));
+  EXPECT_EQ(*date32_from_timestamp, Date32Scalar(2));
+}
+
 TEST(TestTimeScalars, Basics) {
   auto type1 = time32(TimeUnit::MILLI);
   auto type2 = time32(TimeUnit::SECOND);

cpp/src/arrow/util/async_generator.h

@@ -1863,7 +1863,7 @@ constexpr int kDefaultBackgroundQRestart = 16;
 ///
 /// This generator will queue up to max_q blocks
 template <typename T>
-static Result<AsyncGenerator<T>> MakeBackgroundGenerator(
+Result<AsyncGenerator<T>> MakeBackgroundGenerator(
     Iterator<T> iterator, internal::Executor* io_executor,
     int max_q = kDefaultBackgroundMaxQ, int q_restart = kDefaultBackgroundQRestart) {
   if (max_q < q_restart) {
@@ -1887,15 +1887,14 @@ static Result<AsyncGenerator<T>> MakeBackgroundGenerator(
 ///
 /// This generator does not queue
 template <typename T>
-static Result<AsyncGenerator<T>> MakeBlockingGenerator(
-    std::shared_ptr<Iterator<T>> iterator) {
+Result<AsyncGenerator<T>> MakeBlockingGenerator(std::shared_ptr<Iterator<T>> iterator) {
   return [it = std::move(iterator)]() mutable -> Future<T> {
     return Future<T>::MakeFinished(it->Next());
   };
 }
 
 template <typename T>
-static Result<AsyncGenerator<T>> MakeBlockingGenerator(Iterator<T> iterator) {
+Result<AsyncGenerator<T>> MakeBlockingGenerator(Iterator<T> iterator) {
   return MakeBlockingGenerator(std::make_shared<Iterator<T>>(std::move(iterator)));
 }
 

cpp/src/arrow/util/bit_block_counter.h

@@ -425,8 +425,8 @@ class ARROW_EXPORT OptionalBinaryBitBlockCounter {
 // Functional-style bit block visitors.
 
 template <typename VisitNotNull, typename VisitNull>
-static Status VisitBitBlocks(const uint8_t* bitmap, int64_t offset, int64_t length,
-                             VisitNotNull&& visit_not_null, VisitNull&& visit_null) {
+Status VisitBitBlocks(const uint8_t* bitmap, int64_t offset, int64_t length,
+                      VisitNotNull&& visit_not_null, VisitNull&& visit_null) {
   internal::OptionalBitBlockCounter bit_counter(bitmap, offset, length);
   int64_t position = 0;
   while (position < length) {
@@ -453,8 +453,8 @@ static Status VisitBitBlocks(const uint8_t* bitmap, int64_t offset, int64_t leng
 }
 
 template <typename VisitNotNull, typename VisitNull>
-static void VisitBitBlocksVoid(const uint8_t* bitmap, int64_t offset, int64_t length,
-                               VisitNotNull&& visit_not_null, VisitNull&& visit_null) {
+void VisitBitBlocksVoid(const uint8_t* bitmap, int64_t offset, int64_t length,
+                        VisitNotNull&& visit_not_null, VisitNull&& visit_null) {
   internal::OptionalBitBlockCounter bit_counter(bitmap, offset, length);
   int64_t position = 0;
   while (position < length) {
@@ -480,10 +480,10 @@ static void VisitBitBlocksVoid(const uint8_t* bitmap, int64_t offset, int64_t le
 }
 
 template <typename VisitNotNull, typename VisitNull>
-static Status VisitTwoBitBlocks(const uint8_t* left_bitmap, int64_t left_offset,
-                                const uint8_t* right_bitmap, int64_t right_offset,
-                                int64_t length, VisitNotNull&& visit_not_null,
-                                VisitNull&& visit_null) {
+Status VisitTwoBitBlocks(const uint8_t* left_bitmap, int64_t left_offset,
+                         const uint8_t* right_bitmap, int64_t right_offset,
+                         int64_t length, VisitNotNull&& visit_not_null,
+                         VisitNull&& visit_null) {
   if (left_bitmap == NULLPTR || right_bitmap == NULLPTR) {
     // At most one bitmap is present
     if (left_bitmap == NULLPTR) {
@@ -524,10 +524,10 @@ static Status VisitTwoBitBlocks(const uint8_t* left_bitmap, int64_t left_offset,
 }
 
 template <typename VisitNotNull, typename VisitNull>
-static void VisitTwoBitBlocksVoid(const uint8_t* left_bitmap, int64_t left_offset,
-                                  const uint8_t* right_bitmap, int64_t right_offset,
-                                  int64_t length, VisitNotNull&& visit_not_null,
-                                  VisitNull&& visit_null) {
+void VisitTwoBitBlocksVoid(const uint8_t* left_bitmap, int64_t left_offset,
+                           const uint8_t* right_bitmap, int64_t right_offset,
+                           int64_t length, VisitNotNull&& visit_not_null,
+                           VisitNull&& visit_null) {
   if (left_bitmap == NULLPTR || right_bitmap == NULLPTR) {
     // At most one bitmap is present
     if (left_bitmap == NULLPTR) {

cpp/src/arrow/util/future.h

@@ -702,7 +702,7 @@ class WeakFuture {
 /// If a Result<Future> holds an error instead of a Future, construct a finished Future
 /// holding that error.
 template <typename T>
-static Future<T> DeferNotOk(Result<Future<T>> maybe_future) {
+Future<T> DeferNotOk(Result<Future<T>> maybe_future) {
   if (ARROW_PREDICT_FALSE(!maybe_future.ok())) {
     return Future<T>::MakeFinished(std::move(maybe_future).status());
   }

cpp/src/gandiva/engine.cc

@@ -131,6 +131,8 @@ template <typename T>
 arrow::Result<T> AsArrowResult(llvm::Expected<T>& expected,
                                const std::string& error_context) {
   if (!expected) {
+    // NOTE: llvm::handleAllErrors() fails linking with RTTI-disabled LLVM builds
+    // (ARROW-5148)
     return Status::CodeGenError(error_context, llvm::toString(expected.takeError()));
   }
   return std::move(expected.get());
@@ -198,24 +200,34 @@ void AddProcessSymbol(llvm::orc::LLJIT& lljit) {
 }
 
 #ifdef JIT_LINK_SUPPORTED
+#  if LLVM_VERSION_MAJOR < 23
 Result<std::unique_ptr<llvm::jitlink::InProcessMemoryManager>> CreateMemmoryManager() {
   auto maybe_mem_manager = llvm::jitlink::InProcessMemoryManager::Create();
   return AsArrowResult(maybe_mem_manager, "Could not create memory manager: ");
 }
+#  endif
 
 Status UseJITLinkIfEnabled(llvm::orc::LLJITBuilder& jit_builder) {
   static auto maybe_use_jit_link = ::arrow::internal::GetEnvVar("GANDIVA_USE_JIT_LINK");
   if (maybe_use_jit_link.ok()) {
+#  if LLVM_VERSION_MAJOR >= 23
+    jit_builder.setObjectLinkingLayerCreator(
+        [](llvm::orc::ExecutionSession& ES,
+           llvm::jitlink::JITLinkMemoryManager& memory_manager) {
+          return std::make_unique<llvm::orc::ObjectLinkingLayer>(ES, memory_manager);
+        });
+#  else
     ARROW_ASSIGN_OR_RAISE(static auto memory_manager, CreateMemmoryManager());
-#  if LLVM_VERSION_MAJOR >= 21
+#    if LLVM_VERSION_MAJOR >= 21
     jit_builder.setObjectLinkingLayerCreator([&](llvm::orc::ExecutionSession& ES) {
       return std::make_unique<llvm::orc::ObjectLinkingLayer>(ES, *memory_manager);
     });
-#  else
+#    else
     jit_builder.setObjectLinkingLayerCreator(
         [&](llvm::orc::ExecutionSession& ES, const llvm::Triple& TT) {
           return std::make_unique<llvm::orc::ObjectLinkingLayer>(ES, *memory_manager);
         });
+#    endif
 #  endif
   }
   return Status::OK();
@@ -261,13 +273,8 @@ Result<std::unique_ptr<llvm::orc::LLJIT>> BuildJIT(
   return jit;
 }
 
-arrow::Status VerifyAndLinkModule(
-    llvm::Module& dest_module,
-    llvm::Expected<std::unique_ptr<llvm::Module>> src_module_or_error) {
-  ARROW_ASSIGN_OR_RAISE(
-      auto src_ir_module,
-      AsArrowResult(src_module_or_error, "Failed to verify and link module: "));
-
+arrow::Status VerifyAndLinkModule(llvm::Module& dest_module,
+                                  std::unique_ptr<llvm::Module> src_ir_module) {
   src_ir_module->setDataLayout(dest_module.getDataLayout());
 
   std::string error_info;
@@ -282,6 +289,14 @@ arrow::Status VerifyAndLinkModule(
   return Status::OK();
 }
 
+void RemoveBuildTargetAttributes(llvm::Module& module) {
+  for (auto& function : module.functions()) {
+    function.removeFnAttr("target-cpu");
+    function.removeFnAttr("target-features");
+    function.removeFnAttr("tune-cpu");
+  }
+}
+
 }  // namespace
 
 Status Engine::SetLLVMObjectCache(GandivaObjectCache& object_cache) {
@@ -422,10 +437,18 @@ Status Engine::LoadPreCompiledIR() {
   /// Parse the IR module.
   llvm::Expected<std::unique_ptr<llvm::Module>> module_or_error =
       llvm::getOwningLazyBitcodeModule(std::move(buffer), *context());
-  // NOTE: llvm::handleAllErrors() fails linking with RTTI-disabled LLVM builds
-  // (ARROW-5148)
-  ARROW_RETURN_NOT_OK(VerifyAndLinkModule(*module_, std::move(module_or_error)));
-  return Status::OK();
+  ARROW_ASSIGN_OR_RAISE(
+      auto src_ir_module,
+      AsArrowResult(module_or_error, "Failed to verify and link module: "));
+
+  // Built-in bitcode is JIT-compiled on the runtime host. Do not retain the target
+  // selected by Clang when the bitcode was built.
+  // LLVM 23 checks target-feature compatibility even for alwaysinline functions and
+  // prevents inlining on a mismatch. See the LLVM 23 release notes:
+  // https://releases.llvm.org/23.1.0/docs/ReleaseNotes.html#changes-to-the-llvm-ir
+  RemoveBuildTargetAttributes(*src_ir_module);
+
+  return VerifyAndLinkModule(*module_, std::move(src_ir_module));
 }
 
 static llvm::MemoryBufferRef AsLLVMMemoryBuffer(const arrow::Buffer& arrow_buffer) {
@@ -439,7 +462,10 @@ Status Engine::LoadExternalPreCompiledIR() {
   for (const auto& buffer : buffers) {
     auto llvm_memory_buffer_ref = AsLLVMMemoryBuffer(*buffer);
     auto module_or_error = llvm::parseBitcodeFile(llvm_memory_buffer_ref, *context());
-    ARROW_RETURN_NOT_OK(VerifyAndLinkModule(*module_, std::move(module_or_error)));
+    ARROW_ASSIGN_OR_RAISE(
+        auto src_ir_module,
+        AsArrowResult(module_or_error, "Failed to verify and link module: "));
+    ARROW_RETURN_NOT_OK(VerifyAndLinkModule(*module_, std::move(src_ir_module)));
   }
 
   return Status::OK();