apacheGH-37848: [C++][Gandiva] Migrate LLVM JIT engine from MCJIT to …

…ORC v2/LLJIT (apache#39098) ### Rationale for this change Gandiva currently employs MCJIT as its internal JIT engine. However, LLVM has introduced a newer JIT API known as ORC v2/LLJIT since LLVM 7.0, and it has several advantage over MCJIT, in particular, MCJIT is not actively maintained, and is slated for eventual deprecation and removal. ### What changes are included in this PR? * This PR replaces the MCJIT JIT engine with the ORC v2 engine, using the `LLJIT` API. * This PR adds a new JIT linker option `JITLink` (https://llvm.org/docs/JITLink.html), which can be used together with `LLJIT`, for LLVM 14+ on Linux/macOS platform. It is turned off by default but could be turned on with environment variable `GANDIVA_USE_JIT_LINK` ### Are these changes tested? Yes, they are covered by existing unit tests ### Are there any user-facing changes? * `Configuration` class has a new option called `dump_ir`. If users would like to call `DumpIR` API of `Projector` and `Filter`, they have to set the `dump_ir` option first. * Closes: apache#37848 Authored-by: Yue Ni <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
dremio · Jan 4, 2024 · 83cba25 · 83cba25
1 parent ccc674c
commit 83cba25
Show file tree

Hide file tree

Showing 18 changed files with 441 additions and 199 deletions.
diff --git a/cpp/cmake_modules/FindLLVMAlt.cmake b/cpp/cmake_modules/FindLLVMAlt.cmake
@@ -93,8 +93,8 @@ if(LLVM_FOUND)
         debuginfodwarf
         ipo
         linker
-        mcjit
         native
+        orcjit
         target)
     if(LLVM_VERSION_MAJOR GREATER_EQUAL 14)
       list(APPEND LLVM_TARGET_COMPONENTS passes)

diff --git a/cpp/src/gandiva/configuration.h b/cpp/src/gandiva/configuration.h
@@ -37,10 +37,12 @@ class GANDIVA_EXPORT Configuration {
 
   explicit Configuration(bool optimize,
                          std::shared_ptr<FunctionRegistry> function_registry =
-                             gandiva::default_function_registry())
+                             gandiva::default_function_registry(),
+                         bool dump_ir = false)
       : optimize_(optimize),
         target_host_cpu_(true),
-        function_registry_(function_registry) {}
+        function_registry_(std::move(function_registry)),
+        dump_ir_(dump_ir) {}
 
   Configuration() : Configuration(true) {}
 
@@ -50,11 +52,13 @@ class GANDIVA_EXPORT Configuration {
 
   bool optimize() const { return optimize_; }
   bool target_host_cpu() const { return target_host_cpu_; }
+  bool dump_ir() const { return dump_ir_; }
   std::shared_ptr<FunctionRegistry> function_registry() const {
     return function_registry_;
   }
 
   void set_optimize(bool optimize) { optimize_ = optimize; }
+  void set_dump_ir(bool dump_ir) { dump_ir_ = dump_ir; }
   void target_host_cpu(bool target_host_cpu) { target_host_cpu_ = target_host_cpu; }
   void set_function_registry(std::shared_ptr<FunctionRegistry> function_registry) {
     function_registry_ = std::move(function_registry);
@@ -65,6 +69,9 @@ class GANDIVA_EXPORT Configuration {
   bool target_host_cpu_; /* set the mcpu flag to host cpu while compiling llvm ir */
   std::shared_ptr<FunctionRegistry>
       function_registry_; /* function registry that may contain external functions */
+  // flag indicating if IR dumping is needed, defaults to false, and turning it on will
+  // negatively affect performance
+  bool dump_ir_ = false;
 };
 
 /// \brief configuration builder for gandiva
@@ -83,6 +90,12 @@ class GANDIVA_EXPORT ConfigurationBuilder {
     return configuration;
   }
 
+  std::shared_ptr<Configuration> build_with_ir_dumping(bool dump_ir) {
+    std::shared_ptr<Configuration> configuration(
+        new Configuration(true, gandiva::default_function_registry(), dump_ir));
+    return configuration;
+  }
+
   std::shared_ptr<Configuration> build(
       std::shared_ptr<FunctionRegistry> function_registry) {
     std::shared_ptr<Configuration> configuration(