From b3dee6a4cf63d8e824a8d761f4727c2cd91f01b0 Mon Sep 17 00:00:00 2001 From: Pindikura Ravindra Date: Thu, 31 May 2018 11:20:00 +0530 Subject: [PATCH] GDV-45: Make use of the modular features of cmake Separate out the public and private target dependencies. For arrow, export an interface target. This avoids the need to add include dirs for each dependency on arrow. Removed dependency on gtest. Instead, build it as an external project. This is the recommended practice for googletest. For pre-compiled files, generate the bitcode files for each of them independently and then, link them to generate a unified bitcode file. Removed cpplint exceptions since there is no more sourcing of .cc files. Separate out the public include files from private includes, and add them in the dependency list in cmake. pass the bytecode filepath from cmake (instead of /tmp) --- cpp/src/gandiva/CMakeLists.txt | 111 ++---------------- cpp/src/gandiva/README.md | 47 ++++++-- cpp/src/gandiva/codegen/CMakeLists.txt | 96 +++++++++++++++ .../gandiva/{expr => codegen}/annotator.cc | 3 +- cpp/src/gandiva/{expr => codegen}/annotator.h | 8 +- .../{expr => codegen}/annotator_test.cc | 2 +- cpp/src/gandiva/{common => codegen}/arrow.h | 0 cpp/src/gandiva/codegen/bc_file_path.cc.in | 22 ++++ cpp/src/gandiva/codegen/dex.h | 2 +- cpp/src/gandiva/codegen/engine.cc | 9 +- cpp/src/gandiva/codegen/engine.h | 4 +- .../gandiva/{expr => codegen}/eval_batch.h | 2 +- .../gandiva/{expr => codegen}/evaluator.cc | 12 +- cpp/src/gandiva/{expr => codegen}/evaluator.h | 14 +-- .../{expr => codegen}/evaluator_test.cc | 4 +- .../gandiva/{expr => codegen}/expression.h | 4 +- cpp/src/gandiva/codegen/field_descriptor.h | 2 +- cpp/src/gandiva/codegen/func_descriptor.h | 2 +- cpp/src/gandiva/codegen/function_signature.h | 4 +- .../{common => codegen}/gandiva_aliases.h | 0 cpp/src/gandiva/codegen/llvm_generator.cc | 1 + cpp/src/gandiva/codegen/llvm_generator.h | 5 +- .../gandiva/codegen/llvm_generator_test.cc | 7 +- cpp/src/gandiva/codegen/llvm_types.h | 2 +- cpp/src/gandiva/codegen/llvm_types_test.cc | 2 +- cpp/src/gandiva/{common => codegen}/logging.h | 0 cpp/src/gandiva/{expr => codegen}/node.cc | 4 +- cpp/src/gandiva/{expr => codegen}/node.h | 7 +- .../{expr => codegen}/tree_expr_builder.cc | 2 +- .../{expr => codegen}/tree_expr_builder.h | 4 +- .../{expr => codegen}/tree_expr_test.cc | 6 +- cpp/src/gandiva/codegen/value_validity_pair.h | 2 +- cpp/src/gandiva/precompiled/CMakeLists.txt | 53 +++++++++ cpp/src/gandiva/precompiled/arithmetic_ops.cc | 6 + cpp/src/gandiva/precompiled/bitmap.cc | 5 + .../precompiled/{irhelpers.cc => print.cc} | 9 +- cpp/src/gandiva/precompiled/time.cc | 3 + 37 files changed, 290 insertions(+), 176 deletions(-) create mode 100644 cpp/src/gandiva/codegen/CMakeLists.txt rename cpp/src/gandiva/{expr => codegen}/annotator.cc (98%) rename cpp/src/gandiva/{expr => codegen}/annotator.h (94%) rename cpp/src/gandiva/{expr => codegen}/annotator_test.cc (99%) rename cpp/src/gandiva/{common => codegen}/arrow.h (100%) create mode 100644 cpp/src/gandiva/codegen/bc_file_path.cc.in rename cpp/src/gandiva/{expr => codegen}/eval_batch.h (97%) rename cpp/src/gandiva/{expr => codegen}/evaluator.cc (87%) rename cpp/src/gandiva/{expr => codegen}/evaluator.h (87%) rename cpp/src/gandiva/{expr => codegen}/evaluator_test.cc (98%) rename cpp/src/gandiva/{expr => codegen}/expression.h (94%) rename cpp/src/gandiva/{common => codegen}/gandiva_aliases.h (100%) rename cpp/src/gandiva/{common => codegen}/logging.h (100%) rename cpp/src/gandiva/{expr => codegen}/node.cc (98%) rename cpp/src/gandiva/{expr => codegen}/node.h (93%) rename cpp/src/gandiva/{expr => codegen}/tree_expr_builder.cc (98%) rename cpp/src/gandiva/{expr => codegen}/tree_expr_builder.h (97%) rename cpp/src/gandiva/{expr => codegen}/tree_expr_test.cc (97%) create mode 100644 cpp/src/gandiva/precompiled/CMakeLists.txt rename cpp/src/gandiva/precompiled/{irhelpers.cc => print.cc} (83%) diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 4c03f504b19b3..5e973bd68f1af 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -12,106 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -project(gandiva) +cmake_minimum_required(VERSION 3.10) -set(REQD_LLVM_VERSION 6.0) -find_package(LLVM ${REQD_LLVM_VERSION} REQUIRED CONFIG HINTS - /usr/local/opt/llvm - /usr/share) -message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") -message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") -include_directories(${LLVM_INCLUDE_DIRS}) -add_definitions(${LLVM_DEFINITIONS}) +# LLVM/Clang is required by both the subdirs. +find_package(LLVM) -find_library(ARROW_LIB arrow) -message(STATUS "Found arrow : ${ARROW_LIB}") -find_path(ARROW_INCLUDE_DIR arrow/type.h) -message(STATUS "include dir for arrow : ${ARROW_INCLUDE_DIR}") -include_directories(${ARROW_INCLUDE_DIR}) +# Set the path where the byte-code files will be installed. +set(GANDIVA_BC_FILE_NAME irhelpers.bc) +set(GANDIVA_BC_INSTALL_DIR + ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/gandiva) +set(GANDIVA_BC_INSTALL_PATH ${GANDIVA_BC_INSTALL_DIR}/${GANDIVA_BC_FILE_NAME}) +set(GANDIVA_BC_OUTPUT_PATH ${CMAKE_BINARY_DIR}/irhelpers.bc) -find_package(GTest REQUIRED) -include_directories(${GTEST_INCLUDE_DIRS}) - -find_package(Boost REQUIRED) -include_directories(${Boost_INCLUDE_DIRS}) - -include_directories(.) -include_directories(codegen) -include_directories(codegen/cex) -include_directories(codegen/dex) -include_directories(codegen/expr) -include_directories(expr) -include_directories(precompiled) -include_directories(${CMAKE_BINARY_DIR}) - -set(GANDIVA_SRCS - codegen/engine.cc - codegen/function_registry.cc - codegen/llvm_generator.cc - codegen/llvm_types.cc - expr/annotator.cc - expr/evaluator.cc - expr/tree_expr_builder.cc - expr/node.cc) - -set(PRECOMPILED_SOURCED_FILES - precompiled/arithmetic_ops.cc - precompiled/bitmap.cc - precompiled/time.cc) - -SET(PRECOMPILED_SRC precompiled/irhelpers.cc) -#TODO : fix the location of the .bc file, it shouldn't be /tmp. -SET(PRECOMPILED_OUTPUT /tmp/irhelpers.bc) - -find_program(CLANG_EXECUTABLE NAMES clang++-${REQD_LLVM_VERSION} clang) -message(STATUS "Found clang++ ${CLANG_EXECUTABLE}") - -get_filename_component(PRECOMPILED_ABSOLUTE_SRC ${PRECOMPILED_SRC} ABSOLUTE) -add_custom_command( - OUTPUT ${PRECOMPILED_OUTPUT} - COMMAND ${CLANG_EXECUTABLE} -flto -O2 -c ${PRECOMPILED_ABSOLUTE_SRC} -o ${PRECOMPILED_OUTPUT} - DEPENDS ${PRECOMPILED_SRC} ${PRECOMPILED_SOURCED_FILES}) -add_custom_target(precompile-to-ir ALL DEPENDS ${PRECOMPILED_OUTPUT}) - -add_library(gandiva SHARED ${GANDIVA_SRCS}) - -# Find the libraries that correspond to the LLVM components -llvm_map_components_to_libnames(llvm_libs core mcjit native ipo bitreader target linker analysis debuginfodwarf) -# Link against LLVM libraries -target_link_libraries(gandiva ${llvm_libs}) -target_link_libraries(gandiva ${ARROW_LIB}) - -# Test related - -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads REQUIRED) - -function(ADD_GANDIVA_TEST LABEL REL_TEST_NAME) - get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) - set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}") - - add_executable(${TEST_NAME} ${REL_TEST_NAME} ${ARGN}) - if(${LABEL} STREQUAL "integ") - target_link_libraries(${TEST_NAME} gandiva) - endif() - if(${REL_TEST_NAME} MATCHES "llvm") - target_link_libraries(${TEST_NAME} ${llvm_libs}) - endif() - target_link_libraries(${TEST_NAME} ${ARROW_LIB}) - target_link_libraries(${TEST_NAME} ${GTEST_BOTH_LIBRARIES}) - target_link_libraries(${TEST_NAME} Threads::Threads) - add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) - set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LABEL} ${TEST_NAME}) -endfunction() - -#args: label test-file src-files -ADD_GANDIVA_TEST(unittest codegen/dex_llvm_test.cc) -ADD_GANDIVA_TEST(unittest codegen/engine_llvm_test.cc codegen/engine.cc codegen/llvm_types.cc) -ADD_GANDIVA_TEST(unittest codegen/function_signature_test.cc) -ADD_GANDIVA_TEST(unittest codegen/function_registry_test.cc codegen/function_registry.cc) -ADD_GANDIVA_TEST(unittest codegen/llvm_types_test.cc codegen/llvm_types.cc) -ADD_GANDIVA_TEST(unittest codegen/llvm_generator_test.cc codegen/llvm_generator.cc codegen/engine.cc codegen/llvm_types.cc codegen/function_registry.cc expr/annotator.cc) -ADD_GANDIVA_TEST(unittest expr/annotator_test.cc expr/annotator.cc) -ADD_GANDIVA_TEST(unittest expr/tree_expr_test.cc expr/tree_expr_builder.cc expr/node.cc expr/annotator.cc codegen/function_registry.cc) - -ADD_GANDIVA_TEST(integ expr/evaluator_test.cc) +add_subdirectory(codegen) +add_subdirectory(precompiled) diff --git a/cpp/src/gandiva/README.md b/cpp/src/gandiva/README.md index 07e541981e232..679e2e48b0576 100644 --- a/cpp/src/gandiva/README.md +++ b/cpp/src/gandiva/README.md @@ -2,36 +2,57 @@ ## System setup -Gandiva uses CMake as a build configuration system. Currently, it supports -in-source and out-of-source builds with the latter one being preferred. +Gandiva uses CMake as a build configuration system. Currently, it supports +out-of-source builds only. Build Gandiva requires: * A C++11-enabled compiler. On Linux, gcc 4.8 and higher should be sufficient. * CMake -* LLVM +* LLVM * Arrow -* GTest * Boost -On OS X, you can use [Homebrew][1]: +On macOS, you can use [Homebrew][1]: ```shell brew install cmake llvm boost ``` +To install arrow, follow the steps in the [arrow Readme][2]. ## Building Gandiva - mkdir build - cd build - cmake .. - make - +Debug build : + +```shell +git clone https://github.com/dremio/gandiva.git +cd gandiva/cpp +mkdir debug +cd debug +cmake .. +make test +``` + +Release build : + +```shell +git clone https://github.com/dremio/gandiva.git +cd gandiva/cpp +mkdir release +cd release +cmake .. -DCMAKE_BUILD_TYPE=Release +make test +``` + ## Validating code style -We follow the [google cpp code style][2]. To validate compliance, +We follow the [google cpp code style][3]. To validate compliance, - make lint +```shell +cd debug +make lint +``` [1]: https://brew.sh/ -[2]: https://google.github.io/styleguide/cppguide.html +[2]: https://github.com/apache/arrow/tree/master/cpp +[3]: https://google.github.io/styleguide/cppguide.html diff --git a/cpp/src/gandiva/codegen/CMakeLists.txt b/cpp/src/gandiva/codegen/CMakeLists.txt new file mode 100644 index 0000000000000..c0166a26c1591 --- /dev/null +++ b/cpp/src/gandiva/codegen/CMakeLists.txt @@ -0,0 +1,96 @@ +# Copyright (C) 2017-2018 Dremio Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project(gandiva) + +# Find arrow +find_package(ARROW) + +find_package(Boost REQUIRED) + +set(BC_FILE_PATH_CC "${CMAKE_CURRENT_BINARY_DIR}/bc_file_path.cc") +configure_file(bc_file_path.cc.in ${BC_FILE_PATH_CC}) + +add_library(gandiva SHARED + annotator.cc + engine.cc + evaluator.cc + function_registry.cc + llvm_generator.cc + llvm_types.cc + node.cc + tree_expr_builder.cc + ${BC_FILE_PATH_CC}) + +# For users of gandiva library (including integ tests), include-dir is : +# /usr/**/include dir after install, +# cpp/include during build +# For building gandiva library itself, include-dir (in addition to above) is : +# cpp/src +target_include_directories(gandiva + PUBLIC + $ + $ + PRIVATE + ${CMAKE_SOURCE_DIR}/src +) + +# ARROW is a public dependency i.e users of gandiva also will have a dependency on arrow. +target_link_libraries(gandiva + PUBLIC + ARROW::ARROW + PRIVATE + Boost::boost) + +# LLVM is a private dependency i.e users of gandiva will not need to include llvm headers +# or link with llvm libraries. +target_link_llvm(gandiva PRIVATE) + +# Set version for the library. +set(GANDIVA_VERSION_MAJOR 0) +set(GANDIVA_VERSION_MINOR 1) +set(GANDIVA_VERSION_PATCH 0) +set(GANDIVA_VERSION ${GANDIVA_VERSION_MAJOR}.${GANDIVA_VERSION_MINOR}.${GANDIVA_VERSION_PATCH}) + +set_target_properties(gandiva PROPERTIES + VERSION ${GANDIVA_VERSION} + SOVERSION ${GANDIVA_VERSION_MAJOR} +) + +# install for gandiva +include(GNUInstallDirs) + +# install libgandiva +install( + TARGETS gandiva + DESTINATION ${CMAKE_INSTALL_LIBDIR} +) + +# install the header files. +install( + DIRECTORY ${CMAKE_SOURCE_DIR}/include/gandiva + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) + +#args: label test-file src-files +add_gandiva_unit_test(dex_llvm_test.cc) +add_gandiva_unit_test(engine_llvm_test.cc engine.cc llvm_types.cc ${BC_FILE_PATH_CC}) +add_gandiva_unit_test(function_signature_test.cc) +add_gandiva_unit_test(function_registry_test.cc function_registry.cc) +add_gandiva_unit_test(llvm_types_test.cc llvm_types.cc) +add_gandiva_unit_test(llvm_generator_test.cc llvm_generator.cc engine.cc llvm_types.cc function_registry.cc annotator.cc ${BC_FILE_PATH_CC}) +add_gandiva_unit_test(annotator_test.cc annotator.cc) +add_gandiva_unit_test(tree_expr_test.cc tree_expr_builder.cc node.cc annotator.cc function_registry.cc) + +add_gandiva_integ_test(evaluator_test.cc) diff --git a/cpp/src/gandiva/expr/annotator.cc b/cpp/src/gandiva/codegen/annotator.cc similarity index 98% rename from cpp/src/gandiva/expr/annotator.cc rename to cpp/src/gandiva/codegen/annotator.cc index da7188a2b8a96..df4a7d3aadee5 100644 --- a/cpp/src/gandiva/expr/annotator.cc +++ b/cpp/src/gandiva/codegen/annotator.cc @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include #include -#include "expr/annotator.h" +#include "codegen/annotator.h" #include "codegen/field_descriptor.h" namespace gandiva { diff --git a/cpp/src/gandiva/expr/annotator.h b/cpp/src/gandiva/codegen/annotator.h similarity index 94% rename from cpp/src/gandiva/expr/annotator.h rename to cpp/src/gandiva/codegen/annotator.h index edb7d118f61ab..498cccef16c85 100644 --- a/cpp/src/gandiva/expr/annotator.h +++ b/cpp/src/gandiva/codegen/annotator.h @@ -20,10 +20,10 @@ #include #include #include -#include "common/arrow.h" -#include "common/gandiva_aliases.h" -#include "common/logging.h" -#include "expr/eval_batch.h" +#include "gandiva/arrow.h" +#include "gandiva/gandiva_aliases.h" +#include "gandiva/logging.h" +#include "codegen/eval_batch.h" namespace gandiva { diff --git a/cpp/src/gandiva/expr/annotator_test.cc b/cpp/src/gandiva/codegen/annotator_test.cc similarity index 99% rename from cpp/src/gandiva/expr/annotator_test.cc rename to cpp/src/gandiva/codegen/annotator_test.cc index 5655bfe3b4bb8..8ee3b02d90f72 100644 --- a/cpp/src/gandiva/expr/annotator_test.cc +++ b/cpp/src/gandiva/codegen/annotator_test.cc @@ -17,7 +17,7 @@ #include #include #include -#include "expr/annotator.h" +#include "codegen/annotator.h" #include "codegen/field_descriptor.h" namespace gandiva { diff --git a/cpp/src/gandiva/common/arrow.h b/cpp/src/gandiva/codegen/arrow.h similarity index 100% rename from cpp/src/gandiva/common/arrow.h rename to cpp/src/gandiva/codegen/arrow.h diff --git a/cpp/src/gandiva/codegen/bc_file_path.cc.in b/cpp/src/gandiva/codegen/bc_file_path.cc.in new file mode 100644 index 0000000000000..185f4448dc89d --- /dev/null +++ b/cpp/src/gandiva/codegen/bc_file_path.cc.in @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2017-2018 Dremio Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace gandiva { + +// Path to the byte-code file. +extern const char kByteCodeFilePath[] = "${GANDIVA_BC_OUTPUT_PATH}"; + +} // namespace gandiva diff --git a/cpp/src/gandiva/codegen/dex.h b/cpp/src/gandiva/codegen/dex.h index fb72bf76b6c72..730ecff9ae6b0 100644 --- a/cpp/src/gandiva/codegen/dex.h +++ b/cpp/src/gandiva/codegen/dex.h @@ -18,7 +18,7 @@ #include #include -#include "common/gandiva_aliases.h" +#include "gandiva/gandiva_aliases.h" #include "codegen/dex_visitor.h" #include "codegen/field_descriptor.h" #include "codegen/func_descriptor.h" diff --git a/cpp/src/gandiva/codegen/engine.cc b/cpp/src/gandiva/codegen/engine.cc index e518b586c1e38..03e002c427b9b 100644 --- a/cpp/src/gandiva/codegen/engine.cc +++ b/cpp/src/gandiva/codegen/engine.cc @@ -39,10 +39,11 @@ namespace gandiva { -const char Engine::kLibPreCompiledIRDir[] = "/tmp/"; bool Engine::init_once_done_ = false; std::once_flag init_once_flag; +extern const char kByteCodeFilePath[]; + /* * One-time initializations. */ @@ -84,14 +85,12 @@ Engine::Engine() * Handling for pre-compiled IR libraries. */ void Engine::LoadPreCompiledIRFiles() { - std::string fileName = std::string(kLibPreCompiledIRDir) + "irhelpers.bc"; - /// Read from file into memory buffer. llvm::ErrorOr> buffer_or_error = - llvm::MemoryBuffer::getFile(fileName); + llvm::MemoryBuffer::getFile(kByteCodeFilePath); if (!buffer_or_error) { std::stringstream ss; - ss << "Could not load module from IR " << fileName << ": " << + ss << "Could not load module from IR " << kByteCodeFilePath << ": " << buffer_or_error.getError().message(); throw CodeGenException(ss.str()); } diff --git a/cpp/src/gandiva/codegen/engine.h b/cpp/src/gandiva/codegen/engine.h index e8611bd61eedd..a31dde3d9f8e0 100644 --- a/cpp/src/gandiva/codegen/engine.h +++ b/cpp/src/gandiva/codegen/engine.h @@ -23,7 +23,7 @@ #include #include #include -#include "common/logging.h" +#include "gandiva/logging.h" namespace gandiva { @@ -51,8 +51,6 @@ class Engine { void *CompiledFunction(llvm::Function *irFunction); private: - static const char kLibPreCompiledIRDir[]; - // do one time inits. static void InitOnce(); static bool init_once_done_; diff --git a/cpp/src/gandiva/expr/eval_batch.h b/cpp/src/gandiva/codegen/eval_batch.h similarity index 97% rename from cpp/src/gandiva/expr/eval_batch.h rename to cpp/src/gandiva/codegen/eval_batch.h index ac4a9982b9164..0231ed22ab294 100644 --- a/cpp/src/gandiva/expr/eval_batch.h +++ b/cpp/src/gandiva/codegen/eval_batch.h @@ -16,7 +16,7 @@ #ifndef GANDIVA_EXPR_EVALBATCH_H #define GANDIVA_EXPR_EVALBATCH_H -#include "common/gandiva_aliases.h" +#include "gandiva/gandiva_aliases.h" namespace gandiva { diff --git a/cpp/src/gandiva/expr/evaluator.cc b/cpp/src/gandiva/codegen/evaluator.cc similarity index 87% rename from cpp/src/gandiva/expr/evaluator.cc rename to cpp/src/gandiva/codegen/evaluator.cc index e19cabaae968e..753e9b49c5094 100644 --- a/cpp/src/gandiva/expr/evaluator.cc +++ b/cpp/src/gandiva/codegen/evaluator.cc @@ -17,10 +17,20 @@ #include #include #include -#include "expr/evaluator.h" +#include "gandiva/evaluator.h" +#include "codegen/llvm_generator.h" namespace gandiva { +Evaluator::Evaluator(std::unique_ptr llvm_generator, + SchemaPtr schema, + const FieldVector &output_fields, + arrow::MemoryPool *pool) + : llvm_generator_(std::move(llvm_generator)), + schema_(schema), + output_fields_(output_fields), + pool_(pool) {} + // TODO : exceptions std::shared_ptr Evaluator::Make(SchemaPtr schema, const ExpressionVector &exprs, diff --git a/cpp/src/gandiva/expr/evaluator.h b/cpp/src/gandiva/codegen/evaluator.h similarity index 87% rename from cpp/src/gandiva/expr/evaluator.h rename to cpp/src/gandiva/codegen/evaluator.h index f6bc46dad957d..a97bb4dad63ff 100644 --- a/cpp/src/gandiva/expr/evaluator.h +++ b/cpp/src/gandiva/codegen/evaluator.h @@ -19,12 +19,13 @@ #include #include #include -#include "common/arrow.h" -#include "expr/expression.h" -#include "codegen/llvm_generator.h" +#include "gandiva/arrow.h" +#include "gandiva/expression.h" namespace gandiva { +class LLVMGenerator; + /// \brief Evaluator for expressions. /// /// An evaluator is built for a specific schema and vector of expressions. @@ -44,12 +45,9 @@ class Evaluator { Evaluator(std::unique_ptr llvm_generator, SchemaPtr schema, const FieldVector &output_fields, - arrow::MemoryPool *pool) - : llvm_generator_(std::move(llvm_generator)), - schema_(schema), - output_fields_(output_fields), - pool_(pool) {} + arrow::MemoryPool *pool); + /// Allocate an arrow array of length 'length'. ArrayPtr AllocArray(DataTypePtr type, int length); const std::unique_ptr llvm_generator_; diff --git a/cpp/src/gandiva/expr/evaluator_test.cc b/cpp/src/gandiva/codegen/evaluator_test.cc similarity index 98% rename from cpp/src/gandiva/expr/evaluator_test.cc rename to cpp/src/gandiva/codegen/evaluator_test.cc index 938e56dde1810..193ca0c8c6887 100644 --- a/cpp/src/gandiva/expr/evaluator_test.cc +++ b/cpp/src/gandiva/codegen/evaluator_test.cc @@ -19,8 +19,8 @@ #include #include "arrow/memory_pool.h" #include "arrow/test-util.h" -#include "expr/evaluator.h" -#include "expr/tree_expr_builder.h" +#include "gandiva/evaluator.h" +#include "gandiva/tree_expr_builder.h" namespace gandiva { diff --git a/cpp/src/gandiva/expr/expression.h b/cpp/src/gandiva/codegen/expression.h similarity index 94% rename from cpp/src/gandiva/expr/expression.h rename to cpp/src/gandiva/codegen/expression.h index b83e5a54644d8..920f57e616b1d 100644 --- a/cpp/src/gandiva/expr/expression.h +++ b/cpp/src/gandiva/codegen/expression.h @@ -16,8 +16,8 @@ #ifndef GANDIVA_EXPR_EXPRESSION_H #define GANDIVA_EXPR_EXPRESSION_H -#include "common/gandiva_aliases.h" -#include "expr/node.h" +#include "gandiva/gandiva_aliases.h" +#include "gandiva/node.h" namespace gandiva { diff --git a/cpp/src/gandiva/codegen/field_descriptor.h b/cpp/src/gandiva/codegen/field_descriptor.h index 94367ae124425..237307ecb8eee 100644 --- a/cpp/src/gandiva/codegen/field_descriptor.h +++ b/cpp/src/gandiva/codegen/field_descriptor.h @@ -17,7 +17,7 @@ #define GANDIVA_FIELDDESCRIPTOR_H #include -#include "common/arrow.h" +#include "gandiva/arrow.h" namespace gandiva { diff --git a/cpp/src/gandiva/codegen/func_descriptor.h b/cpp/src/gandiva/codegen/func_descriptor.h index c7dcabdaa0200..377d8dd50b8a9 100644 --- a/cpp/src/gandiva/codegen/func_descriptor.h +++ b/cpp/src/gandiva/codegen/func_descriptor.h @@ -18,7 +18,7 @@ #include #include -#include "common/arrow.h" +#include "gandiva/arrow.h" namespace gandiva { diff --git a/cpp/src/gandiva/codegen/function_signature.h b/cpp/src/gandiva/codegen/function_signature.h index 978df267edf85..b7055026c55c5 100644 --- a/cpp/src/gandiva/codegen/function_signature.h +++ b/cpp/src/gandiva/codegen/function_signature.h @@ -19,9 +19,9 @@ #include #include #include -#include "common/arrow.h" -#include "common/logging.h" #include "boost/functional/hash.hpp" +#include "gandiva/arrow.h" +#include "gandiva/logging.h" namespace gandiva { diff --git a/cpp/src/gandiva/common/gandiva_aliases.h b/cpp/src/gandiva/codegen/gandiva_aliases.h similarity index 100% rename from cpp/src/gandiva/common/gandiva_aliases.h rename to cpp/src/gandiva/codegen/gandiva_aliases.h diff --git a/cpp/src/gandiva/codegen/llvm_generator.cc b/cpp/src/gandiva/codegen/llvm_generator.cc index e3c965fa26469..c160718b8080e 100644 --- a/cpp/src/gandiva/codegen/llvm_generator.cc +++ b/cpp/src/gandiva/codegen/llvm_generator.cc @@ -17,6 +17,7 @@ #include #include #include +#include "gandiva/expression.h" #include "codegen/codegen_exception.h" #include "codegen/dex.h" #include "codegen/function_registry.h" diff --git a/cpp/src/gandiva/codegen/llvm_generator.h b/cpp/src/gandiva/codegen/llvm_generator.h index 685342d67f6a6..f0ca03513badb 100644 --- a/cpp/src/gandiva/codegen/llvm_generator.h +++ b/cpp/src/gandiva/codegen/llvm_generator.h @@ -21,7 +21,7 @@ #include #include #include -#include "common/gandiva_aliases.h" +#include "gandiva/gandiva_aliases.h" #include "codegen/dex_visitor.h" #include "codegen/compiled_expr.h" #include "codegen/engine.h" @@ -29,8 +29,7 @@ #include "codegen/value_validity_pair.h" #include "codegen/llvm_types.h" #include "codegen/lvalue.h" -#include "expr/annotator.h" -#include "expr/expression.h" +#include "codegen/annotator.h" namespace gandiva { diff --git a/cpp/src/gandiva/codegen/llvm_generator_test.cc b/cpp/src/gandiva/codegen/llvm_generator_test.cc index 4336607b488c0..659b14c6917fb 100644 --- a/cpp/src/gandiva/codegen/llvm_generator_test.cc +++ b/cpp/src/gandiva/codegen/llvm_generator_test.cc @@ -17,9 +17,12 @@ #include #include #include -#include "codegen/llvm_generator.h" -#include "codegen/function_registry.h" +#include "gandiva/expression.h" #include "codegen/codegen_exception.h" +#include "codegen/dex.h" +#include "codegen/func_descriptor.h" +#include "codegen/function_registry.h" +#include "codegen/llvm_generator.h" namespace gandiva { diff --git a/cpp/src/gandiva/codegen/llvm_types.h b/cpp/src/gandiva/codegen/llvm_types.h index 5a7303095075c..12223718a3fa2 100644 --- a/cpp/src/gandiva/codegen/llvm_types.h +++ b/cpp/src/gandiva/codegen/llvm_types.h @@ -19,7 +19,7 @@ #include #include #include -#include "common/arrow.h" +#include "gandiva/arrow.h" namespace gandiva { diff --git a/cpp/src/gandiva/codegen/llvm_types_test.cc b/cpp/src/gandiva/codegen/llvm_types_test.cc index ef6431ef7d2a4..3d7eebbb3a385 100644 --- a/cpp/src/gandiva/codegen/llvm_types_test.cc +++ b/cpp/src/gandiva/codegen/llvm_types_test.cc @@ -15,7 +15,7 @@ */ #include -#include "codegen/llvm_types.h" +#include "./llvm_types.h" namespace gandiva { diff --git a/cpp/src/gandiva/common/logging.h b/cpp/src/gandiva/codegen/logging.h similarity index 100% rename from cpp/src/gandiva/common/logging.h rename to cpp/src/gandiva/codegen/logging.h diff --git a/cpp/src/gandiva/expr/node.cc b/cpp/src/gandiva/codegen/node.cc similarity index 98% rename from cpp/src/gandiva/expr/node.cc rename to cpp/src/gandiva/codegen/node.cc index cb92488f3ae4f..e0725df29d66e 100644 --- a/cpp/src/gandiva/expr/node.cc +++ b/cpp/src/gandiva/codegen/node.cc @@ -17,11 +17,11 @@ #include #include #include +#include "gandiva/node.h" #include "codegen/dex.h" #include "codegen/function_registry.h" #include "codegen/function_signature.h" -#include "expr/annotator.h" -#include "expr/node.h" +#include "codegen/annotator.h" namespace gandiva { diff --git a/cpp/src/gandiva/expr/node.h b/cpp/src/gandiva/codegen/node.h similarity index 93% rename from cpp/src/gandiva/expr/node.h rename to cpp/src/gandiva/codegen/node.h index c163705c0b2e5..49e63a017c31e 100644 --- a/cpp/src/gandiva/expr/node.h +++ b/cpp/src/gandiva/codegen/node.h @@ -18,11 +18,8 @@ #include #include -#include "common/arrow.h" -#include "common/gandiva_aliases.h" -#include "codegen/func_descriptor.h" -#include "codegen/dex.h" -#include "codegen/value_validity_pair.h" +#include "gandiva/arrow.h" +#include "gandiva/gandiva_aliases.h" namespace gandiva { diff --git a/cpp/src/gandiva/expr/tree_expr_builder.cc b/cpp/src/gandiva/codegen/tree_expr_builder.cc similarity index 98% rename from cpp/src/gandiva/expr/tree_expr_builder.cc rename to cpp/src/gandiva/codegen/tree_expr_builder.cc index f8e5d836a3d01..be94090f805ac 100644 --- a/cpp/src/gandiva/expr/tree_expr_builder.cc +++ b/cpp/src/gandiva/codegen/tree_expr_builder.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "expr/tree_expr_builder.h" +#include "gandiva/tree_expr_builder.h" namespace gandiva { diff --git a/cpp/src/gandiva/expr/tree_expr_builder.h b/cpp/src/gandiva/codegen/tree_expr_builder.h similarity index 97% rename from cpp/src/gandiva/expr/tree_expr_builder.h rename to cpp/src/gandiva/codegen/tree_expr_builder.h index 45da1fd3df8ad..a84c66b568cff 100644 --- a/cpp/src/gandiva/expr/tree_expr_builder.h +++ b/cpp/src/gandiva/codegen/tree_expr_builder.h @@ -18,8 +18,8 @@ #include #include -#include "expr/node.h" -#include "expr/expression.h" +#include "gandiva/node.h" +#include "gandiva/expression.h" namespace gandiva { diff --git a/cpp/src/gandiva/expr/tree_expr_test.cc b/cpp/src/gandiva/codegen/tree_expr_test.cc similarity index 97% rename from cpp/src/gandiva/expr/tree_expr_test.cc rename to cpp/src/gandiva/codegen/tree_expr_test.cc index 88b0985086cb4..6bf25e37f366b 100644 --- a/cpp/src/gandiva/expr/tree_expr_test.cc +++ b/cpp/src/gandiva/codegen/tree_expr_test.cc @@ -15,12 +15,12 @@ */ #include +#include "gandiva/gandiva_aliases.h" +#include "gandiva/tree_expr_builder.h" #include "codegen/dex.h" #include "codegen/function_signature.h" #include "codegen/function_registry.h" -#include "common/gandiva_aliases.h" -#include "expr/annotator.h" -#include "expr/tree_expr_builder.h" +#include "codegen/annotator.h" namespace gandiva { diff --git a/cpp/src/gandiva/codegen/value_validity_pair.h b/cpp/src/gandiva/codegen/value_validity_pair.h index a4519e3ad1cbe..70fef23ddf74b 100644 --- a/cpp/src/gandiva/codegen/value_validity_pair.h +++ b/cpp/src/gandiva/codegen/value_validity_pair.h @@ -17,7 +17,7 @@ #define GANDIVA_VALUEVALIDITYPAIR_H #include -#include "common/gandiva_aliases.h" +#include "gandiva/gandiva_aliases.h" namespace gandiva { diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt new file mode 100644 index 0000000000000..f3c7b2ecafe35 --- /dev/null +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -0,0 +1,53 @@ +# Copyright (C) 2017-2018 Dremio Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project(gandiva) + +set(CLANG_EXECUTABLE ${LLVM_TOOLS_BINARY_DIR}/clang) +set(LINK_EXECUTABLE ${LLVM_TOOLS_BINARY_DIR}/llvm-link) + +set(PRECOMPILED_SRCS + arithmetic_ops.cc + bitmap.cc + time.cc + print.cc) + +# Create bitcode for each of the source files. +foreach(SRC_FILE ${PRECOMPILED_SRCS}) + get_filename_component(SRC_BASE ${SRC_FILE} NAME_WE) + get_filename_component(ABSOLUTE_SRC ${SRC_FILE} ABSOLUTE) + set(BC_FILE ${CMAKE_CURRENT_BINARY_DIR}/${SRC_BASE}.bc) + add_custom_command( + OUTPUT ${BC_FILE} + COMMAND ${CLANG_EXECUTABLE} + -std=c++11 -emit-llvm -O2 -c ${ABSOLUTE_SRC} -o ${BC_FILE} + DEPENDS ${SRC_FILE}) + list(APPEND BC_FILES ${BC_FILE}) +endforeach() + +# link all of the bitcode files into a single bitcode file. +add_custom_command( + OUTPUT ${GANDIVA_BC_OUTPUT_PATH} + COMMAND ${LINK_EXECUTABLE} + -o ${GANDIVA_BC_OUTPUT_PATH} + ${BC_FILES} + DEPENDS ${BC_FILES}) + +add_custom_target(precompiled ALL DEPENDS ${GANDIVA_BC_OUTPUT_PATH}) + +# Install the pre-compiled bitcode file along with gandiva. +install( + FILES ${GANDIVA_BC_OUTPUT_PATH} + DESTINATION ${GANDIVA_BC_INSTALL_DIR} +) diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops.cc b/cpp/src/gandiva/precompiled/arithmetic_ops.cc index 23196c31258b3..8aa026eba7d3e 100644 --- a/cpp/src/gandiva/precompiled/arithmetic_ops.cc +++ b/cpp/src/gandiva/precompiled/arithmetic_ops.cc @@ -14,6 +14,10 @@ * limitations under the License. */ +extern "C" { + +#include "./types.h" + /* * Expand inner macro for all numeric types. */ @@ -94,3 +98,5 @@ CAST_UNARY(castFLOAT8, float32, float64) NUMERIC_AND_BOOL_TYPES(VALIDITY_OP, isnull, !) NUMERIC_AND_BOOL_TYPES(VALIDITY_OP, isnotnull, +) NUMERIC_TYPES(VALIDITY_OP, isnumeric, +) + +} // extern "C" diff --git a/cpp/src/gandiva/precompiled/bitmap.cc b/cpp/src/gandiva/precompiled/bitmap.cc index d297198075555..e3e4c72c10dc1 100644 --- a/cpp/src/gandiva/precompiled/bitmap.cc +++ b/cpp/src/gandiva/precompiled/bitmap.cc @@ -16,6 +16,10 @@ /* BitMap functions */ +extern "C" { + +#include "./types.h" + #define BITS_TO_BYTES(x) ((x + 7) / 8) #define BITS_TO_WORDS(x) ((x + 63) / 64) @@ -40,3 +44,4 @@ void bitMapSetBit(unsigned char *bmap, int position, bool value) { } } +} // extern "C" diff --git a/cpp/src/gandiva/precompiled/irhelpers.cc b/cpp/src/gandiva/precompiled/print.cc similarity index 83% rename from cpp/src/gandiva/precompiled/irhelpers.cc rename to cpp/src/gandiva/precompiled/print.cc index f7797b9af10de..991aeb3f4baf4 100644 --- a/cpp/src/gandiva/precompiled/irhelpers.cc +++ b/cpp/src/gandiva/precompiled/print.cc @@ -14,17 +14,10 @@ * limitations under the License. */ -/* - * All files that need to be pre-compiled to IR should be sourced into this file. - */ -#include - extern "C" { +#include #include "./types.h" -#include "./bitmap.cc" -#include "./arithmetic_ops.cc" -#include "./time.cc" int print_double(char *msg, double val) { return printf(msg, val); diff --git a/cpp/src/gandiva/precompiled/time.cc b/cpp/src/gandiva/precompiled/time.cc index 85d015b247ad6..00d0463f14c55 100644 --- a/cpp/src/gandiva/precompiled/time.cc +++ b/cpp/src/gandiva/precompiled/time.cc @@ -14,6 +14,8 @@ * limitations under the License. */ +extern "C" { + #include #include #include "./types.h" @@ -86,3 +88,4 @@ DATE_TYPES(EXTRACT_HOUR) DATE_TYPES(EXTRACT_MINUTE) +} // extern "C"