Add a workflow that uses the Conan package manager (#997)

.github/workflows/code-coverage.yml

@@ -62,7 +62,7 @@ jobs:
       working-directory: ${{github.workspace}}/build
       run: >
         sudo pip3 install "conan==1.58.0";
-        conan install .. -pr:h ../conanprofiles/clang-libcpp -pr:b ../conanprofiles/clang-libcpp;
+        conan install ../conanprofiles/conanfile-only-boost.txt -pr:h ../conanprofiles/clang-libcpp -pr:b ../conanprofiles/clang-libcpp;
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type

.github/workflows/native-build-conan.yml

@@ -0,0 +1,72 @@
+name: Native build with conan
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  merge_group:
+
+jobs:
+  build:
+    # The CMake configure and build commands are platform-agnostic and should work equally
+    # well on Windows or Mac.  You can convert this to a matrix build if you need
+    # cross-platform coverage.
+    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        warnings: [  "-Wall -Wextra -Werror " ]
+        build-type: [Release]
+    runs-on: ubuntu-22.04
+
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: 'recursive'
+
+    - name: Install dependencies
+      run:  |
+        sudo gem install apt-spy2 && sudo apt-spy2 fix --commit --launchpad --country=US
+        sudo apt-get update
+        sudo apt-get install build-essential 
+
+    - name: Python dependencies
+      run: sudo apt-get install python3-yaml unzip pkg-config python3-icu
+    - name: Create build directory
+      run: mkdir ${{github.workspace}}/build
+    - name: Install and run conan
+      working-directory: ${{github.workspace}}/build
+      run: >
+        sudo pip3 install conan;
+        conan profile detect;
+        conan install .. -pr:b=default -of=. ;
+    - name: Configure CMake
+      # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
+      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.build-type}} -DLOGLEVEL=DEBUG -DCMAKE_TOOLCHAIN_FILE="$(pwd)/build/conan_toolchain.cmake" -DADDITIONAL_COMPILER_FLAGS="${{env.warnings}}" -DUSE_PARALLEL=true -DRUN_EXPENSIVE_TESTS=false -DENABLE_EXPENSIVE_CHECKS=true
+
+    - name: Build
+        # Build your program with the given configuration
+      run: cmake --build ${{github.workspace}}/build --config ${{matrix.build-type}} -- -j $(nproc)
+
+    - name: Test
+      working-directory: ${{github.workspace}}/build/test
+      # Execute tests defined by the CMake configuration.
+      # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
+      run: >
+        source ../conanrun.sh;
+        env CTEST_OUTPUT_ON_FAILURE=1 ctest -C ${{matrix.build-type}} .;
+
+    - name: Running and printing the benchmark examples.
+      working-directory: ${{github.workspace}}/build
+      run: > 
+        source ./conanrun.sh;
+        benchmark/BenchmarkExamples -p;
+
+    - name: E2E
+      run: >
+        source ${{github.workspace}}/build/conanrun.sh;
+        ${{github.workspace}}/e2e/e2e.sh
+

CMakeLists.txt

@@ -80,12 +80,17 @@ find_package(Threads REQUIRED)
 # ICU (for proper collation
 ################################
 find_package(ICU 60 REQUIRED COMPONENTS uc i18n)
-include_directories(${ICU_INCLUDE_DIR})
 
 ###################################
 # JEMALLOC
 ###################################
-if (${JEMALLOC_MANUALLY_INSTALLED})
+
+find_package(jemalloc QUIET)
+if (TARGET jemalloc::jemalloc)
+  MESSAGE(STATUS "Use jemalloc that was installed via conan")
+  link_libraries(jemalloc::jemalloc)
+
+elseif (${JEMALLOC_MANUALLY_INSTALLED})
   link_libraries(jemalloc)
 else()
   find_package(PkgConfig)
@@ -96,14 +101,25 @@ else()
     include_directories(${JEMALLOC_INCLUDE_DIRS})
     link_libraries(${JEMALLOC_LIBRARIES})
   else ()
-    message(FATAL_ERROR "Jemalloc is required, but could not be found via
+    message(WARNING "Jemalloc could not be found via
     pkg-config. If you are sure that you have installed jemalloc on your system
     (e.g. via `apt install libjemalloc-dev` on Ubuntu), you might try rerunning
     cmake with `-DJEMALLOC_MANUALLY_INSTALLED=True`. This is currently necessary
-    on Ubuntu 18.04, where pkg-config does not find jemalloc.")
+    on Ubuntu 18.04, where pkg-config does not find jemalloc. Continuing without jemalloc,
+    this will impact the performance, most notably of the IndexBuilder")
   endif()
 endif()
 
+### ZSTD
+find_package(ZSTD QUIET)
+if (TARGET zstd::libzstd_static)
+    MESSAGE(STATUS "Use zstd that was installed via conan")
+    link_libraries(zstd::libzstd_static)
+else()
+link_libraries(zstd)
+endif()
+
+
 
 ######################################
 # BOOST
@@ -126,7 +142,7 @@ find_package(OpenSSL REQUIRED)
 # `target_link_libraries` that additionally links against the common
 # libraries.
 function (qlever_target_link_libraries target)
-    target_link_libraries(${target} ${ARGN} absl::flat_hash_map absl::flat_hash_set absl::strings absl::str_format)
+    target_link_libraries(${target} ${ARGN} absl::flat_hash_map absl::flat_hash_set absl::strings absl::str_format ICU::uc ICU::i18n)
 endfunction()
 
 

conanfile.txt

@@ -1,6 +1,13 @@
 [requires]
 boost/1.81.0
+icu/73.1
+openssl/3.1.1
+zstd/1.5.5
+# The jemalloc recipe for Conan2 is currently broken, uncomment this line as soon as this is fixed.
+#jemalloc/5.3.0
+
 
 [generators]
 CMakeDeps
-CMakeToolchain
+CMakeToolchain
+VirtualRunEnv

conanprofiles/conanfile-only-boost.txt

@@ -0,0 +1,6 @@
+[requires]
+boost/1.81.0
+
+[generators]
+CMakeDeps
+CMakeToolchain

src/engine/CMakeLists.txt

@@ -41,4 +41,4 @@ add_library(engine
         ../util/Parameters.h RuntimeInformation.cpp CheckUsePatternTrick.cpp CheckUsePatternTrick.h
         VariableToColumnMap.cpp ExportQueryExecutionTrees.cpp )
 
-qlever_target_link_libraries(engine util index parser sparqlExpressions http SortPerformanceEstimator ${ICU_LIBRARIES} Boost::iostreams)
+qlever_target_link_libraries(engine util index parser sparqlExpressions http SortPerformanceEstimator Boost::iostreams)

src/index/CMakeLists.txt

@@ -16,4 +16,4 @@ add_library(index
         CompressedRelation.h CompressedRelation.cpp
         PatternCreator.h PatternCreator.cpp)
 
-qlever_target_link_libraries(index util parser vocabulary compilationInfo ${STXXL_LIBRARIES} ${ICU_LIBRARIES} zstd)
+qlever_target_link_libraries(index util parser vocabulary compilationInfo ${STXXL_LIBRARIES})

src/parser/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_library(rdfEscaping RdfEscaping.h RdfEscaping.cpp)
-qlever_target_link_libraries(rdfEscaping ${ICU_LIBRARIES})
+qlever_target_link_libraries(rdfEscaping)
 
 add_subdirectory(sparqlParser)
 add_subdirectory(data)

src/parser/data/CMakeLists.txt

@@ -1 +1,2 @@
 add_library(parserData Iri.cpp BlankNode.cpp)
+qlever_target_link_libraries(parserData)

src/util/http/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_subdirectory(HttpParser)
 add_library(mediaTypes MediaTypes.h MediaTypes.cpp)
 target_compile_options(mediaTypes PUBLIC -Wno-attributes)
-qlever_target_link_libraries(mediaTypes util ${ICU_LIBRARIES})
+qlever_target_link_libraries(mediaTypes util)
 add_library(http HttpServer.h HttpClient.h HttpClient.cpp HttpUtils.h HttpUtils.cpp UrlParser.h UrlParser.cpp "HttpParser/AcceptHeaderQleverVisitor.h")
 
-qlever_target_link_libraries(http mediaTypes httpParser ${ICU_LIBRARIES} OpenSSL::SSL OpenSSL::Crypto)
+qlever_target_link_libraries(http mediaTypes httpParser OpenSSL::SSL OpenSSL::Crypto)

test/CMakeLists.txt

@@ -82,9 +82,9 @@ endfunction()
 
 addLinkAndDiscoverTest(ValueIdComparatorsTest util)
 
-addLinkAndDiscoverTest(SparqlParserTest parser engine sparqlExpressions ${ICU_LIBRARIES})
+addLinkAndDiscoverTest(SparqlParserTest parser engine sparqlExpressions)
 
-addLinkAndDiscoverTest(StringUtilsTest ${ICU_LIBRARIES})
+addLinkAndDiscoverTest(StringUtilsTest)
 
 addLinkAndDiscoverTest(CacheTest)
 
@@ -95,7 +95,7 @@ addLinkAndDiscoverTestSerial(FileTest)
 
 addLinkAndDiscoverTest(Simple8bTest)
 
-addLinkAndDiscoverTest(ContextFileParserTest parser ${ICU_LIBRARIES})
+addLinkAndDiscoverTest(ContextFileParserTest parser)
 
 addLinkAndDiscoverTest(IndexMetaDataTest index)
 
@@ -131,10 +131,10 @@ addLinkAndDiscoverTest(UnionTest engine)
 
 if (SINGLE_TEST_BINARY)
     target_sources(QLeverAllUnitTestsMain PUBLIC TokenTest.cpp TokenTestCtreHelper.cpp)
-    qlever_target_link_libraries(QLeverAllUnitTestsMain parser re2 util ${ICU_LIBRARIES})
+    qlever_target_link_libraries(QLeverAllUnitTestsMain parser re2 util)
 else()
     add_executable(TokenTest TokenTest.cpp TokenTestCtreHelper.cpp)
-    linkAndDiscoverTest(TokenTest parser re2 util ${ICU_LIBRARIES})
+    linkAndDiscoverTest(TokenTest parser re2 util)
 endif()
 
 addLinkAndDiscoverTest(TurtleParserTest parser re2)
@@ -149,9 +149,9 @@ addLinkAndDiscoverTest(BatchedPipelineTest)
 
 addLinkAndDiscoverTest(TupleHelpersTest)
 
-addLinkAndDiscoverTest(StringSortComparatorTest ${ICU_LIBRARIES})
+addLinkAndDiscoverTest(StringSortComparatorTest)
 
-addLinkAndDiscoverTest(PriorityQueueTest ${ICU_LIBRARIES})
+addLinkAndDiscoverTest(PriorityQueueTest)
 
 addLinkAndDiscoverTest(SynchronizedTest)
 
@@ -203,7 +203,7 @@ addLinkAndDiscoverTest(VocabularyInMemoryTest vocabulary)
 
 addLinkAndDiscoverTest(CompressedVocabularyTest vocabulary)
 
-addLinkAndDiscoverTest(UnicodeVocabularyTest vocabulary ${ICU_LIBRARIES})
+addLinkAndDiscoverTest(UnicodeVocabularyTest vocabulary)
 
 addLinkAndDiscoverTest(CombinedVocabularyTest vocabulary)