From ade183428772725131de48cb87d6918e44c43961 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 30 Nov 2023 00:29:27 +0800 Subject: [PATCH 01/15] Build snappy in the project --- cmake_modules/BaseFunctions.cmake | 2 +- cmake_modules/FindSnappy.cmake | 35 ++++++++++++++++ src/aio/CMakeLists.txt | 7 +++- src/aio/test/CMakeLists.txt | 10 ++++- src/block_service/hdfs/CMakeLists.txt | 7 +++- src/block_service/local/CMakeLists.txt | 6 ++- src/block_service/test/CMakeLists.txt | 25 +++++++----- src/client/test/CMakeLists.txt | 15 ++++--- src/common/test/CMakeLists.txt | 5 ++- src/failure_detector/test/CMakeLists.txt | 19 +++++---- src/geo/bench/CMakeLists.txt | 8 ++-- src/http/test/CMakeLists.txt | 14 ++++--- src/meta/CMakeLists.txt | 31 +++++++------- src/nfs/test/CMakeLists.txt | 12 +++++- src/perf_counter/test/CMakeLists.txt | 8 +++- src/replica/CMakeLists.txt | 29 ++++++++------ src/replica/backup/test/CMakeLists.txt | 5 ++- src/replica/bulk_load/test/CMakeLists.txt | 7 +++- src/replica/duplication/test/CMakeLists.txt | 5 ++- src/replica/storage/simple_kv/CMakeLists.txt | 11 ++++- .../storage/simple_kv/test/CMakeLists.txt | 28 +++++++------ src/replica/test/CMakeLists.txt | 34 +++++++++------- src/runtime/test/CMakeLists.txt | 15 ++++--- src/server/CMakeLists.txt | 33 ++++++++------- src/server/test/CMakeLists.txt | 7 +++- src/shell/CMakeLists.txt | 40 ++++++++++--------- src/test/bench_test/CMakeLists.txt | 8 ++-- .../function_test/bulk_load/CMakeLists.txt | 25 +++++++----- src/test_util/CMakeLists.txt | 7 +++- src/utils/CMakeLists.txt | 7 +++- src/utils/long_adder_bench/CMakeLists.txt | 8 +++- src/utils/test/CMakeLists.txt | 16 +++++--- .../test/nth_element_bench/CMakeLists.txt | 8 +++- src/zookeeper/test/CMakeLists.txt | 17 ++++---- thirdparty/CMakeLists.txt | 30 ++++++++++++++ 35 files changed, 370 insertions(+), 174 deletions(-) create mode 100644 cmake_modules/FindSnappy.cmake diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 1882353c83..27994adf50 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -332,7 +332,7 @@ function(dsn_setup_thirdparty_libs) message(WARNING "Cannot find RocksDB depends cmake modules path, might not find snappy, zstd, lz4") endif() list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DEPENDS_MODULE_PATH}") - find_package(snappy) + find_package(Snappy REQUIRED) find_package(zstd) find_package(lz4) if(USE_JEMALLOC) diff --git a/cmake_modules/FindSnappy.cmake b/cmake_modules/FindSnappy.cmake new file mode 100644 index 0000000000..166e563908 --- /dev/null +++ b/cmake_modules/FindSnappy.cmake @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# - Find SNAPPY (snappy.h, libsnappy.a, libsnappy.so, and libsnappy.so.1) +# This module defines +# SNAPPY_INCLUDE_DIR, directory containing headers +# SNAPPY_SHARED_LIB, path to snappy's shared library +# SNAPPY_STATIC_LIB, path to snappy's static library +# SNAPPY_FOUND, whether snappy has been found + +find_path(SNAPPY_INCLUDE_DIR snappy.h + # make sure we don't accidentally pick up a different version + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) +find_library(SNAPPY_STATIC_LIB libsnappy.a + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Snappy REQUIRED_VARS + SNAPPY_STATIC_LIB SNAPPY_INCLUDE_DIR) diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt index 3754361d06..aead830d3f 100644 --- a/src/aio/CMakeLists.txt +++ b/src/aio/CMakeLists.txt @@ -33,7 +33,12 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + lz4 + zstd + rocksdb + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index b8d4ad675b..2caa885255 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -33,7 +33,15 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio test_utils rocksdb) +set(MY_PROJ_LIBS + gtest + dsn_runtime + dsn_aio + test_utils + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/block_service/hdfs/CMakeLists.txt b/src/block_service/hdfs/CMakeLists.txt index 2bba8b96bb..d453a0568c 100644 --- a/src/block_service/hdfs/CMakeLists.txt +++ b/src/block_service/hdfs/CMakeLists.txt @@ -26,7 +26,12 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS hdfs rocksdb) +set(MY_PROJ_LIBS + hdfs + lz4 + zstd + rocksdb + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/local/CMakeLists.txt b/src/block_service/local/CMakeLists.txt index 9d830f7825..0bc44af012 100644 --- a/src/block_service/local/CMakeLists.txt +++ b/src/block_service/local/CMakeLists.txt @@ -26,7 +26,11 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS rocksdb) +set(MY_PROJ_LIBS + lz4 + zstd + rocksdb + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index 6fbfa9ca11..dab33febd5 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -22,17 +22,20 @@ set(MY_PROJ_NAME dsn_block_service_test) set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn_runtime - dsn_utils - gtest - gtest_main - hdfs - test_utils - rocksdb) + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn_runtime + dsn_utils + gtest + gtest_main + hdfs + test_utils + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt index cfe591ce0f..c4c4e8045e 100644 --- a/src/client/test/CMakeLists.txt +++ b/src/client/test/CMakeLists.txt @@ -22,12 +22,15 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_client - dsn_replication_common - dsn_runtime - dsn_utils - gtest - rocksdb) + dsn_client + dsn_replication_common + dsn_runtime + dsn_utils + gtest + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt index 12f53aa52c..538a9b752f 100644 --- a/src/common/test/CMakeLists.txt +++ b/src/common/test/CMakeLists.txt @@ -31,7 +31,10 @@ set(MY_PROJ_LIBS dsn_replication_common dsn_runtime gtest - rocksdb) + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/failure_detector/test/CMakeLists.txt b/src/failure_detector/test/CMakeLists.txt index 6bd1e05553..4613eed556 100644 --- a/src/failure_detector/test/CMakeLists.txt +++ b/src/failure_detector/test/CMakeLists.txt @@ -34,14 +34,17 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_runtime - dsn_meta_server - dsn_replica_server - dsn_replication_common - dsn.failure_detector - gtest - hashtable - rocksdb) + dsn_runtime + dsn_meta_server + dsn_replica_server + dsn_replication_common + dsn.failure_detector + gtest + hashtable + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt index 6df54a6bcb..e69ef92d65 100644 --- a/src/geo/bench/CMakeLists.txt +++ b/src/geo/bench/CMakeLists.txt @@ -34,9 +34,11 @@ set(MY_PROJ_LIBS s2testing s2 pegasus_client_static - RocksDB::rocksdb - dsn_utils - ) + lz4 + zstd + rocksdb + snappy + dsn_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt index 5ebc142a55..4cf5b44595 100644 --- a/src/http/test/CMakeLists.txt +++ b/src/http/test/CMakeLists.txt @@ -22,12 +22,14 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_http - dsn_runtime - curl - gtest - rocksdb - ) + dsn_http + dsn_runtime + curl + gtest + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 477671acb3..3898163545 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -34,20 +34,23 @@ set(DUPLICATION_SRC set(MY_PROJ_SRC "${DUPLICATION_SRC}") set(MY_PROJ_LIBS - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_dist_cmd - dsn_http - dsn_runtime - dsn_aio - zookeeper - hashtable - hdfs - rocksdb) + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_dist_cmd + dsn_http + dsn_runtime + dsn_aio + zookeeper + hashtable + hdfs + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt index ff591b7e21..7b1ce32806 100644 --- a/src/nfs/test/CMakeLists.txt +++ b/src/nfs/test/CMakeLists.txt @@ -33,7 +33,17 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio dsn_http rocksdb test_utils) +set(MY_PROJ_LIBS + dsn_nfs + dsn_runtime + gtest + dsn_aio + dsn_http + lz4 + zstd + rocksdb + snappy + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/perf_counter/test/CMakeLists.txt b/src/perf_counter/test/CMakeLists.txt index 2e02cf868f..2005cf5c99 100644 --- a/src/perf_counter/test/CMakeLists.txt +++ b/src/perf_counter/test/CMakeLists.txt @@ -33,7 +33,13 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime rocksdb) +set(MY_PROJ_LIBS + gtest + dsn_runtime + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index fcf7891017..d558e86f13 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -57,19 +57,22 @@ set(MY_PROJ_SRC # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS - dsn_replication_common - dsn.failure_detector - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn_nfs - dsn_dist_cmd - dsn_http - dsn_runtime - dsn_aio - dsn_meta_server - rocksdb) +set(MY_PROJ_LIBS + dsn_replication_common + dsn.failure_detector + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn_nfs + dsn_dist_cmd + dsn_http + dsn_runtime + dsn_aio + dsn_meta_server + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::filesystem) diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt index 47158ca93c..664507c7e3 100644 --- a/src/replica/backup/test/CMakeLists.txt +++ b/src/replica/backup/test/CMakeLists.txt @@ -29,7 +29,10 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_utils hashtable gtest - rocksdb) + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 0b0d083851..620c5e0c9c 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -28,9 +28,12 @@ set(MY_PROJ_LIBS dsn_meta_server hashtable gtest test_utils - rocksdb) + lz4 + zstd + rocksdb + snappy) -set(MY_BOOST_LIBS Boost::system Boost::filesystem rocksdb test_utils) +set(MY_BOOST_LIBS Boost::system Boost::filesystem) set(MY_BINPLACES config-test.ini diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index 5cc139f916..9e3723e2dd 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -31,7 +31,10 @@ set(MY_PROJ_LIBS dsn_meta_server hashtable gtest test_utils - rocksdb) + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/storage/simple_kv/CMakeLists.txt b/src/replica/storage/simple_kv/CMakeLists.txt index 8584848f11..c61dcadf15 100644 --- a/src/replica/storage/simple_kv/CMakeLists.txt +++ b/src/replica/storage/simple_kv/CMakeLists.txt @@ -37,7 +37,16 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS}) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable rocksdb) +set(MY_PROJ_LIBS + dsn_replica_server + dsn_meta_server + dsn_client + dsn_runtime + hashtable + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt b/src/replica/storage/simple_kv/test/CMakeLists.txt index a1e0a69b21..f0be3f693e 100644 --- a/src/replica/storage/simple_kv/test/CMakeLists.txt +++ b/src/replica/storage/simple_kv/test/CMakeLists.txt @@ -29,18 +29,22 @@ set(MY_PROJ_NAME dsn.rep_tests.simple_kv) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_runtime - zookeeper - hashtable - gtest - dsn_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_runtime + zookeeper + hashtable + gtest + dsn_utils + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 83af542d10..590ad5139f 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -33,21 +33,25 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_meta_server - dsn_replica_server - dsn.replication.zookeeper_provider - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn.failure_detector - dsn_http - dsn_runtime - zookeeper - hashtable - gtest - test_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replica_server + dsn.replication.zookeeper_provider + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn.failure_detector + dsn_http + dsn_runtime + zookeeper + hashtable + gtest + test_utils + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/runtime/test/CMakeLists.txt b/src/runtime/test/CMakeLists.txt index cd5e0daac8..689610f6f3 100644 --- a/src/runtime/test/CMakeLists.txt +++ b/src/runtime/test/CMakeLists.txt @@ -29,12 +29,15 @@ set(MY_PROJ_NAME dsn_runtime_tests) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest - dsn_runtime - dsn_aio - dsn_meta_server - rocksdb - ) +set(MY_PROJ_LIBS + gtest + dsn_runtime + dsn_aio + dsn_meta_server + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index d19408e276..56adc8d2a2 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -27,21 +27,24 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn.block_service.local - dsn.block_service - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_utils - RocksDB::rocksdb - pegasus_base - pegasus_client_static - event - hashtable - ) + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn.block_service.local + dsn.block_service + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_utils + lz4 + zstd + rocksdb + snappy + pegasus_reporter + pegasus_base + pegasus_client_static + event + hashtable) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt index 8a2fbd6f1c..9efb6d74ec 100644 --- a/src/server/test/CMakeLists.txt +++ b/src/server/test/CMakeLists.txt @@ -45,14 +45,17 @@ set(MY_PROJ_LIBS dsn.failure_detector dsn.replication.zookeeper_provider dsn_utils - RocksDB::rocksdb + lz4 + zstd + rocksdb + snappy pegasus_client_static event pegasus_base gtest gmock hashtable - ) +) add_definitions(-DPEGASUS_UNIT_TEST) add_definitions(-DENABLE_FAIL) diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 909fda9ab6..00fb6d6834 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -28,25 +28,27 @@ set(MY_PROJ_SRC "linenoise/linenoise.c" "sds/sds.c") set(MY_SRC_SEARCH_MODE "GLOB_RECURSE") set(MY_PROJ_LIBS - pegasus_base - dsn.replication.tool - dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn_utils - dsn.block_service.local - dsn.block_service.hdfs - dsn.block_service - dsn.failure_detector - pegasus_client_static - pegasus_geo_lib - RocksDB::rocksdb - absl::flat_hash_set - absl::strings - s2 - hdfs - ) + pegasus_base + dsn.replication.tool + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn_utils + dsn.block_service.local + dsn.block_service.hdfs + dsn.block_service + dsn.failure_detector + pegasus_client_static + pegasus_geo_lib + lz4 + zstd + rocksdb + snappy + absl::flat_hash_set + absl::strings + s2 + hdfs) set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config.ini") diff --git a/src/test/bench_test/CMakeLists.txt b/src/test/bench_test/CMakeLists.txt index 307d543d09..5fe5b58e33 100644 --- a/src/test/bench_test/CMakeLists.txt +++ b/src/test/bench_test/CMakeLists.txt @@ -30,11 +30,13 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS pegasus_client_static dsn_utils - RocksDB::rocksdb + lz4 + zstd + rocksdb + snappy sasl2 gssapi_krb5 - krb5 - ) + krb5) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/test/function_test/bulk_load/CMakeLists.txt b/src/test/function_test/bulk_load/CMakeLists.txt index f9f1976491..f94a04ebcd 100644 --- a/src/test/function_test/bulk_load/CMakeLists.txt +++ b/src/test/function_test/bulk_load/CMakeLists.txt @@ -28,17 +28,20 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_client - dsn_replication_common - dsn_utils - pegasus_client_static - gtest - sasl2 - gssapi_krb5 - krb5 - function_test_utils - test_utils - rocksdb) + dsn_client + dsn_replication_common + dsn_utils + pegasus_client_static + gtest + sasl2 + gssapi_krb5 + krb5 + function_test_utils + test_utils + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt index b1e7ac2dff..da831a6d10 100644 --- a/src/test_util/CMakeLists.txt +++ b/src/test_util/CMakeLists.txt @@ -22,6 +22,11 @@ set(MY_PROJ_NAME test_utils) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest rocksdb) +set(MY_PROJ_LIBS + gtest + lz4 + zstd + rocksdb + snappy) dsn_add_static_library() diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 8c04e3adc4..d327b74dfe 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -31,7 +31,12 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_BOOST_LIBS Boost::system Boost::filesystem) -set(MY_PROJ_LIBS dsn_http rocksdb) +set(MY_PROJ_LIBS + dsn_http + lz4 + zstd + rocksdb + snappy) # Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/utils/long_adder_bench/CMakeLists.txt b/src/utils/long_adder_bench/CMakeLists.txt index d5ace78cac..a775eba4a3 100644 --- a/src/utils/long_adder_bench/CMakeLists.txt +++ b/src/utils/long_adder_bench/CMakeLists.txt @@ -27,7 +27,13 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + dsn_utils + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index 684a5faaf0..d2c33a10c3 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -29,12 +29,16 @@ set(MY_PROJ_NAME dsn_utils_tests) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_http - dsn_runtime - dsn_utils - gtest - test_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_http + dsn_runtime + dsn_utils + gtest + test_utils + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/utils/test/nth_element_bench/CMakeLists.txt b/src/utils/test/nth_element_bench/CMakeLists.txt index 7bf0af804b..e0876cee51 100644 --- a/src/utils/test/nth_element_bench/CMakeLists.txt +++ b/src/utils/test/nth_element_bench/CMakeLists.txt @@ -27,7 +27,13 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + dsn_utils + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index cdb6d4cec2..426244151d 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -33,13 +33,16 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS - dsn.replication.zookeeper_provider - dsn_runtime - zookeeper - hashtable - gtest - rocksdb) +set(MY_PROJ_LIBS + dsn.replication.zookeeper_provider + dsn_runtime + zookeeper + hashtable + gtest + lz4 + zstd + rocksdb + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index e95e24f83e..7f0cad4ff0 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -361,6 +361,34 @@ ExternalProject_Add(jemalloc DOWNLOAD_NO_PROGRESS true ) +set(SNAPPY_OPTIONS + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DBUILD_SHARED_LIBS=OFF + -DSNAPPY_BUILD_TESTS=OFF + -DSNAPPY_BUILD_BENCHMARKS=OFF + -DSNAPPY_FUZZING_BUILD=OFF + -DSNAPPY_INSTALL=ON) +execute_process(COMMAND arch OUTPUT_VARIABLE ARCH_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) +message(STATUS "ARCH_NAME = ${ARCH_NAME}") +if (ARCH_NAME EQUAL "x86_64") + set(SNAPPY_OPTIONS + ${SNAPPY_OPTIONS} + -DSNAPPY_REQUIRE_AVX=ON + -DSNAPPY_REQUIRE_AVX2=ON) +endif () +ExternalProject_Add(snappy + URL ${OSS_URL_PREFIX}/snappy-1.1.10.tar.gz + https://github.com/google/snappy/archive/refs/tags/1.1.10.tar.gz + URL_MD5 70153395ebe6d72febe2cf2e40026a44 + PATCH_COMMAND "" + CMAKE_ARGS ${SNAPPY_OPTIONS} + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) set(ROCKSDB_OPTIONS -DFAIL_ON_WARNINGS=OFF @@ -373,6 +401,8 @@ set(ROCKSDB_OPTIONS -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF -DUSE_RTTI=ON + -DROCKSDB_BUILD_SHARED=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -DWITH_JEMALLOC=${USE_JEMALLOC} -DJEMALLOC_ROOT_DIR=${TP_OUTPUT} From efcdc13ccd31ac3cea0bdbfca08abde2ca024ee7 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 5 Dec 2023 14:00:32 +0800 Subject: [PATCH 02/15] fix snappy more --- .licenserc.yaml | 1 + src/aio/CMakeLists.txt | 2 +- src/aio/test/CMakeLists.txt | 2 +- src/block_service/hdfs/CMakeLists.txt | 2 +- src/block_service/local/CMakeLists.txt | 2 +- src/block_service/test/CMakeLists.txt | 2 +- src/client/test/CMakeLists.txt | 2 +- src/common/test/CMakeLists.txt | 2 +- src/failure_detector/test/CMakeLists.txt | 2 +- src/geo/bench/CMakeLists.txt | 2 +- src/http/test/CMakeLists.txt | 2 +- src/meta/CMakeLists.txt | 2 +- src/nfs/test/CMakeLists.txt | 2 +- src/perf_counter/test/CMakeLists.txt | 2 +- src/replica/CMakeLists.txt | 2 +- src/replica/backup/test/CMakeLists.txt | 2 +- src/replica/bulk_load/test/CMakeLists.txt | 2 +- src/replica/duplication/test/CMakeLists.txt | 2 +- src/replica/storage/simple_kv/CMakeLists.txt | 2 +- .../storage/simple_kv/test/CMakeLists.txt | 2 +- src/replica/test/CMakeLists.txt | 2 +- src/runtime/test/CMakeLists.txt | 2 +- src/server/CMakeLists.txt | 2 +- src/server/test/CMakeLists.txt | 2 +- src/shell/CMakeLists.txt | 2 +- src/test/bench_test/CMakeLists.txt | 2 +- .../function_test/bulk_load/CMakeLists.txt | 2 +- src/test_util/CMakeLists.txt | 2 +- src/utils/CMakeLists.txt | 2 +- src/utils/long_adder_bench/CMakeLists.txt | 2 +- src/utils/test/CMakeLists.txt | 2 +- .../test/nth_element_bench/CMakeLists.txt | 2 +- src/zookeeper/test/CMakeLists.txt | 2 +- thirdparty/CMakeLists.txt | 2 +- .../fix_snappy-Wsign-compare-warning.patch | 26 +++++++++++++++++++ 35 files changed, 60 insertions(+), 33 deletions(-) create mode 100644 thirdparty/fix_snappy-Wsign-compare-warning.patch diff --git a/.licenserc.yaml b/.licenserc.yaml index 3ddf87be61..1c58047590 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -71,6 +71,7 @@ header: - 'thirdparty/fix_libevent_for_macos.patch' - 'thirdparty/fix_prometheus-cpp_limits.patch' - 'thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch' + - 'thirdparty/fix_snappy-Wsign-compare-warning.patch' - 'thirdparty/fix_s2_build_with_absl_and_gtest.patch' - 'thirdparty/fix_thrift_for_cpp11.patch' # TODO(yingchun): shell/* files are import from thirdparties, we can move them to thirdparty later. diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt index aead830d3f..3a27fbe01a 100644 --- a/src/aio/CMakeLists.txt +++ b/src/aio/CMakeLists.txt @@ -35,9 +35,9 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS dsn_runtime + rocksdb lz4 zstd - rocksdb snappy) #Extra files that will be installed diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index 2caa885255..a9378612af 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -38,9 +38,9 @@ set(MY_PROJ_LIBS dsn_runtime dsn_aio test_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/block_service/hdfs/CMakeLists.txt b/src/block_service/hdfs/CMakeLists.txt index d453a0568c..ae821f2bf2 100644 --- a/src/block_service/hdfs/CMakeLists.txt +++ b/src/block_service/hdfs/CMakeLists.txt @@ -28,9 +28,9 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS hdfs + rocksdb lz4 zstd - rocksdb snappy) #Extra files that will be installed diff --git a/src/block_service/local/CMakeLists.txt b/src/block_service/local/CMakeLists.txt index 0bc44af012..8fdd0d53ac 100644 --- a/src/block_service/local/CMakeLists.txt +++ b/src/block_service/local/CMakeLists.txt @@ -27,9 +27,9 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS + rocksdb lz4 zstd - rocksdb snappy) #Extra files that will be installed diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index dab33febd5..b8aaad6520 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -32,9 +32,9 @@ set(MY_PROJ_LIBS gtest_main hdfs test_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt index c4c4e8045e..60f3762760 100644 --- a/src/client/test/CMakeLists.txt +++ b/src/client/test/CMakeLists.txt @@ -27,9 +27,9 @@ set(MY_PROJ_LIBS dsn_runtime dsn_utils gtest + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt index 538a9b752f..fab0926249 100644 --- a/src/common/test/CMakeLists.txt +++ b/src/common/test/CMakeLists.txt @@ -31,9 +31,9 @@ set(MY_PROJ_LIBS dsn_replication_common dsn_runtime gtest + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/failure_detector/test/CMakeLists.txt b/src/failure_detector/test/CMakeLists.txt index 4613eed556..5b5a6832a9 100644 --- a/src/failure_detector/test/CMakeLists.txt +++ b/src/failure_detector/test/CMakeLists.txt @@ -41,9 +41,9 @@ set(MY_PROJ_LIBS dsn.failure_detector gtest hashtable + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt index e69ef92d65..559747b59d 100644 --- a/src/geo/bench/CMakeLists.txt +++ b/src/geo/bench/CMakeLists.txt @@ -34,9 +34,9 @@ set(MY_PROJ_LIBS s2testing s2 pegasus_client_static + rocksdb lz4 zstd - rocksdb snappy dsn_utils) diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt index 4cf5b44595..4400c0183f 100644 --- a/src/http/test/CMakeLists.txt +++ b/src/http/test/CMakeLists.txt @@ -26,9 +26,9 @@ set(MY_PROJ_LIBS dsn_runtime curl gtest + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 3898163545..fcc3a12329 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -47,9 +47,9 @@ set(MY_PROJ_LIBS zookeeper hashtable hdfs + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt index 7b1ce32806..4a22ce4867 100644 --- a/src/nfs/test/CMakeLists.txt +++ b/src/nfs/test/CMakeLists.txt @@ -39,9 +39,9 @@ set(MY_PROJ_LIBS gtest dsn_aio dsn_http + rocksdb lz4 zstd - rocksdb snappy test_utils) diff --git a/src/perf_counter/test/CMakeLists.txt b/src/perf_counter/test/CMakeLists.txt index 2005cf5c99..b2c8b157fc 100644 --- a/src/perf_counter/test/CMakeLists.txt +++ b/src/perf_counter/test/CMakeLists.txt @@ -36,9 +36,9 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS gtest dsn_runtime + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index d558e86f13..5bcd3f57bc 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -69,9 +69,9 @@ set(MY_PROJ_LIBS dsn_runtime dsn_aio dsn_meta_server + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::filesystem) diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt index 664507c7e3..fed60e3120 100644 --- a/src/replica/backup/test/CMakeLists.txt +++ b/src/replica/backup/test/CMakeLists.txt @@ -29,9 +29,9 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_utils hashtable gtest + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 620c5e0c9c..bda1a89819 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -28,9 +28,9 @@ set(MY_PROJ_LIBS dsn_meta_server hashtable gtest test_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index 9e3723e2dd..26e461bd75 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -31,9 +31,9 @@ set(MY_PROJ_LIBS dsn_meta_server hashtable gtest test_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/storage/simple_kv/CMakeLists.txt b/src/replica/storage/simple_kv/CMakeLists.txt index c61dcadf15..a2f8506770 100644 --- a/src/replica/storage/simple_kv/CMakeLists.txt +++ b/src/replica/storage/simple_kv/CMakeLists.txt @@ -43,9 +43,9 @@ set(MY_PROJ_LIBS dsn_client dsn_runtime hashtable + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt b/src/replica/storage/simple_kv/test/CMakeLists.txt index f0be3f693e..0bce41b712 100644 --- a/src/replica/storage/simple_kv/test/CMakeLists.txt +++ b/src/replica/storage/simple_kv/test/CMakeLists.txt @@ -41,9 +41,9 @@ set(MY_PROJ_LIBS hashtable gtest dsn_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 590ad5139f..94b5c89348 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -48,9 +48,9 @@ set(MY_PROJ_LIBS hashtable gtest test_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/runtime/test/CMakeLists.txt b/src/runtime/test/CMakeLists.txt index 689610f6f3..38e52a4f5b 100644 --- a/src/runtime/test/CMakeLists.txt +++ b/src/runtime/test/CMakeLists.txt @@ -34,9 +34,9 @@ set(MY_PROJ_LIBS dsn_runtime dsn_aio dsn_meta_server + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 56adc8d2a2..02f89c4087 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -36,9 +36,9 @@ set(MY_PROJ_LIBS dsn.failure_detector dsn.replication.zookeeper_provider dsn_utils + rocksdb lz4 zstd - rocksdb snappy pegasus_reporter pegasus_base diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt index 9efb6d74ec..2b4c96ac1f 100644 --- a/src/server/test/CMakeLists.txt +++ b/src/server/test/CMakeLists.txt @@ -45,9 +45,9 @@ set(MY_PROJ_LIBS dsn.failure_detector dsn.replication.zookeeper_provider dsn_utils + rocksdb lz4 zstd - rocksdb snappy pegasus_client_static event diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 00fb6d6834..81271006e1 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -41,9 +41,9 @@ set(MY_PROJ_LIBS dsn.failure_detector pegasus_client_static pegasus_geo_lib + rocksdb lz4 zstd - rocksdb snappy absl::flat_hash_set absl::strings diff --git a/src/test/bench_test/CMakeLists.txt b/src/test/bench_test/CMakeLists.txt index 5fe5b58e33..5a52808d8d 100644 --- a/src/test/bench_test/CMakeLists.txt +++ b/src/test/bench_test/CMakeLists.txt @@ -30,9 +30,9 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS pegasus_client_static dsn_utils + rocksdb lz4 zstd - rocksdb snappy sasl2 gssapi_krb5 diff --git a/src/test/function_test/bulk_load/CMakeLists.txt b/src/test/function_test/bulk_load/CMakeLists.txt index f94a04ebcd..ce17323ea7 100644 --- a/src/test/function_test/bulk_load/CMakeLists.txt +++ b/src/test/function_test/bulk_load/CMakeLists.txt @@ -38,9 +38,9 @@ set(MY_PROJ_LIBS krb5 function_test_utils test_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt index da831a6d10..fd4cf1dc16 100644 --- a/src/test_util/CMakeLists.txt +++ b/src/test_util/CMakeLists.txt @@ -24,9 +24,9 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS gtest + rocksdb lz4 zstd - rocksdb snappy) dsn_add_static_library() diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index d327b74dfe..38390f0ed6 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -33,9 +33,9 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem) set(MY_PROJ_LIBS dsn_http + rocksdb lz4 zstd - rocksdb snappy) # Extra files that will be installed diff --git a/src/utils/long_adder_bench/CMakeLists.txt b/src/utils/long_adder_bench/CMakeLists.txt index a775eba4a3..71568c61ae 100644 --- a/src/utils/long_adder_bench/CMakeLists.txt +++ b/src/utils/long_adder_bench/CMakeLists.txt @@ -30,9 +30,9 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS dsn_runtime dsn_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index d2c33a10c3..8502cabb76 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -35,9 +35,9 @@ set(MY_PROJ_LIBS dsn_utils gtest test_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/utils/test/nth_element_bench/CMakeLists.txt b/src/utils/test/nth_element_bench/CMakeLists.txt index e0876cee51..3aae31e4a6 100644 --- a/src/utils/test/nth_element_bench/CMakeLists.txt +++ b/src/utils/test/nth_element_bench/CMakeLists.txt @@ -30,9 +30,9 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS dsn_runtime dsn_utils + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index 426244151d..7dabd1faf5 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -39,9 +39,9 @@ set(MY_PROJ_LIBS zookeeper hashtable gtest + rocksdb lz4 zstd - rocksdb snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 7f0cad4ff0..e43a5fb3a6 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -381,7 +381,7 @@ ExternalProject_Add(snappy URL ${OSS_URL_PREFIX}/snappy-1.1.10.tar.gz https://github.com/google/snappy/archive/refs/tags/1.1.10.tar.gz URL_MD5 70153395ebe6d72febe2cf2e40026a44 - PATCH_COMMAND "" + PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_snappy-Wsign-compare-warning.patch CMAKE_ARGS ${SNAPPY_OPTIONS} BUILD_COMMAND make -j${PARALLEL} INSTALL_COMMAND make install diff --git a/thirdparty/fix_snappy-Wsign-compare-warning.patch b/thirdparty/fix_snappy-Wsign-compare-warning.patch new file mode 100644 index 0000000000..0540d0177c --- /dev/null +++ b/thirdparty/fix_snappy-Wsign-compare-warning.patch @@ -0,0 +1,26 @@ +From 27f34a580be4a3becf5f8c0cba13433f53c21337 Mon Sep 17 00:00:00 2001 +From: Richard O'Grady +Date: Wed, 12 Jul 2023 10:12:01 -0700 +Subject: [PATCH] Fix -Wsign-compare warning + +PiperOrigin-RevId: 547529709 +--- + snappy.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/snappy.cc b/snappy.cc +index 688065b..6473123 100644 +--- a/snappy.cc ++++ b/snappy.cc +@@ -1289,7 +1289,7 @@ std::pair DecompressBranchless( + DeferMemCopy(&deferred_src, &deferred_length, from, len); + } + } while (ip < ip_limit_min_slop && +- (op + deferred_length) < op_limit_min_slop); ++ static_cast(op + deferred_length) < op_limit_min_slop); + exit: + ip--; + assert(ip <= ip_limit); +-- +2.42.1 + From a0eab938f4dd8df3bdacd2b37cb11f2349f0b4c9 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 5 Dec 2023 20:45:01 +0800 Subject: [PATCH 03/15] fix centos 7 --- src/aio/test/CMakeLists.txt | 2 ++ src/perf_counter/test/CMakeLists.txt | 2 ++ src/runtime/ranger/CMakeLists.txt | 14 +++++--------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index a9378612af..2eb4371467 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -34,6 +34,8 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common gtest dsn_runtime dsn_aio diff --git a/src/perf_counter/test/CMakeLists.txt b/src/perf_counter/test/CMakeLists.txt index b2c8b157fc..f924437dec 100644 --- a/src/perf_counter/test/CMakeLists.txt +++ b/src/perf_counter/test/CMakeLists.txt @@ -34,6 +34,8 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common gtest dsn_runtime rocksdb diff --git a/src/runtime/ranger/CMakeLists.txt b/src/runtime/ranger/CMakeLists.txt index 3c3e441a4f..fbbceb570a 100644 --- a/src/runtime/ranger/CMakeLists.txt +++ b/src/runtime/ranger/CMakeLists.txt @@ -17,14 +17,10 @@ set(MY_PROJ_NAME dsn_ranger) -# Search mode for source files under CURRENT project directory? -# "GLOB_RECURSE" for recursive search -# "GLOB" for non-recursive search +set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common) -set(MY_PROJ_LIBS "") - -# Extra files that will be installed -set(MY_BINPLACES "") - -dsn_add_object() +dsn_add_static_library() From 1c3af7022cf22b9103d014eef0cb3a4f2431ea43 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 5 Dec 2023 21:30:08 +0800 Subject: [PATCH 04/15] 12 --- thirdparty/CMakeLists.txt | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index e43a5fb3a6..7e04f8c836 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -364,6 +364,7 @@ ExternalProject_Add(jemalloc set(SNAPPY_OPTIONS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF @@ -389,6 +390,26 @@ ExternalProject_Add(snappy DOWNLOAD_NO_PROGRESS true ) +ExternalProject_Add(zstd + URL ${OSS_URL_PREFIX}/zstd-1.5.5.tar.gz + https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz + URL_MD5 63251602329a106220e0a5ad26ba656f + PATCH_COMMAND "" + CMAKE_COMMAND cmake build/cmake + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_BUILD_TYPE=Release + -DZSTD_BUILD_PROGRAMS=OFF + -DZSTD_BUILD_TESTS=OFF + -DZSTD_BUILD_CONTRIB=OFF + -DZSTD_BUILD_SHARED=OFF + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1 + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) set(ROCKSDB_OPTIONS -DFAIL_ON_WARNINGS=OFF @@ -408,6 +429,7 @@ set(ROCKSDB_OPTIONS -DJEMALLOC_ROOT_DIR=${TP_OUTPUT} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DPORTABLE=${ROCKSDB_PORTABLE}) # Link error on MacOS, disable building encfs plugin. # See https://github.com/pegasus-kv/encfs/issues/4 @@ -424,8 +446,7 @@ ExternalProject_Add(rocksdb COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/encfs COMMAND git clone -b main --depth=1 https://github.com/pegasus-kv/encfs.git ${TP_DIR}/build/Source/rocksdb/plugin/encfs DEPENDS googletest jemalloc - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - ${ROCKSDB_OPTIONS} + CMAKE_ARGS ${ROCKSDB_OPTIONS} DOWNLOAD_EXTRACT_TIMESTAMP true DOWNLOAD_NO_PROGRESS true ) From a0a4f789f003f3a036b9d1f0cc41b1538358f152 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 00:27:03 +0800 Subject: [PATCH 05/15] add zstd and lz4 libs --- cmake_modules/BaseFunctions.cmake | 13 ++++-------- cmake_modules/FindLz4.cmake | 34 +++++++++++++++++++++++++++++++ cmake_modules/FindSnappy.cmake | 3 +-- cmake_modules/FindZstd.cmake | 34 +++++++++++++++++++++++++++++++ thirdparty/CMakeLists.txt | 24 ++++++++++++++++++++-- 5 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 cmake_modules/FindLz4.cmake create mode 100644 cmake_modules/FindZstd.cmake diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 27994adf50..e13f409ce4 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -326,19 +326,14 @@ function(dsn_setup_thirdparty_libs) find_package(fmt REQUIRED) set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default thirdparty libs" FORCE) - # rocksdb - file(GLOB ROCKSDB_DEPENDS_MODULE_PATH ${THIRDPARTY_ROOT}/build/Source/rocksdb/cmake/modules) - if(NOT ROCKSDB_DEPENDS_MODULE_PATH) - message(WARNING "Cannot find RocksDB depends cmake modules path, might not find snappy, zstd, lz4") - endif() - list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DEPENDS_MODULE_PATH}") + # rocksdb and dependent libs + find_package(RocksDB REQUIRED) find_package(Snappy REQUIRED) - find_package(zstd) - find_package(lz4) + find_package(Zstd REQUIRED) + find_package(Lz4 REQUIRED) if(USE_JEMALLOC) find_package(Jemalloc REQUIRED) endif() - find_package(RocksDB REQUIRED) # libhdfs find_package(JNI REQUIRED) diff --git a/cmake_modules/FindLz4.cmake b/cmake_modules/FindLz4.cmake new file mode 100644 index 0000000000..b52d2a63d3 --- /dev/null +++ b/cmake_modules/FindLz4.cmake @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# - Find LZ4 (lz4.h, liblz4.a) +# This module defines +# LZ4_INCLUDE_DIR, directory containing headers +# LZ4_STATIC_LIB, path to lz4's static library +# LZ4_FOUND, whether lz4 has been found + +find_path(LZ4_INCLUDE_DIR lz4.h + # make sure we don't accidentally pick up a different version + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) +find_library(LZ4_STATIC_LIB liblz4.a + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Lz4 REQUIRED_VARS + LZ4_STATIC_LIB LZ4_INCLUDE_DIR) diff --git a/cmake_modules/FindSnappy.cmake b/cmake_modules/FindSnappy.cmake index 166e563908..341445ccf5 100644 --- a/cmake_modules/FindSnappy.cmake +++ b/cmake_modules/FindSnappy.cmake @@ -15,10 +15,9 @@ # specific language governing permissions and limitations # under the License. -# - Find SNAPPY (snappy.h, libsnappy.a, libsnappy.so, and libsnappy.so.1) +# - Find SNAPPY (snappy.h, libsnappy.a) # This module defines # SNAPPY_INCLUDE_DIR, directory containing headers -# SNAPPY_SHARED_LIB, path to snappy's shared library # SNAPPY_STATIC_LIB, path to snappy's static library # SNAPPY_FOUND, whether snappy has been found diff --git a/cmake_modules/FindZstd.cmake b/cmake_modules/FindZstd.cmake new file mode 100644 index 0000000000..258ada4656 --- /dev/null +++ b/cmake_modules/FindZstd.cmake @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# - Find ZSTD (zstd.h, libzstd.a) +# This module defines +# ZSTD_INCLUDE_DIR, directory containing headers +# ZSTD_STATIC_LIB, path to zstd's static library +# ZSTD_FOUND, whether zstd has been found + +find_path(ZSTD_INCLUDE_DIR zstd.h + # make sure we don't accidentally pick up a different version + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) +find_library(ZSTD_STATIC_LIB libzstd.a + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Zstd REQUIRED_VARS + ZSTD_STATIC_LIB ZSTD_INCLUDE_DIR) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 7e04f8c836..ce8f7cbc46 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -395,14 +395,34 @@ ExternalProject_Add(zstd https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz URL_MD5 63251602329a106220e0a5ad26ba656f PATCH_COMMAND "" - CMAKE_COMMAND cmake build/cmake - CMAKE_ARGS + CONFIGURE_COMMAND cmake build/cmake -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_TESTS=OFF -DZSTD_BUILD_CONTRIB=OFF -DZSTD_BUILD_SHARED=OFF + -DZSTD_BUILD_STATIC=ON + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1 + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + +ExternalProject_Add(lz4 + URL ${OSS_URL_PREFIX}/lz4-1.9.4.tar.gz + https://github.com/lz4/lz4/releases/download/v1.9.4/lz4-1.9.4.tar.gz + URL_MD5 e9286adb64040071c5e23498bf753261 + PATCH_COMMAND "" + CONFIGURE_COMMAND cmake build/cmake + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DLZ4_POSITION_INDEPENDENT_LIB=ON + -DCMAKE_BUILD_TYPE=Release + -DLZ4_BUILD_CLI=OFF + -DBUILD_SHARED_LIBS=OFF + -DBUILD_STATIC_LIBS=ON BUILD_COMMAND make -j${PARALLEL} INSTALL_COMMAND make install BUILD_IN_SOURCE 1 From 152caf1d65b9e2a648373483c090d2fcc285602f Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 00:55:23 +0800 Subject: [PATCH 06/15] update docker --- .github/workflows/lint_and_test_cpp.yaml | 3 --- docker/pegasus-build-env/centos7/Dockerfile | 6 ------ docker/pegasus-build-env/ubuntu1804/Dockerfile | 6 ------ docker/pegasus-build-env/ubuntu2004/Dockerfile | 6 ------ docker/pegasus-build-env/ubuntu2204/Dockerfile | 6 ------ thirdparty/CMakeLists.txt | 2 +- 6 files changed, 1 insertion(+), 28 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index a9b7724ebf..85b8eeb09d 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -673,9 +673,6 @@ jobs: run: | # Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md brew install ccache - brew install snappy - brew install lz4 - brew install zstd brew install openssl@1.1 - uses: actions/checkout@v3 - name: Setup cache diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile index 0e89f316b1..193fa8a935 100644 --- a/docker/pegasus-build-env/centos7/Dockerfile +++ b/docker/pegasus-build-env/centos7/Dockerfile @@ -42,12 +42,6 @@ RUN yum -y install centos-release-scl \ which \ openssl-devel \ libaio-devel \ - snappy-devel \ - bzip2-devel \ - zlib \ - zlib-devel \ - libzstd-devel \ - lz4-devel \ bison \ flex \ krb5-devel \ diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile index fb4c9a47ac..e23688dd87 100644 --- a/docker/pegasus-build-env/ubuntu1804/Dockerfile +++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile @@ -30,12 +30,6 @@ RUN apt-get update -y; \ openjdk-8-jdk \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ - zlib1g \ - zlib1g.dev \ patch \ netcat \ wget \ diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile b/docker/pegasus-build-env/ubuntu2004/Dockerfile index 604b5a0b1e..f3197c5ab0 100644 --- a/docker/pegasus-build-env/ubuntu2004/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile @@ -30,12 +30,6 @@ RUN apt-get update -y; \ openjdk-8-jdk \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ - zlib1g \ - zlib1g.dev \ patch \ netcat \ wget \ diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile b/docker/pegasus-build-env/ubuntu2204/Dockerfile index 0eed16c6cf..cdc6912274 100644 --- a/docker/pegasus-build-env/ubuntu2204/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2204/Dockerfile @@ -31,12 +31,6 @@ RUN apt-get update -y; \ pkg-config \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ - zlib1g \ - zlib1g.dev \ patch \ netcat \ wget \ diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index ce8f7cbc46..839a493509 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -465,7 +465,7 @@ ExternalProject_Add(rocksdb PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_rocksdb-cmake-PORTABLE-option.patch COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/encfs COMMAND git clone -b main --depth=1 https://github.com/pegasus-kv/encfs.git ${TP_DIR}/build/Source/rocksdb/plugin/encfs - DEPENDS googletest jemalloc + DEPENDS googletest jemalloc lz4 snappy zstd CMAKE_ARGS ${ROCKSDB_OPTIONS} DOWNLOAD_EXTRACT_TIMESTAMP true DOWNLOAD_NO_PROGRESS true From 1c651781cdd435ae67542cc3b6e9441ed53ba188 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 01:25:51 +0800 Subject: [PATCH 07/15] Update BaseFunctions.cmake --- cmake_modules/BaseFunctions.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index e13f409ce4..0586029eb4 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -328,9 +328,9 @@ function(dsn_setup_thirdparty_libs) # rocksdb and dependent libs find_package(RocksDB REQUIRED) - find_package(Snappy REQUIRED) - find_package(Zstd REQUIRED) - find_package(Lz4 REQUIRED) + find_package(snappy REQUIRED) + find_package(zstd REQUIRED) + find_package(lz4 REQUIRED) if(USE_JEMALLOC) find_package(Jemalloc REQUIRED) endif() From c9db7df7ac343977745d45cbeb539171fd868553 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 01:27:00 +0800 Subject: [PATCH 08/15] Update FindLz4.cmake --- cmake_modules/FindLz4.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake_modules/FindLz4.cmake b/cmake_modules/FindLz4.cmake index b52d2a63d3..21404fdee1 100644 --- a/cmake_modules/FindLz4.cmake +++ b/cmake_modules/FindLz4.cmake @@ -30,5 +30,5 @@ find_library(LZ4_STATIC_LIB liblz4.a NO_SYSTEM_ENVIRONMENT_PATH) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Lz4 REQUIRED_VARS +find_package_handle_standard_args(lz4 REQUIRED_VARS LZ4_STATIC_LIB LZ4_INCLUDE_DIR) From 26ff927cc963a97a0a484c6ddcb06c1cfdc7bfd5 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 08:28:50 +0800 Subject: [PATCH 09/15] 1 --- cmake_modules/{FindLz4.cmake => Findlz4.cmake} | 0 cmake_modules/{FindSnappy.cmake => Findsnappy.cmake} | 0 cmake_modules/{FindZstd.cmake => Findzstd.cmake} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename cmake_modules/{FindLz4.cmake => Findlz4.cmake} (100%) rename cmake_modules/{FindSnappy.cmake => Findsnappy.cmake} (100%) rename cmake_modules/{FindZstd.cmake => Findzstd.cmake} (100%) diff --git a/cmake_modules/FindLz4.cmake b/cmake_modules/Findlz4.cmake similarity index 100% rename from cmake_modules/FindLz4.cmake rename to cmake_modules/Findlz4.cmake diff --git a/cmake_modules/FindSnappy.cmake b/cmake_modules/Findsnappy.cmake similarity index 100% rename from cmake_modules/FindSnappy.cmake rename to cmake_modules/Findsnappy.cmake diff --git a/cmake_modules/FindZstd.cmake b/cmake_modules/Findzstd.cmake similarity index 100% rename from cmake_modules/FindZstd.cmake rename to cmake_modules/Findzstd.cmake From a91a9086c6ffcced6b69bec61f798563f76d39d6 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 08:31:06 +0800 Subject: [PATCH 10/15] 2 --- cmake_modules/BaseFunctions.cmake | 2 +- cmake_modules/Findsnappy.cmake | 2 +- cmake_modules/Findzstd.cmake | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 0586029eb4..16e3a2ffcd 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -327,13 +327,13 @@ function(dsn_setup_thirdparty_libs) set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default thirdparty libs" FORCE) # rocksdb and dependent libs - find_package(RocksDB REQUIRED) find_package(snappy REQUIRED) find_package(zstd REQUIRED) find_package(lz4 REQUIRED) if(USE_JEMALLOC) find_package(Jemalloc REQUIRED) endif() + find_package(RocksDB REQUIRED) # libhdfs find_package(JNI REQUIRED) diff --git a/cmake_modules/Findsnappy.cmake b/cmake_modules/Findsnappy.cmake index 341445ccf5..51a1a5157d 100644 --- a/cmake_modules/Findsnappy.cmake +++ b/cmake_modules/Findsnappy.cmake @@ -30,5 +30,5 @@ find_library(SNAPPY_STATIC_LIB libsnappy.a NO_SYSTEM_ENVIRONMENT_PATH) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Snappy REQUIRED_VARS +find_package_handle_standard_args(snappy REQUIRED_VARS SNAPPY_STATIC_LIB SNAPPY_INCLUDE_DIR) diff --git a/cmake_modules/Findzstd.cmake b/cmake_modules/Findzstd.cmake index 258ada4656..dcbf969782 100644 --- a/cmake_modules/Findzstd.cmake +++ b/cmake_modules/Findzstd.cmake @@ -30,5 +30,5 @@ find_library(ZSTD_STATIC_LIB libzstd.a NO_SYSTEM_ENVIRONMENT_PATH) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Zstd REQUIRED_VARS +find_package_handle_standard_args(zstd REQUIRED_VARS ZSTD_STATIC_LIB ZSTD_INCLUDE_DIR) From cb8ddbbe69d32e7ee7d3d6a1dd47e529a19b8627 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 08:56:10 +0800 Subject: [PATCH 11/15] try --- cmake_modules/BaseFunctions.cmake | 9 ++++++-- cmake_modules/Findlz4.cmake | 34 ------------------------------- cmake_modules/Findsnappy.cmake | 34 ------------------------------- cmake_modules/Findzstd.cmake | 34 ------------------------------- 4 files changed, 7 insertions(+), 104 deletions(-) delete mode 100644 cmake_modules/Findlz4.cmake delete mode 100644 cmake_modules/Findsnappy.cmake delete mode 100644 cmake_modules/Findzstd.cmake diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 16e3a2ffcd..09eb69593f 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -327,13 +327,18 @@ function(dsn_setup_thirdparty_libs) set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default thirdparty libs" FORCE) # rocksdb and dependent libs - find_package(snappy REQUIRED) + file(GLOB ROCKSDB_DEPENDS_MODULE_PATH ${THIRDPARTY_ROOT}/build/Source/rocksdb/cmake/modules) + if(NOT ROCKSDB_DEPENDS_MODULE_PATH) + message(WARNING "Cannot find RocksDB depends cmake modules path, might not find snappy, zstd, lz4") + endif() + list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DEPENDS_MODULE_PATH}") + find_package(Snappy REQUIRED) find_package(zstd REQUIRED) find_package(lz4 REQUIRED) if(USE_JEMALLOC) find_package(Jemalloc REQUIRED) endif() - find_package(RocksDB REQUIRED) + find_package(RocksDB) # libhdfs find_package(JNI REQUIRED) diff --git a/cmake_modules/Findlz4.cmake b/cmake_modules/Findlz4.cmake deleted file mode 100644 index 21404fdee1..0000000000 --- a/cmake_modules/Findlz4.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# - Find LZ4 (lz4.h, liblz4.a) -# This module defines -# LZ4_INCLUDE_DIR, directory containing headers -# LZ4_STATIC_LIB, path to lz4's static library -# LZ4_FOUND, whether lz4 has been found - -find_path(LZ4_INCLUDE_DIR lz4.h - # make sure we don't accidentally pick up a different version - NO_CMAKE_SYSTEM_PATH - NO_SYSTEM_ENVIRONMENT_PATH) -find_library(LZ4_STATIC_LIB liblz4.a - NO_CMAKE_SYSTEM_PATH - NO_SYSTEM_ENVIRONMENT_PATH) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(lz4 REQUIRED_VARS - LZ4_STATIC_LIB LZ4_INCLUDE_DIR) diff --git a/cmake_modules/Findsnappy.cmake b/cmake_modules/Findsnappy.cmake deleted file mode 100644 index 51a1a5157d..0000000000 --- a/cmake_modules/Findsnappy.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# - Find SNAPPY (snappy.h, libsnappy.a) -# This module defines -# SNAPPY_INCLUDE_DIR, directory containing headers -# SNAPPY_STATIC_LIB, path to snappy's static library -# SNAPPY_FOUND, whether snappy has been found - -find_path(SNAPPY_INCLUDE_DIR snappy.h - # make sure we don't accidentally pick up a different version - NO_CMAKE_SYSTEM_PATH - NO_SYSTEM_ENVIRONMENT_PATH) -find_library(SNAPPY_STATIC_LIB libsnappy.a - NO_CMAKE_SYSTEM_PATH - NO_SYSTEM_ENVIRONMENT_PATH) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(snappy REQUIRED_VARS - SNAPPY_STATIC_LIB SNAPPY_INCLUDE_DIR) diff --git a/cmake_modules/Findzstd.cmake b/cmake_modules/Findzstd.cmake deleted file mode 100644 index dcbf969782..0000000000 --- a/cmake_modules/Findzstd.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# - Find ZSTD (zstd.h, libzstd.a) -# This module defines -# ZSTD_INCLUDE_DIR, directory containing headers -# ZSTD_STATIC_LIB, path to zstd's static library -# ZSTD_FOUND, whether zstd has been found - -find_path(ZSTD_INCLUDE_DIR zstd.h - # make sure we don't accidentally pick up a different version - NO_CMAKE_SYSTEM_PATH - NO_SYSTEM_ENVIRONMENT_PATH) -find_library(ZSTD_STATIC_LIB libzstd.a - NO_CMAKE_SYSTEM_PATH - NO_SYSTEM_ENVIRONMENT_PATH) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(zstd REQUIRED_VARS - ZSTD_STATIC_LIB ZSTD_INCLUDE_DIR) From 3db3fd48f7df7b5361d39e893f03d6180c7abb31 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 12:05:20 +0800 Subject: [PATCH 12/15] keei zlib --- docker/pegasus-build-env/centos7/Dockerfile | 2 ++ docker/pegasus-build-env/ubuntu1804/Dockerfile | 2 ++ docker/pegasus-build-env/ubuntu2004/Dockerfile | 2 ++ docker/pegasus-build-env/ubuntu2204/Dockerfile | 2 ++ 4 files changed, 8 insertions(+) diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile index 193fa8a935..c8f8da6640 100644 --- a/docker/pegasus-build-env/centos7/Dockerfile +++ b/docker/pegasus-build-env/centos7/Dockerfile @@ -42,6 +42,8 @@ RUN yum -y install centos-release-scl \ which \ openssl-devel \ libaio-devel \ + zlib \ + zlib-devel \ bison \ flex \ krb5-devel \ diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile index e23688dd87..55189b1997 100644 --- a/docker/pegasus-build-env/ubuntu1804/Dockerfile +++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile @@ -30,6 +30,8 @@ RUN apt-get update -y; \ openjdk-8-jdk \ python3-pip \ libaio-dev \ + zlib1g \ + zlib1g.dev \ patch \ netcat \ wget \ diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile b/docker/pegasus-build-env/ubuntu2004/Dockerfile index f3197c5ab0..73d58e059a 100644 --- a/docker/pegasus-build-env/ubuntu2004/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile @@ -30,6 +30,8 @@ RUN apt-get update -y; \ openjdk-8-jdk \ python3-pip \ libaio-dev \ + zlib1g \ + zlib1g.dev \ patch \ netcat \ wget \ diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile b/docker/pegasus-build-env/ubuntu2204/Dockerfile index cdc6912274..d7efdb2867 100644 --- a/docker/pegasus-build-env/ubuntu2204/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2204/Dockerfile @@ -31,6 +31,8 @@ RUN apt-get update -y; \ pkg-config \ python3-pip \ libaio-dev \ + zlib1g \ + zlib1g.dev \ patch \ netcat \ wget \ From 8f09400f4d65bc506dcaca3b945ecf92c151a11e Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 15:00:25 +0800 Subject: [PATCH 13/15] fix enc test failures --- thirdparty/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 839a493509..c3a9498f76 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -442,8 +442,6 @@ set(ROCKSDB_OPTIONS -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF -DUSE_RTTI=ON - -DROCKSDB_BUILD_SHARED=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -DWITH_JEMALLOC=${USE_JEMALLOC} -DJEMALLOC_ROOT_DIR=${TP_OUTPUT} From 85370b2776b0e61dd9c5a58e8e7cdfb7d709c854 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 6 Dec 2023 16:20:28 +0800 Subject: [PATCH 14/15] update cmake --- cmake_modules/BaseFunctions.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 09eb69593f..dc0e3b28f7 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -326,7 +326,7 @@ function(dsn_setup_thirdparty_libs) find_package(fmt REQUIRED) set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default thirdparty libs" FORCE) - # rocksdb and dependent libs + # rocksdb and dependencies file(GLOB ROCKSDB_DEPENDS_MODULE_PATH ${THIRDPARTY_ROOT}/build/Source/rocksdb/cmake/modules) if(NOT ROCKSDB_DEPENDS_MODULE_PATH) message(WARNING "Cannot find RocksDB depends cmake modules path, might not find snappy, zstd, lz4") @@ -338,7 +338,7 @@ function(dsn_setup_thirdparty_libs) if(USE_JEMALLOC) find_package(Jemalloc REQUIRED) endif() - find_package(RocksDB) + find_package(RocksDB REQUIRED) # libhdfs find_package(JNI REQUIRED) From 586ee17f36d73396dd54f582cfcea658e7f5c857 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 11 Dec 2023 17:33:35 +0800 Subject: [PATCH 15/15] cmake --- src/server/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 02f89c4087..8c8daa4d12 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -40,7 +40,6 @@ set(MY_PROJ_LIBS lz4 zstd snappy - pegasus_reporter pegasus_base pegasus_client_static event