diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index a9b7724ebf..85b8eeb09d 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -673,9 +673,6 @@ jobs: run: | # Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md brew install ccache - brew install snappy - brew install lz4 - brew install zstd brew install openssl@1.1 - uses: actions/checkout@v3 - name: Setup cache diff --git a/.licenserc.yaml b/.licenserc.yaml index 3ddf87be61..1c58047590 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -71,6 +71,7 @@ header: - 'thirdparty/fix_libevent_for_macos.patch' - 'thirdparty/fix_prometheus-cpp_limits.patch' - 'thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch' + - 'thirdparty/fix_snappy-Wsign-compare-warning.patch' - 'thirdparty/fix_s2_build_with_absl_and_gtest.patch' - 'thirdparty/fix_thrift_for_cpp11.patch' # TODO(yingchun): shell/* files are import from thirdparties, we can move them to thirdparty later. diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 1882353c83..dc0e3b28f7 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -326,15 +326,15 @@ function(dsn_setup_thirdparty_libs) find_package(fmt REQUIRED) set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default thirdparty libs" FORCE) - # rocksdb + # rocksdb and dependencies file(GLOB ROCKSDB_DEPENDS_MODULE_PATH ${THIRDPARTY_ROOT}/build/Source/rocksdb/cmake/modules) if(NOT ROCKSDB_DEPENDS_MODULE_PATH) message(WARNING "Cannot find RocksDB depends cmake modules path, might not find snappy, zstd, lz4") endif() list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DEPENDS_MODULE_PATH}") - find_package(snappy) - find_package(zstd) - find_package(lz4) + find_package(Snappy REQUIRED) + find_package(zstd REQUIRED) + find_package(lz4 REQUIRED) if(USE_JEMALLOC) find_package(Jemalloc REQUIRED) endif() diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile index 0e89f316b1..c8f8da6640 100644 --- a/docker/pegasus-build-env/centos7/Dockerfile +++ b/docker/pegasus-build-env/centos7/Dockerfile @@ -42,12 +42,8 @@ RUN yum -y install centos-release-scl \ which \ openssl-devel \ libaio-devel \ - snappy-devel \ - bzip2-devel \ zlib \ zlib-devel \ - libzstd-devel \ - lz4-devel \ bison \ flex \ krb5-devel \ diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile index fb4c9a47ac..55189b1997 100644 --- a/docker/pegasus-build-env/ubuntu1804/Dockerfile +++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile @@ -30,10 +30,6 @@ RUN apt-get update -y; \ openjdk-8-jdk \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ zlib1g \ zlib1g.dev \ patch \ diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile b/docker/pegasus-build-env/ubuntu2004/Dockerfile index 604b5a0b1e..73d58e059a 100644 --- a/docker/pegasus-build-env/ubuntu2004/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile @@ -30,10 +30,6 @@ RUN apt-get update -y; \ openjdk-8-jdk \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ zlib1g \ zlib1g.dev \ patch \ diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile b/docker/pegasus-build-env/ubuntu2204/Dockerfile index 0eed16c6cf..d7efdb2867 100644 --- a/docker/pegasus-build-env/ubuntu2204/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2204/Dockerfile @@ -31,10 +31,6 @@ RUN apt-get update -y; \ pkg-config \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ zlib1g \ zlib1g.dev \ patch \ diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt index 3754361d06..3a27fbe01a 100644 --- a/src/aio/CMakeLists.txt +++ b/src/aio/CMakeLists.txt @@ -33,7 +33,12 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + rocksdb + lz4 + zstd + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index b8d4ad675b..2eb4371467 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -33,7 +33,17 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio test_utils rocksdb) +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common + gtest + dsn_runtime + dsn_aio + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/block_service/hdfs/CMakeLists.txt b/src/block_service/hdfs/CMakeLists.txt index 2bba8b96bb..ae821f2bf2 100644 --- a/src/block_service/hdfs/CMakeLists.txt +++ b/src/block_service/hdfs/CMakeLists.txt @@ -26,7 +26,12 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS hdfs rocksdb) +set(MY_PROJ_LIBS + hdfs + rocksdb + lz4 + zstd + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/local/CMakeLists.txt b/src/block_service/local/CMakeLists.txt index 9d830f7825..8fdd0d53ac 100644 --- a/src/block_service/local/CMakeLists.txt +++ b/src/block_service/local/CMakeLists.txt @@ -26,7 +26,11 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS rocksdb) +set(MY_PROJ_LIBS + rocksdb + lz4 + zstd + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index 6fbfa9ca11..b8aaad6520 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -22,17 +22,20 @@ set(MY_PROJ_NAME dsn_block_service_test) set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn_runtime - dsn_utils - gtest - gtest_main - hdfs - test_utils - rocksdb) + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn_runtime + dsn_utils + gtest + gtest_main + hdfs + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt index cfe591ce0f..60f3762760 100644 --- a/src/client/test/CMakeLists.txt +++ b/src/client/test/CMakeLists.txt @@ -22,12 +22,15 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_client - dsn_replication_common - dsn_runtime - dsn_utils - gtest - rocksdb) + dsn_client + dsn_replication_common + dsn_runtime + dsn_utils + gtest + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt index 12f53aa52c..fab0926249 100644 --- a/src/common/test/CMakeLists.txt +++ b/src/common/test/CMakeLists.txt @@ -31,7 +31,10 @@ set(MY_PROJ_LIBS dsn_replication_common dsn_runtime gtest - rocksdb) + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/failure_detector/test/CMakeLists.txt b/src/failure_detector/test/CMakeLists.txt index 6bd1e05553..5b5a6832a9 100644 --- a/src/failure_detector/test/CMakeLists.txt +++ b/src/failure_detector/test/CMakeLists.txt @@ -34,14 +34,17 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_runtime - dsn_meta_server - dsn_replica_server - dsn_replication_common - dsn.failure_detector - gtest - hashtable - rocksdb) + dsn_runtime + dsn_meta_server + dsn_replica_server + dsn_replication_common + dsn.failure_detector + gtest + hashtable + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt index 6df54a6bcb..559747b59d 100644 --- a/src/geo/bench/CMakeLists.txt +++ b/src/geo/bench/CMakeLists.txt @@ -34,9 +34,11 @@ set(MY_PROJ_LIBS s2testing s2 pegasus_client_static - RocksDB::rocksdb - dsn_utils - ) + rocksdb + lz4 + zstd + snappy + dsn_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt index 5ebc142a55..4400c0183f 100644 --- a/src/http/test/CMakeLists.txt +++ b/src/http/test/CMakeLists.txt @@ -22,12 +22,14 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_http - dsn_runtime - curl - gtest - rocksdb - ) + dsn_http + dsn_runtime + curl + gtest + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 477671acb3..fcc3a12329 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -34,20 +34,23 @@ set(DUPLICATION_SRC set(MY_PROJ_SRC "${DUPLICATION_SRC}") set(MY_PROJ_LIBS - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_dist_cmd - dsn_http - dsn_runtime - dsn_aio - zookeeper - hashtable - hdfs - rocksdb) + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_dist_cmd + dsn_http + dsn_runtime + dsn_aio + zookeeper + hashtable + hdfs + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt index ff591b7e21..4a22ce4867 100644 --- a/src/nfs/test/CMakeLists.txt +++ b/src/nfs/test/CMakeLists.txt @@ -33,7 +33,17 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio dsn_http rocksdb test_utils) +set(MY_PROJ_LIBS + dsn_nfs + dsn_runtime + gtest + dsn_aio + dsn_http + rocksdb + lz4 + zstd + snappy + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/perf_counter/test/CMakeLists.txt b/src/perf_counter/test/CMakeLists.txt index 2e02cf868f..f924437dec 100644 --- a/src/perf_counter/test/CMakeLists.txt +++ b/src/perf_counter/test/CMakeLists.txt @@ -33,7 +33,15 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime rocksdb) +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common + gtest + dsn_runtime + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index fcf7891017..5bcd3f57bc 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -57,19 +57,22 @@ set(MY_PROJ_SRC # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS - dsn_replication_common - dsn.failure_detector - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn_nfs - dsn_dist_cmd - dsn_http - dsn_runtime - dsn_aio - dsn_meta_server - rocksdb) +set(MY_PROJ_LIBS + dsn_replication_common + dsn.failure_detector + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn_nfs + dsn_dist_cmd + dsn_http + dsn_runtime + dsn_aio + dsn_meta_server + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::filesystem) diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt index 47158ca93c..fed60e3120 100644 --- a/src/replica/backup/test/CMakeLists.txt +++ b/src/replica/backup/test/CMakeLists.txt @@ -29,7 +29,10 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_utils hashtable gtest - rocksdb) + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 0b0d083851..bda1a89819 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -28,9 +28,12 @@ set(MY_PROJ_LIBS dsn_meta_server hashtable gtest test_utils - rocksdb) + rocksdb + lz4 + zstd + snappy) -set(MY_BOOST_LIBS Boost::system Boost::filesystem rocksdb test_utils) +set(MY_BOOST_LIBS Boost::system Boost::filesystem) set(MY_BINPLACES config-test.ini diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index 5cc139f916..26e461bd75 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -31,7 +31,10 @@ set(MY_PROJ_LIBS dsn_meta_server hashtable gtest test_utils - rocksdb) + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/storage/simple_kv/CMakeLists.txt b/src/replica/storage/simple_kv/CMakeLists.txt index 8584848f11..a2f8506770 100644 --- a/src/replica/storage/simple_kv/CMakeLists.txt +++ b/src/replica/storage/simple_kv/CMakeLists.txt @@ -37,7 +37,16 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS}) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable rocksdb) +set(MY_PROJ_LIBS + dsn_replica_server + dsn_meta_server + dsn_client + dsn_runtime + hashtable + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt b/src/replica/storage/simple_kv/test/CMakeLists.txt index a1e0a69b21..0bce41b712 100644 --- a/src/replica/storage/simple_kv/test/CMakeLists.txt +++ b/src/replica/storage/simple_kv/test/CMakeLists.txt @@ -29,18 +29,22 @@ set(MY_PROJ_NAME dsn.rep_tests.simple_kv) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_runtime - zookeeper - hashtable - gtest - dsn_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_runtime + zookeeper + hashtable + gtest + dsn_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 83af542d10..94b5c89348 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -33,21 +33,25 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_meta_server - dsn_replica_server - dsn.replication.zookeeper_provider - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn.failure_detector - dsn_http - dsn_runtime - zookeeper - hashtable - gtest - test_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replica_server + dsn.replication.zookeeper_provider + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn.failure_detector + dsn_http + dsn_runtime + zookeeper + hashtable + gtest + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/runtime/ranger/CMakeLists.txt b/src/runtime/ranger/CMakeLists.txt index 3c3e441a4f..fbbceb570a 100644 --- a/src/runtime/ranger/CMakeLists.txt +++ b/src/runtime/ranger/CMakeLists.txt @@ -17,14 +17,10 @@ set(MY_PROJ_NAME dsn_ranger) -# Search mode for source files under CURRENT project directory? -# "GLOB_RECURSE" for recursive search -# "GLOB" for non-recursive search +set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common) -set(MY_PROJ_LIBS "") - -# Extra files that will be installed -set(MY_BINPLACES "") - -dsn_add_object() +dsn_add_static_library() diff --git a/src/runtime/test/CMakeLists.txt b/src/runtime/test/CMakeLists.txt index cd5e0daac8..38e52a4f5b 100644 --- a/src/runtime/test/CMakeLists.txt +++ b/src/runtime/test/CMakeLists.txt @@ -29,12 +29,15 @@ set(MY_PROJ_NAME dsn_runtime_tests) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest - dsn_runtime - dsn_aio - dsn_meta_server - rocksdb - ) +set(MY_PROJ_LIBS + gtest + dsn_runtime + dsn_aio + dsn_meta_server + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index d19408e276..8c8daa4d12 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -27,21 +27,23 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn.block_service.local - dsn.block_service - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_utils - RocksDB::rocksdb - pegasus_base - pegasus_client_static - event - hashtable - ) + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn.block_service.local + dsn.block_service + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_utils + rocksdb + lz4 + zstd + snappy + pegasus_base + pegasus_client_static + event + hashtable) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt index 8a2fbd6f1c..2b4c96ac1f 100644 --- a/src/server/test/CMakeLists.txt +++ b/src/server/test/CMakeLists.txt @@ -45,14 +45,17 @@ set(MY_PROJ_LIBS dsn.failure_detector dsn.replication.zookeeper_provider dsn_utils - RocksDB::rocksdb + rocksdb + lz4 + zstd + snappy pegasus_client_static event pegasus_base gtest gmock hashtable - ) +) add_definitions(-DPEGASUS_UNIT_TEST) add_definitions(-DENABLE_FAIL) diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 909fda9ab6..81271006e1 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -28,25 +28,27 @@ set(MY_PROJ_SRC "linenoise/linenoise.c" "sds/sds.c") set(MY_SRC_SEARCH_MODE "GLOB_RECURSE") set(MY_PROJ_LIBS - pegasus_base - dsn.replication.tool - dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn_utils - dsn.block_service.local - dsn.block_service.hdfs - dsn.block_service - dsn.failure_detector - pegasus_client_static - pegasus_geo_lib - RocksDB::rocksdb - absl::flat_hash_set - absl::strings - s2 - hdfs - ) + pegasus_base + dsn.replication.tool + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn_utils + dsn.block_service.local + dsn.block_service.hdfs + dsn.block_service + dsn.failure_detector + pegasus_client_static + pegasus_geo_lib + rocksdb + lz4 + zstd + snappy + absl::flat_hash_set + absl::strings + s2 + hdfs) set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config.ini") diff --git a/src/test/bench_test/CMakeLists.txt b/src/test/bench_test/CMakeLists.txt index 307d543d09..5a52808d8d 100644 --- a/src/test/bench_test/CMakeLists.txt +++ b/src/test/bench_test/CMakeLists.txt @@ -30,11 +30,13 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS pegasus_client_static dsn_utils - RocksDB::rocksdb + rocksdb + lz4 + zstd + snappy sasl2 gssapi_krb5 - krb5 - ) + krb5) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/test/function_test/bulk_load/CMakeLists.txt b/src/test/function_test/bulk_load/CMakeLists.txt index f9f1976491..ce17323ea7 100644 --- a/src/test/function_test/bulk_load/CMakeLists.txt +++ b/src/test/function_test/bulk_load/CMakeLists.txt @@ -28,17 +28,20 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_client - dsn_replication_common - dsn_utils - pegasus_client_static - gtest - sasl2 - gssapi_krb5 - krb5 - function_test_utils - test_utils - rocksdb) + dsn_client + dsn_replication_common + dsn_utils + pegasus_client_static + gtest + sasl2 + gssapi_krb5 + krb5 + function_test_utils + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt index b1e7ac2dff..fd4cf1dc16 100644 --- a/src/test_util/CMakeLists.txt +++ b/src/test_util/CMakeLists.txt @@ -22,6 +22,11 @@ set(MY_PROJ_NAME test_utils) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest rocksdb) +set(MY_PROJ_LIBS + gtest + rocksdb + lz4 + zstd + snappy) dsn_add_static_library() diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 8c04e3adc4..38390f0ed6 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -31,7 +31,12 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_BOOST_LIBS Boost::system Boost::filesystem) -set(MY_PROJ_LIBS dsn_http rocksdb) +set(MY_PROJ_LIBS + dsn_http + rocksdb + lz4 + zstd + snappy) # Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/utils/long_adder_bench/CMakeLists.txt b/src/utils/long_adder_bench/CMakeLists.txt index d5ace78cac..71568c61ae 100644 --- a/src/utils/long_adder_bench/CMakeLists.txt +++ b/src/utils/long_adder_bench/CMakeLists.txt @@ -27,7 +27,13 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + dsn_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index 684a5faaf0..8502cabb76 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -29,12 +29,16 @@ set(MY_PROJ_NAME dsn_utils_tests) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_http - dsn_runtime - dsn_utils - gtest - test_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_http + dsn_runtime + dsn_utils + gtest + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/utils/test/nth_element_bench/CMakeLists.txt b/src/utils/test/nth_element_bench/CMakeLists.txt index 7bf0af804b..3aae31e4a6 100644 --- a/src/utils/test/nth_element_bench/CMakeLists.txt +++ b/src/utils/test/nth_element_bench/CMakeLists.txt @@ -27,7 +27,13 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + dsn_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index cdb6d4cec2..7dabd1faf5 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -33,13 +33,16 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS - dsn.replication.zookeeper_provider - dsn_runtime - zookeeper - hashtable - gtest - rocksdb) +set(MY_PROJ_LIBS + dsn.replication.zookeeper_provider + dsn_runtime + zookeeper + hashtable + gtest + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index e95e24f83e..c3a9498f76 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -361,6 +361,75 @@ ExternalProject_Add(jemalloc DOWNLOAD_NO_PROGRESS true ) +set(SNAPPY_OPTIONS + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=Release + -DBUILD_SHARED_LIBS=OFF + -DSNAPPY_BUILD_TESTS=OFF + -DSNAPPY_BUILD_BENCHMARKS=OFF + -DSNAPPY_FUZZING_BUILD=OFF + -DSNAPPY_INSTALL=ON) +execute_process(COMMAND arch OUTPUT_VARIABLE ARCH_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) +message(STATUS "ARCH_NAME = ${ARCH_NAME}") +if (ARCH_NAME EQUAL "x86_64") + set(SNAPPY_OPTIONS + ${SNAPPY_OPTIONS} + -DSNAPPY_REQUIRE_AVX=ON + -DSNAPPY_REQUIRE_AVX2=ON) +endif () +ExternalProject_Add(snappy + URL ${OSS_URL_PREFIX}/snappy-1.1.10.tar.gz + https://github.com/google/snappy/archive/refs/tags/1.1.10.tar.gz + URL_MD5 70153395ebe6d72febe2cf2e40026a44 + PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_snappy-Wsign-compare-warning.patch + CMAKE_ARGS ${SNAPPY_OPTIONS} + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + +ExternalProject_Add(zstd + URL ${OSS_URL_PREFIX}/zstd-1.5.5.tar.gz + https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz + URL_MD5 63251602329a106220e0a5ad26ba656f + PATCH_COMMAND "" + CONFIGURE_COMMAND cmake build/cmake + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=Release + -DZSTD_BUILD_PROGRAMS=OFF + -DZSTD_BUILD_TESTS=OFF + -DZSTD_BUILD_CONTRIB=OFF + -DZSTD_BUILD_SHARED=OFF + -DZSTD_BUILD_STATIC=ON + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1 + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + +ExternalProject_Add(lz4 + URL ${OSS_URL_PREFIX}/lz4-1.9.4.tar.gz + https://github.com/lz4/lz4/releases/download/v1.9.4/lz4-1.9.4.tar.gz + URL_MD5 e9286adb64040071c5e23498bf753261 + PATCH_COMMAND "" + CONFIGURE_COMMAND cmake build/cmake + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DLZ4_POSITION_INDEPENDENT_LIB=ON + -DCMAKE_BUILD_TYPE=Release + -DLZ4_BUILD_CLI=OFF + -DBUILD_SHARED_LIBS=OFF + -DBUILD_STATIC_LIBS=ON + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1 + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) set(ROCKSDB_OPTIONS -DFAIL_ON_WARNINGS=OFF @@ -378,6 +447,7 @@ set(ROCKSDB_OPTIONS -DJEMALLOC_ROOT_DIR=${TP_OUTPUT} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DPORTABLE=${ROCKSDB_PORTABLE}) # Link error on MacOS, disable building encfs plugin. # See https://github.com/pegasus-kv/encfs/issues/4 @@ -393,9 +463,8 @@ ExternalProject_Add(rocksdb PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_rocksdb-cmake-PORTABLE-option.patch COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/encfs COMMAND git clone -b main --depth=1 https://github.com/pegasus-kv/encfs.git ${TP_DIR}/build/Source/rocksdb/plugin/encfs - DEPENDS googletest jemalloc - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - ${ROCKSDB_OPTIONS} + DEPENDS googletest jemalloc lz4 snappy zstd + CMAKE_ARGS ${ROCKSDB_OPTIONS} DOWNLOAD_EXTRACT_TIMESTAMP true DOWNLOAD_NO_PROGRESS true ) diff --git a/thirdparty/fix_snappy-Wsign-compare-warning.patch b/thirdparty/fix_snappy-Wsign-compare-warning.patch new file mode 100644 index 0000000000..0540d0177c --- /dev/null +++ b/thirdparty/fix_snappy-Wsign-compare-warning.patch @@ -0,0 +1,26 @@ +From 27f34a580be4a3becf5f8c0cba13433f53c21337 Mon Sep 17 00:00:00 2001 +From: Richard O'Grady +Date: Wed, 12 Jul 2023 10:12:01 -0700 +Subject: [PATCH] Fix -Wsign-compare warning + +PiperOrigin-RevId: 547529709 +--- + snappy.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/snappy.cc b/snappy.cc +index 688065b..6473123 100644 +--- a/snappy.cc ++++ b/snappy.cc +@@ -1289,7 +1289,7 @@ std::pair DecompressBranchless( + DeferMemCopy(&deferred_src, &deferred_length, from, len); + } + } while (ip < ip_limit_min_slop && +- (op + deferred_length) < op_limit_min_slop); ++ static_cast(op + deferred_length) < op_limit_min_slop); + exit: + ip--; + assert(ip <= ip_limit); +-- +2.42.1 +