From e978e4c78b1436aa0ae06915365f85c601781ba7 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 13 Sep 2023 22:28:01 +0800 Subject: [PATCH] feat(thirdparty): upgrade rocksdb to 8.5.3 (#1601) https://github.com/apache/incubator-pegasus/issues/1604 This patch upgrade the `rocksdb` to the latest stable version 8.5.3. Since the 2.5 branch has been created, the new version of rocksdb will only take effect on version 2.6 and later. --- .github/workflows/lint_and_test_cpp.yaml | 12 ++++---- .github/workflows/thirdparty-regular-push.yml | 4 +-- docker/thirdparties-bin/Dockerfile | 2 +- run.sh | 4 +-- src/server/pegasus_event_listener.cpp | 1 + src/server/pegasus_server_impl.cpp | 29 ++++++++++--------- src/server/pegasus_server_impl.h | 2 ++ src/server/pegasus_server_impl_init.cpp | 2 ++ .../test/pegasus_compression_options_test.cpp | 1 + src/utils/alloc.h | 2 ++ thirdparty/CMakeLists.txt | 10 ++----- 11 files changed, 38 insertions(+), 31 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 115f8b124c..e405fa3a47 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -91,7 +91,7 @@ jobs: # Build thirdparties and leave some necessary libraries and source run: | mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* ../scripts/download_hadoop.sh hadoop-bin @@ -151,7 +151,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -296,7 +296,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -438,7 +438,7 @@ jobs: # run: | # rm -f /root/thirdparties-src.zip # mkdir build -# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ +# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ # cmake --build build/ -j $(nproc) # rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* # find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -576,7 +576,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -676,7 +676,7 @@ jobs: run: | export JAVA_HOME="${JAVA_HOME_8_X64}" mkdir -p build - cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} + cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DROCKSDB_PORTABLE=1 cmake --build build/ -j $(sysctl -n hw.physicalcpu) - name: Compilation run: | diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 4fb58b59af..48c04e750a 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -167,7 +167,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin - name: Image digest @@ -203,7 +203,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 USE_JEMALLOC=ON HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index ab91710632..27a0d5146e 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip ARG GITHUB_BRANCH=master ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git -ARG ROCKSDB_PORTABLE=OFF +ARG ROCKSDB_PORTABLE=0 ARG USE_JEMALLOC=OFF ARG HADOOP_BIN_PATH ARG ZOOKEEPER_BIN_PATH diff --git a/run.sh b/run.sh index 75b688a8ae..5ac6a19645 100755 --- a/run.sh +++ b/run.sh @@ -118,7 +118,7 @@ function run_build() ENABLE_GPERF=ON SKIP_THIRDPARTY=NO SANITIZER="" - ROCKSDB_PORTABLE=OFF + ROCKSDB_PORTABLE=0 USE_JEMALLOC=OFF BUILD_TEST=OFF IWYU="" @@ -177,7 +177,7 @@ function run_build() SKIP_THIRDPARTY=YES ;; --enable_rocksdb_portable) - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 ;; --use_jemalloc) ENABLE_GPERF=OFF diff --git a/src/server/pegasus_event_listener.cpp b/src/server/pegasus_event_listener.cpp index e13e2f8635..24faf954e7 100644 --- a/src/server/pegasus_event_listener.cpp +++ b/src/server/pegasus_event_listener.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "common/gpid.h" diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index c348952696..8d6ac8c376 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include @@ -1635,12 +1635,12 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) rocksdb::DBOptions loaded_db_opt; std::vector loaded_cf_descs; rocksdb::ColumnFamilyOptions loaded_data_cf_opts; + rocksdb::ConfigOptions config_options; // Set `ignore_unknown_options` true for forward compatibility. - auto status = rocksdb::LoadLatestOptions(rdb_path, - rocksdb::Env::Default(), - &loaded_db_opt, - &loaded_cf_descs, - /*ignore_unknown_options=*/true); + config_options.ignore_unknown_options = true; + config_options.env = rocksdb::Env::Default(); + auto status = + rocksdb::LoadLatestOptions(config_options, rdb_path, &loaded_db_opt, &loaded_cf_descs); if (!status.ok()) { // Here we ignore an invalid argument error related to `pegasus_data_version` and // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0). @@ -1678,11 +1678,14 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) std::vector column_families( {{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}}); - auto s = rocksdb::CheckOptionsCompatibility(rdb_path, - rocksdb::Env::Default(), - _db_opts, - column_families, - /*ignore_unknown_options=*/true); + rocksdb::ConfigOptions config_options; + config_options.ignore_unknown_options = true; + config_options.ignore_unsupported_options = true; + config_options.sanity_level = + rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible; + config_options.env = rocksdb::Env::Default(); + auto s = + rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts, column_families); if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) { LOG_ERROR_PREFIX("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString()); return dsn::ERR_LOCAL_APP_FAILURE; @@ -2159,7 +2162,7 @@ ::dsn::error_code pegasus_server_impl::copy_checkpoint_to_dir_unsafe(const char {{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}, {META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}}); status = rocksdb::DB::OpenForReadOnly( - rocksdb::DBOptions(), checkpoint_dir, column_families, &handles_opened, &snapshot_db); + _db_opts, checkpoint_dir, column_families, &handles_opened, &snapshot_db); if (!status.ok()) { LOG_ERROR_PREFIX( "OpenForReadOnly from {} failed, error = {}", checkpoint_dir, status.ToString()); @@ -3286,7 +3289,7 @@ ::dsn::error_code pegasus_server_impl::check_column_families(const std::string & *missing_meta_cf = true; *missing_data_cf = true; std::vector column_families; - auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families); + auto s = rocksdb::DB::ListColumnFamilies(_db_opts, path, &column_families); if (!s.ok()) { LOG_ERROR_PREFIX("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString()); if (s.IsCorruption() && diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index c9c5b90ef6..02c6421c7a 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -70,6 +71,7 @@ class WriteBufferManager; namespace dsn { class blob; class message_ex; + namespace replication { class detect_hotkey_request; class detect_hotkey_response; diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index c457ebd9fa..528b2bd8e9 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -422,6 +423,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _rng_rd_opts.rocksdb_iteration_threshold_time_ms = FLAGS_rocksdb_iteration_threshold_time_ms; // init rocksdb::DBOptions + _db_opts.env = rocksdb::Env::Default(); _db_opts.create_if_missing = true; // atomic flush data CF and meta CF, aim to keep consistency of 'last flushed decree' in meta CF // and data in data CF. diff --git a/src/server/test/pegasus_compression_options_test.cpp b/src/server/test/pegasus_compression_options_test.cpp index 9fa38daf66..b33eeeac14 100644 --- a/src/server/test/pegasus_compression_options_test.cpp +++ b/src/server/test/pegasus_compression_options_test.cpp @@ -20,6 +20,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include diff --git a/src/utils/alloc.h b/src/utils/alloc.h index e224eaff6d..542f4efb51 100644 --- a/src/utils/alloc.h +++ b/src/utils/alloc.h @@ -17,6 +17,8 @@ #pragma once +#include + #include "utils/ports.h" // The check for the definition of CACHELINE_SIZE has to be put after including "utils/ports.h", diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 069f21d299..7f128c8940 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -358,15 +358,11 @@ ExternalProject_Add(jemalloc BUILD_IN_SOURCE 1 ) -option(ROCKSDB_PORTABLE "build a portable binary" OFF) +option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) -# The patch name '0879c240' means the patch of rocksdb: -# https://github.com/facebook/rocksdb/commit/0879c240404b00142ba4718f36cd3f2bd537192d ExternalProject_Add(rocksdb - URL ${OSS_URL_PREFIX}/rocksdb-6.6.4.tar.gz - https://github.com/facebook/rocksdb/archive/refs/tags/v6.6.4.tar.gz - URL_MD5 7f7fcca3e96b7d83ef332804c90070c8 - PATCH_COMMAND patch -p1 < ${TP_DIR}/rocksdb_fix_atomic_flush_0879c240.patch + URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.5.3.tar.gz + URL_MD5 f03eac50ec958a21a7cb56183afb8fe4 DEPENDS jemalloc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DFAIL_ON_WARNINGS=OFF