From 1752d6adbe90570787c7e6a6a10073f856c44535 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Sep 2023 20:52:39 +0800 Subject: [PATCH] feat(thirdparty): upgrade rocksdb to 8.3.3 --- .github/workflows/lint_and_test_cpp.yaml | 10 +++---- .github/workflows/thirdparty-regular-push.yml | 4 +-- cmake_modules/BaseFunctions.cmake | 13 +++------ docker/thirdparties-bin/Dockerfile | 2 +- run.sh | 4 +-- src/server/pegasus_server_impl.cpp | 27 ++++++++++--------- src/server/pegasus_server_impl_init.cpp | 1 + thirdparty/CMakeLists.txt | 10 +++---- 8 files changed, 33 insertions(+), 38 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 115f8b124c..2b2814fb75 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -91,7 +91,7 @@ jobs: # Build thirdparties and leave some necessary libraries and source run: | mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* ../scripts/download_hadoop.sh hadoop-bin @@ -151,7 +151,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -296,7 +296,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -438,7 +438,7 @@ jobs: # run: | # rm -f /root/thirdparties-src.zip # mkdir build -# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ +# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ # cmake --build build/ -j $(nproc) # rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* # find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -576,7 +576,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 4fb58b59af..48c04e750a 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -167,7 +167,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin - name: Image digest @@ -203,7 +203,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 USE_JEMALLOC=ON HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 79aba27cb7..5596bbd977 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -204,7 +204,7 @@ function(dsn_setup_compiler_flags) # We want access to the PRI* print format macros. add_definitions(-D__STDC_FORMAT_MACROS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -gdwarf-4" CACHE STRING "" FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -gdwarf-4" CACHE STRING "" FORCE) # -Wall: Enable all warnings. add_compile_options(-Wall) @@ -221,9 +221,6 @@ function(dsn_setup_compiler_flags) # use frame pointers to allow simple stack frame walking for backtraces. # This has a small perf hit but worth it for the ability to profile in production add_compile_options( -fno-omit-frame-pointer) - # -Wno-deprecated-register - # kbr5.h uses the legacy 'register' keyword. - add_compile_options(-Wno-deprecated-register) # -Wno-implicit-float-conversion # Poco/Dynamic/VarHolder.h uses 'unsigned long' to 'float' conversion add_compile_options(-Wno-implicit-float-conversion) @@ -388,11 +385,9 @@ function(dsn_common_setup) set(BUILD_SHARED_LIBS OFF) - include(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG("-std=c++1y" COMPILER_SUPPORTS_CXX1Y) - if(NOT ${COMPILER_SUPPORTS_CXX1Y}) - message(FATAL_ERROR "You need a compiler with C++1y support.") - endif() + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) dsn_setup_system_libs() dsn_setup_compiler_flags() diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index ab91710632..27a0d5146e 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip ARG GITHUB_BRANCH=master ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git -ARG ROCKSDB_PORTABLE=OFF +ARG ROCKSDB_PORTABLE=0 ARG USE_JEMALLOC=OFF ARG HADOOP_BIN_PATH ARG ZOOKEEPER_BIN_PATH diff --git a/run.sh b/run.sh index 75b688a8ae..5ac6a19645 100755 --- a/run.sh +++ b/run.sh @@ -118,7 +118,7 @@ function run_build() ENABLE_GPERF=ON SKIP_THIRDPARTY=NO SANITIZER="" - ROCKSDB_PORTABLE=OFF + ROCKSDB_PORTABLE=0 USE_JEMALLOC=OFF BUILD_TEST=OFF IWYU="" @@ -177,7 +177,7 @@ function run_build() SKIP_THIRDPARTY=YES ;; --enable_rocksdb_portable) - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 ;; --use_jemalloc) ENABLE_GPERF=OFF diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 5bc03eafe9..865f274f7f 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -1634,12 +1634,12 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) rocksdb::DBOptions loaded_db_opt; std::vector loaded_cf_descs; rocksdb::ColumnFamilyOptions loaded_data_cf_opts; + rocksdb::ConfigOptions config_options; // Set `ignore_unknown_options` true for forward compatibility. - auto status = rocksdb::LoadLatestOptions(rdb_path, - rocksdb::Env::Default(), - &loaded_db_opt, - &loaded_cf_descs, - /*ignore_unknown_options=*/true); + config_options.ignore_unknown_options = true; + config_options.env = rocksdb::Env::Default(); + auto status = + rocksdb::LoadLatestOptions(config_options, rdb_path, &loaded_db_opt, &loaded_cf_descs); if (!status.ok()) { // Here we ignore an invalid argument error related to `pegasus_data_version` and // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0). @@ -1677,11 +1677,14 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) std::vector column_families( {{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}}); - auto s = rocksdb::CheckOptionsCompatibility(rdb_path, - rocksdb::Env::Default(), - _db_opts, - column_families, - /*ignore_unknown_options=*/true); + rocksdb::ConfigOptions config_options; + config_options.ignore_unknown_options = true; + config_options.ignore_unsupported_options = true; + config_options.sanity_level = + rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible; + config_options.env = rocksdb::Env::Default(); + auto s = + rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts, column_families); if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) { LOG_ERROR_PREFIX("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString()); return dsn::ERR_LOCAL_APP_FAILURE; @@ -2158,7 +2161,7 @@ ::dsn::error_code pegasus_server_impl::copy_checkpoint_to_dir_unsafe(const char {{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}, {META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}}); status = rocksdb::DB::OpenForReadOnly( - rocksdb::DBOptions(), checkpoint_dir, column_families, &handles_opened, &snapshot_db); + _db_opts, checkpoint_dir, column_families, &handles_opened, &snapshot_db); if (!status.ok()) { LOG_ERROR_PREFIX( "OpenForReadOnly from {} failed, error = {}", checkpoint_dir, status.ToString()); @@ -3285,7 +3288,7 @@ ::dsn::error_code pegasus_server_impl::check_column_families(const std::string & *missing_meta_cf = true; *missing_data_cf = true; std::vector column_families; - auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families); + auto s = rocksdb::DB::ListColumnFamilies(_db_opts, path, &column_families); if (!s.ok()) { LOG_ERROR_PREFIX("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString()); if (s.IsCorruption() && diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index c457ebd9fa..58d08946a1 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -422,6 +422,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _rng_rd_opts.rocksdb_iteration_threshold_time_ms = FLAGS_rocksdb_iteration_threshold_time_ms; // init rocksdb::DBOptions + _db_opts.env = rocksdb::Env::Default(); _db_opts.create_if_missing = true; // atomic flush data CF and meta CF, aim to keep consistency of 'last flushed decree' in meta CF // and data in data CF. diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 577d9bc6ff..c48ebc9eef 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -360,15 +360,11 @@ ExternalProject_Add(jemalloc BUILD_IN_SOURCE 1 ) -option(ROCKSDB_PORTABLE "build a portable binary" OFF) +option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) -# The patch name '0879c240' means the patch of rocksdb: -# https://github.com/facebook/rocksdb/commit/0879c240404b00142ba4718f36cd3f2bd537192d ExternalProject_Add(rocksdb - URL ${OSS_URL_PREFIX}/rocksdb-6.6.4.tar.gz - https://github.com/facebook/rocksdb/archive/refs/tags/v6.6.4.tar.gz - URL_MD5 7f7fcca3e96b7d83ef332804c90070c8 - PATCH_COMMAND patch -p1 < ${TP_DIR}/rocksdb_fix_atomic_flush_0879c240.patch + URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.3.3.tar.gz + URL_MD5 2815555a2ee2d12a7081a91fb988bf24 DEPENDS jemalloc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DFAIL_ON_WARNINGS=OFF