Skip to content

Commit

Permalink
feat(thirdparty): upgrade rocksdb to 8.3.3
Browse files Browse the repository at this point in the history
  • Loading branch information
acelyc111 committed Sep 8, 2023
1 parent be9119e commit c9507f2
Show file tree
Hide file tree
Showing 8 changed files with 33 additions and 38 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/lint_and_test_cpp.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ jobs:
# Build thirdparties and leave some necessary libraries and source
run: |
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
../scripts/download_hadoop.sh hadoop-bin
Expand Down Expand Up @@ -151,7 +151,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
Expand Down Expand Up @@ -296,7 +296,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
Expand Down Expand Up @@ -438,7 +438,7 @@ jobs:
# run: |
# rm -f /root/thirdparties-src.zip
# mkdir build
# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
# cmake --build build/ -j $(nproc)
# rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
# find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
Expand Down Expand Up @@ -576,7 +576,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/thirdparty-regular-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ jobs:
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
OS_VERSION=${{ matrix.osversion }}
ROCKSDB_PORTABLE=ON
ROCKSDB_PORTABLE=1
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
- name: Image digest
Expand Down Expand Up @@ -203,7 +203,7 @@ jobs:
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
OS_VERSION=${{ matrix.osversion }}
ROCKSDB_PORTABLE=ON
ROCKSDB_PORTABLE=1
USE_JEMALLOC=ON
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
Expand Down
13 changes: 4 additions & 9 deletions cmake_modules/BaseFunctions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ function(dsn_setup_compiler_flags)
# We want access to the PRI* print format macros.
add_definitions(-D__STDC_FORMAT_MACROS)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -gdwarf-4" CACHE STRING "" FORCE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -gdwarf-4" CACHE STRING "" FORCE)

# -Wall: Enable all warnings.
add_compile_options(-Wall)
Expand All @@ -221,9 +221,6 @@ function(dsn_setup_compiler_flags)
# use frame pointers to allow simple stack frame walking for backtraces.
# This has a small perf hit but worth it for the ability to profile in production
add_compile_options( -fno-omit-frame-pointer)
# -Wno-deprecated-register
# kbr5.h uses the legacy 'register' keyword.
add_compile_options(-Wno-deprecated-register)
# -Wno-implicit-float-conversion
# Poco/Dynamic/VarHolder.h uses 'unsigned long' to 'float' conversion
add_compile_options(-Wno-implicit-float-conversion)
Expand Down Expand Up @@ -388,11 +385,9 @@ function(dsn_common_setup)

set(BUILD_SHARED_LIBS OFF)

include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG("-std=c++1y" COMPILER_SUPPORTS_CXX1Y)
if(NOT ${COMPILER_SUPPORTS_CXX1Y})
message(FATAL_ERROR "You need a compiler with C++1y support.")
endif()
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)

dsn_setup_system_libs()
dsn_setup_compiler_flags()
Expand Down
2 changes: 1 addition & 1 deletion docker/thirdparties-bin/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip

ARG GITHUB_BRANCH=master
ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
ARG ROCKSDB_PORTABLE=OFF
ARG ROCKSDB_PORTABLE=0
ARG USE_JEMALLOC=OFF
ARG HADOOP_BIN_PATH
ARG ZOOKEEPER_BIN_PATH
Expand Down
4 changes: 2 additions & 2 deletions run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ function run_build()
ENABLE_GPERF=ON
SKIP_THIRDPARTY=NO
SANITIZER=""
ROCKSDB_PORTABLE=OFF
ROCKSDB_PORTABLE=0
USE_JEMALLOC=OFF
BUILD_TEST=OFF
IWYU=""
Expand Down Expand Up @@ -177,7 +177,7 @@ function run_build()
SKIP_THIRDPARTY=YES
;;
--enable_rocksdb_portable)
ROCKSDB_PORTABLE=ON
ROCKSDB_PORTABLE=1
;;
--use_jemalloc)
ENABLE_GPERF=OFF
Expand Down
27 changes: 15 additions & 12 deletions src/server/pegasus_server_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1634,12 +1634,12 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv)
rocksdb::DBOptions loaded_db_opt;
std::vector<rocksdb::ColumnFamilyDescriptor> loaded_cf_descs;
rocksdb::ColumnFamilyOptions loaded_data_cf_opts;
rocksdb::ConfigOptions config_options;
// Set `ignore_unknown_options` true for forward compatibility.
auto status = rocksdb::LoadLatestOptions(rdb_path,
rocksdb::Env::Default(),
&loaded_db_opt,
&loaded_cf_descs,
/*ignore_unknown_options=*/true);
config_options.ignore_unknown_options = true;
config_options.env = rocksdb::Env::Default();
auto status =
rocksdb::LoadLatestOptions(config_options, rdb_path, &loaded_db_opt, &loaded_cf_descs);
if (!status.ok()) {
// Here we ignore an invalid argument error related to `pegasus_data_version` and
// `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0).
Expand Down Expand Up @@ -1677,11 +1677,14 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv)

std::vector<rocksdb::ColumnFamilyDescriptor> column_families(
{{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
auto s = rocksdb::CheckOptionsCompatibility(rdb_path,
rocksdb::Env::Default(),
_db_opts,
column_families,
/*ignore_unknown_options=*/true);
rocksdb::ConfigOptions config_options;
config_options.ignore_unknown_options = true;
config_options.ignore_unsupported_options = true;
config_options.sanity_level =
rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible;
config_options.env = rocksdb::Env::Default();
auto s =
rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts, column_families);
if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) {
LOG_ERROR_PREFIX("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString());
return dsn::ERR_LOCAL_APP_FAILURE;
Expand Down Expand Up @@ -2158,7 +2161,7 @@ ::dsn::error_code pegasus_server_impl::copy_checkpoint_to_dir_unsafe(const char
{{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()},
{META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}});
status = rocksdb::DB::OpenForReadOnly(
rocksdb::DBOptions(), checkpoint_dir, column_families, &handles_opened, &snapshot_db);
_db_opts, checkpoint_dir, column_families, &handles_opened, &snapshot_db);
if (!status.ok()) {
LOG_ERROR_PREFIX(
"OpenForReadOnly from {} failed, error = {}", checkpoint_dir, status.ToString());
Expand Down Expand Up @@ -3285,7 +3288,7 @@ ::dsn::error_code pegasus_server_impl::check_column_families(const std::string &
*missing_meta_cf = true;
*missing_data_cf = true;
std::vector<std::string> column_families;
auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families);
auto s = rocksdb::DB::ListColumnFamilies(_db_opts, path, &column_families);
if (!s.ok()) {
LOG_ERROR_PREFIX("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString());
if (s.IsCorruption() &&
Expand Down
1 change: 1 addition & 0 deletions src/server/pegasus_server_impl_init.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
_rng_rd_opts.rocksdb_iteration_threshold_time_ms = FLAGS_rocksdb_iteration_threshold_time_ms;

// init rocksdb::DBOptions
_db_opts.env = rocksdb::Env::Default();
_db_opts.create_if_missing = true;
// atomic flush data CF and meta CF, aim to keep consistency of 'last flushed decree' in meta CF
// and data in data CF.
Expand Down
10 changes: 3 additions & 7 deletions thirdparty/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -392,15 +392,11 @@ ExternalProject_Add(jemalloc
BUILD_IN_SOURCE 1
)

option(ROCKSDB_PORTABLE "build a portable binary" OFF)
option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0)

# The patch name '0879c240' means the patch of rocksdb:
# https://github.com/facebook/rocksdb/commit/0879c240404b00142ba4718f36cd3f2bd537192d
ExternalProject_Add(rocksdb
URL ${OSS_URL_PREFIX}/rocksdb-6.6.4.tar.gz
https://github.com/facebook/rocksdb/archive/refs/tags/v6.6.4.tar.gz
URL_MD5 7f7fcca3e96b7d83ef332804c90070c8
PATCH_COMMAND patch -p1 < ${TP_DIR}/rocksdb_fix_atomic_flush_0879c240.patch
URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.3.3.tar.gz
URL_MD5 2815555a2ee2d12a7081a91fb988bf24
DEPENDS jemalloc
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
-DFAIL_ON_WARNINGS=OFF
Expand Down

0 comments on commit c9507f2

Please sign in to comment.