feat(thirdparty): upgrade rocksdb to 8.5.3 (apache#1601)

apache#1604 This patch upgrade the `rocksdb` to the latest stable version 8.5.3. Since the 2.5 branch has been created, the new version of rocksdb will only take effect on version 2.6 and later.
acelyc111 · Sep 13, 2023 · e978e4c · e978e4c
1 parent 7cb1599
commit e978e4c
Show file tree

Hide file tree

Showing 11 changed files with 38 additions and 31 deletions.
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
@@ -91,7 +91,7 @@ jobs:
         # Build thirdparties and leave some necessary libraries and source
         run: |
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
           cmake --build build/ -j $(nproc)
           rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
           ../scripts/download_hadoop.sh hadoop-bin
@@ -151,7 +151,7 @@ jobs:
         run: |
           rm -f /root/thirdparties-src.zip
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
           cmake --build build/ -j $(nproc)
           rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
           find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -296,7 +296,7 @@ jobs:
         run: |
           rm -f /root/thirdparties-src.zip
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
           cmake --build build/ -j $(nproc)
           rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
           find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -438,7 +438,7 @@ jobs:
 #        run: |
 #          rm -f /root/thirdparties-src.zip
 #          mkdir build
-#          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+#          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
 #          cmake --build build/ -j $(nproc)
 #          rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
 #          find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -576,7 +576,7 @@ jobs:
         run: |
           rm -f /root/thirdparties-src.zip
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
+          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/
           cmake --build build/ -j $(nproc)
           rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
           find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -676,7 +676,7 @@ jobs:
         run: |
           export JAVA_HOME="${JAVA_HOME_8_X64}"
           mkdir -p build
-          cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR}
+          cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DROCKSDB_PORTABLE=1
           cmake --build build/ -j $(sysctl -n hw.physicalcpu)
       - name: Compilation
         run: |

diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml
@@ -167,7 +167,7 @@ jobs:
           build-args: |
             GITHUB_BRANCH=${{ github.ref_name }}
             OS_VERSION=${{ matrix.osversion }}
-            ROCKSDB_PORTABLE=ON
+            ROCKSDB_PORTABLE=1
             HADOOP_BIN_PATH=hadoop-bin
             ZOOKEEPER_BIN_PATH=zookeeper-bin
       - name: Image digest
@@ -203,7 +203,7 @@ jobs:
           build-args: |
             GITHUB_BRANCH=${{ github.ref_name }}
             OS_VERSION=${{ matrix.osversion }}
-            ROCKSDB_PORTABLE=ON
+            ROCKSDB_PORTABLE=1
             USE_JEMALLOC=ON
             HADOOP_BIN_PATH=hadoop-bin
             ZOOKEEPER_BIN_PATH=zookeeper-bin

diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile
@@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip
 
 ARG GITHUB_BRANCH=master
 ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
-ARG ROCKSDB_PORTABLE=OFF
+ARG ROCKSDB_PORTABLE=0
 ARG USE_JEMALLOC=OFF
 ARG HADOOP_BIN_PATH
 ARG ZOOKEEPER_BIN_PATH

diff --git a/run.sh b/run.sh
@@ -118,7 +118,7 @@ function run_build()
     ENABLE_GPERF=ON
     SKIP_THIRDPARTY=NO
     SANITIZER=""
-    ROCKSDB_PORTABLE=OFF
+    ROCKSDB_PORTABLE=0
     USE_JEMALLOC=OFF
     BUILD_TEST=OFF
     IWYU=""
@@ -177,7 +177,7 @@ function run_build()
                 SKIP_THIRDPARTY=YES
                 ;;
             --enable_rocksdb_portable)
-                ROCKSDB_PORTABLE=ON
+                ROCKSDB_PORTABLE=1
                 ;;
             --use_jemalloc)
                 ENABLE_GPERF=OFF

diff --git a/src/server/pegasus_event_listener.cpp b/src/server/pegasus_event_listener.cpp
@@ -22,6 +22,7 @@
 #include <fmt/core.h>
 #include <rocksdb/compaction_job_stats.h>
 #include <rocksdb/table_properties.h>
+#include <rocksdb/types.h>
 #include <string>
 
 #include "common/gpid.h"

diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp
@@ -22,7 +22,7 @@
 #include <fmt/core.h>
 #include <inttypes.h>
 #include <limits.h>
-#include <rocksdb/cache.h>
+#include <rocksdb/advanced_cache.h>
 #include <rocksdb/convenience.h>
 #include <rocksdb/db.h>
 #include <rocksdb/env.h>
@@ -1635,12 +1635,12 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv)
         rocksdb::DBOptions loaded_db_opt;
         std::vector<rocksdb::ColumnFamilyDescriptor> loaded_cf_descs;
         rocksdb::ColumnFamilyOptions loaded_data_cf_opts;
+        rocksdb::ConfigOptions config_options;
         // Set `ignore_unknown_options` true for forward compatibility.
-        auto status = rocksdb::LoadLatestOptions(rdb_path,
-                                                 rocksdb::Env::Default(),
-                                                 &loaded_db_opt,
-                                                 &loaded_cf_descs,
-                                                 /*ignore_unknown_options=*/true);
+        config_options.ignore_unknown_options = true;
+        config_options.env = rocksdb::Env::Default();
+        auto status =
+            rocksdb::LoadLatestOptions(config_options, rdb_path, &loaded_db_opt, &loaded_cf_descs);
         if (!status.ok()) {
             // Here we ignore an invalid argument error related to `pegasus_data_version` and
             // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0).
@@ -1678,11 +1678,14 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv)
 
     std::vector<rocksdb::ColumnFamilyDescriptor> column_families(
         {{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
-    auto s = rocksdb::CheckOptionsCompatibility(rdb_path,
-                                                rocksdb::Env::Default(),
-                                                _db_opts,
-                                                column_families,
-                                                /*ignore_unknown_options=*/true);
+    rocksdb::ConfigOptions config_options;
+    config_options.ignore_unknown_options = true;
+    config_options.ignore_unsupported_options = true;
+    config_options.sanity_level =
+        rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible;
+    config_options.env = rocksdb::Env::Default();
+    auto s =
+        rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts, column_families);
     if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) {
         LOG_ERROR_PREFIX("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString());
         return dsn::ERR_LOCAL_APP_FAILURE;
@@ -2159,7 +2162,7 @@ ::dsn::error_code pegasus_server_impl::copy_checkpoint_to_dir_unsafe(const char
             {{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()},
              {META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}});
         status = rocksdb::DB::OpenForReadOnly(
-            rocksdb::DBOptions(), checkpoint_dir, column_families, &handles_opened, &snapshot_db);
+            _db_opts, checkpoint_dir, column_families, &handles_opened, &snapshot_db);
         if (!status.ok()) {
             LOG_ERROR_PREFIX(
                 "OpenForReadOnly from {} failed, error = {}", checkpoint_dir, status.ToString());
@@ -3286,7 +3289,7 @@ ::dsn::error_code pegasus_server_impl::check_column_families(const std::string &
     *missing_meta_cf = true;
     *missing_data_cf = true;
     std::vector<std::string> column_families;
-    auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families);
+    auto s = rocksdb::DB::ListColumnFamilies(_db_opts, path, &column_families);
     if (!s.ok()) {
         LOG_ERROR_PREFIX("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString());
         if (s.IsCorruption() &&

diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include <gtest/gtest_prod.h>
+#include <rocksdb/compression_type.h>
 #include <rocksdb/options.h>
 #include <rocksdb/slice.h>
 #include <rocksdb/table.h>
@@ -70,6 +71,7 @@ class WriteBufferManager;
 namespace dsn {
 class blob;
 class message_ex;
+
 namespace replication {
 class detect_hotkey_request;
 class detect_hotkey_response;

diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp
@@ -19,6 +19,7 @@
 
 #include <fmt/core.h>
 #include <rocksdb/cache.h>
+#include <rocksdb/env.h>
 #include <rocksdb/filter_policy.h>
 #include <rocksdb/options.h>
 #include <rocksdb/rate_limiter.h>
@@ -422,6 +423,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
     _rng_rd_opts.rocksdb_iteration_threshold_time_ms = FLAGS_rocksdb_iteration_threshold_time_ms;
 
     // init rocksdb::DBOptions
+    _db_opts.env = rocksdb::Env::Default();
     _db_opts.create_if_missing = true;
     // atomic flush data CF and meta CF, aim to keep consistency of 'last flushed decree' in meta CF
     // and data in data CF.

diff --git a/src/server/test/pegasus_compression_options_test.cpp b/src/server/test/pegasus_compression_options_test.cpp
@@ -20,6 +20,7 @@
 // IWYU pragma: no_include <gtest/gtest-message.h>
 // IWYU pragma: no_include <gtest/gtest-test-part.h>
 #include <gtest/gtest.h>
+#include <rocksdb/compression_type.h>
 #include <rocksdb/db.h>
 #include <rocksdb/options.h>
 #include <map>

diff --git a/src/utils/alloc.h b/src/utils/alloc.h
@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include <algorithm>
+
 #include "utils/ports.h"
 
 // The check for the definition of CACHELINE_SIZE has to be put after including "utils/ports.h",

diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
@@ -358,15 +358,11 @@ ExternalProject_Add(jemalloc
         BUILD_IN_SOURCE 1
         )
 
-option(ROCKSDB_PORTABLE "build a portable binary" OFF)
+option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0)
 
-# The patch name '0879c240' means the patch of rocksdb:
-# https://github.com/facebook/rocksdb/commit/0879c240404b00142ba4718f36cd3f2bd537192d
 ExternalProject_Add(rocksdb
-        URL ${OSS_URL_PREFIX}/rocksdb-6.6.4.tar.gz
-        https://github.com/facebook/rocksdb/archive/refs/tags/v6.6.4.tar.gz
-        URL_MD5 7f7fcca3e96b7d83ef332804c90070c8
-        PATCH_COMMAND patch -p1 < ${TP_DIR}/rocksdb_fix_atomic_flush_0879c240.patch
+        URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.5.3.tar.gz
+        URL_MD5 f03eac50ec958a21a7cb56183afb8fe4
         DEPENDS jemalloc
         CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
         -DFAIL_ON_WARNINGS=OFF