Encrypt demo all

.github/workflows/lint_and_test_cpp.yaml

@@ -91,7 +91,7 @@ jobs:
         # Build thirdparties and leave some necessary libraries and source
         run: |
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
           cmake --build build/ -j $(nproc)
           rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
           ../scripts/download_hadoop.sh hadoop-bin
@@ -151,7 +151,7 @@ jobs:
         run: |
           rm -f /root/thirdparties-src.zip
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
           cmake --build build/ -j $(nproc)
           rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
           find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -195,39 +195,39 @@ jobs:
       matrix:
         test_module:
           - backup_restore_test
-          - base_api_test
-          - base_test
+          - base_api_test # added enc, ing
+          - base_test # ok
           - bulk_load_test
-          - detect_hotspot_test
-          - dsn_aio_test
-          - dsn_block_service_test
-          - dsn_client_test
-          - dsn.failure_detector.tests
-          - dsn_http_test
-          - dsn_meta_state_tests
-          - dsn.meta.test
-          - dsn_nfs_test
-          - dsn_perf_counter_test
-          - dsn_replica_backup_test
-          - dsn_replica_bulk_load_test
-          - dsn_replica_dup_test
-          - dsn_replica_split_test
-          - dsn.replica.test
-          - dsn_replication_common_test
-          - dsn.replication.simple_kv
-          - dsn.rep_tests.simple_kv
-          - dsn_runtime_tests
-          - dsn_utils_tests
-          - dsn.zookeeper.tests
+          - detect_hotspot_test # enc ignored, ok
+          - dsn_aio_test # added enc, ok
+          - dsn_block_service_test # partial added enc, ok
+          - dsn_client_test # ok
+          - dsn.failure_detector.tests # ok
+          - dsn_http_test # ok
+          - dsn_meta_state_tests # added enc, ok
+          - dsn.meta.test # need add enc, ok
+          - dsn_nfs_test # ok, need encrypted test files
+          - dsn_perf_counter_test # ok
+          - dsn_replica_backup_test # added enc, ok
+          - dsn_replica_bulk_load_test # added enc, ok
+          - dsn_replica_dup_test # added enc, ok
+          - dsn_replica_split_test # added enc, ok
+          - dsn.replica.test # added enc, ok
+          - dsn_replication_common_test # added enc, ok
+          - dsn.replication.simple_kv # ok
+          - dsn.rep_tests.simple_kv # need add enc, ok
+          - dsn_runtime_tests # ok
+          - dsn_utils_tests # ok
+          - dsn.zookeeper.tests # ok
           # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after
           # it has been optimized.
 #          - partition_split_test
-          - pegasus_geo_test
-          - pegasus_rproxy_test
-          - pegasus_unit_test
-          - recovery_test
-          - restore_test
-          - throttle_test
+          - pegasus_geo_test # added enc, ok
+          - pegasus_rproxy_test # ok
+          - pegasus_unit_test # added enc, ok
+          - recovery_test -
+          - restore_test -
+          - throttle_test -
     needs: build_Release
     runs-on: ubuntu-latest
     container:
@@ -296,7 +296,7 @@ jobs:
         run: |
           rm -f /root/thirdparties-src.zip
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
           cmake --build build/ -j $(nproc)
           rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
           find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -438,7 +438,7 @@ jobs:
 #        run: |
 #          rm -f /root/thirdparties-src.zip
 #          mkdir build
-#          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+#          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
 #          cmake --build build/ -j $(nproc)
 #          rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
 #          find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -576,7 +576,7 @@ jobs:
         run: |
           rm -f /root/thirdparties-src.zip
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
+          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/
           cmake --build build/ -j $(nproc)
           rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
           find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +

.github/workflows/thirdparty-regular-push.yml

@@ -167,7 +167,7 @@ jobs:
           build-args: |
             GITHUB_BRANCH=${{ github.ref_name }}
             OS_VERSION=${{ matrix.osversion }}
-            ROCKSDB_PORTABLE=ON
+            ROCKSDB_PORTABLE=1
             HADOOP_BIN_PATH=hadoop-bin
             ZOOKEEPER_BIN_PATH=zookeeper-bin
       - name: Image digest
@@ -203,7 +203,7 @@ jobs:
           build-args: |
             GITHUB_BRANCH=${{ github.ref_name }}
             OS_VERSION=${{ matrix.osversion }}
-            ROCKSDB_PORTABLE=ON
+            ROCKSDB_PORTABLE=1
             USE_JEMALLOC=ON
             HADOOP_BIN_PATH=hadoop-bin
             ZOOKEEPER_BIN_PATH=zookeeper-bin

cmake_modules/BaseFunctions.cmake

@@ -204,7 +204,7 @@ function(dsn_setup_compiler_flags)
   # We want access to the PRI* print format macros.
   add_definitions(-D__STDC_FORMAT_MACROS)
 
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -gdwarf-4" CACHE STRING "" FORCE)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -gdwarf-4" CACHE STRING "" FORCE)
 
   #  -Wall: Enable all warnings.
   add_compile_options(-Wall)
@@ -217,6 +217,7 @@ function(dsn_setup_compiler_flags)
   add_compile_options(-Wno-deprecated-declarations)
   add_compile_options(-Wno-inconsistent-missing-override)
   add_compile_options(-Wno-attributes)
+  add_compile_options(-Wno-register)  # kbr5.h uses the legacy 'register' keyword.
   # -fno-omit-frame-pointer
   #   use frame pointers to allow simple stack frame walking for backtraces.
   #   This has a small perf hit but worth it for the ability to profile in production
@@ -389,10 +390,13 @@ function(dsn_common_setup)
   set(BUILD_SHARED_LIBS OFF)
 
   include(CheckCXXCompilerFlag)
-  CHECK_CXX_COMPILER_FLAG("-std=c++1y" COMPILER_SUPPORTS_CXX1Y)
-  if(NOT ${COMPILER_SUPPORTS_CXX1Y})
-    message(FATAL_ERROR "You need a compiler with C++1y support.")
+  CHECK_CXX_COMPILER_FLAG("-std=c++17" COMPILER_SUPPORTS_CXX17)
+  if(NOT ${COMPILER_SUPPORTS_CXX17})
+    message(FATAL_ERROR "You need a compiler with C++17 support.")
   endif()
+  set(CMAKE_CXX_STANDARD 17)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+  set(CMAKE_CXX_EXTENSIONS OFF)
 
   dsn_setup_system_libs()
   dsn_setup_compiler_flags()

docker/thirdparties-bin/Dockerfile

@@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip
 
 ARG GITHUB_BRANCH=master
 ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
-ARG ROCKSDB_PORTABLE=OFF
+ARG ROCKSDB_PORTABLE=0
 ARG USE_JEMALLOC=OFF
 ARG HADOOP_BIN_PATH
 ARG ZOOKEEPER_BIN_PATH

run.sh

@@ -118,7 +118,7 @@ function run_build()
     ENABLE_GPERF=ON
     SKIP_THIRDPARTY=NO
     SANITIZER=""
-    ROCKSDB_PORTABLE=OFF
+    ROCKSDB_PORTABLE=0
     USE_JEMALLOC=OFF
     BUILD_TEST=OFF
     IWYU=""
@@ -177,7 +177,7 @@ function run_build()
                 SKIP_THIRDPARTY=YES
                 ;;
             --enable_rocksdb_portable)
-                ROCKSDB_PORTABLE=ON
+                ROCKSDB_PORTABLE=1
                 ;;
             --use_jemalloc)
                 ENABLE_GPERF=OFF

src/aio/CMakeLists.txt

@@ -33,7 +33,7 @@ set(MY_PROJ_SRC "")
 #"GLOB" for non - recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS dsn_runtime)
+set(MY_PROJ_LIBS dsn_runtime rocksdb)
 
 #Extra files that will be installed
 set(MY_BINPLACES "")

src/aio/aio_provider.h

@@ -27,10 +27,17 @@
 #pragma once
 
 #include <stdint.h>
+#include <memory>
+#include <string>
 
 #include "utils/error_code.h"
 #include "utils/factory_store.h"
 
+namespace rocksdb {
+class RandomAccessFile;
+class RandomRWFile;
+} // namespace rocksdb
+
 namespace dsn {
 
 class aio_context;
@@ -46,6 +53,7 @@ struct linux_fd_t
     inline bool is_invalid() const { return fd == DSN_INVALID_FILE_HANDLE; }
 };
 
+// TODO(yingchun): remove the class
 class aio_provider
 {
 public:
@@ -60,10 +68,11 @@ class aio_provider
     explicit aio_provider(disk_engine *disk);
     virtual ~aio_provider() = default;
 
-    virtual linux_fd_t open(const char *file_name, int flag, int pmode) = 0;
+    virtual std::unique_ptr<rocksdb::RandomAccessFile> open_read_file(const std::string &fname) = 0;
+    virtual std::unique_ptr<rocksdb::RandomRWFile> open_write_file(const std::string &fname) = 0;
 
-    virtual error_code close(linux_fd_t fd) = 0;
-    virtual error_code flush(linux_fd_t fd) = 0;
+    virtual error_code close(rocksdb::RandomRWFile *rwf) = 0;
+    virtual error_code flush(rocksdb::RandomRWFile *rwf) = 0;
     virtual error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0;
     virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0;
 

src/aio/disk_engine.cpp

@@ -26,7 +26,6 @@
 
 #include "disk_engine.h"
 
-#include <algorithm>
 #include <list>
 // IWYU pragma: no_include <string>
 #include <utility>
@@ -46,6 +45,10 @@
 #include "utils/link.h"
 #include "utils/threadpool_code.h"
 
+namespace rocksdb {
+class RandomRWFile;
+} // namespace rocksdb
+
 namespace dsn {
 DEFINE_TASK_CODE_AIO(LPC_AIO_BATCH_WRITE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT)
 
@@ -103,22 +106,26 @@ aio_task *disk_write_queue::unlink_next_workload(void *plength)
     return first;
 }
 
-disk_file::disk_file(linux_fd_t fd) : _fd(fd) {}
+disk_file::disk_file(std::unique_ptr<rocksdb::RandomAccessFile> rf) : _read_file(std::move(rf)) {}
+disk_file::disk_file(std::unique_ptr<rocksdb::RandomRWFile> wf) : _write_file(std::move(wf)) {}
 
 aio_task *disk_file::read(aio_task *tsk)
 {
+    CHECK(_read_file, "");
     tsk->add_ref(); // release on completion, see `on_read_completed`.
     return _read_queue.add_work(tsk, nullptr);
 }
 
 aio_task *disk_file::write(aio_task *tsk, void *ctx)
 {
+    CHECK(_write_file, "");
     tsk->add_ref(); // release on completion
     return _write_queue.add_work(tsk, ctx);
 }
 
 aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size)
 {
+    CHECK(_read_file, "");
     CHECK(wk->next == nullptr, "");
     auto ret = _read_queue.on_work_completed(wk, nullptr);
     wk->enqueue(err, size);
@@ -129,6 +136,7 @@ aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size
 
 aio_task *disk_file::on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size)
 {
+    CHECK(_write_file, "");
     auto ret = _write_queue.on_work_completed(wk, ctx);
 
     while (wk) {

src/aio/disk_engine.h

@@ -30,6 +30,8 @@
 #include <stdint.h>
 #include <memory>
 
+#include <rocksdb/env.h>
+
 #include "aio/aio_task.h"
 #include "aio_provider.h"
 #include "utils/singleton.h"
@@ -56,17 +58,21 @@ class disk_write_queue : public work_queue<aio_task>
 class disk_file
 {
 public:
-    explicit disk_file(linux_fd_t fd);
+    explicit disk_file(std::unique_ptr<rocksdb::RandomAccessFile> rf);
+    explicit disk_file(std::unique_ptr<rocksdb::RandomRWFile> wf);
     aio_task *read(aio_task *tsk);
     aio_task *write(aio_task *tsk, void *ctx);
 
     aio_task *on_read_completed(aio_task *wk, error_code err, size_t size);
     aio_task *on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size);
 
-    linux_fd_t native_handle() const { return _fd; }
+    rocksdb::RandomAccessFile *rfile() const { return _read_file.get(); }
+    rocksdb::RandomRWFile *wfile() const { return _write_file.get(); }
 
 private:
-    linux_fd_t _fd;
+    // TODO(yingchun): unify to use a single RandomRWFile member variable.
+    std::unique_ptr<rocksdb::RandomAccessFile> _read_file;
+    std::unique_ptr<rocksdb::RandomRWFile> _write_file;
     disk_write_queue _write_queue;
     work_queue<aio_task> _read_queue;
 };