Skip to content

Commit

Permalink
feat(encryption): support data encrypt at rest
Browse files Browse the repository at this point in the history
  • Loading branch information
acelyc111 committed Sep 7, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent be9119e commit d02b3e1
Showing 237 changed files with 3,314 additions and 2,162 deletions.
68 changes: 34 additions & 34 deletions .github/workflows/lint_and_test_cpp.yaml
Original file line number Diff line number Diff line change
@@ -91,7 +91,7 @@ jobs:
# Build thirdparties and leave some necessary libraries and source
run: |
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
../scripts/download_hadoop.sh hadoop-bin
@@ -151,7 +151,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -195,39 +195,39 @@ jobs:
matrix:
test_module:
- backup_restore_test
- base_api_test
- base_test
- base_api_test # added enc, ing
- base_test # ok
- bulk_load_test
- detect_hotspot_test
- dsn_aio_test
- dsn_block_service_test
- dsn_client_test
- dsn.failure_detector.tests
- dsn_http_test
- dsn_meta_state_tests
- dsn.meta.test
- dsn_nfs_test
- dsn_perf_counter_test
- dsn_replica_backup_test
- dsn_replica_bulk_load_test
- dsn_replica_dup_test
- dsn_replica_split_test
- dsn.replica.test
- dsn_replication_common_test
- dsn.replication.simple_kv
- dsn.rep_tests.simple_kv
- dsn_runtime_tests
- dsn_utils_tests
- dsn.zookeeper.tests
- detect_hotspot_test # enc ignored, ok
- dsn_aio_test # added enc, ok
- dsn_block_service_test # partial added enc, ok
- dsn_client_test # ok
- dsn.failure_detector.tests # ok
- dsn_http_test # ok
- dsn_meta_state_tests # added enc, ok
- dsn.meta.test # need add enc, ok
- dsn_nfs_test # ok, need encrypted test files
- dsn_perf_counter_test # ok
- dsn_replica_backup_test # added enc, ok
- dsn_replica_bulk_load_test # added enc, ok
- dsn_replica_dup_test # added enc, ok
- dsn_replica_split_test # added enc, ok
- dsn.replica.test # added enc, ok
- dsn_replication_common_test # added enc, ok
- dsn.replication.simple_kv # ok
- dsn.rep_tests.simple_kv # need add enc, ok
- dsn_runtime_tests # ok
- dsn_utils_tests # ok
- dsn.zookeeper.tests # ok
# TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after
# it has been optimized.
# - partition_split_test
- pegasus_geo_test
- pegasus_rproxy_test
- pegasus_unit_test
- recovery_test
- restore_test
- throttle_test
- pegasus_geo_test # added enc, ok
- pegasus_rproxy_test # ok
- pegasus_unit_test # added enc, ok
- recovery_test -
- restore_test -
- throttle_test -
needs: build_Release
runs-on: ubuntu-latest
container:
@@ -296,7 +296,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -438,7 +438,7 @@ jobs:
# run: |
# rm -f /root/thirdparties-src.zip
# mkdir build
# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
# cmake --build build/ -j $(nproc)
# rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
# find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -576,7 +576,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
4 changes: 2 additions & 2 deletions .github/workflows/thirdparty-regular-push.yml
Original file line number Diff line number Diff line change
@@ -167,7 +167,7 @@ jobs:
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
OS_VERSION=${{ matrix.osversion }}
ROCKSDB_PORTABLE=ON
ROCKSDB_PORTABLE=1
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
- name: Image digest
@@ -203,7 +203,7 @@ jobs:
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
OS_VERSION=${{ matrix.osversion }}
ROCKSDB_PORTABLE=ON
ROCKSDB_PORTABLE=1
USE_JEMALLOC=ON
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
2 changes: 1 addition & 1 deletion .licenserc.yaml
Original file line number Diff line number Diff line change
@@ -312,7 +312,7 @@ header:
- 'src/nfs/nfs_client_impl.h'
- 'src/nfs/nfs_code_definition.h'
- 'src/nfs/nfs_node.cpp'
- 'src/nfs/nfs_node_impl.cpp'
- 'src/nfs/nfs_node_simple.cpp'
- 'src/nfs/nfs_node_simple.h'
- 'src/nfs/nfs_server_impl.cpp'
- 'src/nfs/nfs_server_impl.h'
13 changes: 4 additions & 9 deletions cmake_modules/BaseFunctions.cmake
Original file line number Diff line number Diff line change
@@ -204,7 +204,7 @@ function(dsn_setup_compiler_flags)
# We want access to the PRI* print format macros.
add_definitions(-D__STDC_FORMAT_MACROS)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -gdwarf-4" CACHE STRING "" FORCE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -gdwarf-4" CACHE STRING "" FORCE)

# -Wall: Enable all warnings.
add_compile_options(-Wall)
@@ -221,9 +221,6 @@ function(dsn_setup_compiler_flags)
# use frame pointers to allow simple stack frame walking for backtraces.
# This has a small perf hit but worth it for the ability to profile in production
add_compile_options( -fno-omit-frame-pointer)
# -Wno-deprecated-register
# kbr5.h uses the legacy 'register' keyword.
add_compile_options(-Wno-deprecated-register)
# -Wno-implicit-float-conversion
# Poco/Dynamic/VarHolder.h uses 'unsigned long' to 'float' conversion
add_compile_options(-Wno-implicit-float-conversion)
@@ -388,11 +385,9 @@ function(dsn_common_setup)

set(BUILD_SHARED_LIBS OFF)

include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG("-std=c++1y" COMPILER_SUPPORTS_CXX1Y)
if(NOT ${COMPILER_SUPPORTS_CXX1Y})
message(FATAL_ERROR "You need a compiler with C++1y support.")
endif()
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)

dsn_setup_system_libs()
dsn_setup_compiler_flags()
2 changes: 2 additions & 0 deletions docker/pegasus-build-env/centos7/Dockerfile
Original file line number Diff line number Diff line change
@@ -50,6 +50,8 @@ RUN yum -y install centos-release-scl \
lz4-devel \
bison \
flex \
krb5-devel \
cyrus-sasl-devel \
patch; \
yum clean all; \
rm -rf /var/cache/yum;
2 changes: 2 additions & 0 deletions docker/pegasus-build-env/ubuntu1804/Dockerfile
Original file line number Diff line number Diff line change
@@ -50,6 +50,8 @@ RUN apt-get update -y; \
libtool \
libssl-dev \
bison \
libkrb5-dev \
libsasl2-dev \
maven \
flex \
python3-setuptools; \
2 changes: 2 additions & 0 deletions docker/pegasus-build-env/ubuntu2004/Dockerfile
Original file line number Diff line number Diff line change
@@ -50,6 +50,8 @@ RUN apt-get update -y; \
libtool \
libssl-dev \
bison \
libkrb5-dev \
libsasl2-dev \
maven \
flex; \
rm -rf /var/lib/apt/lists/*
2 changes: 2 additions & 0 deletions docker/pegasus-build-env/ubuntu2204/Dockerfile
Original file line number Diff line number Diff line change
@@ -51,6 +51,8 @@ RUN apt-get update -y; \
libtool \
libssl-dev \
bison \
libkrb5-dev \
libsasl2-dev \
maven \
flex; \
rm -rf /var/lib/apt/lists/*
2 changes: 1 addition & 1 deletion docker/thirdparties-bin/Dockerfile
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip

ARG GITHUB_BRANCH=master
ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
ARG ROCKSDB_PORTABLE=OFF
ARG ROCKSDB_PORTABLE=0
ARG USE_JEMALLOC=OFF
ARG HADOOP_BIN_PATH
ARG ZOOKEEPER_BIN_PATH
4 changes: 2 additions & 2 deletions run.sh
Original file line number Diff line number Diff line change
@@ -118,7 +118,7 @@ function run_build()
ENABLE_GPERF=ON
SKIP_THIRDPARTY=NO
SANITIZER=""
ROCKSDB_PORTABLE=OFF
ROCKSDB_PORTABLE=0
USE_JEMALLOC=OFF
BUILD_TEST=OFF
IWYU=""
@@ -177,7 +177,7 @@ function run_build()
SKIP_THIRDPARTY=YES
;;
--enable_rocksdb_portable)
ROCKSDB_PORTABLE=ON
ROCKSDB_PORTABLE=1
;;
--use_jemalloc)
ENABLE_GPERF=OFF
2 changes: 0 additions & 2 deletions scripts/pack_server.sh
Original file line number Diff line number Diff line change
@@ -125,8 +125,6 @@ fi

copy_file ./thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/bin
copy_file ./thirdparty/output/lib/libhdfs* ${pack}/bin
copy_file ./thirdparty/output/lib/libsasl*.so.* ${pack}/bin
copy_file ./thirdparty/output/lib/libcom_err*.so.* ${pack}/bin
copy_file ./scripts/sendmail.sh ${pack}/bin
copy_file ./src/server/config.ini ${pack}/bin
copy_file ./src/server/config.min.ini ${pack}/bin
2 changes: 1 addition & 1 deletion src/aio/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@ set(MY_PROJ_SRC "")
#"GLOB" for non - recursive search
set(MY_SRC_SEARCH_MODE "GLOB")

set(MY_PROJ_LIBS dsn_runtime)
set(MY_PROJ_LIBS dsn_runtime rocksdb)

#Extra files that will be installed
set(MY_BINPLACES "")
16 changes: 12 additions & 4 deletions src/aio/aio_provider.h
Original file line number Diff line number Diff line change
@@ -27,10 +27,17 @@
#pragma once

#include <stdint.h>
#include <memory>
#include <string>

#include "utils/error_code.h"
#include "utils/factory_store.h"

namespace rocksdb {
class RandomAccessFile;
class RandomRWFile;
} // namespace rocksdb

namespace dsn {

class aio_context;
@@ -60,12 +67,13 @@ class aio_provider
explicit aio_provider(disk_engine *disk);
virtual ~aio_provider() = default;

virtual linux_fd_t open(const char *file_name, int flag, int pmode) = 0;
virtual std::unique_ptr<rocksdb::RandomAccessFile> open_read_file(const std::string &fname) = 0;
virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0;

virtual error_code close(linux_fd_t fd) = 0;
virtual error_code flush(linux_fd_t fd) = 0;
virtual std::unique_ptr<rocksdb::RandomRWFile> open_write_file(const std::string &fname) = 0;
virtual error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0;
virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0;
virtual error_code flush(rocksdb::RandomRWFile *rwf) = 0;
virtual error_code close(rocksdb::RandomRWFile *rwf) = 0;

// Submits the aio_task to the underlying disk-io executor.
// This task may not be executed immediately, call `aio_task::wait`
8 changes: 6 additions & 2 deletions src/aio/disk_engine.cpp
Original file line number Diff line number Diff line change
@@ -26,7 +26,6 @@

#include "disk_engine.h"

#include <algorithm>
#include <list>
// IWYU pragma: no_include <string>
#include <utility>
@@ -103,22 +102,26 @@ aio_task *disk_write_queue::unlink_next_workload(void *plength)
return first;
}

disk_file::disk_file(linux_fd_t fd) : _fd(fd) {}
disk_file::disk_file(std::unique_ptr<rocksdb::RandomAccessFile> rf) : _read_file(std::move(rf)) {}
disk_file::disk_file(std::unique_ptr<rocksdb::RandomRWFile> wf) : _write_file(std::move(wf)) {}

aio_task *disk_file::read(aio_task *tsk)
{
CHECK(_read_file, "");
tsk->add_ref(); // release on completion, see `on_read_completed`.
return _read_queue.add_work(tsk, nullptr);
}

aio_task *disk_file::write(aio_task *tsk, void *ctx)
{
CHECK(_write_file, "");
tsk->add_ref(); // release on completion
return _write_queue.add_work(tsk, ctx);
}

aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size)
{
CHECK(_read_file, "");
CHECK(wk->next == nullptr, "");
auto ret = _read_queue.on_work_completed(wk, nullptr);
wk->enqueue(err, size);
@@ -129,6 +132,7 @@ aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size

aio_task *disk_file::on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size)
{
CHECK(_write_file, "");
auto ret = _write_queue.on_work_completed(wk, ctx);

while (wk) {
16 changes: 13 additions & 3 deletions src/aio/disk_engine.h
Original file line number Diff line number Diff line change
@@ -32,9 +32,15 @@

#include "aio/aio_task.h"
#include "aio_provider.h"
#include "rocksdb/env.h"
#include "utils/singleton.h"
#include "utils/work_queue.h"

namespace rocksdb {
class RandomAccessFile;
class RandomRWFile;
} // namespace rocksdb

namespace dsn {
class error_code;

@@ -56,17 +62,21 @@ class disk_write_queue : public work_queue<aio_task>
class disk_file
{
public:
explicit disk_file(linux_fd_t fd);
explicit disk_file(std::unique_ptr<rocksdb::RandomAccessFile> rf);
explicit disk_file(std::unique_ptr<rocksdb::RandomRWFile> wf);
aio_task *read(aio_task *tsk);
aio_task *write(aio_task *tsk, void *ctx);

aio_task *on_read_completed(aio_task *wk, error_code err, size_t size);
aio_task *on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size);

linux_fd_t native_handle() const { return _fd; }
rocksdb::RandomAccessFile *rfile() const { return _read_file.get(); }
rocksdb::RandomRWFile *wfile() const { return _write_file.get(); }

private:
linux_fd_t _fd;
// TODO(yingchun): unify to use a single RandomRWFile member variable.
std::unique_ptr<rocksdb::RandomAccessFile> _read_file;
std::unique_ptr<rocksdb::RandomRWFile> _write_file;
disk_write_queue _write_queue;
work_queue<aio_task> _read_queue;
};
Loading

0 comments on commit d02b3e1

Please sign in to comment.