From d02b3e19630292ebab57661023566df4aa131b22 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 17 Aug 2023 15:16:30 +0800 Subject: [PATCH] feat(encryption): support data encrypt at rest --- .github/workflows/lint_and_test_cpp.yaml | 68 +-- .github/workflows/thirdparty-regular-push.yml | 4 +- .licenserc.yaml | 2 +- cmake_modules/BaseFunctions.cmake | 13 +- docker/pegasus-build-env/centos7/Dockerfile | 2 + .../pegasus-build-env/ubuntu1804/Dockerfile | 2 + .../pegasus-build-env/ubuntu2004/Dockerfile | 2 + .../pegasus-build-env/ubuntu2204/Dockerfile | 2 + docker/thirdparties-bin/Dockerfile | 2 +- run.sh | 4 +- scripts/pack_server.sh | 2 - src/aio/CMakeLists.txt | 2 +- src/aio/aio_provider.h | 16 +- src/aio/disk_engine.cpp | 8 +- src/aio/disk_engine.h | 16 +- src/aio/file_io.cpp | 48 +- src/aio/file_io.h | 10 +- src/aio/native_linux_aio_provider.cpp | 141 +++--- src/aio/native_linux_aio_provider.h | 19 +- src/aio/test/CMakeLists.txt | 2 +- src/aio/test/aio.cpp | 414 ++++++++++++------ src/base/test/utils_test.cpp | 1 - src/block_service/block_service.h | 6 +- src/block_service/directio_writable_file.cpp | 169 ------- src/block_service/directio_writable_file.h | 57 --- src/block_service/fds/fds_service.cpp | 5 +- src/block_service/hdfs/CMakeLists.txt | 8 +- src/block_service/hdfs/hdfs_service.cpp | 182 +++++--- src/block_service/local/CMakeLists.txt | 2 +- src/block_service/local/local_service.cpp | 352 ++++++++------- src/block_service/local/local_service.h | 10 + src/block_service/test/CMakeLists.txt | 3 +- .../test/block_service_manager_test.cpp | 67 +-- src/block_service/test/config-test.ini | 4 +- src/block_service/test/fds_service_test.cpp | 1 + src/block_service/test/hdfs_service_test.cpp | 335 ++++++++------ src/block_service/test/local_service_test.cpp | 58 ++- src/common/fs_manager.h | 6 +- src/common/test/common_test.cpp | 1 + src/common/test/fs_manager_test.cpp | 22 +- src/geo/bench/bench.cpp | 9 +- src/geo/lib/geo_client.cpp | 1 - src/geo/lib/latlng_codec.cpp | 1 + src/geo/test/geo_test.cpp | 5 + src/http/http_server.h | 2 + src/meta/distributed_lock_service_simple.h | 1 + src/meta/dump_file.h | 2 + src/meta/meta_backup_service.h | 2 + src/meta/meta_bulk_load_service.h | 4 + src/meta/meta_state_service_simple.cpp | 128 +++--- src/meta/meta_state_service_simple.h | 1 + src/meta/test/main.cpp | 6 +- src/meta/test/meta_backup_test.cpp | 4 +- src/meta/test/meta_service_test.cpp | 1 - .../test/meta_state/meta_state_service.cpp | 87 ++-- src/nfs/nfs_client_impl.cpp | 11 +- ...{nfs_node_impl.cpp => nfs_node_simple.cpp} | 13 +- src/nfs/nfs_node_simple.h | 14 +- src/nfs/nfs_server_impl.cpp | 106 ++--- src/nfs/nfs_server_impl.h | 12 +- src/nfs/test/CMakeLists.txt | 2 +- src/nfs/test/main.cpp | 168 ++++--- src/perf_counter/test/perf_counter_test.cpp | 1 + .../proxy_ut/redis_proxy_test.cpp | 1 - src/replica/backup/cold_backup_context.cpp | 1 - src/replica/backup/replica_backup_server.cpp | 6 + src/replica/backup/replica_backup_server.h | 1 + .../test/replica_backup_manager_test.cpp | 5 +- src/replica/bulk_load/replica_bulk_loader.cpp | 18 +- src/replica/bulk_load/test/CMakeLists.txt | 2 +- .../test/replica_bulk_loader_test.cpp | 94 ++-- src/replica/duplication/replica_follower.cpp | 1 - src/replica/duplication/test/CMakeLists.txt | 2 + .../test/dup_replica_http_service_test.cpp | 5 +- .../test/duplication_sync_timer_test.cpp | 17 +- .../test/load_from_private_log_test.cpp | 100 +++-- .../duplication/test/mutation_batch_test.cpp | 9 +- .../test/replica_duplicator_manager_test.cpp | 13 +- .../test/replica_duplicator_test.cpp | 11 +- .../test/replica_follower_test.cpp | 13 +- .../duplication/test/ship_mutation_test.cpp | 7 +- src/replica/log_file.cpp | 13 +- src/replica/log_file_stream.h | 1 - src/replica/mutation_cache.cpp | 4 - src/replica/mutation_log_replay.cpp | 2 +- src/replica/replica.cpp | 1 - src/replica/replica.h | 4 +- src/replica/replica_backup.cpp | 5 +- src/replica/replica_http_service.h | 7 + src/replica/replica_learn.cpp | 1 - src/replica/replica_restore.cpp | 50 +-- src/replica/replica_stub.h | 2 +- src/replica/replication_app_base.cpp | 128 ++---- src/replica/replication_app_base.h | 8 +- src/replica/split/test/replica_split_test.cpp | 39 +- src/replica/storage/simple_kv/CMakeLists.txt | 2 +- .../simple_kv/simple_kv.server.impl.cpp | 123 ++++-- .../storage/simple_kv/test/CMakeLists.txt | 3 +- .../storage/simple_kv/test/case-000.ini | 3 + .../storage/simple_kv/test/case-001.ini | 3 + .../storage/simple_kv/test/case-002.ini | 3 + .../storage/simple_kv/test/case-003.ini | 3 + .../storage/simple_kv/test/case-004.ini | 3 + .../storage/simple_kv/test/case-005.ini | 3 + .../storage/simple_kv/test/case-006.ini | 3 + .../storage/simple_kv/test/case-100.ini | 3 + .../storage/simple_kv/test/case-101.ini | 3 + .../storage/simple_kv/test/case-102.ini | 3 + .../storage/simple_kv/test/case-103.ini | 3 + .../storage/simple_kv/test/case-104.ini | 3 + .../storage/simple_kv/test/case-105.ini | 3 + .../storage/simple_kv/test/case-106.ini | 3 + .../storage/simple_kv/test/case-107.ini | 3 + .../storage/simple_kv/test/case-108.ini | 3 + .../storage/simple_kv/test/case-109.ini | 3 + .../storage/simple_kv/test/case-200.ini | 3 + .../storage/simple_kv/test/case-201.ini | 3 + .../storage/simple_kv/test/case-202-0.ini | 3 + .../storage/simple_kv/test/case-202-1.ini | 3 + .../storage/simple_kv/test/case-203-0.ini | 3 + .../storage/simple_kv/test/case-204.ini | 3 + .../storage/simple_kv/test/case-205.ini | 3 + .../storage/simple_kv/test/case-206.ini | 3 + .../storage/simple_kv/test/case-207.ini | 3 + .../storage/simple_kv/test/case-208.ini | 3 + .../storage/simple_kv/test/case-209.ini | 3 + .../storage/simple_kv/test/case-210.ini | 3 + .../storage/simple_kv/test/case-211.ini | 3 + .../storage/simple_kv/test/case-212.ini | 3 + .../storage/simple_kv/test/case-213.ini | 3 + .../storage/simple_kv/test/case-214.ini | 3 + .../storage/simple_kv/test/case-215.ini | 3 + .../storage/simple_kv/test/case-216.ini | 3 + .../storage/simple_kv/test/case-300-0.ini | 3 + .../storage/simple_kv/test/case-300-1.ini | 3 + .../storage/simple_kv/test/case-300-2.ini | 3 + .../storage/simple_kv/test/case-301.ini | 3 + .../storage/simple_kv/test/case-302.ini | 3 + .../storage/simple_kv/test/case-303.ini | 3 + .../storage/simple_kv/test/case-304.ini | 3 + .../storage/simple_kv/test/case-305.ini | 3 + .../storage/simple_kv/test/case-306.ini | 3 + .../storage/simple_kv/test/case-307.ini | 3 + .../storage/simple_kv/test/case-400.ini | 3 + .../storage/simple_kv/test/case-401.ini | 3 + .../storage/simple_kv/test/case-402.ini | 3 + .../storage/simple_kv/test/case-600.ini | 3 + .../storage/simple_kv/test/case-601.ini | 3 + .../storage/simple_kv/test/case-602.ini | 3 + .../storage/simple_kv/test/case-603.ini | 3 + src/replica/storage/simple_kv/test/client.cpp | 1 - src/replica/storage/simple_kv/test/run.sh | 7 + .../simple_kv/test/simple_kv.server.impl.cpp | 130 ++++-- src/replica/test/CMakeLists.txt | 3 +- src/replica/test/log_block_test.cpp | 19 +- src/replica/test/log_file_test.cpp | 5 +- src/replica/test/main.cpp | 25 +- src/replica/test/mutation_log_learn_test.cpp | 7 +- src/replica/test/mutation_log_test.cpp | 99 ++--- src/replica/test/open_replica_test.cpp | 5 +- .../test/replica_disk_migrate_test.cpp | 16 +- src/replica/test/replica_disk_test.cpp | 17 +- .../test/replica_http_service_test.cpp | 6 +- src/replica/test/replica_learn_test.cpp | 7 +- src/replica/test/replica_test.cpp | 64 +-- src/replica/test/replica_test_base.h | 14 +- src/runtime/providers.common.cpp | 1 - src/runtime/rpc/asio_net_provider.cpp | 1 - src/runtime/rpc/asio_rpc_session.cpp | 1 - src/runtime/rpc/rpc_address.h | 2 +- src/runtime/rpc/rpc_host_port.h | 1 + src/runtime/rpc/thrift_message_parser.cpp | 1 - src/runtime/service_engine.cpp | 1 - src/runtime/task/task_engine.cpp | 1 - src/runtime/test/CMakeLists.txt | 1 + src/runtime/test/client_negotiation_test.cpp | 1 - src/runtime/test/pipeline_test.cpp | 1 + src/runtime/test/server_negotiation_test.cpp | 1 - src/runtime/test/task_test.cpp | 10 +- src/server/config.ini | 1 + src/server/config.min.ini | 1 + src/server/pegasus_event_listener.cpp | 1 + src/server/pegasus_server_impl.cpp | 31 +- src/server/pegasus_server_impl.h | 2 + src/server/pegasus_server_impl_init.cpp | 2 + src/server/pegasus_write_service_impl.h | 4 +- src/server/result_writer.cpp | 1 - .../test/capacity_unit_calculator_test.cpp | 35 +- src/server/test/hotkey_collector_test.cpp | 17 +- src/server/test/hotspot_partition_test.cpp | 7 +- .../test/manual_compact_service_test.cpp | 15 +- .../test/pegasus_compression_options_test.cpp | 14 +- .../test/pegasus_mutation_duplicator_test.cpp | 17 +- src/server/test/pegasus_server_impl_test.cpp | 45 +- src/server/test/pegasus_server_test_base.h | 15 +- src/server/test/pegasus_server_write_test.cpp | 5 +- .../test/pegasus_write_service_impl_test.cpp | 15 +- .../test/pegasus_write_service_test.cpp | 15 +- src/server/test/rocksdb_wrapper_test.cpp | 9 +- src/test/bench_test/config.cpp | 4 +- .../function_test/base_api_test/test_copy.cpp | 12 +- .../base_api_test/test_range_read.cpp | 1 - .../bulk_load_test/CMakeLists.txt | 3 +- .../bulk_load_test/test_bulk_load.cpp | 290 +++++++----- src/test/function_test/config.ini | 7 +- src/test_util/CMakeLists.txt | 2 +- src/test_util/test_util.h | 11 + src/utils/CMakeLists.txt | 2 +- src/utils/alloc.h | 12 +- src/utils/command_manager.cpp | 3 +- src/utils/config_api.cpp | 3 + src/utils/configuration.cpp | 34 +- src/utils/env.cpp | 194 ++++++++ src/utils/env.h | 61 +++ src/utils/filesystem.cpp | 229 ++++------ src/utils/filesystem.h | 19 +- src/utils/flags.h | 3 +- src/utils/fmt_logging.h | 11 + src/utils/output_utils.h | 1 + src/utils/simple_logger.cpp | 5 +- src/utils/strings.h | 1 + src/utils/test/CMakeLists.txt | 3 +- src/utils/test/TokenBucketTest.cpp | 2 +- src/utils/test/autoref_ptr_test.cpp | 1 + src/utils/test/env.cpp | 223 +++++++++- src/utils/test/fail_point_test.cpp | 1 + src/utils/test/file_system_test.cpp | 63 ++- src/utils/test/file_utils.cpp | 9 +- src/utils/test/fmt_logging_test.cpp | 1 + src/utils/test/join_point_test.cpp | 1 - src/utils/test/lock.std.cpp | 1 + src/utils/test/main.cpp | 1 + src/utils/test/memutil_test.cpp | 1 + src/utils/test/sema.cpp | 1 + src/utils/test/smart_pointers_test.cpp | 1 + src/utils/test/utils.cpp | 2 +- thirdparty/CMakeLists.txt | 53 +-- 237 files changed, 3314 insertions(+), 2162 deletions(-) delete mode 100644 src/block_service/directio_writable_file.cpp delete mode 100644 src/block_service/directio_writable_file.h rename src/nfs/{nfs_node_impl.cpp => nfs_node_simple.cpp} (93%) create mode 100644 src/utils/env.cpp create mode 100644 src/utils/env.h diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 115f8b124c..a4da42469d 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -91,7 +91,7 @@ jobs: # Build thirdparties and leave some necessary libraries and source run: | mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* ../scripts/download_hadoop.sh hadoop-bin @@ -151,7 +151,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -195,39 +195,39 @@ jobs: matrix: test_module: - backup_restore_test - - base_api_test - - base_test + - base_api_test # added enc, ing + - base_test # ok - bulk_load_test - - detect_hotspot_test - - dsn_aio_test - - dsn_block_service_test - - dsn_client_test - - dsn.failure_detector.tests - - dsn_http_test - - dsn_meta_state_tests - - dsn.meta.test - - dsn_nfs_test - - dsn_perf_counter_test - - dsn_replica_backup_test - - dsn_replica_bulk_load_test - - dsn_replica_dup_test - - dsn_replica_split_test - - dsn.replica.test - - dsn_replication_common_test - - dsn.replication.simple_kv - - dsn.rep_tests.simple_kv - - dsn_runtime_tests - - dsn_utils_tests - - dsn.zookeeper.tests + - detect_hotspot_test # enc ignored, ok + - dsn_aio_test # added enc, ok + - dsn_block_service_test # partial added enc, ok + - dsn_client_test # ok + - dsn.failure_detector.tests # ok + - dsn_http_test # ok + - dsn_meta_state_tests # added enc, ok + - dsn.meta.test # need add enc, ok + - dsn_nfs_test # ok, need encrypted test files + - dsn_perf_counter_test # ok + - dsn_replica_backup_test # added enc, ok + - dsn_replica_bulk_load_test # added enc, ok + - dsn_replica_dup_test # added enc, ok + - dsn_replica_split_test # added enc, ok + - dsn.replica.test # added enc, ok + - dsn_replication_common_test # added enc, ok + - dsn.replication.simple_kv # ok + - dsn.rep_tests.simple_kv # need add enc, ok + - dsn_runtime_tests # ok + - dsn_utils_tests # ok + - dsn.zookeeper.tests # ok # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after # it has been optimized. # - partition_split_test - - pegasus_geo_test - - pegasus_rproxy_test - - pegasus_unit_test - - recovery_test - - restore_test - - throttle_test + - pegasus_geo_test # added enc, ok + - pegasus_rproxy_test # ok + - pegasus_unit_test # added enc, ok + - recovery_test - + - restore_test - + - throttle_test - needs: build_Release runs-on: ubuntu-latest container: @@ -296,7 +296,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -438,7 +438,7 @@ jobs: # run: | # rm -f /root/thirdparties-src.zip # mkdir build -# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ +# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ # cmake --build build/ -j $(nproc) # rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* # find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + @@ -576,7 +576,7 @@ jobs: run: | rm -f /root/thirdparties-src.zip mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/ + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/ cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 152fe15a78..f56b3f4c2d 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -167,7 +167,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin - name: Image digest @@ -203,7 +203,7 @@ jobs: build-args: | GITHUB_BRANCH=${{ github.ref_name }} OS_VERSION=${{ matrix.osversion }} - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 USE_JEMALLOC=ON HADOOP_BIN_PATH=hadoop-bin ZOOKEEPER_BIN_PATH=zookeeper-bin diff --git a/.licenserc.yaml b/.licenserc.yaml index c6f63afd8b..53d316d23b 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -312,7 +312,7 @@ header: - 'src/nfs/nfs_client_impl.h' - 'src/nfs/nfs_code_definition.h' - 'src/nfs/nfs_node.cpp' - - 'src/nfs/nfs_node_impl.cpp' + - 'src/nfs/nfs_node_simple.cpp' - 'src/nfs/nfs_node_simple.h' - 'src/nfs/nfs_server_impl.cpp' - 'src/nfs/nfs_server_impl.h' diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 79aba27cb7..5596bbd977 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -204,7 +204,7 @@ function(dsn_setup_compiler_flags) # We want access to the PRI* print format macros. add_definitions(-D__STDC_FORMAT_MACROS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -gdwarf-4" CACHE STRING "" FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -gdwarf-4" CACHE STRING "" FORCE) # -Wall: Enable all warnings. add_compile_options(-Wall) @@ -221,9 +221,6 @@ function(dsn_setup_compiler_flags) # use frame pointers to allow simple stack frame walking for backtraces. # This has a small perf hit but worth it for the ability to profile in production add_compile_options( -fno-omit-frame-pointer) - # -Wno-deprecated-register - # kbr5.h uses the legacy 'register' keyword. - add_compile_options(-Wno-deprecated-register) # -Wno-implicit-float-conversion # Poco/Dynamic/VarHolder.h uses 'unsigned long' to 'float' conversion add_compile_options(-Wno-implicit-float-conversion) @@ -388,11 +385,9 @@ function(dsn_common_setup) set(BUILD_SHARED_LIBS OFF) - include(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG("-std=c++1y" COMPILER_SUPPORTS_CXX1Y) - if(NOT ${COMPILER_SUPPORTS_CXX1Y}) - message(FATAL_ERROR "You need a compiler with C++1y support.") - endif() + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) dsn_setup_system_libs() dsn_setup_compiler_flags() diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile index 5c536166c1..0e89f316b1 100644 --- a/docker/pegasus-build-env/centos7/Dockerfile +++ b/docker/pegasus-build-env/centos7/Dockerfile @@ -50,6 +50,8 @@ RUN yum -y install centos-release-scl \ lz4-devel \ bison \ flex \ + krb5-devel \ + cyrus-sasl-devel \ patch; \ yum clean all; \ rm -rf /var/cache/yum; diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile index c1e53891af..667b64120e 100644 --- a/docker/pegasus-build-env/ubuntu1804/Dockerfile +++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile @@ -50,6 +50,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex \ python3-setuptools; \ diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile b/docker/pegasus-build-env/ubuntu2004/Dockerfile index 6438fb03bf..604b5a0b1e 100644 --- a/docker/pegasus-build-env/ubuntu2004/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile @@ -50,6 +50,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex; \ rm -rf /var/lib/apt/lists/* diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile b/docker/pegasus-build-env/ubuntu2204/Dockerfile index bab697c786..0eed16c6cf 100644 --- a/docker/pegasus-build-env/ubuntu2204/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2204/Dockerfile @@ -51,6 +51,8 @@ RUN apt-get update -y; \ libtool \ libssl-dev \ bison \ + libkrb5-dev \ + libsasl2-dev \ maven \ flex; \ rm -rf /var/lib/apt/lists/* diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index ab91710632..27a0d5146e 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip ARG GITHUB_BRANCH=master ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git -ARG ROCKSDB_PORTABLE=OFF +ARG ROCKSDB_PORTABLE=0 ARG USE_JEMALLOC=OFF ARG HADOOP_BIN_PATH ARG ZOOKEEPER_BIN_PATH diff --git a/run.sh b/run.sh index 75b688a8ae..5ac6a19645 100755 --- a/run.sh +++ b/run.sh @@ -118,7 +118,7 @@ function run_build() ENABLE_GPERF=ON SKIP_THIRDPARTY=NO SANITIZER="" - ROCKSDB_PORTABLE=OFF + ROCKSDB_PORTABLE=0 USE_JEMALLOC=OFF BUILD_TEST=OFF IWYU="" @@ -177,7 +177,7 @@ function run_build() SKIP_THIRDPARTY=YES ;; --enable_rocksdb_portable) - ROCKSDB_PORTABLE=ON + ROCKSDB_PORTABLE=1 ;; --use_jemalloc) ENABLE_GPERF=OFF diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh index 391f7822f5..d80d6437e2 100755 --- a/scripts/pack_server.sh +++ b/scripts/pack_server.sh @@ -125,8 +125,6 @@ fi copy_file ./thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/bin copy_file ./thirdparty/output/lib/libhdfs* ${pack}/bin -copy_file ./thirdparty/output/lib/libsasl*.so.* ${pack}/bin -copy_file ./thirdparty/output/lib/libcom_err*.so.* ${pack}/bin copy_file ./scripts/sendmail.sh ${pack}/bin copy_file ./src/server/config.ini ${pack}/bin copy_file ./src/server/config.min.ini ${pack}/bin diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt index 45d24cf687..3754361d06 100644 --- a/src/aio/CMakeLists.txt +++ b/src/aio/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime) +set(MY_PROJ_LIBS dsn_runtime rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/aio/aio_provider.h b/src/aio/aio_provider.h index 73848d87be..74fb410bd7 100644 --- a/src/aio/aio_provider.h +++ b/src/aio/aio_provider.h @@ -27,10 +27,17 @@ #pragma once #include +#include +#include #include "utils/error_code.h" #include "utils/factory_store.h" +namespace rocksdb { +class RandomAccessFile; +class RandomRWFile; +} // namespace rocksdb + namespace dsn { class aio_context; @@ -60,12 +67,13 @@ class aio_provider explicit aio_provider(disk_engine *disk); virtual ~aio_provider() = default; - virtual linux_fd_t open(const char *file_name, int flag, int pmode) = 0; + virtual std::unique_ptr open_read_file(const std::string &fname) = 0; + virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; - virtual error_code close(linux_fd_t fd) = 0; - virtual error_code flush(linux_fd_t fd) = 0; + virtual std::unique_ptr open_write_file(const std::string &fname) = 0; virtual error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; - virtual error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) = 0; + virtual error_code flush(rocksdb::RandomRWFile *rwf) = 0; + virtual error_code close(rocksdb::RandomRWFile *rwf) = 0; // Submits the aio_task to the underlying disk-io executor. // This task may not be executed immediately, call `aio_task::wait` diff --git a/src/aio/disk_engine.cpp b/src/aio/disk_engine.cpp index 82748de500..1b104be301 100644 --- a/src/aio/disk_engine.cpp +++ b/src/aio/disk_engine.cpp @@ -26,7 +26,6 @@ #include "disk_engine.h" -#include #include // IWYU pragma: no_include #include @@ -103,22 +102,26 @@ aio_task *disk_write_queue::unlink_next_workload(void *plength) return first; } -disk_file::disk_file(linux_fd_t fd) : _fd(fd) {} +disk_file::disk_file(std::unique_ptr rf) : _read_file(std::move(rf)) {} +disk_file::disk_file(std::unique_ptr wf) : _write_file(std::move(wf)) {} aio_task *disk_file::read(aio_task *tsk) { + CHECK(_read_file, ""); tsk->add_ref(); // release on completion, see `on_read_completed`. return _read_queue.add_work(tsk, nullptr); } aio_task *disk_file::write(aio_task *tsk, void *ctx) { + CHECK(_write_file, ""); tsk->add_ref(); // release on completion return _write_queue.add_work(tsk, ctx); } aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size) { + CHECK(_read_file, ""); CHECK(wk->next == nullptr, ""); auto ret = _read_queue.on_work_completed(wk, nullptr); wk->enqueue(err, size); @@ -129,6 +132,7 @@ aio_task *disk_file::on_read_completed(aio_task *wk, error_code err, size_t size aio_task *disk_file::on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size) { + CHECK(_write_file, ""); auto ret = _write_queue.on_work_completed(wk, ctx); while (wk) { diff --git a/src/aio/disk_engine.h b/src/aio/disk_engine.h index 8eb7846359..9d9fc563a1 100644 --- a/src/aio/disk_engine.h +++ b/src/aio/disk_engine.h @@ -32,9 +32,15 @@ #include "aio/aio_task.h" #include "aio_provider.h" +#include "rocksdb/env.h" #include "utils/singleton.h" #include "utils/work_queue.h" +namespace rocksdb { +class RandomAccessFile; +class RandomRWFile; +} // namespace rocksdb + namespace dsn { class error_code; @@ -56,17 +62,21 @@ class disk_write_queue : public work_queue class disk_file { public: - explicit disk_file(linux_fd_t fd); + explicit disk_file(std::unique_ptr rf); + explicit disk_file(std::unique_ptr wf); aio_task *read(aio_task *tsk); aio_task *write(aio_task *tsk, void *ctx); aio_task *on_read_completed(aio_task *wk, error_code err, size_t size); aio_task *on_write_completed(aio_task *wk, void *ctx, error_code err, size_t size); - linux_fd_t native_handle() const { return _fd; } + rocksdb::RandomAccessFile *rfile() const { return _read_file.get(); } + rocksdb::RandomRWFile *wfile() const { return _write_file.get(); } private: - linux_fd_t _fd; + // TODO(yingchun): unify to use a single RandomRWFile member variable. + std::unique_ptr _read_file; + std::unique_ptr _write_file; disk_write_queue _write_queue; work_queue _read_queue; }; diff --git a/src/aio/file_io.cpp b/src/aio/file_io.cpp index 4f3c10bb54..a4bf26ba85 100644 --- a/src/aio/file_io.cpp +++ b/src/aio/file_io.cpp @@ -26,32 +26,51 @@ #include "aio/file_io.h" +#include // IWYU pragma: no_include #include #include "aio/aio_provider.h" #include "disk_engine.h" +#include "rocksdb/env.h" +#include "utils/fmt_logging.h" namespace dsn { class task_tracker; namespace file { -/*extern*/ disk_file *open(const char *file_name, int flag, int pmode) +/*extern*/ disk_file *open(const std::string &fname, FileOpenType type) { - auto fd = disk_engine::provider().open(file_name, flag, pmode); - if (fd.is_invalid()) { - return nullptr; + switch (type) { + case FileOpenType::kReadOnly: { + auto sf = disk_engine::provider().open_read_file(fname); + if (!sf) { + return nullptr; + } + return new disk_file(std::move(sf)); } - - return new disk_file(fd); + case FileOpenType::kWriteOnly: { + auto wf = disk_engine::provider().open_write_file(fname); + if (!wf) { + return nullptr; + } + return new disk_file(std::move(wf)); + } + default: + CHECK(false, ""); + } + return nullptr; } /*extern*/ error_code close(disk_file *file) { - error_code result = ERR_INVALID_HANDLE; + error_code result = ERR_OK; if (file != nullptr) { - result = disk_engine::provider().close(file->native_handle()); + // A read file is not needed to close. + if (file->wfile()) { + result = disk_engine::provider().close(file->wfile()); + } delete file; file = nullptr; } @@ -60,11 +79,11 @@ namespace file { /*extern*/ error_code flush(disk_file *file) { - if (nullptr != file) { - return disk_engine::provider().flush(file->native_handle()); - } else { + if (file == nullptr || file->wfile() == nullptr) { return ERR_INVALID_HANDLE; } + + return disk_engine::provider().flush(file->wfile()); } /*extern*/ aio_task_ptr read(disk_file *file, @@ -84,7 +103,8 @@ namespace file { cb->get_aio_context()->engine = &disk_engine::instance(); cb->get_aio_context()->dfile = file; - if (!cb->spec().on_aio_call.execute(task::get_current_task(), cb, true)) { + if (!cb->spec().on_aio_call.execute(task::get_current_task(), cb, true) || + file->rfile() == nullptr) { cb->enqueue(ERR_FILE_OPERATION_FAILED, 0); return cb; } @@ -110,6 +130,10 @@ namespace file { cb->get_aio_context()->file_offset = offset; cb->get_aio_context()->type = AIO_Write; cb->get_aio_context()->dfile = file; + if (file->wfile() == nullptr) { + cb->enqueue(ERR_FILE_OPERATION_FAILED, 0); + return cb; + } disk_engine::instance().write(cb); return cb; diff --git a/src/aio/file_io.h b/src/aio/file_io.h index f0b6ffc420..0cef3f1b80 100644 --- a/src/aio/file_io.h +++ b/src/aio/file_io.h @@ -28,6 +28,7 @@ #include #include +#include #include #include "aio/aio_task.h" @@ -47,6 +48,13 @@ class task_tracker; namespace file { +enum class FileOpenType +{ + kReadOnly = 0, + kWriteOnly +}; + +// TODO(yingchun): consider to return a smart pointer /// open file /// /// \param file_name filename of the file. @@ -55,7 +63,7 @@ namespace file { /// /// \return file handle /// -extern disk_file *open(const char *file_name, int flag, int pmode); +extern disk_file *open(const std::string &fname, FileOpenType type); /// close the file handle extern error_code close(disk_file *file); diff --git a/src/aio/native_linux_aio_provider.cpp b/src/aio/native_linux_aio_provider.cpp index 470f3d63ac..7270dff733 100644 --- a/src/aio/native_linux_aio_provider.cpp +++ b/src/aio/native_linux_aio_provider.cpp @@ -26,21 +26,17 @@ #include "native_linux_aio_provider.h" -#include -#include -#include -#include - #include "aio/aio_provider.h" #include "aio/disk_engine.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/service_engine.h" #include "runtime/task/async_calls.h" -#include "utils/fail_point.h" +#include "utils/env.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" #include "utils/ports.h" -#include "utils/safe_strerror_posix.h" -#include "utils/string_view.h" namespace dsn { @@ -48,94 +44,101 @@ native_linux_aio_provider::native_linux_aio_provider(disk_engine *disk) : aio_pr native_linux_aio_provider::~native_linux_aio_provider() {} -linux_fd_t native_linux_aio_provider::open(const char *file_name, int flag, int pmode) +std::unique_ptr +native_linux_aio_provider::open_read_file(const std::string &fname) { - auto fd = ::open(file_name, flag, pmode); - if (fd == DSN_INVALID_FILE_HANDLE) { - LOG_ERROR("create file '{}' failed, err = {}", file_name, utils::safe_strerror(errno)); + std::unique_ptr rfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewRandomAccessFile(fname, &rfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("open read file '{}' failed, err = {}", fname, s.ToString()); } - return linux_fd_t(fd); + return rfile; } -error_code native_linux_aio_provider::close(linux_fd_t fd) +std::unique_ptr +native_linux_aio_provider::open_write_file(const std::string &fname) { - if (fd.is_invalid() || ::close(fd.fd) == 0) { - return ERR_OK; + // rocksdb::NewRandomRWFile() doesn't act as the docs described, it will not create the + // file if it not exists, and an error Status will be returned, so we try to create the + // file by ReopenWritableFile() if it not exist. + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive)->FileExists(fname); + if (!s.ok() && !s.IsNotFound()) { + LOG_ERROR("failed to check whether the file '{}' exist, err = {}", fname, s.ToString()); + return nullptr; + } + + if (s.IsNotFound()) { + std::unique_ptr cfile; + s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->ReopenWritableFile(fname, &cfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("failed to create file '{}', err = {}", fname, s.ToString()); + return nullptr; + } } - LOG_ERROR("close file failed, err = {}", utils::safe_strerror(errno)); - return ERR_FILE_OPERATION_FAILED; + // Open the file for write as RandomRWFile, to support un-sequential write. + std::unique_ptr wfile; + s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewRandomRWFile(fname, &wfile, rocksdb::EnvOptions()); + if (!s.ok()) { + LOG_ERROR("open write file '{}' failed, err = {}", fname, s.ToString()); + } + return wfile; } -error_code native_linux_aio_provider::flush(linux_fd_t fd) +error_code native_linux_aio_provider::close(rocksdb::RandomRWFile *wf) { - if (fd.is_invalid() || ::fsync(fd.fd) == 0) { - return ERR_OK; + auto s = wf->Close(); + if (!s.ok()) { + LOG_ERROR("close file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; } - LOG_ERROR("flush file failed, err = {}", utils::safe_strerror(errno)); - return ERR_FILE_OPERATION_FAILED; + return ERR_OK; +} + +error_code native_linux_aio_provider::flush(rocksdb::RandomRWFile *wf) +{ + auto s = wf->Fsync(); + if (!s.ok()) { + LOG_ERROR("flush file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + + return ERR_OK; } error_code native_linux_aio_provider::write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) { - dsn::error_code resp = ERR_OK; - uint64_t buffer_offset = 0; - do { - // ret is the written data size - auto ret = ::pwrite(aio_ctx.dfile->native_handle().fd, - (char *)aio_ctx.buffer + buffer_offset, - aio_ctx.buffer_size - buffer_offset, - aio_ctx.file_offset + buffer_offset); - if (dsn_unlikely(ret < 0)) { - if (errno == EINTR) { - LOG_WARNING("write failed with errno={} and will retry it.", - utils::safe_strerror(errno)); - continue; - } - resp = ERR_FILE_OPERATION_FAILED; - LOG_ERROR("write failed with errno={}, return {}.", utils::safe_strerror(errno), resp); - return resp; - } - - // mock the `ret` to reproduce the `write incomplete` case in the first write - FAIL_POINT_INJECT_NOT_RETURN_F("aio_pwrite_incomplete", [&](string_view s) -> void { - if (dsn_unlikely(buffer_offset == 0)) { - --ret; - } - }); - - buffer_offset += ret; - if (dsn_unlikely(buffer_offset != aio_ctx.buffer_size)) { - LOG_WARNING( - "write incomplete, request_size={}, total_write_size={}, this_write_size={}, " - "and will retry it.", - aio_ctx.buffer_size, - buffer_offset, - ret); - } - } while (dsn_unlikely(buffer_offset < aio_ctx.buffer_size)); + rocksdb::Slice data((const char *)(aio_ctx.buffer), aio_ctx.buffer_size); + auto s = aio_ctx.dfile->wfile()->Write(aio_ctx.file_offset, data); + if (!s.ok()) { + LOG_ERROR("write file failed, err = {}", s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } - *processed_bytes = buffer_offset; - return resp; + *processed_bytes = aio_ctx.buffer_size; + return ERR_OK; } error_code native_linux_aio_provider::read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) { - auto ret = ::pread(aio_ctx.dfile->native_handle().fd, - aio_ctx.buffer, - aio_ctx.buffer_size, - aio_ctx.file_offset); - if (dsn_unlikely(ret < 0)) { - LOG_WARNING("write failed with errno={} and will retry it.", utils::safe_strerror(errno)); + rocksdb::Slice result; + auto s = aio_ctx.dfile->rfile()->Read( + aio_ctx.file_offset, aio_ctx.buffer_size, &result, (char *)(aio_ctx.buffer)); + if (!s.ok()) { + LOG_ERROR("read file failed, err = {}", s.ToString()); return ERR_FILE_OPERATION_FAILED; } - if (ret == 0) { + + if (result.empty()) { return ERR_HANDLE_EOF; } - *processed_bytes = static_cast(ret); + *processed_bytes = result.size(); return ERR_OK; } diff --git a/src/aio/native_linux_aio_provider.h b/src/aio/native_linux_aio_provider.h index bdb1339b9c..538b808dfa 100644 --- a/src/aio/native_linux_aio_provider.h +++ b/src/aio/native_linux_aio_provider.h @@ -27,11 +27,18 @@ #pragma once #include +#include +#include #include "aio/aio_task.h" #include "aio_provider.h" #include "utils/error_code.h" +namespace rocksdb { +class RandomAccessFile; +class RandomRWFile; +} // namespace rocksdb + namespace dsn { class disk_engine; @@ -41,16 +48,18 @@ class native_linux_aio_provider : public aio_provider explicit native_linux_aio_provider(disk_engine *disk); ~native_linux_aio_provider() override; - linux_fd_t open(const char *file_name, int flag, int pmode) override; - error_code close(linux_fd_t fd) override; - error_code flush(linux_fd_t fd) override; - error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + std::unique_ptr open_read_file(const std::string &fname) override; error_code read(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + std::unique_ptr open_write_file(const std::string &fname) override; + error_code write(const aio_context &aio_ctx, /*out*/ uint64_t *processed_bytes) override; + error_code flush(rocksdb::RandomRWFile *wf) override; + error_code close(rocksdb::RandomRWFile *wf) override; + void submit_aio_task(aio_task *aio) override; aio_context *prepare_aio_context(aio_task *tsk) override { return new aio_context; } -protected: +private: error_code aio_internal(aio_task *aio); }; diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index 4228a01481..357499a9c8 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio) +set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio test_utils rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/aio/test/aio.cpp b/src/aio/test/aio.cpp index 7037996526..36b5517184 100644 --- a/src/aio/test/aio.cpp +++ b/src/aio/test/aio.cpp @@ -24,156 +24,261 @@ * THE SOFTWARE. */ -#include -#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include +#include #include +#include +#include #include #include +#include #include +#include #include "aio/aio_task.h" #include "aio/file_io.h" #include "runtime/task/task_code.h" #include "runtime/tool_api.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" +#include "utils/env.h" #include "utils/error_code.h" -#include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" -#include "utils/strings.h" +#include "utils/test_macros.h" #include "utils/threadpool_code.h" -namespace dsn { -class disk_file; -} // namespace dsn - using namespace ::dsn; DEFINE_THREAD_POOL_CODE(THREAD_POOL_TEST_SERVER) DEFINE_TASK_CODE_AIO(LPC_AIO_TEST, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER); -TEST(core, aio) +class aio_test : public pegasus::encrypt_data_test_base { - fail::setup(); - fail::cfg("aio_pwrite_incomplete", "void()"); - const char *buffer = "hello, world"; - int len = (int)strlen(buffer); +public: + void SetUp() override { utils::filesystem::remove_path(kTestFileName); } - // write - auto fp = file::open("tmp", O_RDWR | O_CREAT | O_BINARY, 0666); + const std::string kTestFileName = "aio_test.txt"; +}; - std::list tasks; - uint64_t offset = 0; +INSTANTIATE_TEST_CASE_P(, aio_test, ::testing::Values(false, true)); - // new write - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::write(fp, buffer, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; - } +TEST_P(aio_test, basic) +{ + const char *kUnitBuffer = "hello, world"; + size_t kUnitBufferLength = strlen(kUnitBuffer); + int kTotalBufferCount = 100; + int kBufferCountPerBatch = 10; + int64_t kFileSize = kUnitBufferLength * kTotalBufferCount; + ASSERT_EQ(0, kTotalBufferCount % kBufferCountPerBatch); + + auto check_callback = [kUnitBufferLength](::dsn::error_code err, size_t n) { + // Use CHECK_* instead of ASSERT_* to exit the tests immediately when error occurs. + CHECK_EQ(ERR_OK, err); + CHECK_EQ(kUnitBufferLength, n); + }; + auto verify_data = [=]() { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + kTestFileName, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(kFileSize, file_size); + + // Create a read file handler. + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); + ASSERT_NE(rfile, nullptr); + + // 1. Check sequential read. + { + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + char read_buffer[kUnitBufferLength + 1]; + read_buffer[kUnitBufferLength] = 0; + auto t = ::dsn::file::read(rfile, + read_buffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + ASSERT_STREQ(kUnitBuffer, read_buffer); + } + } - for (auto &t : tasks) { - t->wait(); - } + // 2. Check concurrent read. + { + uint64_t offset = 0; + std::list tasks; + char read_buffers[kTotalBufferCount][kUnitBufferLength + 1]; + for (int i = 0; i < kTotalBufferCount; i++) { + read_buffers[i][kUnitBufferLength] = 0; + auto t = ::dsn::file::read(rfile, + read_buffers[i], + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + for (int i = 0; i < kTotalBufferCount; i++) { + ASSERT_STREQ(kUnitBuffer, read_buffers[i]); + } + } + ASSERT_EQ(ERR_OK, file::close(rfile)); + }; - // overwrite - offset = 0; - tasks.clear(); - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::write(fp, buffer, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; + // 1. Sequential write. + { + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); } + NO_FATALS(verify_data()); - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); - } + // 2. Un-sequential write. + { + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); - // vector write - tasks.clear(); - std::unique_ptr buffers(new dsn_file_buffer_t[100]); - for (int i = 0; i < 10; i++) { - buffers[i].buffer = static_cast(const_cast(buffer)); - buffers[i].size = len; - } - for (int i = 0; i < 10; i++) { - tasks.push_back(::dsn::file::write_vector( - fp, buffers.get(), 10, offset, LPC_AIO_TEST, nullptr, nullptr)); - offset += 10 * len; - } - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == 10 * len); - } - auto err = file::close(fp); - EXPECT_TRUE(err == ERR_OK); - - // read - char *buffer2 = (char *)alloca((size_t)len); - fp = file::open("tmp", O_RDONLY | O_BINARY, 0); - - // concurrent read - offset = 0; - tasks.clear(); - for (int i = 0; i < 100; i++) { - auto t = ::dsn::file::read(fp, buffer2, len, offset, LPC_AIO_TEST, nullptr, nullptr); - tasks.push_back(t); - offset += len; - } + std::vector offsets; + offsets.reserve(kTotalBufferCount); + for (int i = 0; i < kTotalBufferCount; i++) { + offsets.push_back(i * kUnitBufferLength); + } - for (auto &t : tasks) { - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); - } + std::random_device rd; + std::mt19937 gen(rd()); + std::shuffle(offsets.begin(), offsets.end(), gen); - // sequential read - offset = 0; - tasks.clear(); - for (int i = 0; i < 200; i++) { - buffer2[0] = 'x'; - auto t = ::dsn::file::read(fp, buffer2, len, offset, LPC_AIO_TEST, nullptr, nullptr); - offset += len; - - t->wait(); - EXPECT_TRUE(t->get_transferred_size() == (size_t)len); - EXPECT_TRUE(dsn::utils::mequals(buffer, buffer2, len)); + std::list tasks; + for (const auto &offset : offsets) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); } - - err = file::close(fp); - fail::teardown(); - EXPECT_TRUE(err == ERR_OK); - - utils::filesystem::remove_path("tmp"); + NO_FATALS(verify_data()); + + // 3. Overwrite. + { + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + for (int i = 0; i < kTotalBufferCount; i++) { + auto t = ::dsn::file::write(wfile, + kUnitBuffer, + kUnitBufferLength, + offset, + LPC_AIO_TEST, + nullptr, + check_callback); + offset += kUnitBufferLength; + tasks.push_back(t); + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); + } + NO_FATALS(verify_data()); + + // 4. Vector write. + { + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + uint64_t offset = 0; + std::list tasks; + std::unique_ptr buffers(new dsn_file_buffer_t[kBufferCountPerBatch]); + for (int i = 0; i < kBufferCountPerBatch; i++) { + buffers[i].buffer = static_cast(const_cast(kUnitBuffer)); + buffers[i].size = kUnitBufferLength; + } + for (int i = 0; i < kTotalBufferCount / kBufferCountPerBatch; i++) { + tasks.push_back( + ::dsn::file::write_vector(wfile, + buffers.get(), + kBufferCountPerBatch, + offset, + LPC_AIO_TEST, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(kBufferCountPerBatch * kUnitBufferLength, n); + })); + offset += kBufferCountPerBatch * kUnitBufferLength; + } + for (auto &t : tasks) { + t->wait(); + ASSERT_EQ(kBufferCountPerBatch * kUnitBufferLength, t->get_transferred_size()); + } + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); + } + NO_FATALS(verify_data()); } -TEST(core, aio_share) +TEST_P(aio_test, aio_share) { - auto fp = file::open("tmp", O_WRONLY | O_CREAT | O_BINARY, 0666); - EXPECT_TRUE(fp != nullptr); + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); - auto fp2 = file::open("tmp", O_RDONLY | O_BINARY, 0); - EXPECT_TRUE(fp2 != nullptr); + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); + ASSERT_NE(rfile, nullptr); - file::close(fp); - file::close(fp2); - - utils::filesystem::remove_path("tmp"); + ASSERT_EQ(ERR_OK, file::close(wfile)); + ASSERT_EQ(ERR_OK, file::close(rfile)); } -TEST(core, operation_failed) +TEST_P(aio_test, operation_failed) { - fail::setup(); - fail::cfg("aio_pwrite_incomplete", "void()"); - - auto fp = file::open("tmp_test_file", O_WRONLY, 0600); - EXPECT_TRUE(fp == nullptr); - auto err = std::make_unique(); auto count = std::make_unique(); auto io_callback = [&err, &count](::dsn::error_code e, size_t n) { @@ -181,39 +286,42 @@ TEST(core, operation_failed) *count = n; }; - fp = file::open("tmp_test_file", O_WRONLY | O_CREAT | O_BINARY, 0666); - EXPECT_TRUE(fp != nullptr); - char buffer[512]; - const char *str = "hello file"; - auto t = ::dsn::file::write(fp, str, strlen(str), 0, LPC_AIO_TEST, nullptr, io_callback, 0); + auto wfile = file::open(kTestFileName, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + + char buff[512] = {0}; + const char *kUnitBuffer = "hello file"; + const size_t kUnitBufferLength = strlen(kUnitBuffer); + auto t = ::dsn::file::write( + wfile, kUnitBuffer, kUnitBufferLength, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == strlen(str)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(kUnitBufferLength, *count); - t = ::dsn::file::read(fp, buffer, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(wfile, buff, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_FILE_OPERATION_FAILED); + ASSERT_EQ(ERR_FILE_OPERATION_FAILED, *err); - auto fp2 = file::open("tmp_test_file", O_RDONLY | O_BINARY, 0); - EXPECT_TRUE(fp2 != nullptr); + auto rfile = file::open(kTestFileName, file::FileOpenType::kReadOnly); + ASSERT_NE(nullptr, rfile); - t = ::dsn::file::read(fp2, buffer, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 512, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == strlen(str)); - EXPECT_TRUE(dsn::utils::equals(buffer, str, 10)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(kUnitBufferLength, *count); + ASSERT_STREQ(kUnitBuffer, buff); - t = ::dsn::file::read(fp2, buffer, 5, 0, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 5, 0, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - EXPECT_TRUE(*err == ERR_OK && *count == 5); - EXPECT_TRUE(dsn::utils::equals(buffer, str, 5)); + ASSERT_EQ(ERR_OK, *err); + ASSERT_EQ(5, *count); + ASSERT_STREQ(kUnitBuffer, buff); - t = ::dsn::file::read(fp2, buffer, 512, 100, LPC_AIO_TEST, nullptr, io_callback, 0); + t = ::dsn::file::read(rfile, buff, 512, 100, LPC_AIO_TEST, nullptr, io_callback, 0); t->wait(); - LOG_INFO("error code: {}", *err); - file::close(fp); - file::close(fp2); - fail::teardown(); - - EXPECT_TRUE(utils::filesystem::remove_path("tmp_test_file")); + ASSERT_EQ(ERR_HANDLE_EOF, *err); + ASSERT_EQ(ERR_OK, file::close(wfile)); + ASSERT_EQ(ERR_OK, file::close(rfile)); } DEFINE_TASK_CODE_AIO(LPC_AIO_TEST_READ, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) @@ -223,22 +331,39 @@ struct aio_result dsn::error_code err; size_t sz; }; -TEST(core, dsn_file) + +TEST_P(aio_test, dsn_file) { - int64_t fin_size, fout_size; - ASSERT_TRUE(utils::filesystem::file_size("copy_source.txt", fin_size)); - ASSERT_LT(0, fin_size); + std::string src_file = "copy_source.txt"; + std::string dst_file = "copy_dest.txt"; + if (FLAGS_encrypt_data_at_rest) { + auto s = dsn::utils::encrypt_file(src_file, src_file + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + src_file += ".encrypted"; + + s = dsn::utils::encrypt_file(dst_file, dst_file + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + dst_file += ".encrypted"; + } + + int64_t src_file_size; + ASSERT_TRUE(utils::filesystem::file_size( + src_file, dsn::utils::FileDataType::kSensitive, src_file_size)); + ASSERT_LT(0, src_file_size); + std::string src_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_file, src_file_md5)); + ASSERT_FALSE(src_file_md5.empty()); - dsn::disk_file *fin = file::open("copy_source.txt", O_RDONLY, 0); + auto fin = file::open(src_file, file::FileOpenType::kReadOnly); ASSERT_NE(nullptr, fin); - dsn::disk_file *fout = file::open("copy_dest.txt", O_RDWR | O_CREAT | O_TRUNC, 0666); + auto fout = file::open(dst_file, file::FileOpenType::kWriteOnly); ASSERT_NE(nullptr, fout); - char buffer[1024]; + char kUnitBuffer[1024]; uint64_t offset = 0; while (true) { aio_result rin; aio_task_ptr tin = file::read(fin, - buffer, + kUnitBuffer, 1024, offset, LPC_AIO_TEST_READ, @@ -270,7 +395,7 @@ TEST(core, dsn_file) aio_result rout; aio_task_ptr tout = file::write(fout, - buffer, + kUnitBuffer, rin.sz, offset, LPC_AIO_TEST_WRITE, @@ -296,10 +421,15 @@ TEST(core, dsn_file) offset += rin.sz; } - ASSERT_EQ((uint64_t)fin_size, offset); + ASSERT_EQ((uint64_t)src_file_size, offset); ASSERT_EQ(ERR_OK, file::close(fout)); ASSERT_EQ(ERR_OK, file::close(fin)); - ASSERT_TRUE(utils::filesystem::file_size("copy_dest.txt", fout_size)); - ASSERT_EQ(fin_size, fout_size); + int64_t dst_file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_file, dsn::utils::FileDataType::kSensitive, dst_file_size)); + ASSERT_EQ(src_file_size, dst_file_size); + std::string dst_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_file, dst_file_md5)); + ASSERT_EQ(src_file_md5, dst_file_md5); } diff --git a/src/base/test/utils_test.cpp b/src/base/test/utils_test.cpp index d9922c2f0d..ad7dcc337c 100644 --- a/src/base/test/utils_test.cpp +++ b/src/base/test/utils_test.cpp @@ -20,7 +20,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include diff --git a/src/block_service/block_service.h b/src/block_service/block_service.h index 96f1416445..d351dcf44a 100644 --- a/src/block_service/block_service.h +++ b/src/block_service/block_service.h @@ -238,8 +238,8 @@ struct upload_request */ struct upload_response { - dsn::error_code err; - uint64_t uploaded_size; + dsn::error_code err = ERR_OK; + uint64_t uploaded_size = 0; }; typedef std::function upload_callback; typedef future_task upload_future; @@ -378,6 +378,8 @@ class block_file : public dsn::ref_counter const write_callback &cb, dsn::task_tracker *tracker = nullptr) = 0; + // TODO(yingchun): it seems every read() will read the whole file, consider to read the whole + // file directly. /** * @brief read * @param req, ref {@link #read_request} diff --git a/src/block_service/directio_writable_file.cpp b/src/block_service/directio_writable_file.cpp deleted file mode 100644 index 2c74ae05d2..0000000000 --- a/src/block_service/directio_writable_file.cpp +++ /dev/null @@ -1,169 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "block_service/directio_writable_file.h" - -#include -#include -#include // posix_memalign -#include -#include // getpagesize -#include -#include -#include - -#include "utils/flags.h" -#include "utils/fmt_logging.h" -#include "utils/safe_strerror_posix.h" - -namespace dsn { -namespace dist { -namespace block_service { - -DSN_DEFINE_uint32(replication, - direct_io_buffer_pages, - 64, - "Number of pages we need to set to direct io buffer"); -DSN_TAG_VARIABLE(direct_io_buffer_pages, FT_MUTABLE); - -DSN_DEFINE_bool(replication, - enable_direct_io, - false, - "Whether to enable direct I/O when download files"); -DSN_TAG_VARIABLE(enable_direct_io, FT_MUTABLE); - -const uint32_t g_page_size = getpagesize(); - -direct_io_writable_file::direct_io_writable_file(const std::string &file_path) - : _file_path(file_path), - _fd(-1), - _file_size(0), - _buffer(nullptr), - _buffer_size(FLAGS_direct_io_buffer_pages * g_page_size), - _offset(0) -{ -} - -direct_io_writable_file::~direct_io_writable_file() -{ - if (!_buffer || _fd < 0) { - return; - } - // Here is an ensurance, users shuold call finalize manually - CHECK_EQ_MSG(_offset, 0, "finalize() should be called before destructor"); - - ::free(_buffer); - CHECK_EQ_MSG( - 0, ::close(_fd), "Failed to close {}, err = {}", _file_path, utils::safe_strerror(errno)); -} - -bool direct_io_writable_file::initialize() -{ - if (posix_memalign(&_buffer, g_page_size, _buffer_size) != 0) { - LOG_ERROR("Allocate memaligned buffer failed, err = {}", utils::safe_strerror(errno)); - return false; - } - - int flag = O_WRONLY | O_TRUNC | O_CREAT; -#if !defined(__APPLE__) - flag |= O_DIRECT; -#endif - // TODO(yingchun): there maybe serious error of the disk driver when these system call failed, - // maybe just terminate the process or mark the disk as failed would be better - _fd = ::open(_file_path.c_str(), flag, S_IRUSR | S_IWUSR | S_IRGRP); - if (_fd < 0) { - LOG_ERROR("Failed to open {} with flag {}, err = {}", - _file_path, - flag, - utils::safe_strerror(errno)); - ::free(_buffer); - _buffer = nullptr; - return false; - } - return true; -} - -bool direct_io_writable_file::finalize() -{ - CHECK(_buffer && _fd >= 0, "Initialize the instance first"); - - if (_offset > 0) { - ssize_t written_bytes = ::write(_fd, _buffer, _buffer_size); - if (dsn_unlikely(written_bytes < 0)) { - LOG_ERROR("Failed to write the last chunk, file_path = {}, err = {}", - _file_path, - utils::safe_strerror(errno)); - return false; - } - // TODO(yingchun): would better to retry - if (dsn_unlikely(written_bytes != _buffer_size)) { - LOG_ERROR("Failed to write the last chunk, file_path = {}, data bytes = {}, written " - "bytes = {}", - _file_path, - _buffer_size, - written_bytes); - return false; - } - _offset = 0; - if (::ftruncate(_fd, _file_size) < 0) { - LOG_ERROR("Failed to truncate {}, err = {}", _file_path, utils::safe_strerror(errno)); - return false; - } - } - return true; -} - -bool direct_io_writable_file::write(const char *s, size_t n) -{ - CHECK(_buffer && _fd >= 0, "Initialize the instance first"); - - uint32_t remaining = n; - while (remaining > 0) { - uint32_t bytes = std::min((_buffer_size - _offset), remaining); - memcpy((char *)_buffer + _offset, s, bytes); - _offset += bytes; - remaining -= bytes; - s += bytes; - // buffer is full, flush to file - if (_offset == _buffer_size) { - ssize_t written_bytes = ::write(_fd, _buffer, _buffer_size); - if (dsn_unlikely(written_bytes < 0)) { - LOG_ERROR("Failed to write chunk, file_path = {}, err = {}", - _file_path, - utils::safe_strerror(errno)); - return false; - } - // TODO(yingchun): would better to retry - if (dsn_unlikely(written_bytes != _buffer_size)) { - LOG_ERROR( - "Failed to write chunk, file_path = {}, data bytes = {}, written bytes = {}", - _file_path, - _buffer_size, - written_bytes); - return false; - } - // reset offset - _offset = 0; - } - } - _file_size += n; - return true; -} - -} // namespace block_service -} // namespace dist -} // namespace dsn diff --git a/src/block_service/directio_writable_file.h b/src/block_service/directio_writable_file.h deleted file mode 100644 index d4f99949ff..0000000000 --- a/src/block_service/directio_writable_file.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "utils/ports.h" - -namespace dsn { -namespace dist { -namespace block_service { - -class direct_io_writable_file -{ -public: - explicit direct_io_writable_file(const std::string &file_path); - ~direct_io_writable_file(); - - bool initialize(); - bool write(const char *s, size_t n); - bool finalize(); - -private: - DISALLOW_COPY_AND_ASSIGN(direct_io_writable_file); - - std::string _file_path; - int _fd; - uint32_t _file_size; - - // page size aligned buffer - void *_buffer; - // buffer size - uint32_t _buffer_size; - // buffer offset - uint32_t _offset; -}; - -} // namespace block_service -} // namespace dist -} // namespace dsn diff --git a/src/block_service/fds/fds_service.cpp b/src/block_service/fds/fds_service.cpp index bed4a4fdaa..8be5709ab5 100644 --- a/src/block_service/fds/fds_service.cpp +++ b/src/block_service/fds/fds_service.cpp @@ -39,6 +39,7 @@ #include "utils/TokenBucket.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -606,7 +607,8 @@ dsn::task_ptr fds_file_object::upload(const upload_request &req, const std::string &local_file = req.input_local_name; // get file size int64_t file_sz = 0; - dsn::utils::filesystem::file_size(local_file, file_sz); + dsn::utils::filesystem::file_size( + local_file, dsn::utils::FileDataType::kSensitive, file_sz); upload_response resp; // TODO: we can cache the whole file in buffer, then upload the buffer rather than the @@ -671,6 +673,7 @@ dsn::task_ptr fds_file_object::download(const download_request &req, t->set_tracker(tracker); download_response resp; + // TODO(yingchun): use rocksdb API to implement this. std::shared_ptr handle(new std::ofstream( req.output_local_name, std::ios::binary | std::ios::out | std::ios::trunc)); if (!handle->is_open()) { diff --git a/src/block_service/hdfs/CMakeLists.txt b/src/block_service/hdfs/CMakeLists.txt index 803e85bec3..2bba8b96bb 100644 --- a/src/block_service/hdfs/CMakeLists.txt +++ b/src/block_service/hdfs/CMakeLists.txt @@ -17,20 +17,16 @@ set(MY_PROJ_NAME dsn.block_service.hdfs) -set(DIRECTIO_SRC - ../directio_writable_file.cpp - ) - #Source files under CURRENT project directory will be automatically included. #You can manually set MY_PROJ_SRC to include source files under other directories. -set(MY_PROJ_SRC "${DIRECTIO_SRC}") +set(MY_PROJ_SRC "") #Search mode for source files under CURRENT project directory ? #"GLOB_RECURSE" for recursive search #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS hdfs) +set(MY_PROJ_LIBS hdfs rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/hdfs/hdfs_service.cpp b/src/block_service/hdfs/hdfs_service.cpp index 459108e111..e50a8efe25 100644 --- a/src/block_service/hdfs/hdfs_service.cpp +++ b/src/block_service/hdfs/hdfs_service.cpp @@ -17,19 +17,21 @@ #include #include +#include #include -#include #include #include -#include "block_service/directio_writable_file.h" #include "hdfs/hdfs.h" #include "hdfs_service.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/task/async_calls.h" #include "runtime/task/task.h" #include "utils/TokenBucket.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -37,6 +39,8 @@ #include "utils/safe_strerror_posix.h" #include "utils/strings.h" +DSN_DECLARE_bool(enable_direct_io); + struct hdfsBuilder; namespace dsn { @@ -65,8 +69,6 @@ DSN_DEFINE_uint64(replication, "hdfs write batch size, the default value is 64MB"); DSN_TAG_VARIABLE(hdfs_write_batch_size_bytes, FT_MUTABLE); -DSN_DECLARE_bool(enable_direct_io); - hdfs_service::hdfs_service() { _read_token_bucket.reset(new folly::DynamicTokenBucket()); @@ -108,12 +110,12 @@ error_code hdfs_service::create_fs() hdfsBuilderSetNameNode(builder, _hdfs_name_node.c_str()); _fs = hdfsBuilderConnect(builder); if (!_fs) { - LOG_ERROR("Fail to connect hdfs name node {}, error: {}.", + LOG_ERROR("Fail to connect HDFS name node {}, error: {}.", _hdfs_name_node, utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } - LOG_INFO("Succeed to connect hdfs name node {}.", _hdfs_name_node); + LOG_INFO("Succeed to connect HDFS name node {}.", _hdfs_name_node); return ERR_OK; } @@ -122,10 +124,10 @@ void hdfs_service::close() // This method should be carefully called. // Calls to hdfsDisconnect() by individual threads would terminate // all other connections handed out via hdfsConnect() to the same URI. - LOG_INFO("Try to disconnect hdfs."); + LOG_INFO("Try to disconnect HDFS."); int result = hdfsDisconnect(_fs); if (result == -1) { - LOG_ERROR("Fail to disconnect from the hdfs file system, error: {}.", + LOG_ERROR("Fail to disconnect from the HDFS file system, error: {}.", utils::safe_strerror(errno)); } // Even if there is an error, the resources associated with the hdfsFS will be freed. @@ -134,7 +136,7 @@ void hdfs_service::close() std::string hdfs_service::get_hdfs_entry_name(const std::string &hdfs_path) { - // get exact file name from an hdfs path. + // get exact file name from an HDFS path. int pos = hdfs_path.find_last_of("/"); return hdfs_path.substr(pos + 1); } @@ -305,7 +307,7 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, hdfsFile write_file = hdfsOpenFile(_service->get_fs(), file_name().c_str(), O_WRONLY | O_CREAT, 0, 0, 0); if (!write_file) { - LOG_ERROR("Failed to open hdfs file {} for writting, error: {}.", + LOG_ERROR("Failed to open HDFS file {} for writting, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; @@ -323,7 +325,7 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, (void *)(data + cur_pos), static_cast(write_len)); if (num_written_bytes == -1) { - LOG_ERROR("Failed to write hdfs file {}, error: {}.", + LOG_ERROR("Failed to write HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); hdfsCloseFile(_service->get_fs(), write_file); @@ -333,18 +335,18 @@ error_code hdfs_file_object::write_data_in_batches(const char *data, } if (hdfsHFlush(_service->get_fs(), write_file) != 0) { LOG_ERROR( - "Failed to flush hdfs file {}, error: {}.", file_name(), utils::safe_strerror(errno)); + "Failed to flush HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); hdfsCloseFile(_service->get_fs(), write_file); return ERR_FS_INTERNAL; } written_size = cur_pos; if (hdfsCloseFile(_service->get_fs(), write_file) != 0) { LOG_ERROR( - "Failed to close hdfs file {}, error: {}", file_name(), utils::safe_strerror(errno)); + "Failed to close HDFS file {}, error: {}", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } - LOG_INFO("start to synchronize meta data after successfully wrote data to hdfs"); + LOG_INFO("start to synchronize meta data after successfully wrote data to HDFS"); return get_file_meta(); } @@ -376,23 +378,51 @@ dsn::task_ptr hdfs_file_object::upload(const upload_request &req, add_ref(); auto upload_background = [this, req, t]() { + LOG_INFO("start to upload from '{}' to '{}'", req.input_local_name, file_name()); + upload_response resp; - resp.uploaded_size = 0; - std::ifstream is(req.input_local_name, std::ios::binary | std::ios::in); - if (is.is_open()) { - int64_t file_sz = 0; - dsn::utils::filesystem::file_size(req.input_local_name, file_sz); - std::unique_ptr buffer(new char[file_sz]); - is.read(buffer.get(), file_sz); - is.close(); - resp.err = write_data_in_batches(buffer.get(), file_sz, resp.uploaded_size); - } else { - LOG_ERROR("HDFS upload failed: open local file {} failed when upload to {}, error: {}", - req.input_local_name, - file_name(), - utils::safe_strerror(errno)); - resp.err = dsn::ERR_FILE_OPERATION_FAILED; - } + do { + rocksdb::EnvOptions env_options; + env_options.use_direct_reads = FLAGS_enable_direct_io; + std::unique_ptr rfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(req.input_local_name, &rfile, env_options); + if (!s.ok()) { + LOG_ERROR( + "open local file '{}' failed, err = {}", req.input_local_name, s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + int64_t file_size; + if (!dsn::utils::filesystem::file_size( + req.input_local_name, dsn::utils::FileDataType::kSensitive, file_size)) { + LOG_ERROR("get size of local file '{}' failed", req.input_local_name); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + rocksdb::Slice result; + char scratch[file_size]; + s = rfile->Read(file_size, &result, scratch); + if (!s.ok()) { + LOG_ERROR( + "read local file '{}' failed, err = {}", req.input_local_name, s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + resp.err = write_data_in_batches(result.data(), result.size(), resp.uploaded_size); + if (resp.err != ERR_OK) { + LOG_ERROR("write data to remote '{}' failed, err = {}", file_name(), resp.err); + break; + } + + LOG_INFO("finish to upload from '{}' to '{}', size = {}", + req.input_local_name, + file_name(), + resp.uploaded_size); + } while (false); t->enqueue_with(resp); release_ref(); }; @@ -417,7 +447,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, hdfsFile read_file = hdfsOpenFile(_service->get_fs(), file_name().c_str(), O_RDONLY, 0, 0, 0); if (!read_file) { - LOG_ERROR("Failed to open hdfs file {} for reading, error: {}.", + LOG_ERROR("Failed to open HDFS file {} for reading, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; @@ -446,7 +476,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, cur_pos += num_read_bytes; dst_buf += num_read_bytes; } else if (num_read_bytes == -1) { - LOG_ERROR("Failed to read hdfs file {}, error: {}.", + LOG_ERROR("Failed to read HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); read_success = false; @@ -455,7 +485,7 @@ error_code hdfs_file_object::read_data_in_batches(uint64_t start_pos, } if (hdfsCloseFile(_service->get_fs(), read_file) != 0) { LOG_ERROR( - "Failed to close hdfs file {}, error: {}.", file_name(), utils::safe_strerror(errno)); + "Failed to close HDFS file {}, error: {}.", file_name(), utils::safe_strerror(errno)); return ERR_FS_INTERNAL; } if (read_success) { @@ -504,48 +534,54 @@ dsn::task_ptr hdfs_file_object::download(const download_request &req, auto download_background = [this, req, t]() { download_response resp; resp.downloaded_size = 0; - std::string read_buffer; - size_t read_length = 0; - resp.err = - read_data_in_batches(req.remote_pos, req.remote_length, read_buffer, read_length); - if (resp.err == ERR_OK) { - bool write_succ = false; - if (FLAGS_enable_direct_io) { - auto dio_file = std::make_unique(req.output_local_name); - do { - if (!dio_file->initialize()) { - break; - } - bool wr_ret = dio_file->write(read_buffer.c_str(), read_length); - if (!wr_ret) { - break; - } - if (dio_file->finalize()) { - resp.downloaded_size = read_length; - resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); - write_succ = true; - } - } while (0); - } else { - std::ofstream out(req.output_local_name, - std::ios::binary | std::ios::out | std::ios::trunc); - if (out.is_open()) { - out.write(read_buffer.c_str(), read_length); - out.close(); - resp.downloaded_size = read_length; - resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); - write_succ = true; - } + resp.err = ERR_OK; + bool write_succ = false; + std::string target_file = req.output_local_name; + do { + LOG_INFO("start to download from '{}' to '{}'", file_name(), target_file); + + std::string read_buffer; + size_t read_length = 0; + resp.err = + read_data_in_batches(req.remote_pos, req.remote_length, read_buffer, read_length); + if (resp.err != ERR_OK) { + LOG_ERROR("read data from remote '{}' failed, err = {}", file_name(), resp.err); + break; } - if (!write_succ) { - LOG_ERROR("HDFS download failed: fail to open localfile {} when download {}, " - "error: {}", - req.output_local_name, - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FILE_OPERATION_FAILED; - resp.downloaded_size = 0; + + rocksdb::EnvOptions env_options; + env_options.use_direct_writes = FLAGS_enable_direct_io; + std::unique_ptr wfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewWritableFile(target_file, &wfile, env_options); + if (!s.ok()) { + LOG_ERROR("create local file '{}' failed, err = {}", target_file, s.ToString()); + break; + } + + s = wfile->Append(rocksdb::Slice(read_buffer.data(), read_length)); + if (!s.ok()) { + LOG_ERROR("append local file '{}' failed, err = {}", target_file, s.ToString()); + break; + } + + s = wfile->Fsync(); + if (!s.ok()) { + LOG_ERROR("fsync local file '{}' failed, err = {}", target_file, s.ToString()); + break; } + + resp.downloaded_size = read_length; + resp.file_md5 = utils::string_md5(read_buffer.c_str(), read_length); + write_succ = true; + } while (false); + + if (!write_succ) { + LOG_ERROR("HDFS download failed: fail to write local file {} when download {}", + target_file, + file_name()); + resp.err = ERR_FILE_OPERATION_FAILED; + resp.downloaded_size = 0; } t->enqueue_with(resp); release_ref(); diff --git a/src/block_service/local/CMakeLists.txt b/src/block_service/local/CMakeLists.txt index 0886bece9a..9d830f7825 100644 --- a/src/block_service/local/CMakeLists.txt +++ b/src/block_service/local/CMakeLists.txt @@ -26,7 +26,7 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS "") +set(MY_PROJ_LIBS rocksdb) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/local/local_service.cpp b/src/block_service/local/local_service.cpp index 5eb4b3fba4..ccccea7d82 100644 --- a/src/block_service/local/local_service.cpp +++ b/src/block_service/local/local_service.cpp @@ -15,10 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include -#include +#include #include #include #include @@ -26,20 +23,24 @@ #include #include "local_service.h" -#include "nlohmann/detail/macro_scope.hpp" +#include "nlohmann/json.hpp" #include "nlohmann/json_fwd.hpp" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/task/async_calls.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" -#include "utils/defer.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/fail_point.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/safe_strerror_posix.h" #include "utils/string_view.h" #include "utils/strings.h" +DSN_DECLARE_bool(enable_direct_io); + namespace dsn { class task_tracker; } // namespace dsn @@ -52,22 +53,13 @@ namespace block_service { DEFINE_TASK_CODE(LPC_LOCAL_SERVICE_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE) -struct file_metadata -{ - uint64_t size; - std::string md5; -}; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5) - -bool file_metadata_from_json(std::ifstream &fin, file_metadata &fmeta) noexcept +bool file_metadata_from_json(const std::string &data, file_metadata &fmeta) noexcept { - std::string data; - fin >> data; try { nlohmann::json::parse(data).get_to(fmeta); return true; } catch (nlohmann::json::exception &exp) { - LOG_WARNING("decode meta data from json failed: {} [{}]", exp.what(), data); + LOG_WARNING("decode metadata from json failed: {} [{}]", exp.what(), data); return false; } } @@ -199,7 +191,7 @@ dsn::task_ptr local_service::create_file(const create_file_request &req, if (utils::filesystem::file_exists(file_path) && utils::filesystem::file_exists(meta_file_path)) { - LOG_DEBUG("file({}) already exist", file_path); + LOG_INFO("file({}) already exist", file_path); resp.err = f->load_metadata(); } @@ -276,17 +268,18 @@ error_code local_file_object::load_metadata() return ERR_OK; std::string metadata_path = local_service::get_metafile(file_name()); - std::ifstream is(metadata_path, std::ios::in); - if (!is.is_open()) { - LOG_WARNING( - "load meta data from {} failed, err = {}", metadata_path, utils::safe_strerror(errno)); + std::string data; + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), metadata_path, &data); + if (!s.ok()) { + LOG_WARNING("read file '{}' failed, err = {}", metadata_path, s.ToString()); return ERR_FS_INTERNAL; } - auto cleanup = dsn::defer([&is]() { is.close(); }); file_metadata meta; - bool ans = file_metadata_from_json(is, meta); + bool ans = file_metadata_from_json(data, meta); if (!ans) { + LOG_WARNING("decode metadata '{}' file content failed", metadata_path); return ERR_FS_INTERNAL; } _size = meta.size; @@ -300,16 +293,17 @@ error_code local_file_object::store_metadata() file_metadata meta; meta.md5 = _md5_value; meta.size = _size; - + std::string data = nlohmann::json(meta).dump(); std::string metadata_path = local_service::get_metafile(file_name()); - std::ofstream os(metadata_path, std::ios::out | std::ios::trunc); - if (!os.is_open()) { - LOG_WARNING( - "store to metadata file {} failed, err={}", metadata_path, utils::safe_strerror(errno)); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(data), + metadata_path, + /* should_sync */ true); + if (!s.ok()) { + LOG_WARNING("store to metadata file {} failed, err={}", metadata_path, s.ToString()); return ERR_FS_INTERNAL; } - auto cleanup = dsn::defer([&os]() { os.close(); }); - os << nlohmann::json(meta); return ERR_OK; } @@ -345,24 +339,36 @@ dsn::task_ptr local_file_object::write(const write_request &req, } if (resp.err == ERR_OK) { - LOG_DEBUG("start write file, file = {}", file_name()); + LOG_INFO("start write file, file = {}", file_name()); + + do { + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(req.buffer.data(), req.buffer.length()), + file_name(), + /* should_sync */ true); + if (!s.ok()) { + LOG_WARNING("write file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } - std::ofstream fout(file_name(), std::ifstream::out | std::ifstream::trunc); - if (!fout.is_open()) { - resp.err = ERR_FS_INTERNAL; - } else { - fout.write(req.buffer.data(), req.buffer.length()); resp.written_size = req.buffer.length(); - fout.close(); // Currently we calc the meta data from source data, which save the io bandwidth // a lot, but it is somewhat not correct. _size = resp.written_size; _md5_value = utils::string_md5(req.buffer.data(), req.buffer.length()); + // TODO(yingchun): make store_metadata as a local function, do not depend on the + // member variables (i.e. _size and _md5_value). + auto err = store_metadata(); + if (err != ERR_OK) { + LOG_WARNING("store_metadata failed"); + resp.err = ERR_FS_INTERNAL; + break; + } _has_meta_synced = true; - - store_metadata(); - } + } while (false); } tsk->enqueue_with(resp); release_ref(); @@ -383,38 +389,66 @@ dsn::task_ptr local_file_object::read(const read_request &req, auto read_func = [this, req, tsk]() { read_response resp; - resp.err = ERR_OK; - if (!utils::filesystem::file_exists(file_name()) || - !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { - resp.err = ERR_OBJECT_NOT_FOUND; - } else { - if ((resp.err = load_metadata()) != ERR_OK) { - LOG_WARNING("load meta data of {} failed", file_name()); + do { + if (!utils::filesystem::file_exists(file_name()) || + !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { + LOG_WARNING("data file '{}' or metadata file '{}' not exist", + file_name(), + local_service::get_metafile(file_name())); + resp.err = ERR_OBJECT_NOT_FOUND; + break; + } + + resp.err = load_metadata(); + if (resp.err != ERR_OK) { + LOG_WARNING("load metadata of {} failed", file_name()); + break; + } + + int64_t file_sz = _size; + int64_t total_sz = 0; + if (req.remote_length == -1 || req.remote_length + req.remote_pos > file_sz) { + total_sz = file_sz - req.remote_pos; } else { - int64_t file_sz = _size; - int64_t total_sz = 0; - if (req.remote_length == -1 || req.remote_length + req.remote_pos > file_sz) { - total_sz = file_sz - req.remote_pos; - } else { - total_sz = req.remote_length; - } + total_sz = req.remote_length; + } - LOG_DEBUG("read file({}), size = {}", file_name(), total_sz); - std::string buf; - buf.resize(total_sz + 1); - std::ifstream fin(file_name(), std::ifstream::in); - if (!fin.is_open()) { - resp.err = ERR_FS_INTERNAL; - } else { - fin.seekg(static_cast(req.remote_pos), fin.beg); - fin.read((char *)buf.c_str(), total_sz); - buf[fin.gcount()] = '\0'; - resp.buffer = blob::create_from_bytes(std::move(buf)); - } - fin.close(); + LOG_INFO("start to read file '{}', offset = {}, size = {}", + file_name(), + req.remote_pos, + total_sz); + rocksdb::EnvOptions env_options; + env_options.use_direct_reads = FLAGS_enable_direct_io; + std::unique_ptr sfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(file_name(), &sfile, env_options); + if (!s.ok()) { + LOG_WARNING("open file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } + + s = sfile->Skip(req.remote_pos); + if (!s.ok()) { + LOG_WARNING( + "skip '{}' for {} failed, err = {}", file_name(), req.remote_pos, s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; } - } + rocksdb::Slice result; + std::string buf; + buf.resize(total_sz + 1); + s = sfile->Read(total_sz, &result, buf.data()); + if (!s.ok()) { + LOG_WARNING("read file '{}' failed, err = {}", file_name(), s.ToString()); + resp.err = ERR_FS_INTERNAL; + break; + } + + buf[result.size()] = 0; + resp.buffer = blob::create_from_bytes(std::move(buf)); + } while (false); tsk->enqueue_with(resp); release_ref(); }; @@ -432,58 +466,50 @@ dsn::task_ptr local_file_object::upload(const upload_request &req, upload_future_ptr tsk(new upload_future(code, cb, 0)); tsk->set_tracker(tracker); auto upload_file_func = [this, req, tsk]() { - upload_response resp; - resp.err = ERR_OK; - std::ifstream fin(req.input_local_name, std::ios_base::in); - if (!fin.is_open()) { - LOG_WARNING("open source file {} for read failed, err({})", - req.input_local_name, - utils::safe_strerror(errno)); - resp.err = ERR_FILE_OPERATION_FAILED; - } - - utils::filesystem::create_file(file_name()); - std::ofstream fout(file_name(), std::ios_base::out | std::ios_base::trunc); - if (!fout.is_open()) { - LOG_WARNING("open target file {} for write failed, err({})", - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FS_INTERNAL; - } + LOG_INFO("start to upload from '{}' to '{}'", req.input_local_name, file_name()); - if (resp.err == ERR_OK) { - LOG_DEBUG("start to transfer from src_file({}) to dst_file({})", - req.input_local_name, - file_name()); - int64_t total_sz = 0; - char buf[max_length] = {'\0'}; - while (!fin.eof()) { - fin.read(buf, max_length); - total_sz += fin.gcount(); - fout.write(buf, fin.gcount()); + upload_response resp; + do { + // Create the directory. + std::string path = dsn::utils::filesystem::remove_file_name(file_name()); + if (!dsn::utils::filesystem::create_directory(path)) { + LOG_WARNING("create directory '{}' failed", path); + resp.err = ERR_FILE_OPERATION_FAILED; + break; } - LOG_DEBUG("finish upload file, file = {}, total_size = {}", file_name(), total_sz); - fout.close(); - fin.close(); - - resp.uploaded_size = static_cast(total_sz); - // calc the md5sum by source file for simplicity - _size = total_sz; - error_code res = utils::filesystem::md5sum(req.input_local_name, _md5_value); - if (res == dsn::ERR_OK) { - _has_meta_synced = true; - store_metadata(); - } else { + uint64_t file_size; + auto s = dsn::utils::copy_file(req.input_local_name, file_name(), &file_size); + if (!s.ok()) { + LOG_WARNING("upload from '{}' to '{}' failed, err = {}", + req.input_local_name, + file_name(), + s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + LOG_INFO("finish to upload from '{}' to '{}', size = {}", + req.input_local_name, + file_name(), + file_size); + + resp.uploaded_size = file_size; + _size = file_size; + auto res = utils::filesystem::md5sum(file_name(), _md5_value); + if (res != dsn::ERR_OK) { + LOG_WARNING("calculate md5sum for '{}' failed", file_name()); resp.err = ERR_FS_INTERNAL; + break; } - } else { - if (fin.is_open()) - fin.close(); - if (fout.is_open()) - fout.close(); - } + auto err = store_metadata(); + if (err != ERR_OK) { + LOG_ERROR("store_metadata failed"); + resp.err = ERR_FS_INTERNAL; + break; + } + _has_meta_synced = true; + } while (false); tsk->enqueue_with(resp); release_ref(); }; @@ -505,65 +531,61 @@ dsn::task_ptr local_file_object::download(const download_request &req, download_response resp; resp.err = ERR_OK; std::string target_file = req.output_local_name; - if (target_file.empty()) { - LOG_ERROR( - "download {} failed, because target name({}) is invalid", file_name(), target_file); - resp.err = ERR_INVALID_PARAMETERS; - } - if (resp.err == ERR_OK && !_has_meta_synced) { - if (!utils::filesystem::file_exists(file_name()) || - !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { - resp.err = ERR_OBJECT_NOT_FOUND; + do { + if (target_file.empty()) { + LOG_WARNING("download {} failed, because target name({}) is invalid", + file_name(), + target_file); + resp.err = ERR_INVALID_PARAMETERS; + break; } - } - if (resp.err == ERR_OK) { - std::ifstream fin(file_name(), std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR("open block file({}) failed, err({})", - file_name(), - utils::safe_strerror(errno)); - resp.err = ERR_FS_INTERNAL; + if (!_has_meta_synced) { + if (!utils::filesystem::file_exists(file_name()) || + !utils::filesystem::file_exists(local_service::get_metafile(file_name()))) { + LOG_WARNING("file '{}' or metadata file '{}' not found", + file_name(), + local_service::get_metafile(file_name())); + resp.err = ERR_OBJECT_NOT_FOUND; + break; + } } - std::ofstream fout(target_file, std::ios_base::out | std::ios_base::trunc); - if (!fout.is_open()) { - if (fin.is_open()) - fin.close(); - LOG_ERROR("open target file({}) failed, err({})", - target_file, - utils::safe_strerror(errno)); + LOG_INFO("start to download from '{}' to '{}'", file_name(), target_file); + + // Create the directory. + std::string path = dsn::utils::filesystem::remove_file_name(file_name()); + if (!dsn::utils::filesystem::create_directory(path)) { + LOG_WARNING("create directory '{}' failed", path); resp.err = ERR_FILE_OPERATION_FAILED; + break; } - if (resp.err == ERR_OK) { - LOG_DEBUG( - "start to transfer, src_file({}), dst_file({})", file_name(), target_file); - int64_t total_sz = 0; - char buf[max_length] = {'\0'}; - while (!fin.eof()) { - fin.read(buf, max_length); - total_sz += fin.gcount(); - fout.write(buf, fin.gcount()); - } - LOG_DEBUG("finish download file({}), total_size = {}", target_file, total_sz); - fout.close(); - fin.close(); - resp.downloaded_size = static_cast(total_sz); - - _size = total_sz; - if ((resp.err = utils::filesystem::md5sum(target_file, _md5_value)) != ERR_OK) { - LOG_WARNING("download {} failed when calculate the md5sum of {}", - file_name(), - target_file); - } else { - _has_meta_synced = true; - resp.file_md5 = _md5_value; - } + uint64_t file_size; + auto s = dsn::utils::copy_file(file_name(), target_file, &file_size); + if (!s.ok()) { + LOG_WARNING("download from '{}' to '{}' failed, err = {}", + file_name(), + target_file, + s.ToString()); + resp.err = ERR_FILE_OPERATION_FAILED; + break; + } + + auto res = utils::filesystem::md5sum(target_file, _md5_value); + if (res != dsn::ERR_OK) { + LOG_WARNING("calculate md5sum for {} failed", target_file); + resp.err = ERR_FILE_OPERATION_FAILED; + break; } - } + LOG_INFO("finish download file '{}', size = {}", target_file, file_size); + resp.downloaded_size = file_size; + resp.file_md5 = _md5_value; + _size = file_size; + _has_meta_synced = true; + } while (false); tsk->enqueue_with(resp); release_ref(); }; diff --git a/src/block_service/local/local_service.h b/src/block_service/local/local_service.h index 9816734cf0..a32eebd982 100644 --- a/src/block_service/local/local_service.h +++ b/src/block_service/local/local_service.h @@ -17,6 +17,9 @@ #pragma once +#include +#include +#include #include #include #include @@ -32,6 +35,13 @@ class task_tracker; namespace dist { namespace block_service { +struct file_metadata +{ + int64_t size = 0; + std::string md5; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5) + class local_service : public block_filesystem { public: diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index a8e47d597e..7653a8fe09 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -28,6 +28,7 @@ set(MY_PROJ_LIBS dsn.block_service.fds dsn.block_service.hdfs dsn_runtime + dsn_utils galaxy-fds-sdk-cpp PocoNet PocoFoundation @@ -36,7 +37,7 @@ set(MY_PROJ_LIBS gtest gtest_main hdfs - ) + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/block_service/test/block_service_manager_test.cpp b/src/block_service/test/block_service_manager_test.cpp index 8ead04fd2a..52147b4371 100644 --- a/src/block_service/test/block_service_manager_test.cpp +++ b/src/block_service/test/block_service_manager_test.cpp @@ -15,11 +15,14 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include -#include #include #include #include @@ -29,14 +32,17 @@ #include "block_service/local/local_service.h" #include "block_service_mock.h" #include "metadata_types.h" +#include "test_util/test_util.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" +#include "utils/test_macros.h" namespace dsn { namespace dist { namespace block_service { -class block_service_manager_test : public ::testing::Test +class block_service_manager_test : public pegasus::encrypt_data_test_base { public: block_service_manager_test() @@ -57,15 +63,17 @@ class block_service_manager_test : public ::testing::Test void create_local_file(const std::string &file_name) { std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - utils::filesystem::create_file(whole_name); - std::ofstream test_file; - test_file.open(whole_name); - test_file << "write some data.\n"; - test_file.close(); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice("write some data."), + whole_name, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); _file_meta.name = whole_name; - utils::filesystem::md5sum(whole_name, _file_meta.md5); - utils::filesystem::file_size(whole_name, _file_meta.size); + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(whole_name, _file_meta.md5)); + ASSERT_TRUE(utils::filesystem::file_size( + whole_name, dsn::utils::FileDataType::kSensitive, _file_meta.size)); } void create_remote_file(const std::string &file_name, int64_t size, const std::string &md5) @@ -84,8 +92,9 @@ class block_service_manager_test : public ::testing::Test std::string FILE_NAME = "test_file"; }; -// download_file unit tests -TEST_F(block_service_manager_test, do_download_remote_file_not_exist) +INSTANTIATE_TEST_CASE_P(, block_service_manager_test, ::testing::Values(false, true)); + +TEST_P(block_service_manager_test, remote_file_not_exist) { utils::filesystem::remove_path(LOCAL_DIR); auto fs = std::make_unique(); @@ -93,31 +102,29 @@ TEST_F(block_service_manager_test, do_download_remote_file_not_exist) uint64_t download_size = 0; error_code err = _block_service_manager.download_file( PROVIDER, LOCAL_DIR, FILE_NAME, fs.get(), download_size); - ASSERT_EQ(err, ERR_CORRUPTION); // file does not exist -} - -TEST_F(block_service_manager_test, do_download_same_name_file) -{ - // local file exists, but md5 not matched with remote file - create_local_file(FILE_NAME); - create_remote_file(FILE_NAME, 2333, "md5_not_match"); - uint64_t download_size = 0; - ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST); - ASSERT_EQ(download_size, 0); + ASSERT_EQ(ERR_CORRUPTION, err); } -TEST_F(block_service_manager_test, do_download_file_exist) +TEST_P(block_service_manager_test, local_file_exist) { - create_local_file(FILE_NAME); - create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5); - uint64_t download_size = 0; - ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST); - ASSERT_EQ(download_size, 0); + NO_FATALS(create_local_file(FILE_NAME)); + struct remote_file_info + { + int64_t size; + std::string md5; + } tests[]{{2333, "bad_md5"}, {2333, _file_meta.md5}, {_file_meta.size, "bad_md5"}}; + for (const auto &test : tests) { + // The remote file will be overwritten when repeatedly created. + create_remote_file(FILE_NAME, test.size, test.md5); + uint64_t download_size = 0; + ASSERT_EQ(test_download_file(download_size), ERR_PATH_ALREADY_EXIST); + ASSERT_EQ(download_size, 0); + } } -TEST_F(block_service_manager_test, do_download_succeed) +TEST_P(block_service_manager_test, do_download_succeed) { - create_local_file(FILE_NAME); + NO_FATALS(create_local_file(FILE_NAME)); create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5); // remove local file to mock condition that file not existed std::string file_name = utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME); diff --git a/src/block_service/test/config-test.ini b/src/block_service/test/config-test.ini index c1996d5518..2acb86e2bb 100644 --- a/src/block_service/test/config-test.ini +++ b/src/block_service/test/config-test.ini @@ -53,7 +53,7 @@ max_size = 150 worker_count = 8 [hdfs_test] -test_name_node = -test_backup_path = +test_name_node = +test_backup_path = num_test_file_lines = 4096 num_total_files_for_hdfs_concurrent_test = 64 diff --git a/src/block_service/test/fds_service_test.cpp b/src/block_service/test/fds_service_test.cpp index cb14aa5cd2..d06f3718a1 100644 --- a/src/block_service/test/fds_service_test.cpp +++ b/src/block_service/test/fds_service_test.cpp @@ -147,6 +147,7 @@ void FDSClientTest::TearDown() {} DEFINE_TASK_CODE(lpc_btest, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT) +// TODO(yingchun): add encryption test when FDSClient supports encryption. TEST_F(FDSClientTest, test_basic_operation) { const char *files[] = {"/fdstest/fdstest1/test1/test1", diff --git a/src/block_service/test/hdfs_service_test.cpp b/src/block_service/test/hdfs_service_test.cpp index 6cfcfb379c..6fa2542419 100644 --- a/src/block_service/test/hdfs_service_test.cpp +++ b/src/block_service/test/hdfs_service_test.cpp @@ -15,15 +15,20 @@ // specific language governing permissions and limitations // under the License. +#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include #include #include #include #include -#include +#include #include #include #include @@ -35,27 +40,23 @@ #include "runtime/task/task.h" #include "runtime/task/task_code.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/enum_helper.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" -#include "utils/strings.h" +#include "utils/fmt_logging.h" +#include "utils/test_macros.h" #include "utils/threadpool_code.h" using namespace dsn; using namespace dsn::dist::block_service; -static std::string example_name_node = ""; -static std::string example_backup_path = ""; -// Please modify following paras in 'config-test.ini' to enable hdfs_service_test, -// or hdfs_service_test will be skipped and return true. -DSN_DEFINE_string(hdfs_test, test_name_node, "", "hdfs name node"); -DSN_DEFINE_string(hdfs_test, - test_backup_path, - "", - "path for uploading and downloading test files"); +DSN_DEFINE_string(hdfs_test, test_name_node, "", "hdfs name node"); +DSN_DEFINE_string(hdfs_test, test_backup_path, "", "path for uploading and downloading test files"); DSN_DEFINE_uint32(hdfs_test, num_test_file_lines, 4096, "number of lines in test file"); DSN_DEFINE_uint32(hdfs_test, @@ -65,206 +66,244 @@ DSN_DEFINE_uint32(hdfs_test, DEFINE_TASK_CODE(LPC_TEST_HDFS, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT) -class HDFSClientTest : public testing::Test +class HDFSClientTest : public pegasus::encrypt_data_test_base { protected: - virtual void SetUp() override; - virtual void TearDown() override; - void generate_test_file(const char *filename); - void write_test_files_async(task_tracker *tracker); - std::string name_node; - std::string backup_path; - std::string local_test_dir; - std::string test_data_str; + void generate_test_file(const std::string &filename); + void write_test_files_async(const std::string &local_test_path, task_tracker *tracker); }; -void HDFSClientTest::SetUp() +void HDFSClientTest::generate_test_file(const std::string &filename) { - name_node = FLAGS_test_name_node; - backup_path = FLAGS_test_backup_path; - local_test_dir = "test_dir"; - test_data_str = ""; - for (int i = 0; i < FLAGS_num_test_file_lines; ++i) { - test_data_str += "test"; - } -} - -void HDFSClientTest::TearDown() {} - -void HDFSClientTest::generate_test_file(const char *filename) -{ - // generate a local test file. int lines = FLAGS_num_test_file_lines; - FILE *fp = fopen(filename, "wb"); + std::unique_ptr wfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewWritableFile(filename, &wfile, rocksdb::EnvOptions()); + ASSERT_TRUE(s.ok()) << s.ToString(); for (int i = 0; i < lines; ++i) { - fprintf(fp, "%04d_this_is_a_simple_test_file\n", i); + rocksdb::Slice data(fmt::format("{:04}d_this_is_a_simple_test_file\n", i)); + s = wfile->Append(data); + ASSERT_TRUE(s.ok()) << s.ToString(); } - fclose(fp); + s = wfile->Fsync(); + ASSERT_TRUE(s.ok()) << s.ToString(); } -void HDFSClientTest::write_test_files_async(task_tracker *tracker) +void HDFSClientTest::write_test_files_async(const std::string &local_test_path, + task_tracker *tracker) { - dsn::utils::filesystem::create_directory(local_test_dir); + dsn::utils::filesystem::create_directory(local_test_path); + std::string local_test_data; + for (int i = 0; i < FLAGS_num_test_file_lines; ++i) { + local_test_data += "test"; + } for (int i = 0; i < 100; ++i) { - tasking::enqueue(LPC_TEST_HDFS, tracker, [this, i]() { + tasking::enqueue(LPC_TEST_HDFS, tracker, [&]() { // mock the writing process in hdfs_file_object::download(). - std::string test_file_name = local_test_dir + "/test_file_" + std::to_string(i); - std::ofstream out(test_file_name, std::ios::binary | std::ios::out | std::ios::trunc); - if (out.is_open()) { - out.write(test_data_str.c_str(), test_data_str.length()); - } - out.close(); + std::string test_file_name = local_test_path + "/test_file_" + std::to_string(i); + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(local_test_data), + test_file_name, + /* should_sync */ true); + CHECK(s.ok(), "{}", s.ToString()); }); } } -TEST_F(HDFSClientTest, test_basic_operation) +INSTANTIATE_TEST_CASE_P(, HDFSClientTest, ::testing::Values(false, true)); + +TEST_P(HDFSClientTest, test_hdfs_read_write) +{ + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; + return; + } + + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); + + const std::string kRemoteTestPath = "hdfs_client_test"; + const std::string kRemoteTestRWFile = kRemoteTestPath + "/test_write_file"; + const std::string kTestBuffer = "write_hello_world_for_test"; + const int kTestBufferLength = kTestBuffer.size(); + + // 1. clean up all old file in remote test directory. + printf("clean up all old files.\n"); + remove_path_response rem_resp; + s->remove_path(remove_path_request{kRemoteTestPath, true}, + LPC_TEST_HDFS, + [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, + nullptr) + ->wait(); + ASSERT_TRUE(dsn::ERR_OK == rem_resp.err || dsn::ERR_OBJECT_NOT_FOUND == rem_resp.err); + + // 2. create file. + printf("test write operation.\n"); + create_file_response cf_resp; + s->create_file(create_file_request{kRemoteTestRWFile, false}, + LPC_TEST_HDFS, + [&cf_resp](const create_file_response &r) { cf_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, cf_resp.err); + + // 3. write file. + dsn::blob bb(kTestBuffer.c_str(), 0, kTestBufferLength); + write_response w_resp; + cf_resp.file_handle + ->write(write_request{bb}, + LPC_TEST_HDFS, + [&w_resp](const write_response &w) { w_resp = w; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, w_resp.err); + ASSERT_EQ(kTestBufferLength, w_resp.written_size); + ASSERT_EQ(kTestBufferLength, cf_resp.file_handle->get_size()); + + // 4. read file. + printf("test read just written contents.\n"); + read_response r_resp; + cf_resp.file_handle + ->read(read_request{0, -1}, + LPC_TEST_HDFS, + [&r_resp](const read_response &r) { r_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, r_resp.err); + ASSERT_EQ(kTestBufferLength, r_resp.buffer.length()); + ASSERT_EQ(kTestBuffer, r_resp.buffer.to_string()); + + // 5. partial read. + const uint64_t kOffset = 5; + const int64_t kSize = 10; + cf_resp.file_handle + ->read(read_request{kOffset, kSize}, + LPC_TEST_HDFS, + [&r_resp](const read_response &r) { r_resp = r; }, + nullptr) + ->wait(); + ASSERT_EQ(dsn::ERR_OK, r_resp.err); + ASSERT_EQ(kSize, r_resp.buffer.length()); + ASSERT_EQ(kTestBuffer.substr(kOffset, kSize), r_resp.buffer.to_string()); +} + +TEST_P(HDFSClientTest, test_upload_and_download) { - if (name_node == example_name_node || backup_path == example_backup_path) { + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; return; } - std::vector args = {name_node, backup_path}; - std::shared_ptr s = std::make_shared(); - ASSERT_EQ(dsn::ERR_OK, s->initialize(args)); + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); - std::string local_test_file = "test_file"; - std::string remote_test_file = "hdfs_client_test/test_file"; - int64_t test_file_size = 0; + const std::string kLocalFile = "test_file"; + const std::string kRemoteTestPath = "hdfs_client_test"; + const std::string kRemoteTestFile = kRemoteTestPath + "/test_file"; - generate_test_file(local_test_file.c_str()); - dsn::utils::filesystem::file_size(local_test_file, test_file_size); + // 0. generate test file. + NO_FATALS(generate_test_file(kLocalFile)); + int64_t local_file_size = 0; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kLocalFile, dsn::utils::FileDataType::kSensitive, local_file_size)); + std::string local_file_md5sum; + dsn::utils::filesystem::md5sum(kLocalFile, local_file_md5sum); - // fisrt clean up all old file in test directory. + // 1. clean up all old file in remote test directory. printf("clean up all old files.\n"); remove_path_response rem_resp; - s->remove_path(remove_path_request{"hdfs_client_test", true}, + s->remove_path(remove_path_request{kRemoteTestPath, true}, LPC_TEST_HDFS, [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, nullptr) ->wait(); ASSERT_TRUE(dsn::ERR_OK == rem_resp.err || dsn::ERR_OBJECT_NOT_FOUND == rem_resp.err); - // test upload file. - printf("create and upload: %s.\n", remote_test_file.c_str()); + // 2. create file. + printf("create and upload: %s.\n", kRemoteTestFile.c_str()); create_file_response cf_resp; - s->create_file(create_file_request{remote_test_file, true}, + s->create_file(create_file_request{kRemoteTestFile, true}, LPC_TEST_HDFS, [&cf_resp](const create_file_response &r) { cf_resp = r; }, nullptr) ->wait(); - ASSERT_EQ(cf_resp.err, dsn::ERR_OK); + ASSERT_EQ(dsn::ERR_OK, cf_resp.err); + + // 3. upload file. upload_response u_resp; cf_resp.file_handle - ->upload(upload_request{local_test_file}, + ->upload(upload_request{kLocalFile}, LPC_TEST_HDFS, [&u_resp](const upload_response &r) { u_resp = r; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, u_resp.err); - ASSERT_EQ(test_file_size, cf_resp.file_handle->get_size()); + ASSERT_EQ(local_file_size, cf_resp.file_handle->get_size()); - // test list directory. + // 4. list directory. ls_response l_resp; - s->list_dir(ls_request{"hdfs_client_test"}, + s->list_dir(ls_request{kRemoteTestPath}, LPC_TEST_HDFS, [&l_resp](const ls_response &resp) { l_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, l_resp.err); ASSERT_EQ(1, l_resp.entries->size()); - ASSERT_EQ("test_file", l_resp.entries->at(0).entry_name); + ASSERT_EQ(kLocalFile, l_resp.entries->at(0).entry_name); ASSERT_EQ(false, l_resp.entries->at(0).is_directory); - // test download file. + // 5. download file. download_response d_resp; - printf("test download %s.\n", remote_test_file.c_str()); - s->create_file(create_file_request{remote_test_file, false}, + printf("test download %s.\n", kRemoteTestFile.c_str()); + s->create_file(create_file_request{kRemoteTestFile, false}, LPC_TEST_HDFS, [&cf_resp](const create_file_response &resp) { cf_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_EQ(test_file_size, cf_resp.file_handle->get_size()); - std::string local_file_for_download = "test_file_d"; + ASSERT_EQ(local_file_size, cf_resp.file_handle->get_size()); + std::string kLocalDownloadFile = "test_file_d"; cf_resp.file_handle - ->download(download_request{local_file_for_download, 0, -1}, + ->download(download_request{kLocalDownloadFile, 0, -1}, LPC_TEST_HDFS, [&d_resp](const download_response &resp) { d_resp = resp; }, nullptr) ->wait(); ASSERT_EQ(dsn::ERR_OK, d_resp.err); - ASSERT_EQ(test_file_size, d_resp.downloaded_size); - - // compare local_test_file and local_file_for_download. - int64_t file_size = 0; - dsn::utils::filesystem::file_size(local_file_for_download, file_size); - ASSERT_EQ(test_file_size, file_size); - std::string test_file_md5sum; - dsn::utils::filesystem::md5sum(local_test_file, test_file_md5sum); - std::string downloaded_file_md5sum; - dsn::utils::filesystem::md5sum(local_file_for_download, downloaded_file_md5sum); - ASSERT_EQ(test_file_md5sum, downloaded_file_md5sum); - - // test read and write. - printf("test read write operation.\n"); - std::string test_write_file = "hdfs_client_test/test_write_file"; - s->create_file(create_file_request{test_write_file, false}, - LPC_TEST_HDFS, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - const char *test_buffer = "write_hello_world_for_test"; - int length = strlen(test_buffer); - dsn::blob bb(test_buffer, 0, length); - write_response w_resp; - cf_resp.file_handle - ->write(write_request{bb}, - LPC_TEST_HDFS, - [&w_resp](const write_response &w) { w_resp = w; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, w_resp.err); - ASSERT_EQ(length, w_resp.written_size); - ASSERT_EQ(length, cf_resp.file_handle->get_size()); - printf("test read just written contents.\n"); - read_response r_resp; - cf_resp.file_handle - ->read(read_request{0, -1}, - LPC_TEST_HDFS, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, r_resp.err); - ASSERT_EQ(length, r_resp.buffer.length()); - ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer, length)); - - // test partitial read. - cf_resp.file_handle - ->read(read_request{5, 10}, - LPC_TEST_HDFS, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, r_resp.err); - ASSERT_EQ(10, r_resp.buffer.length()); - ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer + 5, 10)); - - // clean up local test files. - utils::filesystem::remove_path(local_test_file); - utils::filesystem::remove_path(local_file_for_download); + ASSERT_EQ(local_file_size, d_resp.downloaded_size); + + // 6. compare kLocalFile and kLocalDownloadFile. + // 6.1 check file size. + int64_t local_download_file_size = 0; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kLocalDownloadFile, dsn::utils::FileDataType::kSensitive, local_download_file_size)); + ASSERT_EQ(local_file_size, local_download_file_size); + // 6.2 check file md5sum. + std::string local_download_file_md5sum; + dsn::utils::filesystem::md5sum(kLocalDownloadFile, local_download_file_md5sum); + ASSERT_EQ(local_file_md5sum, local_download_file_md5sum); + + // 7. clean up local test files. + utils::filesystem::remove_path(kLocalFile); + utils::filesystem::remove_path(kLocalDownloadFile); } -TEST_F(HDFSClientTest, test_concurrent_upload_download) +TEST_P(HDFSClientTest, test_concurrent_upload_download) { - if (name_node == example_name_node || backup_path == example_backup_path) { + if (strlen(FLAGS_test_name_node) == 0 || strlen(FLAGS_test_backup_path) == 0) { + // TODO(yingchun): use GTEST_SKIP after upgrading gtest. + std::cout << "Set hdfs_test.* configs in config-test.ini to enable hdfs_service_test." + << std::endl; return; } - std::vector args = {name_node, backup_path}; - std::shared_ptr s = std::make_shared(); - ASSERT_EQ(dsn::ERR_OK, s->initialize(args)); + auto s = std::make_shared(); + ASSERT_EQ(dsn::ERR_OK, s->initialize({FLAGS_test_name_node, FLAGS_test_backup_path})); int total_files = FLAGS_num_total_files_for_hdfs_concurrent_test; std::vector local_file_names; @@ -282,11 +321,12 @@ TEST_F(HDFSClientTest, test_concurrent_upload_download) // generate test files. for (int i = 0; i < total_files; ++i) { std::string file_name = "randomfile" + std::to_string(i); - generate_test_file(file_name.c_str()); + NO_FATALS(generate_test_file(file_name)); int64_t file_size = 0; - dsn::utils::filesystem::file_size(file_name, file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + file_name, dsn::utils::FileDataType::kSensitive, file_size)); std::string md5sum; - dsn::utils::filesystem::md5sum(file_name, md5sum); + ASSERT_EQ(ERR_OK, dsn::utils::filesystem::md5sum(file_name, md5sum)); local_file_names.emplace_back(file_name); remote_file_names.emplace_back("hdfs_concurrent_test/" + file_name); @@ -385,13 +425,16 @@ TEST_F(HDFSClientTest, test_concurrent_upload_download) } } -TEST_F(HDFSClientTest, test_rename_path_while_writing) +TEST_P(HDFSClientTest, test_rename_path_while_writing) { + std::string kLocalTestPath = "test_dir"; + task_tracker tracker; - write_test_files_async(&tracker); + write_test_files_async(kLocalTestPath, &tracker); usleep(100); - std::string rename_dir = "rename_dir." + std::to_string(dsn_now_ms()); + + std::string kLocalRenamedTestPath = "rename_dir." + std::to_string(dsn_now_ms()); // rename succeed but writing failed. - ASSERT_TRUE(dsn::utils::filesystem::rename_path(local_test_dir, rename_dir)); + ASSERT_TRUE(dsn::utils::filesystem::rename_path(kLocalTestPath, kLocalRenamedTestPath)); tracker.cancel_outstanding_tasks(); } diff --git a/src/block_service/test/local_service_test.cpp b/src/block_service/test/local_service_test.cpp index e355a1b281..6f63f0ffa2 100644 --- a/src/block_service/test/local_service_test.cpp +++ b/src/block_service/test/local_service_test.cpp @@ -24,11 +24,17 @@ #include #include #include -#include +#include +#include +#include #include +#include +#include +#include #include #include "block_service/local/local_service.h" +#include "utils/env.h" #include "utils/error_code.h" namespace dsn { @@ -41,16 +47,19 @@ TEST(local_service, store_metadata) { local_file_object file("a.txt"); error_code ec = file.store_metadata(); - ASSERT_EQ(ec, ERR_OK); + ASSERT_EQ(ERR_OK, ec); auto meta_file_path = local_service::get_metafile(file.file_name()); ASSERT_TRUE(boost::filesystem::exists(meta_file_path)); - std::ifstream ifs(meta_file_path); - nlohmann::json j; - ifs >> j; - ASSERT_EQ(j["md5"], ""); - ASSERT_EQ(j["size"], 0); + std::string data; + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), meta_file_path, &data); + ASSERT_TRUE(s.ok()) << s.ToString(); + + nlohmann::json j = nlohmann::json::parse(data); + ASSERT_EQ("", j["md5"]); + ASSERT_EQ(0, j["size"]); } TEST(local_service, load_metadata) @@ -59,30 +68,41 @@ TEST(local_service, load_metadata) auto meta_file_path = local_service::get_metafile(file.file_name()); { - std::ofstream ofs(meta_file_path); nlohmann::json j({{"md5", "abcde"}, {"size", 5}}); - ofs << j; - ofs.close(); + std::string data = j.dump(); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(data), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); - ASSERT_EQ(file.load_metadata(), ERR_OK); - ASSERT_EQ(file.get_md5sum(), "abcde"); - ASSERT_EQ(file.get_size(), 5); + ASSERT_EQ(ERR_OK, file.load_metadata()); + ASSERT_EQ("abcde", file.get_md5sum()); + ASSERT_EQ(5, file.get_size()); } { - std::ofstream ofs(meta_file_path); - ofs << "invalid json string"; - ofs.close(); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice("invalid json string"), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); local_file_object file2("a.txt"); ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL); } { - std::ofstream ofs(meta_file_path); nlohmann::json j({{"md5", "abcde"}, {"no such key", "illegal"}}); - ofs << j; - ofs.close(); + std::string data = j.dump(); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(data), + meta_file_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); local_file_object file2("a.txt"); ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL); diff --git a/src/common/fs_manager.h b/src/common/fs_manager.h index 65fb0243ac..a8985f22ef 100644 --- a/src/common/fs_manager.h +++ b/src/common/fs_manager.h @@ -172,10 +172,10 @@ class fs_manager friend class replica_disk_migrator; friend class replica_disk_test_base; friend class open_replica_test; - FRIEND_TEST(fs_manager, find_best_dir_for_new_replica); - FRIEND_TEST(fs_manager, get_dir_node); + FRIEND_TEST(fs_manager_test, find_best_dir_for_new_replica); + FRIEND_TEST(fs_manager_test, get_dir_node); FRIEND_TEST(open_replica_test, open_replica_add_decree_and_ballot_check); - FRIEND_TEST(replica_error_test, test_auto_trash_of_corruption); + FRIEND_TEST(replica_test, test_auto_trash_of_corruption); }; } // replication } // dsn diff --git a/src/common/test/common_test.cpp b/src/common/test/common_test.cpp index dc2bdccaac..c2bf31713a 100644 --- a/src/common/test/common_test.cpp +++ b/src/common/test/common_test.cpp @@ -22,6 +22,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include namespace dsn { TEST(duplication_common, get_current_cluster_name) diff --git a/src/common/test/fs_manager_test.cpp b/src/common/test/fs_manager_test.cpp index 542b122f1c..6027c649d6 100644 --- a/src/common/test/fs_manager_test.cpp +++ b/src/common/test/fs_manager_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -33,6 +34,7 @@ #include "common/gpid.h" #include "common/replication_other_types.h" #include "metadata_types.h" +#include "test_util/test_util.h" #include "utils/fail_point.h" #include "utils/filesystem.h" @@ -47,7 +49,13 @@ TEST(dir_node, replica_dir) ASSERT_EQ("path/1.0.test", dn.replica_dir("test", gpid(1, 0))); } -TEST(fs_manager, initialize) +class fs_manager_test : public pegasus::encrypt_data_test_base +{ +}; + +INSTANTIATE_TEST_CASE_P(, fs_manager_test, ::testing::Values(false, true)); + +TEST_P(fs_manager_test, initialize) { fail::setup(); struct broken_disk_test @@ -69,7 +77,7 @@ TEST(fs_manager, initialize) fail::teardown(); } -TEST(fs_manager, dir_update_disk_status) +TEST_P(fs_manager_test, dir_update_disk_status) { struct update_disk_status { @@ -94,7 +102,7 @@ TEST(fs_manager, dir_update_disk_status) } } -TEST(fs_manager, get_dir_node) +TEST_P(fs_manager_test, get_dir_node) { fs_manager fm; fm.initialize({"./data1"}, {"data1"}); @@ -115,7 +123,7 @@ TEST(fs_manager, get_dir_node) ASSERT_EQ(nullptr, fm.get_dir_node(base_dir + "/data2/replica1")); } -TEST(fs_manager, find_replica_dir) +TEST_P(fs_manager_test, find_replica_dir) { fs_manager fm; fm.initialize({"./data1", "./data2", "./data3"}, {"data1", "data2", "data3"}); @@ -137,7 +145,7 @@ TEST(fs_manager, find_replica_dir) ASSERT_EQ(dn, dn1); } -TEST(fs_manager, create_replica_dir_if_necessary) +TEST_P(fs_manager_test, create_replica_dir_if_necessary) { fs_manager fm; @@ -154,7 +162,7 @@ TEST(fs_manager, create_replica_dir_if_necessary) ASSERT_EQ("data1", dn->tag); } -TEST(fs_manager, create_child_replica_dir) +TEST_P(fs_manager_test, create_child_replica_dir) { fs_manager fm; fm.initialize({"./data1", "./data2", "./data3"}, {"data1", "data2", "data3"}); @@ -174,7 +182,7 @@ TEST(fs_manager, create_child_replica_dir) ASSERT_EQ(dir, child_dir); } -TEST(fs_manager, find_best_dir_for_new_replica) +TEST_P(fs_manager_test, find_best_dir_for_new_replica) { // dn1 | 1.0, 1.1 +1.6 // dn2 | 1.2, 1.3 +1.7 2.0 diff --git a/src/geo/bench/bench.cpp b/src/geo/bench/bench.cpp index e65714b7ad..18324e7673 100644 --- a/src/geo/bench/bench.cpp +++ b/src/geo/bench/bench.cpp @@ -34,6 +34,7 @@ #include "geo/lib/geo_client.h" #include "geo/lib/latlng_codec.h" +#include "utils/env.h" #include "utils/errors.h" #include "utils/fmt_logging.h" #include "utils/string_conv.h" @@ -76,10 +77,12 @@ int main(int argc, char **argv) } } + // TODO(yingchun): the benchmark can not exit normally, we need to fix it later. pegasus::geo::geo_client my_geo( "config.ini", cluster_name.c_str(), app_name.c_str(), geo_app_name.c_str()); - if (!my_geo.set_max_level(max_level).is_ok()) { - std::cerr << "set_max_level failed" << std::endl; + auto err = my_geo.set_max_level(max_level); + if (!err.is_ok()) { + std::cerr << "set_max_level failed, err: " << err << std::endl; return -1; } @@ -108,7 +111,7 @@ int main(int argc, char **argv) RESULT_COUNT }; auto statistics = rocksdb::CreateDBStatistics(); - rocksdb::Env *env = rocksdb::Env::Default(); + rocksdb::Env *env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); uint64_t start = env->NowNanos(); std::atomic count(test_count); dsn::utils::notify_event get_completed; diff --git a/src/geo/lib/geo_client.cpp b/src/geo/lib/geo_client.cpp index 335afda62e..0d4927695e 100644 --- a/src/geo/lib/geo_client.cpp +++ b/src/geo/lib/geo_client.cpp @@ -32,7 +32,6 @@ #include #include #include -#include #include #include #include diff --git a/src/geo/lib/latlng_codec.cpp b/src/geo/lib/latlng_codec.cpp index 8a74618bc6..171db59c58 100644 --- a/src/geo/lib/latlng_codec.cpp +++ b/src/geo/lib/latlng_codec.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "utils/error_code.h" #include "utils/errors.h" diff --git a/src/geo/test/geo_test.cpp b/src/geo/test/geo_test.cpp index f0777a9b03..e9df7a422a 100644 --- a/src/geo/test/geo_test.cpp +++ b/src/geo/test/geo_test.cpp @@ -18,6 +18,7 @@ */ #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -55,6 +56,10 @@ namespace geo { DSN_DECLARE_int32(min_level); +// TODO(yingchun): it doesn't make sense to derive from pegasus::encrypt_data_test_base to test +// encryption or non-encryption senarios, because the Pegasus cluster has been started with a +// fixed value of FLAGS_encrypt_data_at_rest. +// We can test the senarios after clearing and restarting the cluster. class geo_client_test : public ::testing::Test { public: diff --git a/src/http/http_server.h b/src/http/http_server.h index b182947e41..c84055261c 100644 --- a/src/http/http_server.h +++ b/src/http/http_server.h @@ -148,6 +148,8 @@ class http_server_base : public http_service // ``` extern http_call ®ister_http_call(std::string full_path); +extern void deregister_http_call(const std::string &full_path); + // Starts serving HTTP requests. // The internal HTTP server will reuse the rDSN server port. extern void start_http_server(); diff --git a/src/meta/distributed_lock_service_simple.h b/src/meta/distributed_lock_service_simple.h index a2da463b49..59b2bde5e6 100644 --- a/src/meta/distributed_lock_service_simple.h +++ b/src/meta/distributed_lock_service_simple.h @@ -53,6 +53,7 @@ namespace dsn { namespace dist { +// Only for test purpose. class distributed_lock_service_simple : public distributed_lock_service { public: diff --git a/src/meta/dump_file.h b/src/meta/dump_file.h index 4e8018ca58..062ffd1fcc 100644 --- a/src/meta/dump_file.h +++ b/src/meta/dump_file.h @@ -63,6 +63,8 @@ struct block_header uint32_t crc32; }; +// TODO(yingchun): use rocksdb APIs to unify the file operations. +// A tool to dump app_states of meta server to local file, used by remote command "meta.dump". class dump_file { public: diff --git a/src/meta/meta_backup_service.h b/src/meta/meta_backup_service.h index e6563dbcc2..6c1e8be95a 100644 --- a/src/meta/meta_backup_service.h +++ b/src/meta/meta_backup_service.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include #include @@ -46,6 +47,7 @@ namespace dsn { class message_ex; class rpc_address; + namespace dist { namespace block_service { class block_filesystem; diff --git a/src/meta/meta_bulk_load_service.h b/src/meta/meta_bulk_load_service.h index ba151757a8..c411d87f44 100644 --- a/src/meta/meta_bulk_load_service.h +++ b/src/meta/meta_bulk_load_service.h @@ -97,6 +97,10 @@ struct bulk_load_info int32_t app_id; std::string app_name; int32_t partition_count; + bulk_load_info(int32_t id = 0, const std::string &name = "", int32_t pcount = 0) + : app_id(id), app_name(name), partition_count(pcount) + { + } DEFINE_JSON_SERIALIZATION(app_id, app_name, partition_count) }; diff --git a/src/meta/meta_state_service_simple.cpp b/src/meta/meta_state_service_simple.cpp index f2db0da0a7..aa54612032 100644 --- a/src/meta/meta_state_service_simple.cpp +++ b/src/meta/meta_state_service_simple.cpp @@ -26,8 +26,6 @@ #include "meta_state_service_simple.h" -#include -#include #include #include #include @@ -35,14 +33,17 @@ #include #include "aio/file_io.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/service_app.h" #include "runtime/task/async_calls.h" #include "runtime/task/task.h" #include "utils/autoref_ptr.h" #include "utils/binary_reader.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/strings.h" #include "utils/utils.h" @@ -228,7 +229,7 @@ error_code meta_state_service_simple::apply_transaction( default: CHECK(false, "unsupported operation"); } - CHECK_EQ_MSG(ec, ERR_OK, "unexpected error when applying"); + CHECK_EQ_MSG(ERR_OK, ec, "unexpected error when applying"); } return ERR_OK; @@ -242,57 +243,77 @@ error_code meta_state_service_simple::initialize(const std::vector _offset = 0; std::string log_path = dsn::utils::filesystem::path_combine(work_dir, "meta_state_service.log"); if (utils::filesystem::file_exists(log_path)) { - if (FILE *fd = fopen(log_path.c_str(), "rb")) { - for (;;) { - log_header header; - if (fread(&header, sizeof(log_header), 1, fd) != 1) { - break; - } - if (header.magic != log_header::default_magic) { - break; - } - std::shared_ptr buffer(dsn::utils::make_shared_array(header.size)); - if (fread(buffer.get(), header.size, 1, fd) != 1) { - break; - } - _offset += sizeof(header) + header.size; - binary_reader reader(blob(buffer, (int)header.size)); - int op_type = 0; - reader.read(op_type); - - switch (static_cast(op_type)) { - case operation_type::create_node: { - std::string node; - blob data; - create_node_log::parse(reader, node, data); - create_node_internal(node, data); - break; - } - case operation_type::delete_node: { - std::string node; - bool recursively_delete; - delete_node_log::parse(reader, node, recursively_delete); - delete_node_internal(node, recursively_delete); - break; - } - case operation_type::set_data: { - std::string node; - blob data; - set_data_log::parse(reader, node, data); - set_data_internal(node, data); - break; - } - default: - // The log is complete but its content is modified by cosmic ray. This is - // unacceptable - CHECK(false, "meta state server log corrupted"); - } + std::unique_ptr log_file; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(log_path, &log_file, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", log_path, s.ToString()); + + while (true) { + static const int kLogHeaderSize = sizeof(log_header); + static const int kDefaultMagic = 0xdeadbeef; + rocksdb::Slice result; + + // Read header. + char scratch[kLogHeaderSize] = {0}; + s = log_file->PositionedRead(_offset, kLogHeaderSize, &result, scratch); + CHECK(s.ok(), "read log file '{}' header failed, err = {}", log_path, s.ToString()); + if (result.empty()) { + LOG_INFO("read EOF of log file '{}'", log_path); + break; + } + log_header *header = reinterpret_cast(scratch); + if (header->magic != kDefaultMagic) { + LOG_WARNING("read log file '{}' header with bad magic {}, skip the left!", + log_path, + header->magic); + break; + } + _offset += kLogHeaderSize; + + // Read body. + std::shared_ptr buffer(dsn::utils::make_shared_array(header->size)); + s = log_file->PositionedRead(_offset, header->size, &result, buffer.get()); + CHECK(s.ok(), + "read log file '{}' header with bad body, err = {}", + log_path, + s.ToString()); + _offset += header->size; + + binary_reader reader(blob(buffer, header->size)); + int op_type = 0; + CHECK_EQ(sizeof(op_type), reader.read(op_type)); + + switch (static_cast(op_type)) { + case operation_type::create_node: { + std::string node; + blob data; + create_node_log::parse(reader, node, data); + create_node_internal(node, data); + break; + } + case operation_type::delete_node: { + std::string node; + bool recursively_delete; + delete_node_log::parse(reader, node, recursively_delete); + delete_node_internal(node, recursively_delete); + break; + } + case operation_type::set_data: { + std::string node; + blob data; + set_data_log::parse(reader, node, data); + set_data_internal(node, data); + break; + } + default: + // The log is complete but its content is modified by cosmic ray. This is + // unacceptable + CHECK(false, "meta state server log corrupted"); } - fclose(fd); } } - _log = file::open(log_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + _log = file::open(log_path, file::FileOpenType::kWriteOnly); if (!_log) { LOG_ERROR("open file failed: {}", log_path); return ERR_FILE_OPERATION_FAILED; @@ -507,8 +528,9 @@ task_ptr meta_state_service_simple::get_children(const std::string &node, meta_state_service_simple::~meta_state_service_simple() { - _tracker.cancel_outstanding_tasks(); - file::close(_log); + _tracker.wait_outstanding_tasks(); + CHECK_EQ(ERR_OK, file::flush(_log)); + CHECK_EQ(ERR_OK, file::close(_log)); for (const auto &kv : _quick_map) { if ("/" != kv.first) { diff --git a/src/meta/meta_state_service_simple.h b/src/meta/meta_state_service_simple.h index f26e33e174..697b5b159a 100644 --- a/src/meta/meta_state_service_simple.h +++ b/src/meta/meta_state_service_simple.h @@ -66,6 +66,7 @@ DEFINE_TASK_CODE_AIO(LPC_META_STATE_SERVICE_SIMPLE_INTERNAL, TASK_PRIORITY_HIGH, THREAD_POOL_DEFAULT); +// Only for test purpose. class meta_state_service_simple : public meta_state_service { public: diff --git a/src/meta/test/main.cpp b/src/meta/test/main.cpp index fd82dd1779..e410572bcc 100644 --- a/src/meta/test/main.cpp +++ b/src/meta/test/main.cpp @@ -63,7 +63,11 @@ TEST(meta, state_sync) { g_app->state_sync_test(); } TEST(meta, update_configuration) { g_app->update_configuration_test(); } -TEST(meta, balancer_validator) { g_app->balancer_validator(); } +TEST(meta, balancer_validator) +{ + // TODO(yingchun): this test last too long time, optimize it! + g_app->balancer_validator(); +} TEST(meta, apply_balancer) { g_app->apply_balancer_test(); } diff --git a/src/meta/test/meta_backup_test.cpp b/src/meta/test/meta_backup_test.cpp index 212da5aea0..3692e0724f 100644 --- a/src/meta/test/meta_backup_test.cpp +++ b/src/meta/test/meta_backup_test.cpp @@ -38,6 +38,7 @@ #include "meta_test_base.h" #include "runtime/api_layer1.h" #include "runtime/rpc/rpc_address.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/fail_point.h" #include "utils/filesystem.h" @@ -108,7 +109,8 @@ class backup_service_test : public meta_test_base cold_backup::get_app_metadata_file(backup_root, app->app_name, app_id, backup_id); int64_t metadata_file_size = 0; - if (!dsn::utils::filesystem::file_size(metadata_file, metadata_file_size)) { + if (!dsn::utils::filesystem::file_size( + metadata_file, dsn::utils::FileDataType::kSensitive, metadata_file_size)) { return false; } return metadata_file_size > 0; diff --git a/src/meta/test/meta_service_test.cpp b/src/meta/test/meta_service_test.cpp index bf993c0dde..ba57043940 100644 --- a/src/meta/test/meta_service_test.cpp +++ b/src/meta/test/meta_service_test.cpp @@ -28,7 +28,6 @@ #include "meta/meta_service.h" #include "meta_admin_types.h" #include "meta_test_base.h" -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/meta/test/meta_state/meta_state_service.cpp b/src/meta/test/meta_state/meta_state_service.cpp index a286c5067a..2bc59246dc 100644 --- a/src/meta/test/meta_state/meta_state_service.cpp +++ b/src/meta/test/meta_state/meta_state_service.cpp @@ -27,6 +27,7 @@ #include "meta/meta_state_service.h" #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -36,12 +37,18 @@ #include "meta/meta_state_service_simple.h" #include "meta/meta_state_service_zookeeper.h" +#include "runtime/service_app.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" +#include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/threadpool_code.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace dsn; using namespace dsn::dist; @@ -50,8 +57,8 @@ DEFINE_TASK_CODE(META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, TASK_PRIORITY_HIGH, TH typedef std::function service_creator_func; typedef std::function service_deleter_func; -#define expect_ok [](error_code ec) { EXPECT_TRUE(ec == ERR_OK); } -#define expect_err [](error_code ec) { EXPECT_FALSE(ec == ERR_OK); } +#define expect_ok [](error_code ec) { CHECK_EQ(ERR_OK, ec); } +#define expect_err [](error_code ec) { CHECK_NE(ERR_OK, ec); } void provider_basic_test(const service_creator_func &service_creator, const service_deleter_func &service_deleter) @@ -70,9 +77,9 @@ void provider_basic_test(const service_creator_func &service_creator, service->get_children("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const std::vector &children) { - CHECK(ec == ERR_OK && children.size() == 1 && - *children.begin() == "1", - "unexpected child"); + CHECK_EQ(ERR_OK, ec); + CHECK_EQ(1, children.size()); + CHECK_EQ("1", *children.begin()); }); service->node_exist("/1/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok)->wait(); service->delete_node("/1", false, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) @@ -107,11 +114,11 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const dsn::blob &value) { - expect_ok(ec); + CHECK_EQ(ERR_OK, ec); dsn::binary_reader reader(value); int read_value = 0; reader.read(read_value); - CHECK_EQ(read_value, 0xdeadbeef); + CHECK_EQ(0xdeadbeef, read_value); }) ->wait(); writer = dsn::binary_writer(); @@ -124,27 +131,26 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/1", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const dsn::blob &value) { - expect_ok(ec); + CHECK_EQ(ERR_OK, ec); dsn::binary_reader reader(value); int read_value = 0; reader.read(read_value); - CHECK_EQ(read_value, 0xbeefdead); + CHECK_EQ(0xbeefdead, read_value); }) ->wait(); } - // clean the node created in previos code-block, to support test in next round + // clean the node created in previous code-block, to support test in next round { service->delete_node("/1", false, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok) ->wait(); } - typedef dsn::dist::meta_state_service::transaction_entries TEntries; // transaction op { // basic dsn::binary_writer writer; writer.write(0xdeadbeef); - std::shared_ptr entries = service->new_transaction_entries(5); + auto entries = service->new_transaction_entries(5); entries->create_node("/2"); entries->create_node("/2/2"); entries->create_node("/2/3"); @@ -155,11 +161,11 @@ void provider_basic_test(const service_creator_func &service_creator, entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok); tsk->wait(); for (unsigned int i = 0; i < 5; ++i) { - EXPECT_TRUE(entries->get_result(i) == ERR_OK); + ASSERT_EQ(ERR_OK, entries->get_result(i)); } // an invalid operation will stop whole transaction - entries = service->new_transaction_entries(5); + entries = service->new_transaction_entries(4); entries->create_node("/3"); entries->create_node("/4"); entries->delete_node("/2"); // delete a non empty dir @@ -168,11 +174,12 @@ void provider_basic_test(const service_creator_func &service_creator, service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) ->wait(); error_code err[4] = {ERR_OK, ERR_OK, ERR_INVALID_PARAMETERS, ERR_INCONSISTENT_STATE}; - for (unsigned int i = 0; i < 4; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 4; ++i) { + ASSERT_EQ(err[i], entries->get_result(i)); + } // another invalid transaction - entries = service->new_transaction_entries(5); + entries = service->new_transaction_entries(4); entries->create_node("/3"); entries->create_node("/4"); entries->delete_node("/5"); // delete a non exist dir @@ -182,8 +189,9 @@ void provider_basic_test(const service_creator_func &service_creator, err[2] = ERR_OBJECT_NOT_FOUND; service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_err) ->wait(); - for (unsigned int i = 0; i < 4; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 4; ++i) { + ASSERT_EQ(err[i], entries->get_result(i)); + } } // check replay with transaction @@ -195,7 +203,9 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_children("/2", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const std::vector &children) { - ASSERT_TRUE(children.size() == 1 && children[0] == "2"); + CHECK_EQ(ERR_OK, ec); + CHECK_EQ(1, children.size()); + CHECK_EQ("2", children[0]); }) ->wait(); @@ -203,27 +213,27 @@ void provider_basic_test(const service_creator_func &service_creator, ->get_data("/2", META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, [](error_code ec, const blob &value) { - ASSERT_TRUE(ec == ERR_OK); + CHECK_EQ(ERR_OK, ec); binary_reader reader(value); int content_value; reader.read(content_value); - ASSERT_TRUE(content_value == 0xdeadbeef); + CHECK_EQ(0xdeadbeef, content_value); }) ->wait(); } // delete the nodes created just now, using transaction delete { - std::shared_ptr entries = service->new_transaction_entries(2); + auto entries = service->new_transaction_entries(2); entries->delete_node("/2/2"); entries->delete_node("/2"); service->submit_transaction(entries, META_STATE_SERVICE_SIMPLE_TEST_CALLBACK, expect_ok) ->wait(); - error_code err[2] = {ERR_OK, ERR_OK}; - for (unsigned int i = 0; i < 2; ++i) - EXPECT_EQ(err[i], entries->get_result(i)); + for (unsigned int i = 0; i < 2; ++i) { + ASSERT_EQ(ERR_OK, entries->get_result(i)); + } } service_deleter(service); @@ -235,7 +245,7 @@ void recursively_create_node_callback(meta_state_service *service, int current_layer, error_code ec) { - ASSERT_TRUE(ec == ERR_OK); + ASSERT_EQ(ERR_OK, ec); if (current_layer <= 0) return; @@ -279,31 +289,40 @@ void provider_recursively_create_delete_test(const service_creator_func &creator deleter(service); } -#undef expect_ok -#undef expect_err +class meta_state_service_test : public pegasus::encrypt_data_test_base +{ +}; + +INSTANTIATE_TEST_CASE_P(, meta_state_service_test, ::testing::Values(false, true)); -TEST(meta_state_service, simple) +TEST_P(meta_state_service_test, simple) { auto simple_service_creator = [] { meta_state_service_simple *svc = new meta_state_service_simple(); - svc->initialize({}); + auto err = svc->initialize({}); + CHECK_EQ(ERR_OK, err); return svc; }; auto simple_service_deleter = [](meta_state_service *simple_svc) { delete simple_svc; }; provider_basic_test(simple_service_creator, simple_service_deleter); provider_recursively_create_delete_test(simple_service_creator, simple_service_deleter); + + std::string log_path = dsn::utils::filesystem::path_combine( + service_app::current_service_app_info().data_dir, "meta_state_service.log"); + ASSERT_TRUE(dsn::utils::filesystem::remove_path(log_path)); } -TEST(meta_state_service, zookeeper) +TEST_P(meta_state_service_test, zookeeper) { auto zookeeper_service_creator = [] { meta_state_service_zookeeper *svc = new meta_state_service_zookeeper(); - svc->initialize({}); + auto err = svc->initialize({}); + CHECK_EQ(ERR_OK, err); return svc; }; auto zookeeper_service_deleter = [](meta_state_service *zookeeper_svc) { - ASSERT_EQ(zookeeper_svc->finalize(), ERR_OK); + ASSERT_EQ(ERR_OK, zookeeper_svc->finalize()); }; provider_basic_test(zookeeper_service_creator, zookeeper_service_deleter); diff --git a/src/nfs/nfs_client_impl.cpp b/src/nfs/nfs_client_impl.cpp index c0ced7b8c6..32f87d72d4 100644 --- a/src/nfs/nfs_client_impl.cpp +++ b/src/nfs/nfs_client_impl.cpp @@ -26,8 +26,7 @@ #include "nfs_client_impl.h" -// IWYU pragma: no_include -#include +#include #include #include "nfs/nfs_code_definition.h" @@ -38,7 +37,6 @@ #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" -#include "utils/ports.h" #include "utils/string_conv.h" #include "utils/token_buckets.h" @@ -460,8 +458,7 @@ void nfs_client_impl::continue_write() // double check zauto_lock l(fc->user_req->user_req_lock); if (!fc->file_holder->file_handle) { - fc->file_holder->file_handle = - file::open(file_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + fc->file_holder->file_handle = file::open(file_path, file::FileOpenType::kWriteOnly); } } @@ -470,6 +467,10 @@ void nfs_client_impl::continue_write() LOG_ERROR("open file {} failed", file_path); handle_completion(fc->user_req, ERR_FILE_OPERATION_FAILED); } else { + LOG_DEBUG("nfs: copy to file {} [{}, {}]", + file_path, + reqc->response.offset, + reqc->response.offset + reqc->response.size); zauto_lock l(reqc->lock); if (reqc->is_valid) { reqc->local_write_task = file::write(fc->file_holder->file_handle, diff --git a/src/nfs/nfs_node_impl.cpp b/src/nfs/nfs_node_simple.cpp similarity index 93% rename from src/nfs/nfs_node_impl.cpp rename to src/nfs/nfs_node_simple.cpp index 387547fd88..bb334a5085 100644 --- a/src/nfs/nfs_node_impl.cpp +++ b/src/nfs/nfs_node_simple.cpp @@ -80,12 +80,17 @@ void nfs_node_simple::register_async_rpc_handler_for_test() error_code nfs_node_simple::stop() { - delete _server; - _server = nullptr; + if (_server != nullptr) { + _server->close_service(); - delete _client; - _client = nullptr; + delete _server; + _server = nullptr; + } + if (_client != nullptr) { + delete _client; + _client = nullptr; + } return ERR_OK; } diff --git a/src/nfs/nfs_node_simple.h b/src/nfs/nfs_node_simple.h index 2376b1e34e..15e2344168 100644 --- a/src/nfs/nfs_node_simple.h +++ b/src/nfs/nfs_node_simple.h @@ -34,14 +34,24 @@ */ #pragma once -#include "runtime/tool_api.h" +#include + #include "nfs/nfs_node.h" +#include "utils/error_code.h" namespace dsn { +class aio_task; +template +class rpc_replier; + namespace service { -class nfs_service_impl; +class copy_request; +class copy_response; +class get_file_size_request; +class get_file_size_response; class nfs_client_impl; +class nfs_service_impl; class nfs_node_simple : public nfs_node { diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index cadacba1a3..c4bcf7a2e4 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -26,12 +26,10 @@ #include "nfs/nfs_server_impl.h" -#include -#include -#include #include #include #include +#include #include #include "nfs/nfs_code_definition.h" @@ -39,10 +37,9 @@ #include "runtime/api_layer1.h" #include "runtime/task/async_calls.h" #include "utils/TokenBucket.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/flags.h" -#include "utils/ports.h" -#include "utils/safe_strerror_posix.h" #include "utils/string_conv.h" #include "utils/utils.h" @@ -90,38 +87,35 @@ void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, dsn::utils::filesystem::path_combine(request.source_dir, request.file_name); disk_file *dfile = nullptr; - { + do { zauto_lock l(_handles_map_lock); auto it = _handles_map.find(file_path); // find file handle cache first - if (it == _handles_map.end()) { - dfile = file::open(file_path.c_str(), O_RDONLY | O_BINARY, 0); - if (dfile != nullptr) { - auto fh = std::make_shared(); - fh->file_handle = dfile; - fh->file_access_count = 1; - fh->last_access_time = dsn_now_ms(); - _handles_map.insert(std::make_pair(file_path, std::move(fh))); + dfile = file::open(file_path, file::FileOpenType::kReadOnly); + if (dfile == nullptr) { + LOG_ERROR("[nfs_service] open file {} failed", file_path); + ::dsn::service::copy_response resp; + resp.error = ERR_OBJECT_NOT_FOUND; + reply(resp); + return; } - } else { - dfile = it->second->file_handle; - it->second->file_access_count++; - it->second->last_access_time = dsn_now_ms(); - } - } - - LOG_DEBUG( - "nfs: copy file {} [{}, {}]", file_path, request.offset, request.offset + request.size); - if (dfile == nullptr) { - LOG_ERROR("[nfs_service] open file {} failed", file_path); - ::dsn::service::copy_response resp; - resp.error = ERR_OBJECT_NOT_FOUND; - reply(resp); - return; - } - - std::shared_ptr cp = std::make_shared(std::move(reply)); + auto fh = std::make_shared(); + fh->file_handle = dfile; + it = _handles_map.insert(std::make_pair(file_path, std::move(fh))).first; + } + dfile = it->second->file_handle; + it->second->file_access_count++; + it->second->last_access_time = dsn_now_ms(); + } while (false); + + CHECK_NOTNULL(dfile, ""); + LOG_DEBUG("nfs: copy from file {} [{}, {}]", + file_path, + request.offset, + request.offset + request.size); + + auto cp = std::make_shared(std::move(reply)); cp->bb = blob(dsn::utils::make_shared_array(request.size), request.size); cp->dst_dir = request.dst_dir; cp->source_disk_tag = request.source_disk_tag; @@ -182,58 +176,53 @@ void nfs_service_impl::on_get_file_size( { get_file_size_response resp; error_code err = ERR_OK; - std::vector file_list; std::string folder = request.source_dir; + // TODO(yingchun): refactor the following code! if (request.file_list.size() == 0) // return all file size in the destination file folder { if (!dsn::utils::filesystem::directory_exists(folder)) { LOG_ERROR("[nfs_service] directory {} not exist", folder); err = ERR_OBJECT_NOT_FOUND; } else { + std::vector file_list; if (!dsn::utils::filesystem::get_subfiles(folder, file_list, true)) { LOG_ERROR("[nfs_service] get subfiles of directory {} failed", folder); err = ERR_FILE_OPERATION_FAILED; } else { - for (auto &fpath : file_list) { - // TODO: using uint64 instead as file ma - // Done + for (const auto &fpath : file_list) { int64_t sz; - if (!dsn::utils::filesystem::file_size(fpath, sz)) { + // TODO(yingchun): check if there are any files that are not sensitive (not + // encrypted). + if (!dsn::utils::filesystem::file_size( + fpath, dsn::utils::FileDataType::kSensitive, sz)) { LOG_ERROR("[nfs_service] get size of file {} failed", fpath); err = ERR_FILE_OPERATION_FAILED; break; } - resp.size_list.push_back((uint64_t)sz); + resp.size_list.push_back(sz); resp.file_list.push_back( fpath.substr(request.source_dir.length(), fpath.length() - 1)); } - file_list.clear(); } } } else // return file size in the request file folder { - for (size_t i = 0; i < request.file_list.size(); i++) { - std::string file_path = - dsn::utils::filesystem::path_combine(folder, request.file_list[i]); - - struct stat st; - if (0 != ::stat(file_path.c_str(), &st)) { - LOG_ERROR("[nfs_service] get stat of file {} failed, err = {}", - file_path, - dsn::utils::safe_strerror(errno)); - err = ERR_OBJECT_NOT_FOUND; + for (const auto &file_name : request.file_list) { + std::string file_path = dsn::utils::filesystem::path_combine(folder, file_name); + int64_t sz; + // TODO(yingchun): check if there are any files that are not sensitive (not encrypted). + if (!dsn::utils::filesystem::file_size( + file_path, dsn::utils::FileDataType::kSensitive, sz)) { + LOG_ERROR("[nfs_service] get size of file {} failed", file_path); + err = ERR_FILE_OPERATION_FAILED; break; } - // TODO: using int64 instead as file may exceed the size of 32bit - // Done - uint64_t size = st.st_size; - - resp.size_list.push_back(size); - resp.file_list.push_back((folder + request.file_list[i]) - .substr(request.source_dir.length(), - (folder + request.file_list[i]).length() - 1)); + resp.size_list.push_back(sz); + resp.file_list.push_back( + (folder + file_name) + .substr(request.source_dir.length(), (folder + file_name).length() - 1)); } } @@ -253,8 +242,9 @@ void nfs_service_impl::close_file() // release out-of-date file handle dsn_now_ms() - fptr->last_access_time > (uint64_t)FLAGS_file_close_expire_time_ms) { LOG_DEBUG("nfs: close file handle {}", it->first); it = _handles_map.erase(it); - } else + } else { it++; + } } } diff --git a/src/nfs/nfs_server_impl.h b/src/nfs/nfs_server_impl.h index 9ba1134040..ece68ecb33 100644 --- a/src/nfs/nfs_server_impl.h +++ b/src/nfs/nfs_server_impl.h @@ -66,7 +66,6 @@ class nfs_service_impl : public ::dsn::serverlet void register_cli_commands(); - // TODO(yingchun): seems nobody call it, can be removed? void close_service() { unregister_rpc_handler(RPC_NFS_COPY); @@ -107,14 +106,9 @@ class nfs_service_impl : public ::dsn::serverlet struct file_handle_info_on_server { - disk_file *file_handle; - int32_t file_access_count; // concurrent r/w count - uint64_t last_access_time; // last touch time - - file_handle_info_on_server() - : file_handle(nullptr), file_access_count(0), last_access_time(0) - { - } + disk_file *file_handle = nullptr; + int32_t file_access_count = 0; // concurrent r/w count + uint64_t last_access_time = 0; // last touch time ~file_handle_info_on_server() { diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt index 735bb29bd6..d3f26cc006 100644 --- a/src/nfs/test/CMakeLists.txt +++ b/src/nfs/test/CMakeLists.txt @@ -33,7 +33,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio) +set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio rocksdb test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/nfs/test/main.cpp b/src/nfs/test/main.cpp index 445c4732fe..ca5aa48670 100644 --- a/src/nfs/test/main.cpp +++ b/src/nfs/test/main.cpp @@ -24,13 +24,18 @@ * THE SOFTWARE. */ +#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include -#include +#include +#include #include #include +#include #include #include "aio/aio_task.h" @@ -40,11 +45,16 @@ #include "runtime/rpc/rpc_address.h" #include "runtime/task/task_code.h" #include "runtime/tool_api.h" +#include "test_util/test_util.h" #include "utils/autoref_ptr.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/threadpool_code.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace dsn; DEFINE_TASK_CODE_AIO(LPC_AIO_TEST_NFS, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) @@ -54,36 +64,61 @@ struct aio_result size_t sz; }; -TEST(nfs, basic) +class nfs_test : public pegasus::encrypt_data_test_base +{ +}; + +INSTANTIATE_TEST_CASE_P(, nfs_test, ::testing::Values(false, true)); + +TEST_P(nfs_test, basic) { - std::unique_ptr nfs(dsn::nfs_node::create()); + auto nfs = dsn::nfs_node::create(); nfs->start(); nfs->register_async_rpc_handler_for_test(); dsn::gpid fake_pid = gpid(1, 0); - utils::filesystem::remove_path("nfs_test_dir"); - utils::filesystem::remove_path("nfs_test_dir_copy"); + // Prepare the destination directory. + std::string dst_dir = "nfs_test_dir"; + ASSERT_TRUE(utils::filesystem::remove_path(dst_dir)); + ASSERT_FALSE(utils::filesystem::directory_exists(dst_dir)); + ASSERT_TRUE(utils::filesystem::create_directory(dst_dir)); + ASSERT_TRUE(utils::filesystem::directory_exists(dst_dir)); - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir")); - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir_copy")); - - ASSERT_TRUE(utils::filesystem::create_directory("nfs_test_dir")); - ASSERT_TRUE(utils::filesystem::directory_exists("nfs_test_dir")); + // Prepare the source files information. + std::vector src_filenames({"nfs_test_file1", "nfs_test_file2"}); + if (FLAGS_encrypt_data_at_rest) { + for (int i = 0; i < src_filenames.size(); i++) { + auto s = dsn::utils::encrypt_file(src_filenames[i], src_filenames[i] + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + src_filenames[i] += ".encrypted"; + } + } + std::vector src_file_sizes; + std::vector src_file_md5s; + for (const auto &src_filename : src_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + src_filename, dsn::utils::FileDataType::kSensitive, file_size)); + src_file_sizes.push_back(file_size); + std::string src_file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(src_filename, src_file_md5)); + src_file_md5s.emplace_back(std::move(src_file_md5)); + } + // copy files to the destination directory. { - // copy nfs_test_file1 nfs_test_file2 nfs_test_dir - ASSERT_FALSE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_FALSE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - std::vector files{"nfs_test_file1", "nfs_test_file2"}; + // The destination directory is empty before copying. + std::vector dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(dst_dir, dst_filenames, true)); + ASSERT_TRUE(dst_filenames.empty()); aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_files(dsn::rpc_address("localhost", 20101), "default", ".", - files, + src_filenames, "default", - "nfs_test_dir", + dst_dir, fake_pid, false, false, @@ -100,32 +135,32 @@ TEST(nfs, basic) ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - int64_t sz1, sz2; - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_file1", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file1", sz2)); - ASSERT_EQ(sz1, sz2); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_file2", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file2", sz2)); - ASSERT_EQ(sz1, sz2); + // The destination files equal to the source files after copying. + ASSERT_TRUE(utils::filesystem::get_subfiles(dst_dir, dst_filenames, true)); + std::sort(dst_filenames.begin(), dst_filenames.end()); + ASSERT_EQ(src_filenames.size(), dst_filenames.size()); + int i = 0; + for (const auto &dst_filename : dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } + // copy files to the destination directory, files will be overwritten. { - // copy files again, overwrite - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir/nfs_test_file2")); - - std::vector files{"nfs_test_file1", "nfs_test_file2"}; - aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_files(dsn::rpc_address("localhost", 20101), "default", ".", - files, + src_filenames, "default", - "nfs_test_dir", + dst_dir, fake_pid, true, false, @@ -141,22 +176,42 @@ TEST(nfs, basic) ASSERT_EQ(r.err, t->error()); ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); + // this is only true for simulator if (dsn::tools::get_current_tool()->name() == "simulator") { ASSERT_EQ(1, t->get_count()); } + + // The destination files equal to the source files after overwrite copying. + std::vector dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(dst_dir, dst_filenames, true)); + std::sort(dst_filenames.begin(), dst_filenames.end()); + ASSERT_EQ(src_filenames.size(), dst_filenames.size()); + int i = 0; + for (const auto &dst_filename : dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } + // copy files from dst_dir to new_dst_dir. { - // copy nfs_test_dir nfs_test_dir_copy - ASSERT_FALSE(utils::filesystem::directory_exists("nfs_test_dir_copy")); + std::string new_dst_dir = "nfs_test_dir_copy"; + ASSERT_TRUE(utils::filesystem::remove_path(new_dst_dir)); + ASSERT_FALSE(utils::filesystem::directory_exists(new_dst_dir)); aio_result r; dsn::aio_task_ptr t = nfs->copy_remote_directory(dsn::rpc_address("localhost", 20101), "default", - "nfs_test_dir", + dst_dir, "default", - "nfs_test_dir_copy", + new_dst_dir, fake_pid, false, false, @@ -173,22 +228,25 @@ TEST(nfs, basic) ASSERT_EQ(ERR_OK, r.err); ASSERT_EQ(r.sz, t->get_transferred_size()); - ASSERT_TRUE(utils::filesystem::directory_exists("nfs_test_dir_copy")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir_copy/nfs_test_file1")); - ASSERT_TRUE(utils::filesystem::file_exists("nfs_test_dir_copy/nfs_test_file2")); - - std::vector sub1, sub2; - ASSERT_TRUE(utils::filesystem::get_subfiles("nfs_test_dir", sub1, true)); - ASSERT_TRUE(utils::filesystem::get_subfiles("nfs_test_dir_copy", sub2, true)); - ASSERT_EQ(sub1.size(), sub2.size()); - - int64_t sz1, sz2; - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file1", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir_copy/nfs_test_file1", sz2)); - ASSERT_EQ(sz1, sz2); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir/nfs_test_file2", sz1)); - ASSERT_TRUE(utils::filesystem::file_size("nfs_test_dir_copy/nfs_test_file2", sz2)); - ASSERT_EQ(sz1, sz2); + // The new_dst_dir will be created automatically. + ASSERT_TRUE(utils::filesystem::directory_exists(new_dst_dir)); + + std::vector new_dst_filenames; + ASSERT_TRUE(utils::filesystem::get_subfiles(new_dst_dir, new_dst_filenames, true)); + std::sort(new_dst_filenames.begin(), new_dst_filenames.end()); + ASSERT_EQ(src_filenames.size(), new_dst_filenames.size()); + + int i = 0; + for (const auto &new_dst_filename : new_dst_filenames) { + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + new_dst_filename, dsn::utils::FileDataType::kSensitive, file_size)); + ASSERT_EQ(src_file_sizes[i], file_size); + std::string file_md5; + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(new_dst_filename, file_md5)); + ASSERT_EQ(src_file_md5s[i], file_md5); + i++; + } } nfs->stop(); diff --git a/src/perf_counter/test/perf_counter_test.cpp b/src/perf_counter/test/perf_counter_test.cpp index f83679bd45..0758ef8071 100644 --- a/src/perf_counter/test/perf_counter_test.cpp +++ b/src/perf_counter/test/perf_counter_test.cpp @@ -38,6 +38,7 @@ // IWYU pragma: no_include #include #include +#include #include #include #include diff --git a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp index 2706ae772f..400c41b4ce 100644 --- a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp +++ b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/src/replica/backup/cold_backup_context.cpp b/src/replica/backup/cold_backup_context.cpp index f9063a8966..9dd6138c69 100644 --- a/src/replica/backup/cold_backup_context.cpp +++ b/src/replica/backup/cold_backup_context.cpp @@ -17,7 +17,6 @@ #include "cold_backup_context.h" -#include #include #include #include diff --git a/src/replica/backup/replica_backup_server.cpp b/src/replica/backup/replica_backup_server.cpp index 3ed7c878c5..23de8972b3 100644 --- a/src/replica/backup/replica_backup_server.cpp +++ b/src/replica/backup/replica_backup_server.cpp @@ -51,6 +51,12 @@ replica_backup_server::replica_backup_server(const replica_stub *rs) : _stub(rs) }); } +replica_backup_server::~replica_backup_server() +{ + dsn_rpc_unregiser_handler(RPC_COLD_BACKUP); + dsn_rpc_unregiser_handler(RPC_CLEAR_COLD_BACKUP); +} + void replica_backup_server::on_cold_backup(backup_rpc rpc) { const backup_request &request = rpc.request(); diff --git a/src/replica/backup/replica_backup_server.h b/src/replica/backup/replica_backup_server.h index c2786fcbda..7d30f78cc1 100644 --- a/src/replica/backup/replica_backup_server.h +++ b/src/replica/backup/replica_backup_server.h @@ -30,6 +30,7 @@ class replica_backup_server { public: explicit replica_backup_server(const replica_stub *rs); + ~replica_backup_server(); private: void on_cold_backup(backup_rpc rpc); diff --git a/src/replica/backup/test/replica_backup_manager_test.cpp b/src/replica/backup/test/replica_backup_manager_test.cpp index f01ecfc196..ec3fb957f3 100644 --- a/src/replica/backup/test/replica_backup_manager_test.cpp +++ b/src/replica/backup/test/replica_backup_manager_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -39,7 +40,9 @@ class replica_backup_manager_test : public replica_test_base } }; -TEST_F(replica_backup_manager_test, clear_cold_backup) +INSTANTIATE_TEST_CASE_P(, replica_backup_manager_test, ::testing::Values(false, true)); + +TEST_P(replica_backup_manager_test, clear_cold_backup) { std::string policy_name = "test_policy"; diff --git a/src/replica/bulk_load/replica_bulk_loader.cpp b/src/replica/bulk_load/replica_bulk_loader.cpp index 60f3e5a043..3af6ef4998 100644 --- a/src/replica/bulk_load/replica_bulk_loader.cpp +++ b/src/replica/bulk_load/replica_bulk_loader.cpp @@ -16,6 +16,7 @@ // under the License. #include +#include #include #include #include @@ -38,12 +39,14 @@ #include "replica/replica_stub.h" #include "replica/replication_app_base.h" #include "replica_bulk_loader.h" +#include "rocksdb/env.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" #include "runtime/task/async_calls.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/chrono_literals.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" @@ -497,7 +500,8 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, // We are not sure if the file was cached by system. And we couldn't // afford the io overhead which is cased by reading file in verify_file(), // so if file exist we just verify file size - if (utils::filesystem::verify_file_size(file_name, f_meta.size)) { + if (utils::filesystem::verify_file_size( + file_name, dsn::utils::FileDataType::kSensitive, f_meta.size)) { // local file exist and is verified ec = ERR_OK; f_size = f_meta.size; @@ -520,7 +524,8 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, if (ec == ERR_OK && !verified) { if (!f_meta.md5.empty() && f_md5 != f_meta.md5) { ec = ERR_CORRUPTION; - } else if (!utils::filesystem::verify_file_size(file_name, f_meta.size)) { + } else if (!utils::filesystem::verify_file_size( + file_name, dsn::utils::FileDataType::kSensitive, f_meta.size)) { ec = ERR_CORRUPTION; } } @@ -559,10 +564,11 @@ void replica_bulk_loader::download_sst_file(const std::string &remote_dir, error_code replica_bulk_loader::parse_bulk_load_metadata(const std::string &fname) { std::string buf; - error_code ec = utils::filesystem::read_file(fname, buf); - if (ec != ERR_OK) { - LOG_ERROR_PREFIX("read file {} failed, error = {}", fname, ec); - return ec; + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &buf); + if (!s.ok()) { + LOG_ERROR_PREFIX("read file {} failed, error = {}", fname, s.ToString()); + return ERR_FILE_OPERATION_FAILED; } blob bb = blob::create_from_bytes(std::move(buf)); diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 77081196c5..c4540758a1 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -29,7 +29,7 @@ set(MY_PROJ_LIBS dsn_meta_server gtest ) -set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) +set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex rocksdb test_utils) set(MY_BINPLACES config-test.ini diff --git a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp index 6b94754e85..8aa40a8e8c 100644 --- a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp +++ b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp @@ -17,10 +17,13 @@ #include "replica/bulk_load/replica_bulk_loader.h" +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include +#include +#include +#include #include // IWYU pragma: keep #include #include @@ -34,6 +37,7 @@ #include "runtime/rpc/rpc_address.h" #include "runtime/task/task_tracker.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" @@ -251,15 +255,16 @@ class replica_bulk_loader_test : public replica_test_base void create_local_file(const std::string &file_name) { std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - utils::filesystem::create_file(whole_name); - std::ofstream test_file; - test_file.open(whole_name); - test_file << "write some data.\n"; - test_file.close(); - + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice("write some data.\n"), + whole_name, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); _file_meta.name = whole_name; utils::filesystem::md5sum(whole_name, _file_meta.md5); - utils::filesystem::file_size(whole_name, _file_meta.size); + utils::filesystem::file_size( + whole_name, dsn::utils::FileDataType::kSensitive, _file_meta.size); } error_code create_local_metadata_file() @@ -269,21 +274,16 @@ class replica_bulk_loader_test : public replica_test_base _metadata.file_total_size = _file_meta.size; std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); - utils::filesystem::create_file(whole_name); - std::ofstream os(whole_name.c_str(), - (std::ofstream::out | std::ios::binary | std::ofstream::trunc)); - if (!os.is_open()) { - LOG_ERROR("open file {} failed", whole_name); - return ERR_FILE_OPERATION_FAILED; - } - blob bb = json::json_forwarder::encode(_metadata); - os.write((const char *)bb.data(), (std::streamsize)bb.length()); - if (os.bad()) { + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(bb.data(), bb.length()), + whole_name, + /* should_sync */ true); + if (!s.ok()) { LOG_ERROR("write file {} failed", whole_name); return ERR_FILE_OPERATION_FAILED; } - os.close(); return ERR_OK; } @@ -452,14 +452,16 @@ class replica_bulk_loader_test : public replica_test_base std::string FILE_NAME = "test_sst_file"; }; +INSTANTIATE_TEST_CASE_P(, replica_bulk_loader_test, ::testing::Values(false, true)); + // on_bulk_load unit tests -TEST_F(replica_bulk_loader_test, on_bulk_load_not_primary) +TEST_P(replica_bulk_loader_test, on_bulk_load_not_primary) { create_bulk_load_request(bulk_load_status::BLS_DOWNLOADING); ASSERT_EQ(test_on_bulk_load(), ERR_INVALID_STATE); } -TEST_F(replica_bulk_loader_test, on_bulk_load_ballot_change) +TEST_P(replica_bulk_loader_test, on_bulk_load_ballot_change) { create_bulk_load_request(bulk_load_status::BLS_DOWNLOADING, BALLOT + 1); mock_primary_states(); @@ -467,7 +469,7 @@ TEST_F(replica_bulk_loader_test, on_bulk_load_ballot_change) } // on_group_bulk_load unit tests -TEST_F(replica_bulk_loader_test, on_group_bulk_load_test) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_test) { struct test_struct { @@ -494,7 +496,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_test) } // start_downloading unit tests -TEST_F(replica_bulk_loader_test, start_downloading_test) +TEST_P(replica_bulk_loader_test, start_downloading_test) { // Test cases: // - stub concurrent downloading count excceed @@ -521,7 +523,7 @@ TEST_F(replica_bulk_loader_test, start_downloading_test) } // start_downloading unit tests -TEST_F(replica_bulk_loader_test, rollback_to_downloading_test) +TEST_P(replica_bulk_loader_test, rollback_to_downloading_test) { fail::cfg("replica_bulk_loader_download_files", "return()"); struct test_struct @@ -541,23 +543,23 @@ TEST_F(replica_bulk_loader_test, rollback_to_downloading_test) } // parse_bulk_load_metadata unit tests -TEST_F(replica_bulk_loader_test, bulk_load_metadata_not_exist) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_not_exist) { ASSERT_EQ(test_parse_bulk_load_metadata("path_not_exist"), ERR_FILE_OPERATION_FAILED); } -TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_corrupt) { // create file can not parse as bulk_load_metadata structure utils::filesystem::create_directory(LOCAL_DIR); create_local_file(METADATA); std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); error_code ec = test_parse_bulk_load_metadata(metadata_file_name); - ASSERT_EQ(ec, ERR_CORRUPTION); + ASSERT_EQ(ERR_CORRUPTION, ec); utils::filesystem::remove_path(LOCAL_DIR); } -TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) { utils::filesystem::create_directory(LOCAL_DIR); error_code ec = create_local_metadata_file(); @@ -571,7 +573,7 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) } // finish download test -TEST_F(replica_bulk_loader_test, finish_download_test) +TEST_P(replica_bulk_loader_test, finish_download_test) { mock_downloading_progress(100, 50, 50); stub->set_bulk_load_downloading_count(3); @@ -582,7 +584,7 @@ TEST_F(replica_bulk_loader_test, finish_download_test) } // start ingestion test -TEST_F(replica_bulk_loader_test, start_ingestion_test) +TEST_P(replica_bulk_loader_test, start_ingestion_test) { mock_group_progress(bulk_load_status::BLS_DOWNLOADED); test_start_ingestion(); @@ -590,7 +592,7 @@ TEST_F(replica_bulk_loader_test, start_ingestion_test) } // handle_bulk_load_finish unit tests -TEST_F(replica_bulk_loader_test, bulk_load_finish_test) +TEST_P(replica_bulk_loader_test, bulk_load_finish_test) { // Test cases // - bulk load succeed @@ -673,7 +675,7 @@ TEST_F(replica_bulk_loader_test, bulk_load_finish_test) } // pause_bulk_load unit tests -TEST_F(replica_bulk_loader_test, pause_bulk_load_test) +TEST_P(replica_bulk_loader_test, pause_bulk_load_test) { const int32_t stub_downloading_count = 3; // Test cases: @@ -704,7 +706,7 @@ TEST_F(replica_bulk_loader_test, pause_bulk_load_test) } // report_group_download_progress unit tests -TEST_F(replica_bulk_loader_test, report_group_download_progress_test) +TEST_P(replica_bulk_loader_test, report_group_download_progress_test) { struct test_struct { @@ -729,7 +731,7 @@ TEST_F(replica_bulk_loader_test, report_group_download_progress_test) } // report_group_ingestion_status unit tests -TEST_F(replica_bulk_loader_test, report_group_ingestion_status_test) +TEST_P(replica_bulk_loader_test, report_group_ingestion_status_test) { struct ingestion_struct @@ -803,27 +805,27 @@ TEST_F(replica_bulk_loader_test, report_group_ingestion_status_test) } // report_group_context_clean_flag unit tests -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_in_unhealthy_state) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_in_unhealthy_state) { // _primary_states.membership.secondaries is empty mock_replica_config(partition_status::PS_PRIMARY); ASSERT_FALSE(test_report_group_cleaned_up()); } -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_not_cleaned_up) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_not_cleaned_up) { mock_group_cleanup_flag(bulk_load_status::BLS_SUCCEED, true, false); ASSERT_FALSE(test_report_group_cleaned_up()); } -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_all_cleaned_up) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_all_cleaned_up) { mock_group_cleanup_flag(bulk_load_status::BLS_INVALID, true, true); ASSERT_TRUE(test_report_group_cleaned_up()); } // report_group_is_paused unit tests -TEST_F(replica_bulk_loader_test, report_group_is_paused_test) +TEST_P(replica_bulk_loader_test, report_group_is_paused_test) { struct test_struct { @@ -837,21 +839,21 @@ TEST_F(replica_bulk_loader_test, report_group_is_paused_test) } // on_group_bulk_load_reply unit tests -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_downloading_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_downloading_error) { mock_group_progress(bulk_load_status::BLS_DOWNLOADING, 30, 30, 60); test_on_group_bulk_load_reply(bulk_load_status::BLS_DOWNLOADING, BALLOT, ERR_BUSY); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_downloaded_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_downloaded_error) { mock_group_progress(bulk_load_status::BLS_DOWNLOADED); test_on_group_bulk_load_reply(bulk_load_status::BLS_DOWNLOADED, BALLOT, ERR_INVALID_STATE); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) { mock_group_ingestion_states(ingestion_status::IS_RUNNING, ingestion_status::IS_SUCCEED); test_on_group_bulk_load_reply( @@ -859,7 +861,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) { mock_group_cleanup_flag(bulk_load_status::BLS_SUCCEED); test_on_group_bulk_load_reply( @@ -867,14 +869,14 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_failed_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_failed_error) { mock_group_ingestion_states(ingestion_status::IS_RUNNING, ingestion_status::IS_SUCCEED); test_on_group_bulk_load_reply(bulk_load_status::BLS_FAILED, BALLOT, ERR_OK, ERR_TIMEOUT); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) { mock_group_progress(bulk_load_status::BLS_PAUSED, 100, 50, 10); test_on_group_bulk_load_reply( @@ -882,7 +884,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) { mock_group_cleanup_flag(bulk_load_status::BLS_INVALID, true, false); test_on_group_bulk_load_reply(bulk_load_status::BLS_CANCELED, BALLOT, ERR_OBJECT_NOT_FOUND); @@ -890,7 +892,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) } // validate_status unit test -TEST_F(replica_bulk_loader_test, validate_status_test) +TEST_P(replica_bulk_loader_test, validate_status_test) { struct validate_struct { diff --git a/src/replica/duplication/replica_follower.cpp b/src/replica/duplication/replica_follower.cpp index f9c2a996d1..f5a29b94dd 100644 --- a/src/replica/duplication/replica_follower.cpp +++ b/src/replica/duplication/replica_follower.cpp @@ -20,7 +20,6 @@ #include "replica_follower.h" #include -#include #include #include #include diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index 09619343ec..d60a6782de 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -30,6 +30,8 @@ set(MY_PROJ_LIBS dsn_meta_server zookeeper hashtable gtest + test_utils + rocksdb ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/duplication/test/dup_replica_http_service_test.cpp b/src/replica/duplication/test/dup_replica_http_service_test.cpp index 17fe0f1c7f..80b2fb87c3 100644 --- a/src/replica/duplication/test/dup_replica_http_service_test.cpp +++ b/src/replica/duplication/test/dup_replica_http_service_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -39,7 +40,9 @@ class dup_replica_http_service_test : public duplication_test_base { }; -TEST_F(dup_replica_http_service_test, query_duplication_handler) +INSTANTIATE_TEST_CASE_P(, dup_replica_http_service_test, ::testing::Values(false, true)); + +TEST_P(dup_replica_http_service_test, query_duplication_handler) { auto pri = stub->add_primary_replica(1, 1); diff --git a/src/replica/duplication/test/duplication_sync_timer_test.cpp b/src/replica/duplication/test/duplication_sync_timer_test.cpp index d12c7f50dd..c680ddcbea 100644 --- a/src/replica/duplication/test/duplication_sync_timer_test.cpp +++ b/src/replica/duplication/test/duplication_sync_timer_test.cpp @@ -17,6 +17,7 @@ #include "replica/duplication/duplication_sync_timer.h" +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -376,19 +377,21 @@ class duplication_sync_timer_test : public duplication_test_base std::unique_ptr dup_sync; }; -TEST_F(duplication_sync_timer_test, duplication_sync) { test_duplication_sync(); } +INSTANTIATE_TEST_CASE_P(, duplication_sync_timer_test, ::testing::Values(false, true)); -TEST_F(duplication_sync_timer_test, update_duplication_map) { test_update_duplication_map(); } +TEST_P(duplication_sync_timer_test, duplication_sync) { test_duplication_sync(); } -TEST_F(duplication_sync_timer_test, update_on_non_primary) { test_update_on_non_primary(); } +TEST_P(duplication_sync_timer_test, update_duplication_map) { test_update_duplication_map(); } -TEST_F(duplication_sync_timer_test, update_confirmed_points) { test_update_confirmed_points(); } +TEST_P(duplication_sync_timer_test, update_on_non_primary) { test_update_on_non_primary(); } -TEST_F(duplication_sync_timer_test, on_duplication_sync_reply) { test_on_duplication_sync_reply(); } +TEST_P(duplication_sync_timer_test, update_confirmed_points) { test_update_confirmed_points(); } -TEST_F(duplication_sync_timer_test, replica_status_transition) { test_replica_status_transition(); } +TEST_P(duplication_sync_timer_test, on_duplication_sync_reply) { test_on_duplication_sync_reply(); } -TEST_F(duplication_sync_timer_test, receive_illegal_duplication_status) +TEST_P(duplication_sync_timer_test, replica_status_transition) { test_replica_status_transition(); } + +TEST_P(duplication_sync_timer_test, receive_illegal_duplication_status) { test_receive_illegal_duplication_status(); } diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index 342669f63c..9380cd9afc 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -15,16 +15,18 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include +#include #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include #include -#include +#include "aio/aio_task.h" +#include "aio/file_io.h" #include "common/gpid.h" #include "common/replication.codes.h" #include "common/replication_other_types.h" @@ -44,6 +46,7 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/chrono_literals.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/errors.h" #include "utils/fail_point.h" @@ -58,7 +61,6 @@ #include #include #include -#include #include #include #include @@ -68,6 +70,7 @@ #include "duplication_test_base.h" #include "replica/duplication/load_from_private_log.h" #include "replica/mutation_log_utils.h" +#include "test_util/test_util.h" namespace dsn { namespace replication { @@ -273,78 +276,88 @@ class load_from_private_log_test : public duplication_test_base std::unique_ptr duplicator; }; -TEST_F(load_from_private_log_test, find_log_file_to_start) { test_find_log_file_to_start(); } +INSTANTIATE_TEST_CASE_P(, load_from_private_log_test, ::testing::Values(false, true)); -TEST_F(load_from_private_log_test, start_duplication_10000_4MB) +TEST_P(load_from_private_log_test, find_log_file_to_start) { test_find_log_file_to_start(); } + +TEST_P(load_from_private_log_test, start_duplication_10000_4MB) { test_start_duplication(10000, 4); } -TEST_F(load_from_private_log_test, start_duplication_50000_4MB) +TEST_P(load_from_private_log_test, start_duplication_50000_4MB) { test_start_duplication(50000, 4); } -TEST_F(load_from_private_log_test, start_duplication_10000_1MB) +TEST_P(load_from_private_log_test, start_duplication_10000_1MB) { test_start_duplication(10000, 1); } -TEST_F(load_from_private_log_test, start_duplication_50000_1MB) +TEST_P(load_from_private_log_test, start_duplication_50000_1MB) { test_start_duplication(50000, 1); } -TEST_F(load_from_private_log_test, start_duplication_100000_4MB) +TEST_P(load_from_private_log_test, start_duplication_100000_4MB) { test_start_duplication(100000, 4); } // Ensure replica_duplicator can correctly handle real-world log file -TEST_F(load_from_private_log_test, handle_real_private_log) +TEST_P(load_from_private_log_test, handle_real_private_log) { + std::vector log_files({"log.1.0.handle_real_private_log", + "log.1.0.handle_real_private_log2", + "log.1.0.all_loaded_are_write_empties"}); + if (FLAGS_encrypt_data_at_rest) { + for (int i = 0; i < log_files.size(); i++) { + auto s = dsn::utils::encrypt_file(log_files[i], log_files[i] + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + log_files[i] += ".encrypted"; + } + } + struct test_data { - std::string fname; int puts; int total; gpid id; } tests[] = { // PUT, PUT, PUT, EMPTY, PUT, EMPTY, EMPTY - {"log.1.0.handle_real_private_log", 4, 6, gpid(1, 4)}, + {4, 6, gpid(1, 4)}, // EMPTY, PUT, EMPTY - {"log.1.0.handle_real_private_log2", 1, 2, gpid(1, 4)}, + {1, 2, gpid(1, 4)}, // EMPTY, EMPTY, EMPTY - {"log.1.0.all_loaded_are_write_empties", 0, 2, gpid(1, 5)}, + {0, 2, gpid(1, 5)}, }; - for (auto tt : tests) { + ASSERT_EQ(log_files.size(), sizeof(tests) / sizeof(test_data)); + for (int i = 0; i < log_files.size(); i++) { // reset replica to specified gpid duplicator.reset(nullptr); - _replica = create_mock_replica(stub.get(), tt.id.get_app_id(), tt.id.get_partition_index()); + _replica = create_mock_replica( + stub.get(), tests[i].id.get_app_id(), tests[i].id.get_partition_index()); // Update '_log_dir' to the corresponding replica created above. _log_dir = _replica->dir(); ASSERT_TRUE(utils::filesystem::path_exists(_log_dir)) << _log_dir; // Copy the log file to '_log_dir' - boost::filesystem::path file(tt.fname); - ASSERT_TRUE(dsn::utils::filesystem::file_exists(tt.fname)) << tt.fname; - boost::system::error_code ec; - boost::filesystem::copy_file( - file, _log_dir + "/log.1.0", boost::filesystem::copy_option::overwrite_if_exists, ec); - ASSERT_TRUE(!ec) << ec.value() << ", " << ec.category().name() << ", " << ec.message(); + auto s = dsn::utils::copy_file(log_files[i], _log_dir + "/log.1.0"); + ASSERT_TRUE(s.ok()) << s.ToString(); // Start to verify. - load_and_wait_all_entries_loaded(tt.puts, tt.total, tt.id, 1, 0); + load_and_wait_all_entries_loaded(tests[i].puts, tests[i].total, tests[i].id, 1, 0); } } -TEST_F(load_from_private_log_test, restart_duplication) { test_restart_duplication(); } +TEST_P(load_from_private_log_test, restart_duplication) { test_restart_duplication(); } -TEST_F(load_from_private_log_test, ignore_useless) +TEST_P(load_from_private_log_test, ignore_useless) { utils::filesystem::remove_path(_log_dir); @@ -410,7 +423,9 @@ class load_fail_mode_test : public load_from_private_log_test std::unique_ptr end_stage; }; -TEST_F(load_fail_mode_test, fail_skip) +INSTANTIATE_TEST_CASE_P(, load_fail_mode_test, ::testing::Values(false, true)); + +TEST_P(load_fail_mode_test, fail_skip) { duplicator->update_fail_mode(duplication_fail_mode::FAIL_SKIP); ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); @@ -428,7 +443,7 @@ TEST_F(load_fail_mode_test, fail_skip) ASSERT_GT(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); } -TEST_F(load_fail_mode_test, fail_slow) +TEST_P(load_fail_mode_test, fail_slow) { duplicator->update_fail_mode(duplication_fail_mode::FAIL_SLOW); ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); @@ -447,16 +462,31 @@ TEST_F(load_fail_mode_test, fail_slow) ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); } -TEST_F(load_fail_mode_test, fail_skip_real_corrupted_file) +TEST_P(load_fail_mode_test, fail_skip_real_corrupted_file) { { // inject some bad data in the middle of the first file std::string log_path = _log_dir + "/log.1.0"; - auto file_size = boost::filesystem::file_size(log_path); - int fd = open(log_path.c_str(), O_WRONLY); + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + log_path, dsn::utils::FileDataType::kSensitive, file_size)); + auto wfile = file::open(log_path, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + const char buf[] = "xxxxxx"; - auto written_size = pwrite(fd, buf, sizeof(buf), file_size / 2); - ASSERT_EQ(written_size, sizeof(buf)); - close(fd); + auto buff_len = sizeof(buf); + auto t = ::dsn::file::write(wfile, + buf, + buff_len, + file_size / 2, + LPC_AIO_IMMEDIATE_CALLBACK, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(buff_len, n); + }); + t->wait(); + ASSERT_EQ(ERR_OK, ::dsn::file::flush(wfile)); + ASSERT_EQ(ERR_OK, ::dsn::file::close(wfile)); } duplicator->update_fail_mode(duplication_fail_mode::FAIL_SKIP); diff --git a/src/replica/duplication/test/mutation_batch_test.cpp b/src/replica/duplication/test/mutation_batch_test.cpp index dbc09a8627..a41aa8ab8d 100644 --- a/src/replica/duplication/test/mutation_batch_test.cpp +++ b/src/replica/duplication/test/mutation_batch_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -55,7 +56,9 @@ class mutation_batch_test : public duplication_test_base } }; -TEST_F(mutation_batch_test, add_mutation_if_valid) +INSTANTIATE_TEST_CASE_P(, mutation_batch_test, ::testing::Values(false, true)); + +TEST_P(mutation_batch_test, add_mutation_if_valid) { auto duplicator = create_test_duplicator(0); mutation_batch batcher(duplicator.get()); @@ -85,7 +88,7 @@ TEST_F(mutation_batch_test, add_mutation_if_valid) ASSERT_EQ(result.size(), 2); } -TEST_F(mutation_batch_test, ignore_non_idempotent_write) +TEST_P(mutation_batch_test, ignore_non_idempotent_write) { auto duplicator = create_test_duplicator(0); mutation_batch batcher(duplicator.get()); @@ -98,7 +101,7 @@ TEST_F(mutation_batch_test, ignore_non_idempotent_write) ASSERT_EQ(result.size(), 0); } -TEST_F(mutation_batch_test, mutation_buffer_commit) +TEST_P(mutation_batch_test, mutation_buffer_commit) { auto duplicator = create_test_duplicator(0); mutation_batch batcher(duplicator.get()); diff --git a/src/replica/duplication/test/replica_duplicator_manager_test.cpp b/src/replica/duplication/test/replica_duplicator_manager_test.cpp index 94d09aeddd..93aed6dd8b 100644 --- a/src/replica/duplication/test/replica_duplicator_manager_test.cpp +++ b/src/replica/duplication/test/replica_duplicator_manager_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -184,24 +185,26 @@ class replica_duplicator_manager_test : public duplication_test_base } }; -TEST_F(replica_duplicator_manager_test, get_duplication_confirms) +INSTANTIATE_TEST_CASE_P(, replica_duplicator_manager_test, ::testing::Values(false, true)); + +TEST_P(replica_duplicator_manager_test, get_duplication_confirms) { test_get_duplication_confirms(); } -TEST_F(replica_duplicator_manager_test, set_confirmed_decree_non_primary) +TEST_P(replica_duplicator_manager_test, set_confirmed_decree_non_primary) { test_set_confirmed_decree_non_primary(); } -TEST_F(replica_duplicator_manager_test, remove_non_existed_duplications) +TEST_P(replica_duplicator_manager_test, remove_non_existed_duplications) { test_remove_non_existed_duplications(); } -TEST_F(replica_duplicator_manager_test, min_confirmed_decree) { test_min_confirmed_decree(); } +TEST_P(replica_duplicator_manager_test, min_confirmed_decree) { test_min_confirmed_decree(); } -TEST_F(replica_duplicator_manager_test, update_checkpoint_prepared) +TEST_P(replica_duplicator_manager_test, update_checkpoint_prepared) { auto r = stub->add_primary_replica(2, 1); duplication_entry ent; diff --git a/src/replica/duplication/test/replica_duplicator_test.cpp b/src/replica/duplication/test/replica_duplicator_test.cpp index 4d60c81b21..87fcd374b9 100644 --- a/src/replica/duplication/test/replica_duplicator_test.cpp +++ b/src/replica/duplication/test/replica_duplicator_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -138,11 +139,13 @@ class replica_duplicator_test : public duplication_test_base } }; -TEST_F(replica_duplicator_test, new_duplicator) { test_new_duplicator(); } +INSTANTIATE_TEST_CASE_P(, replica_duplicator_test, ::testing::Values(false, true)); -TEST_F(replica_duplicator_test, pause_start_duplication) { test_pause_start_duplication(); } +TEST_P(replica_duplicator_test, new_duplicator) { test_new_duplicator(); } -TEST_F(replica_duplicator_test, duplication_progress) +TEST_P(replica_duplicator_test, pause_start_duplication) { test_pause_start_duplication(); } + +TEST_P(replica_duplicator_test, duplication_progress) { auto duplicator = create_test_duplicator(); ASSERT_EQ(duplicator->progress().last_decree, 0); // start duplication from empty plog @@ -171,7 +174,7 @@ TEST_F(replica_duplicator_test, duplication_progress) ASSERT_TRUE(duplicator_for_checkpoint->progress().checkpoint_has_prepared); } -TEST_F(replica_duplicator_test, prapre_dup) +TEST_P(replica_duplicator_test, prapre_dup) { auto duplicator = create_test_duplicator(invalid_decree, 100); replica()->update_expect_last_durable_decree(100); diff --git a/src/replica/duplication/test/replica_follower_test.cpp b/src/replica/duplication/test/replica_follower_test.cpp index f013d61693..a961c36048 100644 --- a/src/replica/duplication/test/replica_follower_test.cpp +++ b/src/replica/duplication/test/replica_follower_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -116,7 +117,9 @@ class replica_follower_test : public duplication_test_base mock_replica_ptr _mock_replica; }; -TEST_F(replica_follower_test, test_init_master_info) +INSTANTIATE_TEST_CASE_P(, replica_follower_test, ::testing::Values(false, true)); + +TEST_P(replica_follower_test, test_init_master_info) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, @@ -140,7 +143,7 @@ TEST_F(replica_follower_test, test_init_master_info) ASSERT_FALSE(_mock_replica->is_duplication_follower()); } -TEST_F(replica_follower_test, test_duplicate_checkpoint) +TEST_P(replica_follower_test, test_duplicate_checkpoint) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, @@ -160,7 +163,7 @@ TEST_F(replica_follower_test, test_duplicate_checkpoint) ASSERT_EQ(follower->duplicate_checkpoint(), ERR_BUSY); } -TEST_F(replica_follower_test, test_async_duplicate_checkpoint_from_master_replica) +TEST_P(replica_follower_test, test_async_duplicate_checkpoint_from_master_replica) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, @@ -182,7 +185,7 @@ TEST_F(replica_follower_test, test_async_duplicate_checkpoint_from_master_replic fail::teardown(); } -TEST_F(replica_follower_test, test_update_master_replica_config) +TEST_P(replica_follower_test, test_update_master_replica_config) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, @@ -229,7 +232,7 @@ TEST_F(replica_follower_test, test_update_master_replica_config) ASSERT_EQ(master_replica_config(follower).pid, p.pid); } -TEST_F(replica_follower_test, test_nfs_copy_checkpoint) +TEST_P(replica_follower_test, test_nfs_copy_checkpoint) { _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterClusterKey, "master"); _app_info.envs.emplace(duplication_constants::kDuplicationEnvMasterMetasKey, diff --git a/src/replica/duplication/test/ship_mutation_test.cpp b/src/replica/duplication/test/ship_mutation_test.cpp index d6356418f6..29e17f2a8a 100644 --- a/src/replica/duplication/test/ship_mutation_test.cpp +++ b/src/replica/duplication/test/ship_mutation_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -105,14 +106,16 @@ class ship_mutation_test : public duplication_test_base std::unique_ptr duplicator; }; -TEST_F(ship_mutation_test, ship_mutation_tuple_set) { test_ship_mutation_tuple_set(); } +INSTANTIATE_TEST_CASE_P(, ship_mutation_test, ::testing::Values(false, true)); + +TEST_P(ship_mutation_test, ship_mutation_tuple_set) { test_ship_mutation_tuple_set(); } void retry(pipeline::base *base) { base->schedule([base]() { retry(base); }, 10_s); } -TEST_F(ship_mutation_test, pause) +TEST_P(ship_mutation_test, pause) { auto shipper = mock_ship_mutation(); diff --git a/src/replica/log_file.cpp b/src/replica/log_file.cpp index 7362aa7c41..c9daf853ba 100644 --- a/src/replica/log_file.cpp +++ b/src/replica/log_file.cpp @@ -26,7 +26,6 @@ #include "log_file.h" -#include #include #include #include @@ -42,6 +41,7 @@ #include "utils/binary_writer.h" #include "utils/blob.h" #include "utils/crc.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" @@ -99,7 +99,7 @@ log_file::~log_file() { close(); } return nullptr; } - disk_file *hfile = file::open(path, O_RDONLY | O_BINARY, 0); + disk_file *hfile = file::open(path, file::FileOpenType::kReadOnly); if (!hfile) { err = ERR_FILE_OPERATION_FAILED; LOG_WARNING("open log file {} failed", path); @@ -155,7 +155,7 @@ log_file::~log_file() { close(); } return nullptr; } - disk_file *hfile = file::open(path, O_RDWR | O_CREAT | O_BINARY, 0666); + disk_file *hfile = file::open(path, file::FileOpenType::kWriteOnly); if (!hfile) { LOG_WARNING("create log {} failed", path); return nullptr; @@ -179,7 +179,9 @@ log_file::log_file( if (is_read) { int64_t sz; - CHECK(dsn::utils::filesystem::file_size(_path, sz), "fail to get file size of {}.", _path); + CHECK(dsn::utils::filesystem::file_size(_path, dsn::utils::FileDataType::kSensitive, sz), + "fail to get file size of {}.", + _path); _end_offset += sz; } } @@ -268,6 +270,7 @@ aio_task_ptr log_file::commit_log_block(log_block &block, log_appender pending(offset, block); return commit_log_blocks(pending, evt, tracker, std::move(callback), hash); } + aio_task_ptr log_file::commit_log_blocks(log_appender &pending, dsn::task_code evt, dsn::task_tracker *tracker, @@ -333,7 +336,7 @@ aio_task_ptr log_file::commit_log_blocks(log_appender &pending, hash); } - if (utils::FLAGS_enable_latency_tracer) { + if (dsn_unlikely(utils::FLAGS_enable_latency_tracer)) { tsk->_tracer->set_parent_point_name("commit_pending_mutations"); tsk->_tracer->set_description("log"); for (const auto &mutation : pending.mutations()) { diff --git a/src/replica/log_file_stream.h b/src/replica/log_file_stream.h index 150b6e824b..b31a5047c6 100644 --- a/src/replica/log_file_stream.h +++ b/src/replica/log_file_stream.h @@ -34,7 +34,6 @@ namespace dsn { namespace replication { -// log_file::file_streamer class log_file::file_streamer { public: diff --git a/src/replica/mutation_cache.cpp b/src/replica/mutation_cache.cpp index c97c9b0451..6444cdacb3 100644 --- a/src/replica/mutation_cache.cpp +++ b/src/replica/mutation_cache.cpp @@ -26,11 +26,7 @@ #include "mutation_cache.h" -// HACK: iwyu suggests each time vector[] is used. -// https://github.com/include-what-you-use/include-what-you-use/issues/166 -// TODO(yingchun): remove this pragma by using mapping.imp // IWYU pragma: no_include -#include #include "consensus_types.h" #include "mutation.h" diff --git a/src/replica/mutation_log_replay.cpp b/src/replica/mutation_log_replay.cpp index 261ff4706a..1c95da91fe 100644 --- a/src/replica/mutation_log_replay.cpp +++ b/src/replica/mutation_log_replay.cpp @@ -140,7 +140,7 @@ namespace replication { if (log == nullptr) { if (err == ERR_HANDLE_EOF || err == ERR_INCOMPLETE_DATA || err == ERR_INVALID_PARAMETERS) { - LOG_DEBUG("skip file {} during log replay", fpath); + LOG_INFO("skip file {} during log replay", fpath); continue; } else { return err; diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index f159d7699e..d9de3e8441 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include diff --git a/src/replica/replica.h b/src/replica/replica.h index 3933784da0..dece645950 100644 --- a/src/replica/replica.h +++ b/src/replica/replica.h @@ -450,7 +450,7 @@ class replica : public serverlet, public ref_counter, public replica_ba ///////////////////////////////////////////////////////////////// // replica restore from backup - bool read_cold_backup_metadata(const std::string &file, cold_backup_metadata &backup_metadata); + bool read_cold_backup_metadata(const std::string &fname, cold_backup_metadata &backup_metadata); // checkpoint on cold backup media maybe contain useless file, // we should abandon these file base cold_backup_metadata bool remove_useless_file_under_chkpt(const std::string &chkpt_dir, @@ -555,7 +555,7 @@ class replica : public serverlet, public ref_counter, public replica_ba friend class ::pegasus::server::pegasus_server_test_base; friend class ::pegasus::server::rocksdb_wrapper_test; FRIEND_TEST(replica_disk_test, disk_io_error_test); - FRIEND_TEST(replica_error_test, test_auto_trash_of_corruption); + FRIEND_TEST(replica_test, test_auto_trash_of_corruption); // replica configuration, updated by update_local_configuration ONLY replica_configuration _config; diff --git a/src/replica/replica_backup.cpp b/src/replica/replica_backup.cpp index 50a4b45f20..d6c452be9d 100644 --- a/src/replica/replica_backup.cpp +++ b/src/replica/replica_backup.cpp @@ -51,6 +51,7 @@ #include "runtime/api_layer1.h" #include "runtime/task/async_calls.h" #include "utils/autoref_ptr.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -386,10 +387,12 @@ statistic_file_infos_under_dir(const std::string &dir, total_size = 0; file_infos.clear(); + // TODO(yingchun): check if there are any files that are not sensitive (not encrypted). for (std::string &file : sub_files) { std::pair file_info; - if (!utils::filesystem::file_size(file, file_info.second)) { + if (!utils::filesystem::file_size( + file, dsn::utils::FileDataType::kSensitive, file_info.second)) { LOG_ERROR("get file size of {} failed", file); return false; } diff --git a/src/replica/replica_http_service.h b/src/replica/replica_http_service.h index d02707441d..1550eea8c7 100644 --- a/src/replica/replica_http_service.h +++ b/src/replica/replica_http_service.h @@ -51,6 +51,13 @@ class replica_http_service : public http_server_base "ip:port/replica/maual_compaction?app_id="); } + ~replica_http_service() + { + deregister_http_call("replica/duplication"); + deregister_http_call("replica/data_version"); + deregister_http_call("replica/manual_compaction"); + } + std::string path() const override { return "replica"; } void query_duplication_handler(const http_request &req, http_response &resp); diff --git a/src/replica/replica_learn.cpp b/src/replica/replica_learn.cpp index 75fc9c5894..bc111da55a 100644 --- a/src/replica/replica_learn.cpp +++ b/src/replica/replica_learn.cpp @@ -46,7 +46,6 @@ #include #include -#include "aio/aio_task.h" #include "common/fs_manager.h" #include "common/gpid.h" #include "common/replication.codes.h" diff --git a/src/replica/replica_restore.cpp b/src/replica/replica_restore.cpp index 1a37479777..6b9fe82588 100644 --- a/src/replica/replica_restore.cpp +++ b/src/replica/replica_restore.cpp @@ -17,6 +17,8 @@ #include #include +#include +#include #include #include #include @@ -49,10 +51,10 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/utils.h" using namespace dsn::dist::block_service; @@ -93,41 +95,26 @@ bool replica::remove_useless_file_under_chkpt(const std::string &chkpt_dir, return true; } -bool replica::read_cold_backup_metadata(const std::string &file, +bool replica::read_cold_backup_metadata(const std::string &fname, cold_backup_metadata &backup_metadata) { - if (!::dsn::utils::filesystem::file_exists(file)) { + if (!::dsn::utils::filesystem::file_exists(fname)) { LOG_ERROR_PREFIX( - "checkpoint on remote storage media is damaged, coz file({}) doesn't exist", file); + "checkpoint on remote storage media is damaged, coz file({}) doesn't exist", fname); return false; } - int64_t file_sz = 0; - if (!::dsn::utils::filesystem::file_size(file, file_sz)) { - LOG_ERROR_PREFIX("get file({}) size failed", file); - return false; - } - std::shared_ptr buf = utils::make_shared_array(file_sz + 1); - std::ifstream fin(file, std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR_PREFIX("open file({}) failed", file); + std::string data; + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &data); + if (!s.ok()) { + LOG_ERROR_PREFIX("read file '{}' failed, err = {}", fname, s.ToString()); return false; } - fin.read(buf.get(), file_sz); - CHECK_EQ_MSG(file_sz, - fin.gcount(), - "{}: read file({}) failed, need {}, but read {}", - name(), - file, - file_sz, - fin.gcount()); - fin.close(); - - buf.get()[fin.gcount()] = '\0'; - blob bb; - bb.assign(std::move(buf), 0, file_sz); - if (!::dsn::json::json_forwarder::decode(bb, backup_metadata)) { - LOG_ERROR_PREFIX("file({}) under checkpoint is damaged", file); + + if (!::dsn::json::json_forwarder::decode( + blob::create_from_bytes(std::move(data)), backup_metadata)) { + LOG_ERROR_PREFIX("file({}) under checkpoint is damaged", fname); return false; } return true; @@ -160,7 +147,12 @@ error_code replica::download_checkpoint(const configuration_restore_request &req const std::string file_name = utils::filesystem::path_combine(local_chkpt_dir, f_meta.name); if (download_err == ERR_OK || download_err == ERR_PATH_ALREADY_EXIST) { - if (!utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) { + // TODO(yingchun): check if there are any files that are not sensitive (not + // encrypted). + if (!utils::filesystem::verify_file(file_name, + dsn::utils::FileDataType::kSensitive, + f_meta.md5, + f_meta.size)) { download_err = ERR_CORRUPTION; } else if (download_err == ERR_PATH_ALREADY_EXIST) { download_err = ERR_OK; diff --git a/src/replica/replica_stub.h b/src/replica/replica_stub.h index b6f54c4850..1072925b4e 100644 --- a/src/replica/replica_stub.h +++ b/src/replica/replica_stub.h @@ -417,7 +417,7 @@ class replica_stub : public serverlet, public ref_counter friend class replica_follower_test; friend class replica_http_service_test; FRIEND_TEST(open_replica_test, open_replica_add_decree_and_ballot_check); - FRIEND_TEST(replica_error_test, test_auto_trash_of_corruption); + FRIEND_TEST(replica_test, test_auto_trash_of_corruption); FRIEND_TEST(replica_test, test_clear_on_failure); typedef std::unordered_map opening_replicas; diff --git a/src/replica/replication_app_base.cpp b/src/replica/replication_app_base.cpp index db92bdf17e..7bb4a7423b 100644 --- a/src/replica/replication_app_base.cpp +++ b/src/replica/replication_app_base.cpp @@ -25,16 +25,14 @@ */ #include -#include +#include +#include #include -#include #include #include #include #include -#include "aio/aio_task.h" -#include "aio/file_io.h" #include "common/bulk_load_common.h" #include "common/duplication_common.h" #include "common/replica_envs.h" @@ -50,68 +48,43 @@ #include "runtime/rpc/serialization.h" #include "runtime/task/task_code.h" #include "runtime/task/task_spec.h" -#include "runtime/task/task_tracker.h" -#include "utils/autoref_ptr.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/factory_store.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" -#include "utils/ports.h" #include "utils/string_view.h" -#include "utils/threadpool_code.h" -#include "utils/utils.h" namespace dsn { -class disk_file; namespace replication { const std::string replica_init_info::kInitInfo = ".init-info"; -DEFINE_TASK_CODE_AIO(LPC_AIO_INFO_WRITE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) - namespace { -error_code write_blob_to_file(const std::string &file, const blob &data) +error_code write_blob_to_file(const std::string &fname, const blob &data) { - std::string tmp_file = file + ".tmp"; - disk_file *hfile = file::open(tmp_file.c_str(), O_WRONLY | O_CREAT | O_BINARY | O_TRUNC, 0666); + // TODO(yingchun): consider not encrypt the meta files. + std::string tmp_fname = fname + ".tmp"; + auto cleanup = defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); }); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(data.data(), data.length()), + tmp_fname, + /* should_sync */ true); LOG_AND_RETURN_NOT_TRUE( - ERROR, hfile, ERR_FILE_OPERATION_FAILED, "open file {} failed", tmp_file); - auto cleanup = defer([tmp_file]() { utils::filesystem::remove_path(tmp_file); }); - - error_code err; - size_t sz = 0; - task_tracker tracker; - aio_task_ptr tsk = file::write(hfile, - data.data(), - data.length(), - 0, - LPC_AIO_INFO_WRITE, - &tracker, - [&err, &sz](error_code e, size_t s) { - err = e; - sz = s; - }, - 0); - CHECK_NOTNULL(tsk, "create file::write task failed"); - tracker.wait_outstanding_tasks(); - file::flush(hfile); - file::close(hfile); - LOG_AND_RETURN_NOT_OK(ERROR, err, "write file {} failed", tmp_file); - CHECK_EQ(data.length(), sz); - // TODO(yingchun): need fsync too？ + ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "write file {} failed", tmp_fname); LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::rename_path(tmp_file, file), + utils::filesystem::rename_path(tmp_fname, fname), ERR_FILE_OPERATION_FAILED, "move file from {} to {} failed", - tmp_file, - file); - + tmp_fname, + fname); return ERR_OK; } } // namespace @@ -146,38 +119,24 @@ error_code replica_init_info::store(const std::string &dir) return ERR_OK; } -error_code replica_init_info::load_json(const std::string &file) +error_code replica_init_info::load_json(const std::string &fname) { - std::ifstream is(file, std::ios::binary); - LOG_AND_RETURN_NOT_TRUE( - ERROR, is.is_open(), ERR_FILE_OPERATION_FAILED, "open file {} failed", file); - - int64_t sz = 0; + std::string data; + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &data); + LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname); LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::file_size(std::string(file), sz), + json::json_forwarder::decode( + blob::create_from_bytes(std::move(data)), *this), ERR_FILE_OPERATION_FAILED, - "get file size of {} failed", - file); - - std::shared_ptr buffer(utils::make_shared_array(sz)); - is.read((char *)buffer.get(), sz); - LOG_AND_RETURN_NOT_TRUE( - ERROR, !is.bad(), ERR_FILE_OPERATION_FAILED, "read file {} failed", file); - is.close(); - - LOG_AND_RETURN_NOT_TRUE( - ERROR, - json::json_forwarder::decode(blob(buffer, sz), *this), - ERR_FILE_OPERATION_FAILED, - "decode json from file {} failed", - file); - + "decode json from file {} failed", + fname); return ERR_OK; } -error_code replica_init_info::store_json(const std::string &file) +error_code replica_init_info::store_json(const std::string &fname) { - return write_blob_to_file(file, json::json_forwarder::encode(*this)); + return write_blob_to_file(fname, json::json_forwarder::encode(*this)); } std::string replica_init_info::to_string() @@ -190,35 +149,22 @@ std::string replica_init_info::to_string() return oss.str(); } -error_code replica_app_info::load(const std::string &file) +error_code replica_app_info::load(const std::string &fname) { - std::ifstream is(file, std::ios::binary); - LOG_AND_RETURN_NOT_TRUE( - ERROR, is.is_open(), ERR_FILE_OPERATION_FAILED, "open file {} failed", file); - - int64_t sz = 0; - LOG_AND_RETURN_NOT_TRUE(ERROR, - utils::filesystem::file_size(std::string(file), sz), - ERR_FILE_OPERATION_FAILED, - "get file size of {} failed", - file); - - std::shared_ptr buffer(utils::make_shared_array(sz)); - is.read((char *)buffer.get(), sz); - is.close(); - - binary_reader reader(blob(buffer, sz)); - int magic; + std::string data; + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &data); + LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname); + binary_reader reader(blob::create_from_bytes(std::move(data))); + int magic = 0; unmarshall(reader, magic, DSF_THRIFT_BINARY); - LOG_AND_RETURN_NOT_TRUE( - ERROR, magic == 0xdeadbeef, ERR_INVALID_DATA, "data in file {} is invalid (magic)", file); - + ERROR, magic == 0xdeadbeef, ERR_INVALID_DATA, "data in file {} is invalid (magic)", fname); unmarshall(reader, *_app, DSF_THRIFT_JSON); return ERR_OK; } -error_code replica_app_info::store(const std::string &file) +error_code replica_app_info::store(const std::string &fname) { binary_writer writer; int magic = 0xdeadbeef; @@ -238,7 +184,7 @@ error_code replica_app_info::store(const std::string &file) marshall(writer, tmp, DSF_THRIFT_JSON); } - return write_blob_to_file(file, writer.get_buffer()); + return write_blob_to_file(fname, writer.get_buffer()); } /*static*/ diff --git a/src/replica/replication_app_base.h b/src/replica/replication_app_base.h index e48ef8a5dd..29d28f90fd 100644 --- a/src/replica/replication_app_base.h +++ b/src/replica/replication_app_base.h @@ -75,8 +75,8 @@ class replica_init_info std::string to_string(); private: - error_code load_json(const std::string &file); - error_code store_json(const std::string &file); + error_code load_json(const std::string &fname); + error_code store_json(const std::string &fname); }; class replica_app_info @@ -86,8 +86,8 @@ class replica_app_info public: replica_app_info(app_info *app) { _app = app; } - error_code load(const std::string &file); - error_code store(const std::string &file); + error_code load(const std::string &fname); + error_code store(const std::string &fname); }; /// The store engine interface of Pegasus. diff --git a/src/replica/split/test/replica_split_test.cpp b/src/replica/split/test/replica_split_test.cpp index 4f12f91e00..c0edc3a013 100644 --- a/src/replica/split/test/replica_split_test.cpp +++ b/src/replica/split/test/replica_split_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -558,8 +559,10 @@ class replica_split_test : public replica_test_base learn_state _mock_learn_state; }; +INSTANTIATE_TEST_CASE_P(, replica_split_test, ::testing::Values(false, true)); + // parent_start_split tests -TEST_F(replica_split_test, parent_start_split_tests) +TEST_P(replica_split_test, parent_start_split_tests) { fail::cfg("replica_stub_create_child_replica_if_not_found", "return()"); fail::cfg("replica_child_init_replica", "return()"); @@ -595,7 +598,7 @@ TEST_F(replica_split_test, parent_start_split_tests) } // child_init_replica test -TEST_F(replica_split_test, child_init_replica_test) +TEST_P(replica_split_test, child_init_replica_test) { fail::cfg("replica_stub_split_replica_exec", "return()"); test_child_init_replica(); @@ -605,7 +608,7 @@ TEST_F(replica_split_test, child_init_replica_test) } // parent_check_states tests -TEST_F(replica_split_test, parent_check_states_tests) +TEST_P(replica_split_test, parent_check_states_tests) { fail::cfg("replica_stub_split_replica_exec", "return()"); @@ -625,7 +628,7 @@ TEST_F(replica_split_test, parent_check_states_tests) } // child_copy_prepare_list test -TEST_F(replica_split_test, copy_prepare_list_succeed) +TEST_P(replica_split_test, copy_prepare_list_succeed) { fail::cfg("replica_stub_split_replica_exec", "return()"); fail::cfg("replica_child_learn_states", "return()"); @@ -642,7 +645,7 @@ TEST_F(replica_split_test, copy_prepare_list_succeed) } // child_learn_states tests -TEST_F(replica_split_test, child_learn_states_tests) +TEST_P(replica_split_test, child_learn_states_tests) { generate_child(); @@ -674,7 +677,7 @@ TEST_F(replica_split_test, child_learn_states_tests) } // child_apply_private_logs test -TEST_F(replica_split_test, child_apply_private_logs_succeed) +TEST_P(replica_split_test, child_apply_private_logs_succeed) { fail::cfg("mutation_log_replay_succeed", "return()"); fail::cfg("replication_app_base_apply_mutation", "return()"); @@ -688,7 +691,7 @@ TEST_F(replica_split_test, child_apply_private_logs_succeed) } // child_catch_up_states tests -TEST_F(replica_split_test, child_catch_up_states_tests) +TEST_P(replica_split_test, child_catch_up_states_tests) { fail::cfg("replica_child_notify_catch_up", "return()"); fail::cfg("replication_app_base_apply_mutation", "return()"); @@ -713,7 +716,7 @@ TEST_F(replica_split_test, child_catch_up_states_tests) } // parent_handle_child_catch_up tests -TEST_F(replica_split_test, parent_handle_catch_up_test) +TEST_P(replica_split_test, parent_handle_catch_up_test) { fail::cfg("replica_parent_check_sync_point_commit", "return()"); ballot WRONG_BALLOT = 1; @@ -739,7 +742,7 @@ TEST_F(replica_split_test, parent_handle_catch_up_test) } // update_child_group_partition_count tests -TEST_F(replica_split_test, update_child_group_partition_count_test) +TEST_P(replica_split_test, update_child_group_partition_count_test) { fail::cfg("replica_parent_update_partition_count_request", "return()"); generate_child(); @@ -775,7 +778,7 @@ TEST_F(replica_split_test, update_child_group_partition_count_test) } // on_update_child_group_partition_count tests -TEST_F(replica_split_test, child_update_partition_count_test) +TEST_P(replica_split_test, child_update_partition_count_test) { ballot WRONG_BALLOT = INIT_BALLOT + 1; generate_child(); @@ -802,7 +805,7 @@ TEST_F(replica_split_test, child_update_partition_count_test) } // on_update_child_group_partition_count_reply tests -TEST_F(replica_split_test, parent_on_update_partition_reply_test) +TEST_P(replica_split_test, parent_on_update_partition_reply_test) { fail::cfg("replica_register_child_on_meta", "return()"); generate_child(); @@ -836,7 +839,7 @@ TEST_F(replica_split_test, parent_on_update_partition_reply_test) } // register_child test -TEST_F(replica_split_test, register_child_test) +TEST_P(replica_split_test, register_child_test) { fail::cfg("replica_parent_send_register_request", "return()"); test_register_child_on_meta(); @@ -845,7 +848,7 @@ TEST_F(replica_split_test, register_child_test) } // register_child_reply tests -TEST_F(replica_split_test, register_child_reply_test) +TEST_P(replica_split_test, register_child_reply_test) { fail::cfg("replica_init_group_check", "return()"); fail::cfg("replica_broadcast_group_check", "return()"); @@ -878,7 +881,7 @@ TEST_F(replica_split_test, register_child_reply_test) } // trigger_primary_parent_split unit test -TEST_F(replica_split_test, trigger_primary_parent_split_test) +TEST_P(replica_split_test, trigger_primary_parent_split_test) { fail::cfg("replica_broadcast_group_check", "return()"); generate_child(); @@ -923,7 +926,7 @@ TEST_F(replica_split_test, trigger_primary_parent_split_test) } // trigger_secondary_parent_split unit test -TEST_F(replica_split_test, secondary_handle_split_test) +TEST_P(replica_split_test, secondary_handle_split_test) { generate_child(); @@ -970,7 +973,7 @@ TEST_F(replica_split_test, secondary_handle_split_test) } } -TEST_F(replica_split_test, primary_parent_handle_stop_test) +TEST_P(replica_split_test, primary_parent_handle_stop_test) { fail::cfg("replica_parent_send_notify_stop_request", "return()"); // Test cases: @@ -1002,7 +1005,7 @@ TEST_F(replica_split_test, primary_parent_handle_stop_test) } } -TEST_F(replica_split_test, query_child_state_reply_test) +TEST_P(replica_split_test, query_child_state_reply_test) { fail::cfg("replica_init_group_check", "return()"); fail::cfg("replica_broadcast_group_check", "return()"); @@ -1014,7 +1017,7 @@ TEST_F(replica_split_test, query_child_state_reply_test) ASSERT_TRUE(primary_parent_not_in_split()); } -TEST_F(replica_split_test, check_partition_hash_test) +TEST_P(replica_split_test, check_partition_hash_test) { uint64_t send_to_parent_after_split = 1; uint64_t send_to_child_after_split = 9; diff --git a/src/replica/storage/simple_kv/CMakeLists.txt b/src/replica/storage/simple_kv/CMakeLists.txt index cb7d328fc1..667b4ab862 100644 --- a/src/replica/storage/simple_kv/CMakeLists.txt +++ b/src/replica/storage/simple_kv/CMakeLists.txt @@ -37,7 +37,7 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS}) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable) +set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp index b6f0a00e3f..5cc8d8d608 100644 --- a/src/replica/storage/simple_kv/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/simple_kv.server.impl.cpp @@ -35,25 +35,35 @@ #include "simple_kv.server.impl.h" +#include #include +#include #include #include #include -#include #include -#include +#include #include #include +#include "aio/aio_task.h" +#include "aio/file_io.h" +#include "common/replication.codes.h" #include "consensus_types.h" #include "replica/storage/simple_kv/simple_kv.server.h" +#include "rocksdb/env.h" +#include "rocksdb/status.h" #include "runtime/serverlet.h" #include "simple_kv_types.h" +#include "utils/autoref_ptr.h" +#include "utils/binary_reader.h" +#include "utils/blob.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/utils.h" namespace dsn { -class blob; namespace replication { class replica; @@ -176,37 +186,56 @@ void simple_kv_service_impl::recover(const std::string &name, int64_t version) { zauto_lock l(_lock); - std::ifstream is(name.c_str(), std::ios::binary); - if (!is.is_open()) - return; + std::unique_ptr rfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", name, s.ToString()); _store.clear(); - uint64_t count; - int magic; - - is.read((char *)&count, sizeof(count)); - is.read((char *)&magic, sizeof(magic)); + // Read header. + uint64_t count = 0; + int magic = 0; + rocksdb::Slice result; + static const uint64_t kHeaderSize = sizeof(count) + sizeof(magic); + char buff[kHeaderSize] = {0}; + s = rfile->Read(kHeaderSize, &result, buff); + CHECK(s.ok(), "read header failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + binary_reader reader(blob(buff, 0, kHeaderSize)); + CHECK_EQ(sizeof(count), reader.read(count)); + CHECK_EQ(sizeof(magic), reader.read(magic)); CHECK_EQ_MSG(magic, 0xdeadbeef, "invalid checkpoint"); + // Read kv pairs. for (uint64_t i = 0; i < count; i++) { - std::string key; - std::string value; - - uint32_t sz; - is.read((char *)&sz, (uint32_t)sizeof(sz)); - key.resize(sz); - - is.read((char *)&key[0], sz); - - is.read((char *)&sz, (uint32_t)sizeof(sz)); - value.resize(sz); - - is.read((char *)&value[0], sz); - + // Read key. + uint32_t sz = 0; + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read key size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr key_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, key_buffer.get()); + CHECK(s.ok(), "read key failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string key = result.ToString(); + + // Read value. + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read value size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr value_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, value_buffer.get()); + CHECK(s.ok(), "read value failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string value = result.ToString(); + + // Store the kv pair. _store[key] = value; } - is.close(); } ::dsn::error_code simple_kv_service_impl::sync_checkpoint() @@ -222,29 +251,43 @@ ::dsn::error_code simple_kv_service_impl::sync_checkpoint() return ERR_OK; } - std::ofstream os(name, std::ios::binary); + std::string fname = fmt::format("{}/checkpoint.{}", data_dir(), last_commit); + auto wfile = file::open(fname, file::FileOpenType::kWriteOnly); + CHECK_NOTNULL(wfile, ""); +#define WRITE_DATA_SIZE(data, size) \ + do { \ + auto tsk = ::dsn::file::write( \ + wfile, (char *)&data, size, offset, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr); \ + tsk->wait(); \ + offset += size; \ + } while (false) + +#define WRITE_DATA(data) WRITE_DATA_SIZE(data, sizeof(data)) + + uint64_t offset = 0; uint64_t count = (uint64_t)_store.size(); - int magic = 0xdeadbeef; + WRITE_DATA(count); - os.write((const char *)&count, (uint32_t)sizeof(count)); - os.write((const char *)&magic, (uint32_t)sizeof(magic)); + int magic = 0xdeadbeef; + WRITE_DATA(magic); - for (auto it = _store.begin(); it != _store.end(); ++it) { - const std::string &k = it->first; + for (const auto &kv : _store) { + const std::string &k = kv.first; uint32_t sz = (uint32_t)k.length(); + WRITE_DATA(sz); + WRITE_DATA_SIZE(k[0], sz); - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&k[0], sz); - - const std::string &v = it->second; + const std::string &v = kv.second; sz = (uint32_t)v.length(); - - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&v[0], sz); + WRITE_DATA(sz); + WRITE_DATA_SIZE(v[0], sz); } +#undef WRITE_DATA +#undef WRITE_DATA_SIZE - os.close(); + CHECK_EQ(ERR_OK, file::flush(wfile)); + CHECK_EQ(ERR_OK, file::close(wfile)); // TODO: gc checkpoints set_last_durable_decree(last_commit); diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt b/src/replica/storage/simple_kv/test/CMakeLists.txt index 1d64070fb6..de86358d09 100644 --- a/src/replica/storage/simple_kv/test/CMakeLists.txt +++ b/src/replica/storage/simple_kv/test/CMakeLists.txt @@ -39,7 +39,8 @@ set(MY_PROJ_LIBS dsn_replica_server zookeeper hashtable gtest - ) + dsn_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/storage/simple_kv/test/case-000.ini b/src/replica/storage/simple_kv/test/case-000.ini index a05a95e53f..4cb2b55450 100644 --- a/src/replica/storage/simple_kv/test/case-000.ini +++ b/src/replica/storage/simple_kv/test/case-000.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-001.ini b/src/replica/storage/simple_kv/test/case-001.ini index 4f214ef80b..08c42f699a 100644 --- a/src/replica/storage/simple_kv/test/case-001.ini +++ b/src/replica/storage/simple_kv/test/case-001.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-002.ini b/src/replica/storage/simple_kv/test/case-002.ini index d75bbc1013..e883afcc03 100644 --- a/src/replica/storage/simple_kv/test/case-002.ini +++ b/src/replica/storage/simple_kv/test/case-002.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-003.ini b/src/replica/storage/simple_kv/test/case-003.ini index 8d60d50d66..5ee053ad4d 100644 --- a/src/replica/storage/simple_kv/test/case-003.ini +++ b/src/replica/storage/simple_kv/test/case-003.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-004.ini b/src/replica/storage/simple_kv/test/case-004.ini index 5c74f7e281..a2eebf4ee6 100644 --- a/src/replica/storage/simple_kv/test/case-004.ini +++ b/src/replica/storage/simple_kv/test/case-004.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-005.ini b/src/replica/storage/simple_kv/test/case-005.ini index 4c8c84b324..1b75197419 100644 --- a/src/replica/storage/simple_kv/test/case-005.ini +++ b/src/replica/storage/simple_kv/test/case-005.ini @@ -148,6 +148,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-006.ini b/src/replica/storage/simple_kv/test/case-006.ini index 18e782b779..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-006.ini +++ b/src/replica/storage/simple_kv/test/case-006.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-100.ini b/src/replica/storage/simple_kv/test/case-100.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-100.ini +++ b/src/replica/storage/simple_kv/test/case-100.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-101.ini b/src/replica/storage/simple_kv/test/case-101.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-101.ini +++ b/src/replica/storage/simple_kv/test/case-101.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-102.ini b/src/replica/storage/simple_kv/test/case-102.ini index 8c35047783..4ac4bcc5f8 100644 --- a/src/replica/storage/simple_kv/test/case-102.ini +++ b/src/replica/storage/simple_kv/test/case-102.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-103.ini b/src/replica/storage/simple_kv/test/case-103.ini index 64a643dd0a..3d70ea9c5e 100644 --- a/src/replica/storage/simple_kv/test/case-103.ini +++ b/src/replica/storage/simple_kv/test/case-103.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-104.ini b/src/replica/storage/simple_kv/test/case-104.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-104.ini +++ b/src/replica/storage/simple_kv/test/case-104.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-105.ini b/src/replica/storage/simple_kv/test/case-105.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-105.ini +++ b/src/replica/storage/simple_kv/test/case-105.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-106.ini b/src/replica/storage/simple_kv/test/case-106.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-106.ini +++ b/src/replica/storage/simple_kv/test/case-106.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-107.ini b/src/replica/storage/simple_kv/test/case-107.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-107.ini +++ b/src/replica/storage/simple_kv/test/case-107.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-108.ini b/src/replica/storage/simple_kv/test/case-108.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-108.ini +++ b/src/replica/storage/simple_kv/test/case-108.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-109.ini b/src/replica/storage/simple_kv/test/case-109.ini index d95415e373..9a1d88c697 100644 --- a/src/replica/storage/simple_kv/test/case-109.ini +++ b/src/replica/storage/simple_kv/test/case-109.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-200.ini b/src/replica/storage/simple_kv/test/case-200.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-200.ini +++ b/src/replica/storage/simple_kv/test/case-200.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-201.ini b/src/replica/storage/simple_kv/test/case-201.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-201.ini +++ b/src/replica/storage/simple_kv/test/case-201.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-202-0.ini b/src/replica/storage/simple_kv/test/case-202-0.ini index a25eb9145e..dabdf01544 100644 --- a/src/replica/storage/simple_kv/test/case-202-0.ini +++ b/src/replica/storage/simple_kv/test/case-202-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-202-1.ini b/src/replica/storage/simple_kv/test/case-202-1.ini index a25eb9145e..dabdf01544 100644 --- a/src/replica/storage/simple_kv/test/case-202-1.ini +++ b/src/replica/storage/simple_kv/test/case-202-1.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-203-0.ini b/src/replica/storage/simple_kv/test/case-203-0.ini index 82d693aa9b..42c9c006c9 100644 --- a/src/replica/storage/simple_kv/test/case-203-0.ini +++ b/src/replica/storage/simple_kv/test/case-203-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-204.ini b/src/replica/storage/simple_kv/test/case-204.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-204.ini +++ b/src/replica/storage/simple_kv/test/case-204.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-205.ini b/src/replica/storage/simple_kv/test/case-205.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-205.ini +++ b/src/replica/storage/simple_kv/test/case-205.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-206.ini b/src/replica/storage/simple_kv/test/case-206.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-206.ini +++ b/src/replica/storage/simple_kv/test/case-206.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-207.ini b/src/replica/storage/simple_kv/test/case-207.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-207.ini +++ b/src/replica/storage/simple_kv/test/case-207.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-208.ini b/src/replica/storage/simple_kv/test/case-208.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-208.ini +++ b/src/replica/storage/simple_kv/test/case-208.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-209.ini b/src/replica/storage/simple_kv/test/case-209.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-209.ini +++ b/src/replica/storage/simple_kv/test/case-209.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-210.ini b/src/replica/storage/simple_kv/test/case-210.ini index 23784c15f0..a9dd9383cc 100644 --- a/src/replica/storage/simple_kv/test/case-210.ini +++ b/src/replica/storage/simple_kv/test/case-210.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-211.ini b/src/replica/storage/simple_kv/test/case-211.ini index 23784c15f0..a9dd9383cc 100644 --- a/src/replica/storage/simple_kv/test/case-211.ini +++ b/src/replica/storage/simple_kv/test/case-211.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-212.ini b/src/replica/storage/simple_kv/test/case-212.ini index 4582c07878..a61a428f21 100644 --- a/src/replica/storage/simple_kv/test/case-212.ini +++ b/src/replica/storage/simple_kv/test/case-212.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-213.ini b/src/replica/storage/simple_kv/test/case-213.ini index 103a599a63..660afcfff9 100644 --- a/src/replica/storage/simple_kv/test/case-213.ini +++ b/src/replica/storage/simple_kv/test/case-213.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-214.ini b/src/replica/storage/simple_kv/test/case-214.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-214.ini +++ b/src/replica/storage/simple_kv/test/case-214.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-215.ini b/src/replica/storage/simple_kv/test/case-215.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-215.ini +++ b/src/replica/storage/simple_kv/test/case-215.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-216.ini b/src/replica/storage/simple_kv/test/case-216.ini index 9fde171e66..eb1a97a05e 100644 --- a/src/replica/storage/simple_kv/test/case-216.ini +++ b/src/replica/storage/simple_kv/test/case-216.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-300-0.ini b/src/replica/storage/simple_kv/test/case-300-0.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-0.ini +++ b/src/replica/storage/simple_kv/test/case-300-0.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-300-1.ini b/src/replica/storage/simple_kv/test/case-300-1.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-1.ini +++ b/src/replica/storage/simple_kv/test/case-300-1.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-300-2.ini b/src/replica/storage/simple_kv/test/case-300-2.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-300-2.ini +++ b/src/replica/storage/simple_kv/test/case-300-2.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-301.ini b/src/replica/storage/simple_kv/test/case-301.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-301.ini +++ b/src/replica/storage/simple_kv/test/case-301.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-302.ini b/src/replica/storage/simple_kv/test/case-302.ini index 38d3a831f1..fdfb947769 100644 --- a/src/replica/storage/simple_kv/test/case-302.ini +++ b/src/replica/storage/simple_kv/test/case-302.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-303.ini b/src/replica/storage/simple_kv/test/case-303.ini index 04932bc943..db032f0a27 100644 --- a/src/replica/storage/simple_kv/test/case-303.ini +++ b/src/replica/storage/simple_kv/test/case-303.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-304.ini b/src/replica/storage/simple_kv/test/case-304.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-304.ini +++ b/src/replica/storage/simple_kv/test/case-304.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-305.ini b/src/replica/storage/simple_kv/test/case-305.ini index 04932bc943..db032f0a27 100644 --- a/src/replica/storage/simple_kv/test/case-305.ini +++ b/src/replica/storage/simple_kv/test/case-305.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-306.ini b/src/replica/storage/simple_kv/test/case-306.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-306.ini +++ b/src/replica/storage/simple_kv/test/case-306.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-307.ini b/src/replica/storage/simple_kv/test/case-307.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-307.ini +++ b/src/replica/storage/simple_kv/test/case-307.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-400.ini b/src/replica/storage/simple_kv/test/case-400.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-400.ini +++ b/src/replica/storage/simple_kv/test/case-400.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-401.ini b/src/replica/storage/simple_kv/test/case-401.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-401.ini +++ b/src/replica/storage/simple_kv/test/case-401.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-402.ini b/src/replica/storage/simple_kv/test/case-402.ini index 0f601799a1..69b06e6724 100644 --- a/src/replica/storage/simple_kv/test/case-402.ini +++ b/src/replica/storage/simple_kv/test/case-402.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-600.ini b/src/replica/storage/simple_kv/test/case-600.ini index 18e782b779..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-600.ini +++ b/src/replica/storage/simple_kv/test/case-600.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-601.ini b/src/replica/storage/simple_kv/test/case-601.ini index 9a25f26cc8..20ee3c8bf0 100644 --- a/src/replica/storage/simple_kv/test/case-601.ini +++ b/src/replica/storage/simple_kv/test/case-601.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-602.ini b/src/replica/storage/simple_kv/test/case-602.ini index 18e782b779..4a8d09e289 100644 --- a/src/replica/storage/simple_kv/test/case-602.ini +++ b/src/replica/storage/simple_kv/test/case-602.ini @@ -149,6 +149,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/case-603.ini b/src/replica/storage/simple_kv/test/case-603.ini index 1531c037fe..563b86adf8 100644 --- a/src/replica/storage/simple_kv/test/case-603.ini +++ b/src/replica/storage/simple_kv/test/case-603.ini @@ -151,6 +151,9 @@ arguments = localhost:34601 [meta_server] server_list = localhost:34601 +[pegasus.server] +encrypt_data_at_rest = false + [replication.app] app_name = simple_kv.instance0 app_type = simple_kv diff --git a/src/replica/storage/simple_kv/test/client.cpp b/src/replica/storage/simple_kv/test/client.cpp index b576986080..905cdb080c 100644 --- a/src/replica/storage/simple_kv/test/client.cpp +++ b/src/replica/storage/simple_kv/test/client.cpp @@ -36,7 +36,6 @@ #include "client.h" #include -#include #include #include #include diff --git a/src/replica/storage/simple_kv/test/run.sh b/src/replica/storage/simple_kv/test/run.sh index b3daa127d1..17affedf79 100755 --- a/src/replica/storage/simple_kv/test/run.sh +++ b/src/replica/storage/simple_kv/test/run.sh @@ -112,6 +112,13 @@ else fi if [ ! -z "${cases}" ]; then + OLD_TEST_OPTS=${TEST_OPTS} + TEST_OPTS=${OLD_TEST_OPTS},encrypt_data_at_rest=false + for id in ${cases}; do + run_case ${id} + echo + done + TEST_OPTS=${OLD_TEST_OPTS},encrypt_data_at_rest=true for id in ${cases}; do run_case ${id} echo diff --git a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp index 4e2d1e4e78..31338567d7 100644 --- a/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp +++ b/src/replica/storage/simple_kv/test/simple_kv.server.impl.cpp @@ -25,23 +25,36 @@ */ #include "simple_kv.server.impl.h" +#include #include #include #include #include -#include -#include +#include #include #include +#include "aio/aio_task.h" +#include "aio/file_io.h" #include "consensus_types.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/serverlet.h" +#include "runtime/task/task_code.h" #include "simple_kv_types.h" +#include "utils/autoref_ptr.h" +#include "utils/binary_reader.h" +#include "utils/blob.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/threadpool_code.h" +#include "utils/utils.h" +// TODO(yingchun): most of the code are the same as +// src/replica/storage/simple_kv/simple_kv.server.impl.cpp, unify the code! namespace dsn { -class blob; namespace replication { class replica; @@ -54,6 +67,8 @@ namespace dsn { namespace replication { namespace test { +DEFINE_TASK_CODE(LPC_AIO_IMMEDIATE_CALLBACK, TASK_PRIORITY_COMMON, dsn::THREAD_POOL_DEFAULT) + bool simple_kv_service_impl::s_simple_kv_open_fail = false; bool simple_kv_service_impl::s_simple_kv_close_fail = false; bool simple_kv_service_impl::s_simple_kv_get_checkpoint_fail = false; @@ -174,34 +189,54 @@ void simple_kv_service_impl::recover(const std::string &name, int64_t version) { dsn::zauto_lock l(_lock); - std::ifstream is(name.c_str(), std::ios::binary); - if (!is.is_open()) - return; + std::unique_ptr rfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(name, &rfile, rocksdb::EnvOptions()); + CHECK(s.ok(), "open log file '{}' failed, err = {}", name, s.ToString()); _store.clear(); - uint64_t count; - int magic; - - is.read((char *)&count, sizeof(count)); - is.read((char *)&magic, sizeof(magic)); + // Read header. + uint64_t count = 0; + int magic = 0; + rocksdb::Slice result; + static const uint64_t kHeaderSize = sizeof(count) + sizeof(magic); + char buff[kHeaderSize] = {0}; + s = rfile->Read(kHeaderSize, &result, buff); + CHECK(s.ok(), "read header failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + binary_reader reader(blob(buff, 0, kHeaderSize)); + CHECK_EQ(sizeof(count), reader.read(count)); + CHECK_EQ(sizeof(magic), reader.read(magic)); CHECK_EQ_MSG(magic, 0xdeadbeef, "invalid checkpoint"); + // Read kv pairs. for (uint64_t i = 0; i < count; i++) { - std::string key; - std::string value; - - uint32_t sz; - is.read((char *)&sz, (uint32_t)sizeof(sz)); - key.resize(sz); - - is.read((char *)&key[0], sz); - - is.read((char *)&sz, (uint32_t)sizeof(sz)); - value.resize(sz); - - is.read((char *)&value[0], sz); - + // Read key. + uint32_t sz = 0; + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read key size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr key_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, key_buffer.get()); + CHECK(s.ok(), "read key failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string key = result.ToString(); + + // Read value. + s = rfile->Read(sizeof(sz), &result, (char *)&sz); + CHECK(s.ok(), "read value size failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + + std::shared_ptr value_buffer(dsn::utils::make_shared_array(sz)); + s = rfile->Read(sz, &result, value_buffer.get()); + CHECK(s.ok(), "read value failed, err = {}", s.ToString()); + CHECK(!result.empty(), "read EOF of file '{}'", name); + std::string value = result.ToString(); + + // Store the kv pair. _store[key] = value; } } @@ -218,30 +253,43 @@ ::dsn::error_code simple_kv_service_impl::sync_checkpoint() return ERR_OK; } - // TODO: should use async write instead - char name[256]; - sprintf(name, "%s/checkpoint.%" PRId64, data_dir().c_str(), last_commit); - std::ofstream os(name, std::ios::binary); + std::string fname = fmt::format("{}/checkpoint.{}", data_dir(), last_commit); + auto wfile = file::open(fname, file::FileOpenType::kWriteOnly); + CHECK_NOTNULL(wfile, ""); +#define WRITE_DATA_SIZE(data, size) \ + do { \ + auto tsk = ::dsn::file::write( \ + wfile, (char *)&data, size, offset, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr); \ + tsk->wait(); \ + offset += size; \ + } while (false) + +#define WRITE_DATA(data) WRITE_DATA_SIZE(data, sizeof(data)) + + uint64_t offset = 0; uint64_t count = (uint64_t)_store.size(); - int magic = 0xdeadbeef; + WRITE_DATA(count); - os.write((const char *)&count, (uint32_t)sizeof(count)); - os.write((const char *)&magic, (uint32_t)sizeof(magic)); + int magic = 0xdeadbeef; + WRITE_DATA(magic); - for (auto it = _store.begin(); it != _store.end(); ++it) { - const std::string &k = it->first; + for (const auto &kv : _store) { + const std::string &k = kv.first; uint32_t sz = (uint32_t)k.length(); + WRITE_DATA(sz); + WRITE_DATA_SIZE(k[0], sz); - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&k[0], sz); - - const std::string &v = it->second; + const std::string &v = kv.second; sz = (uint32_t)v.length(); - - os.write((const char *)&sz, (uint32_t)sizeof(sz)); - os.write((const char *)&v[0], sz); + WRITE_DATA(sz); + WRITE_DATA_SIZE(v[0], sz); } +#undef WRITE_DATA +#undef WRITE_DATA_SIZE + + CHECK_EQ(ERR_OK, file::flush(wfile)); + CHECK_EQ(ERR_OK, file::close(wfile)); set_last_durable_decree(last_commit); LOG_INFO("simple_kv_service_impl create checkpoint succeed, " diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 3b82d28e12..587826baad 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -47,7 +47,8 @@ set(MY_PROJ_LIBS dsn_meta_server zookeeper hashtable gtest - test_utils) + test_utils + rocksdb) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/test/log_block_test.cpp b/src/replica/test/log_block_test.cpp index 5e1029ba9b..ada8829e0b 100644 --- a/src/replica/test/log_block_test.cpp +++ b/src/replica/test/log_block_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -39,7 +40,9 @@ class log_block_test : public replica_test_base { }; -TEST_F(log_block_test, constructor) +INSTANTIATE_TEST_CASE_P(, log_block_test, ::testing::Values(false, true)); + +TEST_P(log_block_test, constructor) { log_block block(1); ASSERT_EQ(block.data().size(), 1); @@ -47,7 +50,7 @@ TEST_F(log_block_test, constructor) ASSERT_EQ(block.start_offset(), 1); } -TEST_F(log_block_test, log_block_header) +TEST_P(log_block_test, log_block_header) { log_block block(10); auto hdr = (log_block_header *)block.front().data(); @@ -60,7 +63,9 @@ class log_appender_test : public replica_test_base { }; -TEST_F(log_appender_test, constructor) +INSTANTIATE_TEST_CASE_P(, log_appender_test, ::testing::Values(false, true)); + +TEST_P(log_appender_test, constructor) { log_block block; binary_writer temp_writer; @@ -75,7 +80,7 @@ TEST_F(log_appender_test, constructor) ASSERT_EQ(appender.callbacks().size(), 0); } -TEST_F(log_appender_test, append_mutation) +TEST_P(log_appender_test, append_mutation) { log_appender appender(10); for (int i = 0; i < 5; i++) { @@ -88,7 +93,7 @@ TEST_F(log_appender_test, append_mutation) ASSERT_EQ(appender.blob_count(), 1 + 5 * 2); } -TEST_F(log_appender_test, log_block_not_full) +TEST_P(log_appender_test, log_block_not_full) { log_appender appender(10); for (int i = 0; i < 5; i++) { @@ -106,7 +111,7 @@ TEST_F(log_appender_test, log_block_not_full) ASSERT_EQ(block.data().size(), 1 + 5 * 2); } -TEST_F(log_appender_test, log_block_full) +TEST_P(log_appender_test, log_block_full) { log_appender appender(10); for (int i = 0; i < 1024; i++) { // more than DEFAULT_MAX_BLOCK_BYTES @@ -130,7 +135,7 @@ TEST_F(log_appender_test, log_block_full) ASSERT_EQ(sz, appender.size()); } -TEST_F(log_appender_test, read_log_block) +TEST_P(log_appender_test, read_log_block) { log_appender appender(10); for (int i = 0; i < 1024; i++) { // more than DEFAULT_MAX_BLOCK_BYTES diff --git a/src/replica/test/log_file_test.cpp b/src/replica/test/log_file_test.cpp index 5e0e44cd9b..e0736f15cc 100644 --- a/src/replica/test/log_file_test.cpp +++ b/src/replica/test/log_file_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -56,7 +57,9 @@ class log_file_test : public replica_test_base size_t _start_offset{10}; }; -TEST_F(log_file_test, commit_log_blocks) +INSTANTIATE_TEST_CASE_P(, log_file_test, ::testing::Values(false, true)); + +TEST_P(log_file_test, commit_log_blocks) { // write one block auto appender = std::make_shared(_start_offset); diff --git a/src/replica/test/main.cpp b/src/replica/test/main.cpp index 61d0f998a1..6e414ffddf 100644 --- a/src/replica/test/main.cpp +++ b/src/replica/test/main.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include #include #include #include @@ -24,27 +25,35 @@ #include "replication_service_test_app.h" #include "runtime/app_model.h" #include "runtime/service_app.h" +#include "test_util/test_util.h" #include "utils/error_code.h" int gtest_flags = 0; int gtest_ret = 0; replication_service_test_app *app; -TEST(cold_backup_context, check_backup_on_remote) { app->check_backup_on_remote_test(); } +class cold_backup_context_test : public pegasus::encrypt_data_test_base +{ +}; + +TEST_P(cold_backup_context_test, check_backup_on_remote) { app->check_backup_on_remote_test(); } -TEST(cold_backup_context, read_current_chkpt_file) { app->read_current_chkpt_file_test(); } +TEST_P(cold_backup_context_test, read_current_chkpt_file) { app->read_current_chkpt_file_test(); } -TEST(cold_backup_context, remote_chkpt_dir_exist) { app->remote_chkpt_dir_exist_test(); } +TEST_P(cold_backup_context_test, remote_chkpt_dir_exist) { app->remote_chkpt_dir_exist_test(); } -TEST(cold_backup_context, upload_checkpoint_to_remote) { app->upload_checkpoint_to_remote_test(); } +TEST_P(cold_backup_context_test, upload_checkpoint_to_remote) +{ + app->upload_checkpoint_to_remote_test(); +} -TEST(cold_backup_context, read_backup_metadata) { app->read_backup_metadata_test(); } +TEST_P(cold_backup_context_test, read_backup_metadata) { app->read_backup_metadata_test(); } -TEST(cold_backup_context, on_upload_chkpt_dir) { app->on_upload_chkpt_dir_test(); } +TEST_P(cold_backup_context_test, on_upload_chkpt_dir) { app->on_upload_chkpt_dir_test(); } -TEST(cold_backup_context, write_metadata_file) { app->write_backup_metadata_test(); } +TEST_P(cold_backup_context_test, write_metadata_file) { app->write_backup_metadata_test(); } -TEST(cold_backup_context, write_current_chkpt_file) { app->write_current_chkpt_file_test(); } +TEST_P(cold_backup_context_test, write_current_chkpt_file) { app->write_current_chkpt_file_test(); } error_code replication_service_test_app::start(const std::vector &args) { diff --git a/src/replica/test/mutation_log_learn_test.cpp b/src/replica/test/mutation_log_learn_test.cpp index 6c2fdf71ac..b14701b443 100644 --- a/src/replica/test/mutation_log_learn_test.cpp +++ b/src/replica/test/mutation_log_learn_test.cpp @@ -24,6 +24,7 @@ * THE SOFTWARE. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -56,11 +57,13 @@ class message_ex; namespace replication { -class mutation_log_test : public replica_test_base +class mutation_log_learn_test : public replica_test_base { }; -TEST_F(mutation_log_test, learn) +INSTANTIATE_TEST_CASE_P(, mutation_log_learn_test, ::testing::Values(false, true)); + +TEST_P(mutation_log_learn_test, learn) { std::chrono::steady_clock clock; gpid gpid(1, 1); diff --git a/src/replica/test/mutation_log_test.cpp b/src/replica/test/mutation_log_test.cpp index 6bca13336a..de396f899c 100644 --- a/src/replica/test/mutation_log_test.cpp +++ b/src/replica/test/mutation_log_test.cpp @@ -26,14 +26,15 @@ #include "replica/mutation_log.h" +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include #include "aio/aio_task.h" +#include "aio/file_io.h" #include "backup_types.h" #include "common/replication.codes.h" #include "consensus_types.h" @@ -42,9 +43,11 @@ #include "replica/mutation.h" #include "replica/test/mock_utils.h" #include "replica_test_base.h" +#include "test_util/test_util.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/ports.h" @@ -56,43 +59,32 @@ class message_ex; using namespace ::dsn; using namespace ::dsn::replication; -static void copy_file(const char *from_file, const char *to_file, int64_t to_size = -1) +static void overwrite_file(const char *file, int offset, const void *buf, int size) { - int64_t from_size; - ASSERT_TRUE(dsn::utils::filesystem::file_size(from_file, from_size)); - ASSERT_LE(to_size, from_size); - FILE *from = fopen(from_file, "rb"); - ASSERT_TRUE(from != nullptr); - FILE *to = fopen(to_file, "wb"); - ASSERT_TRUE(to != nullptr); - if (to_size == -1) - to_size = from_size; - if (to_size > 0) { - std::unique_ptr buf(new char[to_size]); - auto n = fread(buf.get(), 1, to_size, from); - ASSERT_EQ(to_size, n); - n = fwrite(buf.get(), 1, to_size, to); - ASSERT_EQ(to_size, n); - } - int r = fclose(from); - ASSERT_EQ(0, r); - r = fclose(to); - ASSERT_EQ(0, r); + auto wfile = file::open(file, file::FileOpenType::kWriteOnly); + ASSERT_NE(wfile, nullptr); + auto t = ::dsn::file::write(wfile, + (const char *)buf, + size, + offset, + LPC_AIO_IMMEDIATE_CALLBACK, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(size, n); + }); + t->wait(); + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); } -static void overwrite_file(const char *file, int offset, const void *buf, int size) +class replication_test : public pegasus::encrypt_data_test_base { - FILE *f = fopen(file, "r+b"); - ASSERT_TRUE(f != nullptr); - int r = fseek(f, offset, SEEK_SET); - ASSERT_EQ(0, r); - size_t n = fwrite(buf, 1, size, f); - ASSERT_EQ(size, n); - r = fclose(f); - ASSERT_EQ(0, r); -} +}; + +INSTANTIATE_TEST_CASE_P(, replication_test, ::testing::Values(false, true)); -TEST(replication, log_file) +TEST_P(replication_test, log_file) { replica_log_info_map mdecrees; gpid gpid(1, 0); @@ -188,7 +180,7 @@ TEST(replication, log_file) // bad file data: empty file ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.0")); - copy_file(fpath.c_str(), "log.1.0", 0); + dsn::utils::copy_file_by_size(fpath, "log.1.0", 0); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.0")); lf = log_file::open_read("log.1.0", err); ASSERT_TRUE(lf == nullptr); @@ -198,7 +190,7 @@ TEST(replication, log_file) // bad file data: incomplete log_block_header ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.1")); - copy_file(fpath.c_str(), "log.1.1", sizeof(log_block_header) - 1); + dsn::utils::copy_file_by_size(fpath, "log.1.1", sizeof(log_block_header) - 1); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.1")); lf = log_file::open_read("log.1.1", err); ASSERT_TRUE(lf == nullptr); @@ -208,7 +200,7 @@ TEST(replication, log_file) // bad file data: bad log_block_header (magic = 0xfeadbeef) ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.2")); - copy_file(fpath.c_str(), "log.1.2"); + dsn::utils::copy_file_by_size(fpath, "log.1.2"); int32_t bad_magic = 0xfeadbeef; overwrite_file("log.1.2", FIELD_OFFSET(log_block_header, magic), &bad_magic, sizeof(bad_magic)); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.2")); @@ -220,7 +212,7 @@ TEST(replication, log_file) // bad file data: bad log_block_header (crc check failed) ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.3")); - copy_file(fpath.c_str(), "log.1.3"); + dsn::utils::copy_file_by_size(fpath, "log.1.3"); int32_t bad_crc = 0; overwrite_file("log.1.3", FIELD_OFFSET(log_block_header, body_crc), &bad_crc, sizeof(bad_crc)); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.3")); @@ -232,14 +224,13 @@ TEST(replication, log_file) // bad file data: incomplete block body ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4")); - copy_file(fpath.c_str(), "log.1.4", sizeof(log_block_header) + 1); + dsn::utils::copy_file_by_size(fpath, "log.1.4", sizeof(log_block_header) + 1); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4")); lf = log_file::open_read("log.1.4", err); ASSERT_TRUE(lf == nullptr); ASSERT_EQ(ERR_INCOMPLETE_DATA, err); ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4")); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4.removed")); - ASSERT_TRUE(dsn::utils::filesystem::rename_path("log.1.4.removed", "log.1.4")); // read the file for test offset = 100; @@ -249,7 +240,7 @@ TEST(replication, log_file) ASSERT_EQ(1, lf->index()); ASSERT_EQ(100, lf->start_offset()); int64_t sz; - ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, sz)); + ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, dsn::utils::FileDataType::kSensitive, sz)); ASSERT_EQ(lf->start_offset() + sz, lf->end_offset()); // read data @@ -452,20 +443,22 @@ class mutation_log_test : public replica_test_base } }; -TEST_F(mutation_log_test, replay_single_file_1000) { test_replay_single_file(1000); } +INSTANTIATE_TEST_CASE_P(, mutation_log_test, ::testing::Values(false, true)); + +TEST_P(mutation_log_test, replay_single_file_1000) { test_replay_single_file(1000); } -TEST_F(mutation_log_test, replay_single_file_2000) { test_replay_single_file(2000); } +TEST_P(mutation_log_test, replay_single_file_2000) { test_replay_single_file(2000); } -TEST_F(mutation_log_test, replay_single_file_5000) { test_replay_single_file(5000); } +TEST_P(mutation_log_test, replay_single_file_5000) { test_replay_single_file(5000); } -TEST_F(mutation_log_test, replay_single_file_10000) { test_replay_single_file(10000); } +TEST_P(mutation_log_test, replay_single_file_10000) { test_replay_single_file(10000); } -TEST_F(mutation_log_test, replay_single_file_1) { test_replay_single_file(1); } +TEST_P(mutation_log_test, replay_single_file_1) { test_replay_single_file(1); } -TEST_F(mutation_log_test, replay_single_file_10) { test_replay_single_file(10); } +TEST_P(mutation_log_test, replay_single_file_10) { test_replay_single_file(10); } // mutation_log::open -TEST_F(mutation_log_test, open) +TEST_P(mutation_log_test, open) { std::vector mutations; @@ -498,13 +491,13 @@ TEST_F(mutation_log_test, open) } } -TEST_F(mutation_log_test, replay_multiple_files_10000_1mb) { test_replay_multiple_files(10000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_10000_1mb) { test_replay_multiple_files(10000, 1); } -TEST_F(mutation_log_test, replay_multiple_files_20000_1mb) { test_replay_multiple_files(20000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_20000_1mb) { test_replay_multiple_files(20000, 1); } -TEST_F(mutation_log_test, replay_multiple_files_50000_1mb) { test_replay_multiple_files(50000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_50000_1mb) { test_replay_multiple_files(50000, 1); } -TEST_F(mutation_log_test, replay_start_decree) +TEST_P(mutation_log_test, replay_start_decree) { // decree ranges from [1, 30) generate_multiple_log_files(3); @@ -517,7 +510,7 @@ TEST_F(mutation_log_test, replay_start_decree) ASSERT_EQ(mlog->get_log_file_map().size(), 3); } -TEST_F(mutation_log_test, reset_from) +TEST_P(mutation_log_test, reset_from) { std::vector expected; { // writing logs @@ -565,7 +558,7 @@ TEST_F(mutation_log_test, reset_from) // multi-threaded testing. ensure reset_from will wait until // all previous writes complete. -TEST_F(mutation_log_test, reset_from_while_writing) +TEST_P(mutation_log_test, reset_from_while_writing) { std::vector expected; { // writing logs diff --git a/src/replica/test/open_replica_test.cpp b/src/replica/test/open_replica_test.cpp index b4cb088f62..1fa4ebe6fc 100644 --- a/src/replica/test/open_replica_test.cpp +++ b/src/replica/test/open_replica_test.cpp @@ -16,6 +16,7 @@ // under the License. #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -46,7 +47,9 @@ class open_replica_test : public replica_test_base ~open_replica_test() { dsn::utils::filesystem::remove_path("./tmp_dir"); } }; -TEST_F(open_replica_test, open_replica_add_decree_and_ballot_check) +INSTANTIATE_TEST_CASE_P(, open_replica_test, ::testing::Values(false, true)); + +TEST_P(open_replica_test, open_replica_add_decree_and_ballot_check) { app_info ai; ai.app_type = "replica"; diff --git a/src/replica/test/replica_disk_migrate_test.cpp b/src/replica/test/replica_disk_migrate_test.cpp index 024b039cef..b3a796163a 100644 --- a/src/replica/test/replica_disk_migrate_test.cpp +++ b/src/replica/test/replica_disk_migrate_test.cpp @@ -19,9 +19,11 @@ #include #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include @@ -151,7 +153,9 @@ class replica_disk_migrate_test : public replica_disk_test_base } }; -TEST_F(replica_disk_migrate_test, on_migrate_replica) +INSTANTIATE_TEST_CASE_P(, replica_disk_migrate_test, ::testing::Values(false, true)); + +TEST_P(replica_disk_migrate_test, on_migrate_replica) { auto &request = *fake_migrate_rpc.mutable_request(); auto &response = fake_migrate_rpc.response(); @@ -171,7 +175,7 @@ TEST_F(replica_disk_migrate_test, on_migrate_replica) ASSERT_EQ(response.err, ERR_OK); } -TEST_F(replica_disk_migrate_test, migrate_disk_replica_check) +TEST_P(replica_disk_migrate_test, migrate_disk_replica_check) { auto &request = *fake_migrate_rpc.mutable_request(); auto &response = fake_migrate_rpc.response(); @@ -230,7 +234,7 @@ TEST_F(replica_disk_migrate_test, migrate_disk_replica_check) ASSERT_EQ(response.err, ERR_OK); } -TEST_F(replica_disk_migrate_test, disk_migrate_replica_run) +TEST_P(replica_disk_migrate_test, disk_migrate_replica_run) { auto &request = *fake_migrate_rpc.mutable_request(); @@ -295,7 +299,7 @@ TEST_F(replica_disk_migrate_test, disk_migrate_replica_run) ASSERT_EQ(replica_ptr->disk_migrator()->status(), disk_migration_status::IDLE); } -TEST_F(replica_disk_migrate_test, disk_migrate_replica_close) +TEST_P(replica_disk_migrate_test, disk_migrate_replica_close) { auto &request = *fake_migrate_rpc.mutable_request(); request.pid = dsn::gpid(app_info_1.app_id, 2); @@ -310,7 +314,7 @@ TEST_F(replica_disk_migrate_test, disk_migrate_replica_close) ASSERT_TRUE(close_current_replica(fake_migrate_rpc)); } -TEST_F(replica_disk_migrate_test, disk_migrate_replica_update) +TEST_P(replica_disk_migrate_test, disk_migrate_replica_update) { auto &request = *fake_migrate_rpc.mutable_request(); request.pid = dsn::gpid(app_info_1.app_id, 3); @@ -366,7 +370,7 @@ TEST_F(replica_disk_migrate_test, disk_migrate_replica_update) // Test load from new replica dir failed, then fall back to load from origin dir succeed, // and then mark the "new" replica dir as ".gar". -TEST_F(replica_disk_migrate_test, disk_migrate_replica_open) +TEST_P(replica_disk_migrate_test, disk_migrate_replica_open) { gpid test_pid(app_info_1.app_id, 4); diff --git a/src/replica/test/replica_disk_test.cpp b/src/replica/test/replica_disk_test.cpp index 2ecef5adc7..3bf7459347 100644 --- a/src/replica/test/replica_disk_test.cpp +++ b/src/replica/test/replica_disk_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -86,7 +87,9 @@ class replica_disk_test : public replica_disk_test_base } }; -TEST_F(replica_disk_test, on_query_disk_info_all_app) +INSTANTIATE_TEST_CASE_P(, replica_disk_test, ::testing::Values(false, true)); + +TEST_P(replica_disk_test, on_query_disk_info_all_app) { generate_fake_rpc(); stub->on_query_disk_info(fake_query_disk_rpc); @@ -160,7 +163,7 @@ TEST_F(replica_disk_test, on_query_disk_info_all_app) } } -TEST_F(replica_disk_test, on_query_disk_info_app_not_existed) +TEST_P(replica_disk_test, on_query_disk_info_app_not_existed) { generate_fake_rpc(); query_disk_info_request &request = *fake_query_disk_rpc.mutable_request(); @@ -169,7 +172,7 @@ TEST_F(replica_disk_test, on_query_disk_info_app_not_existed) ASSERT_EQ(fake_query_disk_rpc.response().err, ERR_OBJECT_NOT_FOUND); } -TEST_F(replica_disk_test, on_query_disk_info_one_app) +TEST_P(replica_disk_test, on_query_disk_info_one_app) { generate_fake_rpc(); query_disk_info_request &request = *fake_query_disk_rpc.mutable_request(); @@ -195,7 +198,7 @@ TEST_F(replica_disk_test, on_query_disk_info_one_app) } } -TEST_F(replica_disk_test, gc_disk_useless_dir) +TEST_P(replica_disk_test, gc_disk_useless_dir) { FLAGS_gc_disk_error_replica_interval_seconds = 1; FLAGS_gc_disk_garbage_replica_interval_seconds = 1; @@ -237,7 +240,7 @@ TEST_F(replica_disk_test, gc_disk_useless_dir) ASSERT_EQ(report.error_replica_count, 2); } -TEST_F(replica_disk_test, disk_status_test) +TEST_P(replica_disk_test, disk_status_test) { struct disk_status_test { @@ -261,7 +264,7 @@ TEST_F(replica_disk_test, disk_status_test) dn->status = disk_status::NORMAL; } -TEST_F(replica_disk_test, add_new_disk_test) +TEST_P(replica_disk_test, add_new_disk_test) { // Test case: // - invalid params @@ -291,7 +294,7 @@ TEST_F(replica_disk_test, add_new_disk_test) } } -TEST_F(replica_disk_test, disk_io_error_test) +TEST_P(replica_disk_test, disk_io_error_test) { // Disable failure detector to avoid connecting with meta server which is not started. FLAGS_fd_disabled = true; diff --git a/src/replica/test/replica_http_service_test.cpp b/src/replica/test/replica_http_service_test.cpp index 94c682302f..4598a5f3f0 100644 --- a/src/replica/test/replica_http_service_test.cpp +++ b/src/replica/test/replica_http_service_test.cpp @@ -16,6 +16,7 @@ // under the License. #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -86,7 +87,10 @@ class replica_http_service_test : public replica_test_base std::unique_ptr _http_svc; }; -TEST_F(replica_http_service_test, update_config_handler) +// Non-encryption related, just test the 'false' parameter. +INSTANTIATE_TEST_CASE_P(, replica_http_service_test, ::testing::Values(false)); + +TEST_P(replica_http_service_test, update_config_handler) { // Test the default value. test_check_config("config_sync_interval_ms", "30000"); diff --git a/src/replica/test/replica_learn_test.cpp b/src/replica/test/replica_learn_test.cpp index 8ff02993a1..8cf8e5b662 100644 --- a/src/replica/test/replica_learn_test.cpp +++ b/src/replica/test/replica_learn_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -180,9 +181,11 @@ class replica_learn_test : public duplication_test_base } }; -TEST_F(replica_learn_test, get_learn_start_decree) { test_get_learn_start_decree(); } +INSTANTIATE_TEST_CASE_P(, replica_learn_test, ::testing::Values(false, true)); -TEST_F(replica_learn_test, get_max_gced_decree_for_learn) { test_get_max_gced_decree_for_learn(); } +TEST_P(replica_learn_test, get_learn_start_decree) { test_get_learn_start_decree(); } + +TEST_P(replica_learn_test, get_max_gced_decree_for_learn) { test_get_max_gced_decree_for_learn(); } } // namespace replication } // namespace dsn diff --git a/src/replica/test/replica_test.cpp b/src/replica/test/replica_test.cpp index f1ff0acc9d..9b7aea83b4 100644 --- a/src/replica/test/replica_test.cpp +++ b/src/replica/test/replica_test.cpp @@ -16,7 +16,7 @@ // under the License. #include -#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -54,7 +54,6 @@ #include "replica/test/mock_utils.h" #include "replica_test_base.h" #include "runtime/api_layer1.h" -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_message.h" @@ -62,6 +61,7 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -186,7 +186,8 @@ class replica_test : public replica_test_base cold_backup::get_current_chkpt_file(backup_root, req.app_name, req.pid, req.backup_id); ASSERT_TRUE(dsn::utils::filesystem::file_exists(current_chkpt_file)); int64_t size = 0; - dsn::utils::filesystem::file_size(current_chkpt_file, size); + dsn::utils::filesystem::file_size( + current_chkpt_file, dsn::utils::FileDataType::kSensitive, size); ASSERT_LT(0, size); } @@ -242,7 +243,7 @@ class replica_test : public replica_test_base // load new max_replica_count from file auto err = replica_info.load(path); - ASSERT_EQ(err, ERR_OK); + ASSERT_EQ(ERR_OK, err); ASSERT_EQ(info, _mock_replica->_app_info); std::cout << "the loaded new app_info is " << info << std::endl; @@ -257,6 +258,8 @@ class replica_test : public replica_test_base std::cout << "the loaded original app_info is " << info << std::endl; } + void test_auto_trash(error_code ec); + public: dsn::app_info _app_info; dsn::gpid _pid; @@ -268,7 +271,9 @@ class replica_test : public replica_test_base const std::string _policy_name; }; -TEST_F(replica_test, write_size_limited) +INSTANTIATE_TEST_CASE_P(, replica_test, ::testing::Values(false, true)); + +TEST_P(replica_test, write_size_limited) { int count = 100; struct dsn::message_header header; @@ -288,7 +293,7 @@ TEST_F(replica_test, write_size_limited) ASSERT_EQ(get_write_size_exceed_threshold_count(), count); } -TEST_F(replica_test, backup_request_qps) +TEST_P(replica_test, backup_request_qps) { // create backup request struct dsn::message_header header; @@ -307,7 +312,7 @@ TEST_F(replica_test, backup_request_qps) ASSERT_GT(get_table_level_backup_request_qps(), 0); } -TEST_F(replica_test, query_data_version_test) +TEST_P(replica_test, query_data_version_test) { replica_http_service http_svc(stub.get()); struct query_data_version_test @@ -334,7 +339,7 @@ TEST_F(replica_test, query_data_version_test) } } -TEST_F(replica_test, query_compaction_test) +TEST_P(replica_test, query_compaction_test) { replica_http_service http_svc(stub.get()); struct query_compaction_test @@ -362,7 +367,7 @@ TEST_F(replica_test, query_compaction_test) } } -TEST_F(replica_test, update_validate_partition_hash_test) +TEST_P(replica_test, update_validate_partition_hash_test) { struct update_validate_partition_hash_test { @@ -385,7 +390,7 @@ TEST_F(replica_test, update_validate_partition_hash_test) } } -TEST_F(replica_test, update_allow_ingest_behind_test) +TEST_P(replica_test, update_allow_ingest_behind_test) { struct update_allow_ingest_behind_test { @@ -408,24 +413,26 @@ TEST_F(replica_test, update_allow_ingest_behind_test) } } -TEST_F(replica_test, test_replica_backup_and_restore) +TEST_P(replica_test, test_replica_backup_and_restore) { // TODO(yingchun): this test last too long time, optimize it! + return; test_on_cold_backup(); auto err = test_find_valid_checkpoint(); ASSERT_EQ(ERR_OK, err); } -TEST_F(replica_test, test_replica_backup_and_restore_with_specific_path) +TEST_P(replica_test, test_replica_backup_and_restore_with_specific_path) { // TODO(yingchun): this test last too long time, optimize it! + return; std::string user_specified_path = "test/backup"; test_on_cold_backup(user_specified_path); auto err = test_find_valid_checkpoint(user_specified_path); ASSERT_EQ(ERR_OK, err); } -TEST_F(replica_test, test_trigger_manual_emergency_checkpoint) +TEST_P(replica_test, test_trigger_manual_emergency_checkpoint) { ASSERT_EQ(_mock_replica->trigger_manual_emergency_checkpoint(100), ERR_OK); ASSERT_TRUE(is_checkpointing()); @@ -452,7 +459,7 @@ TEST_F(replica_test, test_trigger_manual_emergency_checkpoint) _mock_replica->tracker()->wait_outstanding_tasks(); } -TEST_F(replica_test, test_query_last_checkpoint_info) +TEST_P(replica_test, test_query_last_checkpoint_info) { // test no exist gpid auto req = std::make_unique(); @@ -475,7 +482,7 @@ TEST_F(replica_test, test_query_last_checkpoint_info) ASSERT_STR_CONTAINS(resp.base_local_dir, "/data/checkpoint.100"); } -TEST_F(replica_test, test_clear_on_failure) +TEST_P(replica_test, test_clear_on_failure) { // Clear up the remaining state. auto *dn = stub->get_fs_manager()->find_replica_dir(_app_info.app_type, _pid); @@ -498,26 +505,18 @@ TEST_F(replica_test, test_clear_on_failure) ASSERT_FALSE(has_gpid(_pid)); } -class replica_error_test : public replica_test, public testing::WithParamInterface -{ -}; - -INSTANTIATE_TEST_CASE_P(, - replica_error_test, - ::testing::Values(ERR_RDB_CORRUPTION, ERR_DISK_IO_ERROR)); - -TEST_P(replica_error_test, test_auto_trash_of_corruption) +void replica_test::test_auto_trash(error_code ec) { - const auto ec = GetParam(); - // The replica path will only be moved to error path when encounter ERR_RDB_CORRUPTION error. + // The replica path will only be moved to error path when encounter ERR_RDB_CORRUPTION + // error. bool moved_to_err_path = (ec == ERR_RDB_CORRUPTION); // Clear up the remaining state. auto *dn = stub->get_fs_manager()->find_replica_dir(_app_info.app_type, _pid); if (dn != nullptr) { dsn::utils::filesystem::remove_path(dn->replica_dir(_app_info.app_type, _pid)); + dn->holding_replicas.clear(); } - dn->holding_replicas.clear(); // Disable failure detector to avoid connecting with meta server which is not started. FLAGS_fd_disabled = true; @@ -565,7 +564,14 @@ TEST_P(replica_error_test, test_auto_trash_of_corruption) } } -TEST_F(replica_test, update_deny_client_test) +TEST_P(replica_test, test_auto_trash_of_corruption) +{ + NO_FATALS(test_auto_trash(ERR_RDB_CORRUPTION)); +} + +TEST_P(replica_test, test_auto_trash_of_io_error) { NO_FATALS(test_auto_trash(ERR_DISK_IO_ERROR)); } + +TEST_P(replica_test, update_deny_client_test) { struct update_deny_client_test { @@ -584,7 +590,7 @@ TEST_F(replica_test, update_deny_client_test) } } -TEST_F(replica_test, test_update_app_max_replica_count) { test_update_app_max_replica_count(); } +TEST_P(replica_test, test_update_app_max_replica_count) { test_update_app_max_replica_count(); } } // namespace replication } // namespace dsn diff --git a/src/replica/test/replica_test_base.h b/src/replica/test/replica_test_base.h index 3aed53c809..17bb9adf14 100644 --- a/src/replica/test/replica_test_base.h +++ b/src/replica/test/replica_test_base.h @@ -26,20 +26,20 @@ #pragma once -#include "utils/smart_pointers.h" -#include "replica/replication_app_base.h" -#include "utils/filesystem.h" -#include "utils/errors.h" #include -#include "replica/replica_stub.h" - #include "mock_utils.h" +#include "replica/replication_app_base.h" +#include "replica/replica_stub.h" +#include "test_util/test_util.h" +#include "utils/errors.h" +#include "utils/filesystem.h" +#include "utils/smart_pointers.h" namespace dsn { namespace replication { -class replica_stub_test_base : public ::testing::Test +class replica_stub_test_base : public pegasus::encrypt_data_test_base { public: replica_stub_test_base() { stub = std::make_unique(); } diff --git a/src/runtime/providers.common.cpp b/src/runtime/providers.common.cpp index 52fe571f68..b3fc676680 100644 --- a/src/runtime/providers.common.cpp +++ b/src/runtime/providers.common.cpp @@ -33,7 +33,6 @@ * xxxx-xx-xx, author, fix bug about xxx */ -#include #include #include "runtime/env_provider.h" diff --git a/src/runtime/rpc/asio_net_provider.cpp b/src/runtime/rpc/asio_net_provider.cpp index 7677d071cb..f80eb205cd 100644 --- a/src/runtime/rpc/asio_net_provider.cpp +++ b/src/runtime/rpc/asio_net_provider.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include diff --git a/src/runtime/rpc/asio_rpc_session.cpp b/src/runtime/rpc/asio_rpc_session.cpp index a8518cb261..5006a257f3 100644 --- a/src/runtime/rpc/asio_rpc_session.cpp +++ b/src/runtime/rpc/asio_rpc_session.cpp @@ -28,7 +28,6 @@ #include // IWYU pragma: keep // IWYU pragma: no_include -#include #include #include #include diff --git a/src/runtime/rpc/rpc_address.h b/src/runtime/rpc/rpc_address.h index 7b8cc68201..e3ea8422b6 100644 --- a/src/runtime/rpc/rpc_address.h +++ b/src/runtime/rpc/rpc_address.h @@ -27,12 +27,12 @@ #pragma once #include // IWYU pragma: keep - #include #include #include #include #include +#include namespace apache { namespace thrift { diff --git a/src/runtime/rpc/rpc_host_port.h b/src/runtime/rpc/rpc_host_port.h index bfeba0123e..a5461a9cda 100644 --- a/src/runtime/rpc/rpc_host_port.h +++ b/src/runtime/rpc/rpc_host_port.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "runtime/rpc/rpc_address.h" diff --git a/src/runtime/rpc/thrift_message_parser.cpp b/src/runtime/rpc/thrift_message_parser.cpp index e67c2ad0ce..5d702c3c53 100644 --- a/src/runtime/rpc/thrift_message_parser.cpp +++ b/src/runtime/rpc/thrift_message_parser.cpp @@ -27,7 +27,6 @@ #include "thrift_message_parser.h" #include -#include #include #include #include diff --git a/src/runtime/service_engine.cpp b/src/runtime/service_engine.cpp index 90d42a68b7..ee0c742d81 100644 --- a/src/runtime/service_engine.cpp +++ b/src/runtime/service_engine.cpp @@ -27,7 +27,6 @@ #include "service_engine.h" #include -#include #include #include #include diff --git a/src/runtime/task/task_engine.cpp b/src/runtime/task/task_engine.cpp index c729a3b611..11ac1b03fd 100644 --- a/src/runtime/task/task_engine.cpp +++ b/src/runtime/task/task_engine.cpp @@ -28,7 +28,6 @@ // IWYU pragma: no_include #include -#include #include #include diff --git a/src/runtime/test/CMakeLists.txt b/src/runtime/test/CMakeLists.txt index c0146eb73b..01f17e8c50 100644 --- a/src/runtime/test/CMakeLists.txt +++ b/src/runtime/test/CMakeLists.txt @@ -33,6 +33,7 @@ set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio dsn_meta_server + rocksdb ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/runtime/test/client_negotiation_test.cpp b/src/runtime/test/client_negotiation_test.cpp index 00eae954be..74dab238ac 100644 --- a/src/runtime/test/client_negotiation_test.cpp +++ b/src/runtime/test/client_negotiation_test.cpp @@ -22,7 +22,6 @@ #include #include -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/runtime/test/pipeline_test.cpp b/src/runtime/test/pipeline_test.cpp index 22ae223f66..bdc1e5a868 100644 --- a/src/runtime/test/pipeline_test.cpp +++ b/src/runtime/test/pipeline_test.cpp @@ -27,6 +27,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include "common/replication.codes.h" diff --git a/src/runtime/test/server_negotiation_test.cpp b/src/runtime/test/server_negotiation_test.cpp index 907ed0ca8a..0ed7471642 100644 --- a/src/runtime/test/server_negotiation_test.cpp +++ b/src/runtime/test/server_negotiation_test.cpp @@ -23,7 +23,6 @@ #include #include -#include "runtime/rpc/network.h" #include "runtime/rpc/network.sim.h" #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_holder.h" diff --git a/src/runtime/test/task_test.cpp b/src/runtime/test/task_test.cpp index ba4f3d8875..1ef5516a14 100644 --- a/src/runtime/test/task_test.cpp +++ b/src/runtime/test/task_test.cpp @@ -17,7 +17,6 @@ #include "runtime/task/task.h" -#include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -26,8 +25,11 @@ #include "aio/file_io.h" #include "runtime/task/task_code.h" #include "runtime/task/task_spec.h" +#include "utils/flags.h" #include "utils/threadpool_code.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace dsn { class disk_file; @@ -66,7 +68,10 @@ class task_test : public ::testing::Test static void test_signal_finished_task() { - disk_file *fp = file::open("config-test.ini", O_RDONLY | O_BINARY, 0); + // config-test.ini is not encrypted, so set FLAGS_encrypt_data_at_rest = false on force. + FLAGS_encrypt_data_at_rest = false; + + disk_file *fp = file::open("config-test.ini", file::FileOpenType::kReadOnly); // this aio task is enqueued into read-queue of disk_engine char buffer[128]; @@ -80,6 +85,7 @@ class task_test : public ::testing::Test // signal a finished task won't cause failure t->signal_waiters(); // signal_waiters may return false t->signal_waiters(); + ASSERT_EQ(ERR_OK, file::close(fp)); } }; diff --git a/src/server/config.ini b/src/server/config.ini index 2d2b7d19d6..5be55337ee 100644 --- a/src/server/config.ini +++ b/src/server/config.ini @@ -400,6 +400,7 @@ stateful = true # The HTTP port exposed to Prometheus for pulling metrics from pegasus server. prometheus_port = 9091 + encrypt_data_at_rest = false [pegasus.collector] available_detect_app = stat diff --git a/src/server/config.min.ini b/src/server/config.min.ini index 947cc5c91e..02a1f68f08 100644 --- a/src/server/config.min.ini +++ b/src/server/config.min.ini @@ -158,6 +158,7 @@ perf_counter_sink = # The HTTP port exposed to Prometheus for pulling metrics from pegasus server. prometheus_port = @PROMETHEUS_PORT@ + encrypt_data_at_rest = false [pegasus.collector] available_detect_app = stat diff --git a/src/server/pegasus_event_listener.cpp b/src/server/pegasus_event_listener.cpp index e5414d8b79..41201da7a7 100644 --- a/src/server/pegasus_event_listener.cpp +++ b/src/server/pegasus_event_listener.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 5bc03eafe9..55748d1f38 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -22,10 +22,9 @@ #include #include #include -#include +#include #include #include -#include #include #include #include @@ -76,6 +75,7 @@ #include "utils/blob.h" #include "utils/chrono_literals.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" @@ -1634,12 +1634,12 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) rocksdb::DBOptions loaded_db_opt; std::vector loaded_cf_descs; rocksdb::ColumnFamilyOptions loaded_data_cf_opts; + rocksdb::ConfigOptions config_options; // Set `ignore_unknown_options` true for forward compatibility. - auto status = rocksdb::LoadLatestOptions(rdb_path, - rocksdb::Env::Default(), - &loaded_db_opt, - &loaded_cf_descs, - /*ignore_unknown_options=*/true); + config_options.ignore_unknown_options = true; + config_options.env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); + auto status = + rocksdb::LoadLatestOptions(config_options, rdb_path, &loaded_db_opt, &loaded_cf_descs); if (!status.ok()) { // Here we ignore an invalid argument error related to `pegasus_data_version` and // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0). @@ -1677,11 +1677,14 @@ dsn::error_code pegasus_server_impl::start(int argc, char **argv) std::vector column_families( {{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}}); - auto s = rocksdb::CheckOptionsCompatibility(rdb_path, - rocksdb::Env::Default(), - _db_opts, - column_families, - /*ignore_unknown_options=*/true); + rocksdb::ConfigOptions config_options; + config_options.ignore_unknown_options = true; + config_options.ignore_unsupported_options = true; + config_options.sanity_level = + rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible; + config_options.env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); + auto s = + rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts, column_families); if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) { LOG_ERROR_PREFIX("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString()); return dsn::ERR_LOCAL_APP_FAILURE; @@ -2158,7 +2161,7 @@ ::dsn::error_code pegasus_server_impl::copy_checkpoint_to_dir_unsafe(const char {{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}, {META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}}); status = rocksdb::DB::OpenForReadOnly( - rocksdb::DBOptions(), checkpoint_dir, column_families, &handles_opened, &snapshot_db); + _db_opts, checkpoint_dir, column_families, &handles_opened, &snapshot_db); if (!status.ok()) { LOG_ERROR_PREFIX( "OpenForReadOnly from {} failed, error = {}", checkpoint_dir, status.ToString()); @@ -3285,7 +3288,7 @@ ::dsn::error_code pegasus_server_impl::check_column_families(const std::string & *missing_meta_cf = true; *missing_data_cf = true; std::vector column_families; - auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families); + auto s = rocksdb::DB::ListColumnFamilies(_db_opts, path, &column_families); if (!s.ok()) { LOG_ERROR_PREFIX("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString()); if (s.IsCorruption() && diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index c9c5b90ef6..02c6421c7a 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -70,6 +71,7 @@ class WriteBufferManager; namespace dsn { class blob; class message_ex; + namespace replication { class detect_hotkey_request; class detect_hotkey_response; diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index c457ebd9fa..855d6c6710 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -51,6 +51,7 @@ #include "server/pegasus_read_service.h" #include "server/pegasus_server_write.h" // IWYU pragma: keep #include "server/range_read_limiter.h" +#include "utils/env.h" #include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/strings.h" @@ -422,6 +423,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _rng_rd_opts.rocksdb_iteration_threshold_time_ms = FLAGS_rocksdb_iteration_threshold_time_ms; // init rocksdb::DBOptions + _db_opts.env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); _db_opts.create_if_missing = true; // atomic flush data CF and meta CF, aim to keep consistency of 'last flushed decree' in meta CF // and data in data CF. diff --git a/src/server/pegasus_write_service_impl.h b/src/server/pegasus_write_service_impl.h index b75b379a3f..e150fa7829 100644 --- a/src/server/pegasus_write_service_impl.h +++ b/src/server/pegasus_write_service_impl.h @@ -28,6 +28,7 @@ #include "pegasus_write_service.h" #include "rocksdb_wrapper.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/filesystem.h" #include "utils/string_conv.h" #include "utils/strings.h" @@ -75,7 +76,8 @@ inline dsn::error_code get_external_files_path(const std::string &bulk_load_dir, for (const auto &f_meta : metadata.files) { const auto &file_name = dsn::utils::filesystem::path_combine(bulk_load_dir, f_meta.name); if (verify_before_ingest && - !dsn::utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) { + !dsn::utils::filesystem::verify_file( + file_name, dsn::utils::FileDataType::kSensitive, f_meta.md5, f_meta.size)) { break; } files_path.emplace_back(file_name); diff --git a/src/server/result_writer.cpp b/src/server/result_writer.cpp index 16ec2886bb..d4e9ebeb74 100644 --- a/src/server/result_writer.cpp +++ b/src/server/result_writer.cpp @@ -21,7 +21,6 @@ #include #include -#include #include #include "pegasus/client.h" diff --git a/src/server/test/capacity_unit_calculator_test.cpp b/src/server/test/capacity_unit_calculator_test.cpp index cdfe1bddd4..5893d356b3 100644 --- a/src/server/test/capacity_unit_calculator_test.cpp +++ b/src/server/test/capacity_unit_calculator_test.cpp @@ -17,13 +17,14 @@ * under the License. */ -#include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include +#include +#include #include #include #include @@ -156,9 +157,11 @@ class capacity_unit_calculator_test : public pegasus_server_test_base } }; -TEST_F(capacity_unit_calculator_test, init) { test_init(); } +INSTANTIATE_TEST_CASE_P(, capacity_unit_calculator_test, ::testing::Values(false, true)); + +TEST_P(capacity_unit_calculator_test, init) { test_init(); } -TEST_F(capacity_unit_calculator_test, get) +TEST_P(capacity_unit_calculator_test, get) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -198,7 +201,7 @@ TEST_F(capacity_unit_calculator_test, get) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, multi_get) +TEST_P(capacity_unit_calculator_test, multi_get) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -230,7 +233,7 @@ TEST_F(capacity_unit_calculator_test, multi_get) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, scan) +TEST_P(capacity_unit_calculator_test, scan) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -265,7 +268,7 @@ TEST_F(capacity_unit_calculator_test, scan) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, sortkey_count) +TEST_P(capacity_unit_calculator_test, sortkey_count) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -281,7 +284,7 @@ TEST_F(capacity_unit_calculator_test, sortkey_count) } } -TEST_F(capacity_unit_calculator_test, ttl) +TEST_P(capacity_unit_calculator_test, ttl) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); msg->header->context.u.is_backup_request = false; @@ -297,7 +300,7 @@ TEST_F(capacity_unit_calculator_test, ttl) } } -TEST_F(capacity_unit_calculator_test, put) +TEST_P(capacity_unit_calculator_test, put) { for (int i = 0; i < MAX_ROCKSDB_STATUS_CODE; i++) { _cal->add_put_cu(i, key, dsn::blob::create_from_bytes(std::string(4097, ' '))); @@ -311,7 +314,7 @@ TEST_F(capacity_unit_calculator_test, put) } } -TEST_F(capacity_unit_calculator_test, remove) +TEST_P(capacity_unit_calculator_test, remove) { for (int i = 0; i < MAX_ROCKSDB_STATUS_CODE; i++) { _cal->add_remove_cu(i, key); @@ -325,7 +328,7 @@ TEST_F(capacity_unit_calculator_test, remove) } } -TEST_F(capacity_unit_calculator_test, multi_put) +TEST_P(capacity_unit_calculator_test, multi_put) { std::vector<::dsn::apps::key_value> kvs; @@ -347,7 +350,7 @@ TEST_F(capacity_unit_calculator_test, multi_put) } } -TEST_F(capacity_unit_calculator_test, multi_remove) +TEST_P(capacity_unit_calculator_test, multi_remove) { std::vector<::dsn::blob> keys; @@ -369,7 +372,7 @@ TEST_F(capacity_unit_calculator_test, multi_remove) } } -TEST_F(capacity_unit_calculator_test, incr) +TEST_P(capacity_unit_calculator_test, incr) { for (int i = 0; i < MAX_ROCKSDB_STATUS_CODE; i++) { _cal->add_incr_cu(i, key); @@ -387,7 +390,7 @@ TEST_F(capacity_unit_calculator_test, incr) } } -TEST_F(capacity_unit_calculator_test, check_and_set) +TEST_P(capacity_unit_calculator_test, check_and_set) { dsn::blob cas_hash_key = dsn::blob::create_from_bytes("hash_key"); dsn::blob check_sort_key = dsn::blob::create_from_bytes("check_sort_key"); @@ -419,7 +422,7 @@ TEST_F(capacity_unit_calculator_test, check_and_set) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, check_and_mutate) +TEST_P(capacity_unit_calculator_test, check_and_mutate) { dsn::blob cam_hash_key = dsn::blob::create_from_bytes("hash_key"); dsn::blob check_sort_key = dsn::blob::create_from_bytes("check_sort_key"); @@ -456,7 +459,7 @@ TEST_F(capacity_unit_calculator_test, check_and_mutate) _cal->reset(); } -TEST_F(capacity_unit_calculator_test, backup_request_bytes) +TEST_P(capacity_unit_calculator_test, backup_request_bytes) { dsn::message_ptr msg = dsn::message_ex::create_request(RPC_TEST, static_cast(1000), 1, 1); diff --git a/src/server/test/hotkey_collector_test.cpp b/src/server/test/hotkey_collector_test.cpp index 7b784a6d2a..3d69678a90 100644 --- a/src/server/test/hotkey_collector_test.cpp +++ b/src/server/test/hotkey_collector_test.cpp @@ -17,6 +17,7 @@ #include "server/hotkey_collector.h" +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -122,7 +123,9 @@ class coarse_collector_test : public pegasus_server_test_base dsn::task_tracker _tracker; }; -TEST_F(coarse_collector_test, coarse_collector) +INSTANTIATE_TEST_CASE_P(, coarse_collector_test, ::testing::Values(false, true)); + +TEST_P(coarse_collector_test, coarse_collector) { detect_hotkey_result result; @@ -177,7 +180,9 @@ class fine_collector_test : public pegasus_server_test_base dsn::task_tracker _tracker; }; -TEST_F(fine_collector_test, fine_collector) +INSTANTIATE_TEST_CASE_P(, fine_collector_test, ::testing::Values(false, true)); + +TEST_P(fine_collector_test, fine_collector) { detect_hotkey_result result; @@ -285,13 +290,15 @@ class hotkey_collector_test : public pegasus_server_test_base dsn::task_tracker _tracker; }; -TEST_F(hotkey_collector_test, hotkey_type) +INSTANTIATE_TEST_CASE_P(, hotkey_collector_test, ::testing::Values(false, true)); + +TEST_P(hotkey_collector_test, hotkey_type) { ASSERT_EQ(get_collector_type(get_read_collector()), dsn::replication::hotkey_type::READ); ASSERT_EQ(get_collector_type(get_write_collector()), dsn::replication::hotkey_type::WRITE); } -TEST_F(hotkey_collector_test, state_transform) +TEST_P(hotkey_collector_test, state_transform) { auto collector = get_read_collector(); ASSERT_EQ(get_collector_stat(collector), hotkey_collector_state::STOPPED); @@ -359,7 +366,7 @@ TEST_F(hotkey_collector_test, state_transform) _tracker.wait_outstanding_tasks(); } -TEST_F(hotkey_collector_test, data_completeness) +TEST_P(hotkey_collector_test, data_completeness) { dsn::replication::detect_hotkey_response resp; on_detect_hotkey(generate_control_rpc(dsn::replication::hotkey_type::READ, diff --git a/src/server/test/hotspot_partition_test.cpp b/src/server/test/hotspot_partition_test.cpp index d978fc831b..97f4252bc7 100644 --- a/src/server/test/hotspot_partition_test.cpp +++ b/src/server/test/hotspot_partition_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -122,7 +123,9 @@ class hotspot_partition_test : public pegasus_server_test_base void clear_calculator_histories() { calculator._partitions_stat_histories.clear(); } }; -TEST_F(hotspot_partition_test, hotspot_partition_policy) +INSTANTIATE_TEST_CASE_P(, hotspot_partition_test, ::testing::Values(false, true)); + +TEST_P(hotspot_partition_test, hotspot_partition_policy) { // Insert normal scenario data to test std::vector test_rows = generate_row_data(); @@ -175,7 +178,7 @@ TEST_F(hotspot_partition_test, hotspot_partition_policy) clear_calculator_histories(); } -TEST_F(hotspot_partition_test, send_detect_hotkey_request) +TEST_P(hotspot_partition_test, send_detect_hotkey_request) { const int READ_HOT_PARTITION = 7; const int WRITE_HOT_PARTITION = 0; diff --git a/src/server/test/manual_compact_service_test.cpp b/src/server/test/manual_compact_service_test.cpp index d5d3f8b7c7..ff980dfd62 100644 --- a/src/server/test/manual_compact_service_test.cpp +++ b/src/server/test/manual_compact_service_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -110,7 +111,9 @@ class manual_compact_service_test : public pegasus_server_test_base } }; -TEST_F(manual_compact_service_test, check_compact_disabled) +INSTANTIATE_TEST_CASE_P(, manual_compact_service_test, ::testing::Values(false, true)); + +TEST_P(manual_compact_service_test, check_compact_disabled) { std::map envs; check_compact_disabled(envs, false); @@ -134,7 +137,7 @@ TEST_F(manual_compact_service_test, check_compact_disabled) check_compact_disabled(envs, false); } -TEST_F(manual_compact_service_test, check_once_compact) +TEST_P(manual_compact_service_test, check_once_compact) { // suppose compacted at 1500000000 set_compact_time(compacted_ts); @@ -167,7 +170,7 @@ TEST_F(manual_compact_service_test, check_once_compact) check_once_compact(envs, true); } -TEST_F(manual_compact_service_test, check_periodic_compact) +TEST_P(manual_compact_service_test, check_periodic_compact) { std::map envs; @@ -245,7 +248,7 @@ TEST_F(manual_compact_service_test, check_periodic_compact) check_periodic_compact(envs, false); } -TEST_F(manual_compact_service_test, extract_manual_compact_opts) +TEST_P(manual_compact_service_test, extract_manual_compact_opts) { // init _db max level set_num_level(7); @@ -283,7 +286,7 @@ TEST_F(manual_compact_service_test, extract_manual_compact_opts) ASSERT_EQ(out.target_level, -1); } -TEST_F(manual_compact_service_test, check_manual_compact_state_0_interval) +TEST_P(manual_compact_service_test, check_manual_compact_state_0_interval) { FLAGS_manual_compact_min_interval_seconds = 0; @@ -299,7 +302,7 @@ TEST_F(manual_compact_service_test, check_manual_compact_state_0_interval) check_manual_compact_state(false, "2nd start not ok"); } -TEST_F(manual_compact_service_test, check_manual_compact_state_1h_interval) +TEST_P(manual_compact_service_test, check_manual_compact_state_1h_interval) { FLAGS_manual_compact_min_interval_seconds = 3600; diff --git a/src/server/test/pegasus_compression_options_test.cpp b/src/server/test/pegasus_compression_options_test.cpp index 9fa38daf66..16f9689bdf 100644 --- a/src/server/test/pegasus_compression_options_test.cpp +++ b/src/server/test/pegasus_compression_options_test.cpp @@ -17,9 +17,11 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include @@ -100,7 +102,9 @@ class pegasus_compression_options_test : public pegasus_server_test_base } }; -TEST_F(pegasus_compression_options_test, compression_type_convert_ok) +INSTANTIATE_TEST_CASE_P(, pegasus_compression_options_test, ::testing::Values(false, true)); + +TEST_P(pegasus_compression_options_test, compression_type_convert_ok) { compression_type_convert("none", none); compression_type_convert("snappy", snappy); @@ -108,7 +112,7 @@ TEST_F(pegasus_compression_options_test, compression_type_convert_ok) compression_type_convert("zstd", zstd); } -TEST_F(pegasus_compression_options_test, compression_type_convert_not_support) +TEST_P(pegasus_compression_options_test, compression_type_convert_not_support) { rocksdb::CompressionType tmp_type; ASSERT_FALSE(compression_str_to_type("not_support_zip", tmp_type)); @@ -121,7 +125,7 @@ TEST_F(pegasus_compression_options_test, compression_type_convert_not_support) ASSERT_EQ("", compression_type_to_str(rocksdb::kDisableCompressionOption)); } -TEST_F(pegasus_compression_options_test, compression_types_convert_ok) +TEST_P(pegasus_compression_options_test, compression_types_convert_ok) { // Old style. compression_types_convert_ok("none", {none, none, none, none, none, none, none}); @@ -142,7 +146,7 @@ TEST_F(pegasus_compression_options_test, compression_types_convert_ok) {none, lz4, snappy, zstd, lz4, snappy, zstd}); } -TEST_F(pegasus_compression_options_test, compression_types_convert_fail) +TEST_P(pegasus_compression_options_test, compression_types_convert_fail) { // Old style. compression_types_convert_fail("none1"); @@ -156,7 +160,7 @@ TEST_F(pegasus_compression_options_test, compression_types_convert_fail) compression_types_convert_fail("per_levelsnappy"); } -TEST_F(pegasus_compression_options_test, check_rocksdb_compression_types_default) +TEST_P(pegasus_compression_options_test, check_rocksdb_compression_types_default) { start(); check_db_compression_types({none, none, lz4, lz4, lz4, lz4}, "start with default"); diff --git a/src/server/test/pegasus_mutation_duplicator_test.cpp b/src/server/test/pegasus_mutation_duplicator_test.cpp index 168793236f..3d143e1893 100644 --- a/src/server/test/pegasus_mutation_duplicator_test.cpp +++ b/src/server/test/pegasus_mutation_duplicator_test.cpp @@ -20,6 +20,7 @@ #include "server/pegasus_mutation_duplicator.h" #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -288,7 +289,9 @@ class pegasus_mutation_duplicator_test : public pegasus_server_test_base } }; -TEST_F(pegasus_mutation_duplicator_test, get_hash_from_request) +INSTANTIATE_TEST_CASE_P(, pegasus_mutation_duplicator_test, ::testing::Values(false, true)); + +TEST_P(pegasus_mutation_duplicator_test, get_hash_from_request) { std::string hash_key("hash"); std::string sort_key("sort"); @@ -336,7 +339,7 @@ TEST_F(pegasus_mutation_duplicator_test, get_hash_from_request) // Verifies that calls on `get_hash_key_from_request` won't make // message unable to read. (if `get_hash_key_from_request` doesn't // copy the message internally, it will.) -TEST_F(pegasus_mutation_duplicator_test, read_after_get_hash_key) +TEST_P(pegasus_mutation_duplicator_test, read_after_get_hash_key) { std::string hash_key("hash"); std::string sort_key("sort"); @@ -358,18 +361,18 @@ TEST_F(pegasus_mutation_duplicator_test, read_after_get_hash_key) ASSERT_EQ(rpc.request().key.to_string(), raw_key.to_string()); } -TEST_F(pegasus_mutation_duplicator_test, duplicate) { test_duplicate(); } +TEST_P(pegasus_mutation_duplicator_test, duplicate) { test_duplicate(); } -TEST_F(pegasus_mutation_duplicator_test, duplicate_failed) { test_duplicate_failed(); } +TEST_P(pegasus_mutation_duplicator_test, duplicate_failed) { test_duplicate_failed(); } -TEST_F(pegasus_mutation_duplicator_test, duplicate_isolated_hashkeys) +TEST_P(pegasus_mutation_duplicator_test, duplicate_isolated_hashkeys) { test_duplicate_isolated_hashkeys(); } -TEST_F(pegasus_mutation_duplicator_test, create_duplicator) { test_create_duplicator(); } +TEST_P(pegasus_mutation_duplicator_test, create_duplicator) { test_create_duplicator(); } -TEST_F(pegasus_mutation_duplicator_test, duplicate_duplicate) +TEST_P(pegasus_mutation_duplicator_test, duplicate_duplicate) { replica_base replica(dsn::gpid(1, 1), "fake_replica", "temp"); auto duplicator = new_mutation_duplicator(&replica, "onebox2", "temp"); diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp index 9776571ae9..90c2276b60 100644 --- a/src/server/test/pegasus_server_impl_test.cpp +++ b/src/server/test/pegasus_server_impl_test.cpp @@ -21,6 +21,7 @@ #include #include #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -124,10 +125,10 @@ class pegasus_server_impl_test : public pegasus_server_test_base } } - start(all_test_envs); + ASSERT_EQ(dsn::ERR_OK, start(all_test_envs)); if (is_restart) { - _server->stop(false); - start(); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); + ASSERT_EQ(dsn::ERR_OK, start()); } std::map query_envs; @@ -143,22 +144,24 @@ class pegasus_server_impl_test : public pegasus_server_test_base } }; -TEST_F(pegasus_server_impl_test, test_table_level_slow_query) +INSTANTIATE_TEST_CASE_P(, pegasus_server_impl_test, ::testing::Values(false, true)); + +TEST_P(pegasus_server_impl_test, test_table_level_slow_query) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); test_table_level_slow_query(); } -TEST_F(pegasus_server_impl_test, default_data_version) +TEST_P(pegasus_server_impl_test, default_data_version) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(_server->_pegasus_data_version, 1); } -TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options) +TEST_P(pegasus_server_impl_test, test_open_db_with_latest_options) { // open a new db with no app env. - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_NORMAL, _server->_usage_scenario); // set bulk_load scenario for the db. ASSERT_TRUE(_server->set_usage_scenario(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD)); @@ -167,51 +170,51 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options) ASSERT_EQ(1000000000, opts.level0_file_num_compaction_trigger); ASSERT_EQ(true, opts.disable_auto_compactions); // reopen the db. - _server->stop(false); - start(); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); ASSERT_EQ(opts.level0_file_num_compaction_trigger, _server->_db->GetOptions().level0_file_num_compaction_trigger); ASSERT_EQ(opts.disable_auto_compactions, _server->_db->GetOptions().disable_auto_compactions); } -TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs) +TEST_P(pegasus_server_impl_test, test_open_db_with_app_envs) { std::map envs; envs[ROCKSDB_ENV_USAGE_SCENARIO_KEY] = ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD; - start(envs); + ASSERT_EQ(dsn::ERR_OK, start(envs)); ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario); } -TEST_F(pegasus_server_impl_test, test_open_db_with_rocksdb_envs) +TEST_P(pegasus_server_impl_test, test_open_db_with_rocksdb_envs) { // Hint: Verify the set_rocksdb_options_before_creating function by boolean is_restart=false. test_open_db_with_rocksdb_envs(false); } -TEST_F(pegasus_server_impl_test, test_restart_db_with_rocksdb_envs) +TEST_P(pegasus_server_impl_test, test_restart_db_with_rocksdb_envs) { // Hint: Verify the reset_rocksdb_options function by boolean is_restart=true. test_open_db_with_rocksdb_envs(true); } -TEST_F(pegasus_server_impl_test, test_stop_db_twice) +TEST_P(pegasus_server_impl_test, test_stop_db_twice) { - start(); + ASSERT_EQ(dsn::ERR_OK, start()); ASSERT_TRUE(_server->_is_open); ASSERT_TRUE(_server->_db != nullptr); - _server->stop(false); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); ASSERT_FALSE(_server->_is_open); ASSERT_TRUE(_server->_db == nullptr); // stop again - _server->stop(false); + ASSERT_EQ(dsn::ERR_OK, _server->stop(false)); ASSERT_FALSE(_server->_is_open); ASSERT_TRUE(_server->_db == nullptr); } -TEST_F(pegasus_server_impl_test, test_update_user_specified_compaction) +TEST_P(pegasus_server_impl_test, test_update_user_specified_compaction) { _server->_user_specified_compaction = ""; std::map envs; @@ -225,7 +228,7 @@ TEST_F(pegasus_server_impl_test, test_update_user_specified_compaction) ASSERT_EQ(user_specified_compaction, _server->_user_specified_compaction); } -TEST_F(pegasus_server_impl_test, test_load_from_duplication_data) +TEST_P(pegasus_server_impl_test, test_load_from_duplication_data) { auto origin_file = fmt::format("{}/{}", _server->duplication_dir(), "checkpoint"); dsn::utils::filesystem::create_directory(_server->duplication_dir()); diff --git a/src/server/test/pegasus_server_test_base.h b/src/server/test/pegasus_server_test_base.h index 7b1fe225d3..5cea01f3de 100644 --- a/src/server/test/pegasus_server_test_base.h +++ b/src/server/test/pegasus_server_test_base.h @@ -24,9 +24,13 @@ #include #include #include "common/fs_manager.h" +#include "utils/flags.h" #include "replica/replica_stub.h" +#include "test_util/test_util.h" #include "utils/filesystem.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace pegasus { namespace server { @@ -39,7 +43,7 @@ class mock_pegasus_server_impl : public pegasus_server_impl MOCK_CONST_METHOD0(is_duplication_follower, bool()); }; -class pegasus_server_test_base : public ::testing::Test +class pegasus_server_test_base : public pegasus::encrypt_data_test_base { public: pegasus_server_test_base() @@ -49,7 +53,14 @@ class pegasus_server_test_base : public ::testing::Test _replica_stub = new dsn::replication::replica_stub(); _replica_stub->get_fs_manager()->initialize({"test_dir"}, {"test_tag"}); - _gpid = dsn::gpid(100, 1); + // Use different gpid for encryption and non-encryption test to avoid reopening a rocksdb + // instance with different encryption option. + if (FLAGS_encrypt_data_at_rest) { + _gpid = dsn::gpid(100, 0); + } else { + _gpid = dsn::gpid(100, 1); + } + dsn::app_info app_info; app_info.app_type = "pegasus"; diff --git a/src/server/test/pegasus_server_write_test.cpp b/src/server/test/pegasus_server_write_test.cpp index 0e8c571147..abe048460e 100644 --- a/src/server/test/pegasus_server_write_test.cpp +++ b/src/server/test/pegasus_server_write_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -139,7 +140,9 @@ class pegasus_server_write_test : public pegasus_server_test_base } }; -TEST_F(pegasus_server_write_test, batch_writes) { test_batch_writes(); } +INSTANTIATE_TEST_CASE_P(, pegasus_server_write_test, ::testing::Values(false, true)); + +TEST_P(pegasus_server_write_test, batch_writes) { test_batch_writes(); } } // namespace server } // namespace pegasus diff --git a/src/server/test/pegasus_write_service_impl_test.cpp b/src/server/test/pegasus_write_service_impl_test.cpp index f466f9f3ac..93aee6cdb2 100644 --- a/src/server/test/pegasus_write_service_impl_test.cpp +++ b/src/server/test/pegasus_write_service_impl_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -86,7 +87,9 @@ class incr_test : public pegasus_write_service_impl_test dsn::apps::incr_response resp; }; -TEST_F(incr_test, incr_on_absent_record) +INSTANTIATE_TEST_CASE_P(, incr_test, ::testing::Values(false, true)); + +TEST_P(incr_test, incr_on_absent_record) { // ensure key is absent db_get_context get_ctx; @@ -101,7 +104,7 @@ TEST_F(incr_test, incr_on_absent_record) ASSERT_TRUE(get_ctx.found); } -TEST_F(incr_test, negative_incr_and_zero_incr) +TEST_P(incr_test, negative_incr_and_zero_incr) { req.increment = -100; ASSERT_EQ(0, _write_impl->incr(0, req, resp)); @@ -116,7 +119,7 @@ TEST_F(incr_test, negative_incr_and_zero_incr) ASSERT_EQ(resp.new_value, -101); } -TEST_F(incr_test, invalid_incr) +TEST_P(incr_test, invalid_incr) { single_set(req.key, dsn::blob::create_from_bytes("abc")); @@ -133,7 +136,7 @@ TEST_F(incr_test, invalid_incr) ASSERT_EQ(resp.new_value, 100); } -TEST_F(incr_test, fail_on_get) +TEST_P(incr_test, fail_on_get) { dsn::fail::setup(); dsn::fail::cfg("db_get", "100%1*return()"); @@ -146,7 +149,7 @@ TEST_F(incr_test, fail_on_get) dsn::fail::teardown(); } -TEST_F(incr_test, fail_on_put) +TEST_P(incr_test, fail_on_put) { dsn::fail::setup(); dsn::fail::cfg("db_write_batch_put", "100%1*return()"); @@ -159,7 +162,7 @@ TEST_F(incr_test, fail_on_put) dsn::fail::teardown(); } -TEST_F(incr_test, incr_on_expire_record) +TEST_P(incr_test, incr_on_expire_record) { // make the key expired req.expire_ts_seconds = 1; diff --git a/src/server/test/pegasus_write_service_test.cpp b/src/server/test/pegasus_write_service_test.cpp index 8971076bb0..95f5b723d2 100644 --- a/src/server/test/pegasus_write_service_test.cpp +++ b/src/server/test/pegasus_write_service_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -223,13 +224,15 @@ class pegasus_write_service_test : public pegasus_server_test_base } }; -TEST_F(pegasus_write_service_test, multi_put) { test_multi_put(); } +INSTANTIATE_TEST_CASE_P(, pegasus_write_service_test, ::testing::Values(false, true)); -TEST_F(pegasus_write_service_test, multi_remove) { test_multi_remove(); } +TEST_P(pegasus_write_service_test, multi_put) { test_multi_put(); } -TEST_F(pegasus_write_service_test, batched_writes) { test_batched_writes(); } +TEST_P(pegasus_write_service_test, multi_remove) { test_multi_remove(); } -TEST_F(pegasus_write_service_test, duplicate_not_batched) +TEST_P(pegasus_write_service_test, batched_writes) { test_batched_writes(); } + +TEST_P(pegasus_write_service_test, duplicate_not_batched) { std::string hash_key = "hash_key"; constexpr int kv_num = 100; @@ -279,7 +282,7 @@ TEST_F(pegasus_write_service_test, duplicate_not_batched) } } -TEST_F(pegasus_write_service_test, duplicate_batched) +TEST_P(pegasus_write_service_test, duplicate_batched) { std::string hash_key = "hash_key"; constexpr int kv_num = 100; @@ -313,7 +316,7 @@ TEST_F(pegasus_write_service_test, duplicate_batched) } } -TEST_F(pegasus_write_service_test, illegal_duplicate_request) +TEST_P(pegasus_write_service_test, illegal_duplicate_request) { std::string hash_key = "hash_key"; std::string sort_key = "sort_key"; diff --git a/src/server/test/rocksdb_wrapper_test.cpp b/src/server/test/rocksdb_wrapper_test.cpp index 24d88e6564..0bfc9270f8 100644 --- a/src/server/test/rocksdb_wrapper_test.cpp +++ b/src/server/test/rocksdb_wrapper_test.cpp @@ -17,6 +17,7 @@ * under the License. */ +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include @@ -102,7 +103,9 @@ class rocksdb_wrapper_test : public pegasus_server_test_base } }; -TEST_F(rocksdb_wrapper_test, get) +INSTANTIATE_TEST_CASE_P(, rocksdb_wrapper_test, ::testing::Values(false, true)); + +TEST_P(rocksdb_wrapper_test, get) { // not found db_get_context get_ctx1; @@ -134,7 +137,7 @@ TEST_F(rocksdb_wrapper_test, get) ASSERT_EQ(user_value, value); } -TEST_F(rocksdb_wrapper_test, put_verify_timetag) +TEST_P(rocksdb_wrapper_test, put_verify_timetag) { set_app_duplicating(); @@ -211,7 +214,7 @@ TEST_F(rocksdb_wrapper_test, put_verify_timetag) } // verify timetag on data version v0 -TEST_F(rocksdb_wrapper_test, verify_timetag_compatible_with_version_0) +TEST_P(rocksdb_wrapper_test, verify_timetag_compatible_with_version_0) { const_cast(_rocksdb_wrapper->_pegasus_data_version) = 0; // old version diff --git a/src/test/bench_test/config.cpp b/src/test/bench_test/config.cpp index e307c79bb7..8276142162 100644 --- a/src/test/bench_test/config.cpp +++ b/src/test/bench_test/config.cpp @@ -19,11 +19,11 @@ #include "config.h" -#include +#include "utils/env.h" namespace pegasus { namespace test { -config::config() { env = rocksdb::Env::Default(); } +config::config() { env = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); } } // namespace test } // namespace pegasus diff --git a/src/test/function_test/base_api_test/test_copy.cpp b/src/test/function_test/base_api_test/test_copy.cpp index 910a07fff6..d2155f12f0 100644 --- a/src/test/function_test/base_api_test/test_copy.cpp +++ b/src/test/function_test/base_api_test/test_copy.cpp @@ -98,9 +98,9 @@ class copy_data_test : public test_util ASSERT_EQ(dsn::ERR_OK, ddl_client_->create_app( destination_app_name, "pegasus", default_partitions, 3, {}, false)); - srouce_client_ = + source_client_ = pegasus_client_factory::get_client(cluster_name_.c_str(), source_app_name.c_str()); - ASSERT_NE(nullptr, srouce_client_); + ASSERT_NE(nullptr, source_client_); destination_client_ = pegasus_client_factory::get_client(cluster_name_.c_str(), destination_app_name.c_str()); ASSERT_NE(nullptr, destination_client_); @@ -132,7 +132,7 @@ class copy_data_test : public test_util while (expect_data_[empty_hash_key].size() < 1000) { sort_key = random_string(); value = random_string(); - ASSERT_EQ(PERR_OK, srouce_client_->set(empty_hash_key, sort_key, value)) + ASSERT_EQ(PERR_OK, source_client_->set(empty_hash_key, sort_key, value)) << "hash_key=" << hash_key << ", sort_key=" << sort_key; expect_data_[empty_hash_key][sort_key] = value; } @@ -142,7 +142,7 @@ class copy_data_test : public test_util while (expect_data_[hash_key].size() < 10) { sort_key = random_string(); value = random_string(); - ASSERT_EQ(PERR_OK, srouce_client_->set(hash_key, sort_key, value)) + ASSERT_EQ(PERR_OK, source_client_->set(hash_key, sort_key, value)) << "hash_key=" << hash_key << ", sort_key=" << sort_key; expect_data_[hash_key][sort_key] = value; } @@ -163,7 +163,7 @@ class copy_data_test : public test_util char buffer_[256]; map> expect_data_; - pegasus_client *srouce_client_; + pegasus_client *source_client_; pegasus_client *destination_client_; }; const char copy_data_test::CCH[] = @@ -176,7 +176,7 @@ TEST_F(copy_data_test, EMPTY_HASH_KEY_COPY) pegasus_client::scan_options options; options.return_expire_ts = true; vector raw_scanners; - ASSERT_EQ(PERR_OK, srouce_client_->get_unordered_scanners(INT_MAX, options, raw_scanners)); + ASSERT_EQ(PERR_OK, source_client_->get_unordered_scanners(INT_MAX, options, raw_scanners)); LOG_INFO("open source app scanner succeed, partition_count = {}", raw_scanners.size()); diff --git a/src/test/function_test/base_api_test/test_range_read.cpp b/src/test/function_test/base_api_test/test_range_read.cpp index 1a0742fcb4..5827d8da83 100644 --- a/src/test/function_test/base_api_test/test_range_read.cpp +++ b/src/test/function_test/base_api_test/test_range_read.cpp @@ -109,7 +109,6 @@ class range_read_test : public test_util std::map expect_kvs_; }; -// TODO(yingchun): use TEST_P to refact TEST_F(range_read_test, multiget_test) { pegasus::pegasus_client::multi_get_options options; diff --git a/src/test/function_test/bulk_load_test/CMakeLists.txt b/src/test/function_test/bulk_load_test/CMakeLists.txt index 92a677ddf8..a6eac08515 100644 --- a/src/test/function_test/bulk_load_test/CMakeLists.txt +++ b/src/test/function_test/bulk_load_test/CMakeLists.txt @@ -37,7 +37,8 @@ set(MY_PROJ_LIBS gssapi_krb5 krb5 function_test_utils - ) + rocksdb + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/test/function_test/bulk_load_test/test_bulk_load.cpp b/src/test/function_test/bulk_load_test/test_bulk_load.cpp index 83b2dd6ecc..acd14a88df 100644 --- a/src/test/function_test/bulk_load_test/test_bulk_load.cpp +++ b/src/test/function_test/bulk_load_test/test_bulk_load.cpp @@ -15,10 +15,14 @@ // specific language governing permissions and limitations // under the License. +#include // IWYU pragma: no_include // IWYU pragma: no_include #include -#include +#include +#include +#include +#include #include #include #include @@ -29,17 +33,28 @@ #include #include "base/pegasus_const.h" +#include "block_service/local/local_service.h" #include "bulk_load_types.h" +#include "client/partition_resolver.h" #include "client/replication_ddl_client.h" +#include "common/json_helper.h" #include "include/pegasus/client.h" #include "include/pegasus/error.h" +#include "meta/meta_bulk_load_service.h" #include "meta_admin_types.h" -#include "metadata_types.h" +#include "rocksdb/env.h" #include "test/function_test/utils/test_util.h" +#include "utils/blob.h" +#include "utils/enum_helper.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/errors.h" #include "utils/filesystem.h" -#include "utils/utils.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" +#include "utils/test_macros.h" + +DSN_DECLARE_bool(encrypt_data_at_rest); using namespace ::dsn; using namespace ::dsn::replication; @@ -55,12 +70,12 @@ using std::string; /// - `bulk_load_root` sub-directory stores right data /// - Please do not rename any files or directories under this folder /// -/// The app who is executing bulk load: -/// - app_name is `temp`, app_id is 2, partition_count is 8 +/// The app to test bulk load functionality: +/// - partition count should be 8 /// /// Data: -/// hashkey: hashi sortkey: sorti value: newValue i=[0, 1000] -/// hashkey: hashkeyj sortkey: sortkeyj value: newValue j=[0, 1000] +/// hashkey: hash${i} sortkey: sort${i} value: newValue i=[0, 1000] +/// hashkey: hashkey${j} sortkey: sortkey${j} value: newValue j=[0, 1000] /// class bulk_load_test : public test_util { @@ -68,81 +83,114 @@ class bulk_load_test : public test_util bulk_load_test() : test_util(map({{"rocksdb.allow_ingest_behind", "true"}})) { TRICKY_CODE_TO_AVOID_LINK_ERROR; - bulk_load_local_root_ = - utils::filesystem::path_combine("onebox/block_service/local_service/", LOCAL_ROOT); + BULK_LOAD_LOCAL_APP_ROOT = + fmt::format("{}/{}/{}", LOCAL_BULK_LOAD_ROOT, CLUSTER, app_name_); } void SetUp() override { test_util::SetUp(); - ASSERT_NO_FATAL_FAILURE(copy_bulk_load_files()); + NO_FATALS(copy_bulk_load_files()); } void TearDown() override { ASSERT_EQ(ERR_OK, ddl_client_->drop_app(app_name_, 0)); - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("rm -rf onebox/block_service")); + NO_FATALS(run_cmd_from_project_root("rm -rf " + LOCAL_BULK_LOAD_ROOT)); } - void copy_bulk_load_files() + void generate_bulk_load_info(const bulk_load_info &bli, const std::string &bulk_load_info_path) { - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("mkdir -p onebox/block_service")); - ASSERT_NO_FATAL_FAILURE( - run_cmd_from_project_root("mkdir -p onebox/block_service/local_service")); - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root( - "cp -r src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/" + - LOCAL_ROOT + " onebox/block_service/local_service")); - string cmd = "echo '{\"app_id\":" + std::to_string(app_id_) + - ",\"app_name\":\"temp\",\"partition_count\":8}' > " - "onebox/block_service/local_service/bulk_load_root/cluster/temp/" - "bulk_load_info"; - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root(cmd)); + blob value = dsn::json::json_forwarder::encode(bli); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(value.data(), value.length()), + bulk_load_info_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); } - error_code start_bulk_load(bool ingest_behind = false) + void generate_bulk_load_info_meta(const std::string &bulk_load_info_path) { - auto err_resp = - ddl_client_->start_bulk_load(app_name_, CLUSTER, PROVIDER, LOCAL_ROOT, ingest_behind); - return err_resp.get_value().err; + dist::block_service::file_metadata fm; + ASSERT_TRUE(utils::filesystem::file_size( + bulk_load_info_path, dsn::utils::FileDataType::kSensitive, fm.size)); + ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(bulk_load_info_path, fm.md5)); + std::string value = nlohmann::json(fm).dump(); + string bulk_load_info_meta_path = + fmt::format("{}/cluster/{}/.bulk_load_info.meta", LOCAL_BULK_LOAD_ROOT, app_name_); + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(value), + bulk_load_info_meta_path, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); } - void remove_file(const string &file_path) + void copy_bulk_load_files() + { + // TODO(yingchun): remove + // src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/mock_bulk_load_info + // Prepare bulk load files. + // The source data has 8 partitions. + ASSERT_EQ(8, partition_count_); + NO_FATALS(run_cmd_from_project_root("mkdir -p " + LOCAL_BULK_LOAD_ROOT)); + NO_FATALS(run_cmd_from_project_root( + fmt::format("cp -r {}/{} {}", SOURCE_FILES_ROOT, BULK_LOAD, LOCAL_SERVICE_ROOT))); + if (FLAGS_encrypt_data_at_rest) { + std::vector src_files; + ASSERT_TRUE(dsn::utils::filesystem::get_subfiles(LOCAL_SERVICE_ROOT, src_files, true)); + for (const auto &src_file : src_files) { + auto s = dsn::utils::encrypt_file(src_file); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + } + + // Generate 'bulk_load_info'. + string bulk_load_info_path = + fmt::format("{}/cluster/{}/bulk_load_info", LOCAL_BULK_LOAD_ROOT, app_name_); + NO_FATALS(generate_bulk_load_info(bulk_load_info(app_id_, app_name_, partition_count_), + bulk_load_info_path)); + + // Generate '.bulk_load_info.meta'. + NO_FATALS(generate_bulk_load_info_meta(bulk_load_info_path)); + } + + error_code start_bulk_load(bool ingest_behind = false) { - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root("rm " + file_path)); + return ddl_client_->start_bulk_load(app_name_, CLUSTER, PROVIDER, BULK_LOAD, ingest_behind) + .get_value() + .err; } - void replace_bulk_load_info() + void remove_file(const string &file_path) { - string cmd = "cp -R " - "src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files/" - "mock_bulk_load_info/. " + - bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + "/"; - ASSERT_NO_FATAL_FAILURE(run_cmd_from_project_root(cmd)); + NO_FATALS(run_cmd_from_project_root("rm " + file_path)); } void update_allow_ingest_behind(const string &allow_ingest_behind) { - // update app envs - std::vector keys; - keys.emplace_back(ROCKSDB_ALLOW_INGEST_BEHIND); - std::vector values; - values.emplace_back(allow_ingest_behind); - ASSERT_EQ(ERR_OK, ddl_client_->set_app_envs(app_name_, keys, values).get_value().err); - std::cout << "sleep 31s to wait app_envs update" << std::endl; + ASSERT_EQ( + ERR_OK, + ddl_client_ + ->set_app_envs(app_name_, {ROCKSDB_ALLOW_INGEST_BEHIND}, {allow_ingest_behind}) + .get_value() + .err); + LOG_INFO("sleep 31s to wait app_envs update"); std::this_thread::sleep_for(std::chrono::seconds(31)); } - bulk_load_status::type wait_bulk_load_finish(int64_t seconds) + bulk_load_status::type wait_bulk_load_finish(int64_t remain_seconds) { int64_t sleep_time = 5; error_code err = ERR_OK; bulk_load_status::type last_status = bulk_load_status::BLS_INVALID; // when bulk load end, err will be ERR_INVALID_STATE - while (seconds > 0 && err == ERR_OK) { - sleep_time = sleep_time > seconds ? seconds : sleep_time; - seconds -= sleep_time; - std::cout << "sleep " << sleep_time << "s to query bulk status" << std::endl; + while (remain_seconds > 0 && err == ERR_OK) { + sleep_time = sleep_time > remain_seconds ? remain_seconds : sleep_time; + remain_seconds -= sleep_time; + LOG_INFO("sleep {}s to query bulk status", sleep_time); std::this_thread::sleep_for(std::chrono::seconds(sleep_time)); auto resp = ddl_client_->query_bulk_load(app_name_).get_value(); @@ -156,21 +204,21 @@ class bulk_load_test : public test_util void verify_bulk_load_data() { - ASSERT_NO_FATAL_FAILURE(verify_data("hashkey", "sortkey")); - ASSERT_NO_FATAL_FAILURE(verify_data(HASHKEY_PREFIX, SORTKEY_PREFIX)); + NO_FATALS(verify_data("hashkey", "sortkey")); + NO_FATALS(verify_data(HASHKEY_PREFIX, SORTKEY_PREFIX)); } void verify_data(const string &hashkey_prefix, const string &sortkey_prefix) { - const string &expected_value = VALUE; for (int i = 0; i < COUNT; ++i) { string hash_key = hashkey_prefix + std::to_string(i); + LOG_INFO("Start to verify hashkey: {}", hash_key); for (int j = 0; j < COUNT; ++j) { string sort_key = sortkey_prefix + std::to_string(j); string act_value; ASSERT_EQ(PERR_OK, client_->get(hash_key, sort_key, act_value)) << hash_key << "," << sort_key; - ASSERT_EQ(expected_value, act_value) << hash_key << "," << sort_key; + ASSERT_EQ(VALUE, act_value) << hash_key << "," << sort_key; } } } @@ -211,72 +259,87 @@ class bulk_load_test : public test_util } protected: - string bulk_load_local_root_; - - const string LOCAL_ROOT = "bulk_load_root"; + string BULK_LOAD_LOCAL_APP_ROOT; + const string SOURCE_FILES_ROOT = + "src/test/function_test/bulk_load_test/pegasus-bulk-load-function-test-files"; + const string LOCAL_SERVICE_ROOT = "onebox/block_service/local_service"; + const string LOCAL_BULK_LOAD_ROOT = "onebox/block_service/local_service/bulk_load_root"; + const string BULK_LOAD = "bulk_load_root"; const string CLUSTER = "cluster"; const string PROVIDER = "local_service"; - const string HASHKEY_PREFIX = "hash"; const string SORTKEY_PREFIX = "sort"; const string VALUE = "newValue"; - const int32_t COUNT = 1000; + const int32_t COUNT = 100; }; -/// -/// case1: lack of `bulk_load_info` file -/// case2: `bulk_load_info` file inconsistent with app_info -/// -TEST_F(bulk_load_test, bulk_load_test_failed) +// Test bulk load failed because `bulk_load_info` file is missing +TEST_F(bulk_load_test, bulk_load_test_missing_bulk_load_info) { - // bulk load failed because `bulk_load_info` file is missing - ASSERT_NO_FATAL_FAILURE( - remove_file(bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + "/bulk_load_info")); + NO_FATALS(remove_file(BULK_LOAD_LOCAL_APP_ROOT + "/bulk_load_info")); ASSERT_EQ(ERR_OBJECT_NOT_FOUND, start_bulk_load()); +} - // bulk load failed because `bulk_load_info` file inconsistent with current app_info - ASSERT_NO_FATAL_FAILURE(replace_bulk_load_info()); - ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load()); +// Test bulk load failed because `bulk_load_info` file inconsistent with current app_info +TEST_F(bulk_load_test, bulk_load_test_inconsistent_bulk_load_info) +{ + // Only 'app_id' and 'partition_count' will be checked. + bulk_load_info tests[] = {{app_id_ + 1, app_name_, partition_count_}, + {app_id_, app_name_, partition_count_ * 2}}; + for (const auto &test : tests) { + // Generate inconsistent 'bulk_load_info'. + string bulk_load_info_path = + fmt::format("{}/cluster/{}/bulk_load_info", LOCAL_BULK_LOAD_ROOT, app_name_); + NO_FATALS(generate_bulk_load_info(test, bulk_load_info_path)); + + // Generate '.bulk_load_info.meta'. + NO_FATALS(generate_bulk_load_info_meta(bulk_load_info_path)); + + ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load()) << test.app_id << "," << test.app_name + << "," << test.partition_count; + } +} + +// Test bulk load failed because partition[0] `bulk_load_metadata` file is missing +TEST_F(bulk_load_test, bulk_load_test_missing_p0_bulk_load_metadata) +{ + NO_FATALS(remove_file(BULK_LOAD_LOCAL_APP_ROOT + "/0/bulk_load_metadata")); + ASSERT_EQ(ERR_OK, start_bulk_load()); + ASSERT_EQ(bulk_load_status::BLS_FAILED, wait_bulk_load_finish(300)); } /// -/// case1: lack of `bulk_load_metadata` file -/// case2: bulk load succeed with data verfied -/// case3: bulk load data consistent: +/// case1: bulk load succeed with data verfied +/// case2: bulk load data consistent: /// - old data will be overrided by bulk load data /// - get/set/del succeed after bulk load /// TEST_F(bulk_load_test, bulk_load_tests) { - // bulk load failed because partition[0] `bulk_load_metadata` file is missing - ASSERT_NO_FATAL_FAILURE(remove_file(bulk_load_local_root_ + "/" + CLUSTER + "/" + app_name_ + - "/0/bulk_load_metadata")); - ASSERT_EQ(ERR_OK, start_bulk_load()); - // bulk load will get FAILED - ASSERT_EQ(bulk_load_status::BLS_FAILED, wait_bulk_load_finish(300)); - - // recover complete files - ASSERT_NO_FATAL_FAILURE(copy_bulk_load_files()); - // write old data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); + NO_FATALS(operate_data(operation::SET, "oldValue", 10)); + NO_FATALS(operate_data(operation::GET, "oldValue", 10)); ASSERT_EQ(ERR_OK, start_bulk_load()); ASSERT_EQ(bulk_load_status::BLS_SUCCEED, wait_bulk_load_finish(300)); - std::cout << "Start to verify data..." << std::endl; - ASSERT_NO_FATAL_FAILURE(verify_bulk_load_data()); - - // value overide by bulk_loaded_data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, VALUE, 10)); - - // write data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "valueAfterBulkLoad", 20)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "valueAfterBulkLoad", 20)); - - // del data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::DEL, "", 15)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::NO_VALUE, "", 15)); + LOG_INFO("Start to verify data..."); + NO_FATALS(verify_bulk_load_data()); + + // values have been overwritten by bulk_load_data + LOG_INFO("Start to GET data..."); + NO_FATALS(operate_data(operation::GET, VALUE, 10)); + + // write new data succeed after bulk load + LOG_INFO("Start to SET data..."); + NO_FATALS(operate_data(operation::SET, "valueAfterBulkLoad", 20)); + LOG_INFO("Start to GET data..."); + NO_FATALS(operate_data(operation::GET, "valueAfterBulkLoad", 20)); + + // delete data succeed after bulk load + LOG_INFO("Start to DEL data..."); + NO_FATALS(operate_data(operation::DEL, "", 15)); + LOG_INFO("Start to NO_VALUE data..."); + NO_FATALS(operate_data(operation::NO_VALUE, "", 15)); } /// @@ -288,30 +351,35 @@ TEST_F(bulk_load_test, bulk_load_tests) /// TEST_F(bulk_load_test, bulk_load_ingest_behind_tests) { - ASSERT_NO_FATAL_FAILURE(update_allow_ingest_behind("false")); + NO_FATALS(update_allow_ingest_behind("false")); // app envs allow_ingest_behind = false, request ingest_behind = true ASSERT_EQ(ERR_INCONSISTENT_STATE, start_bulk_load(true)); - ASSERT_NO_FATAL_FAILURE(update_allow_ingest_behind("true")); + NO_FATALS(update_allow_ingest_behind("true")); // write old data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); + NO_FATALS(operate_data(operation::SET, "oldValue", 10)); + NO_FATALS(operate_data(operation::GET, "oldValue", 10)); ASSERT_EQ(ERR_OK, start_bulk_load(true)); ASSERT_EQ(bulk_load_status::BLS_SUCCEED, wait_bulk_load_finish(300)); - std::cout << "Start to verify data..." << std::endl; - // value overide by bulk_loaded_data - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "oldValue", 10)); - ASSERT_NO_FATAL_FAILURE(verify_data("hashkey", "sortkey")); - - // write data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::SET, "valueAfterBulkLoad", 20)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::GET, "valueAfterBulkLoad", 20)); - - // del data after bulk load succeed - ASSERT_NO_FATAL_FAILURE(operate_data(operation::DEL, "", 15)); - ASSERT_NO_FATAL_FAILURE(operate_data(operation::NO_VALUE, "", 15)); + // values have been overwritten by bulk_load_data + LOG_INFO("Start to GET data..."); + NO_FATALS(operate_data(operation::GET, "oldValue", 10)); + LOG_INFO("Start to verify data..."); + NO_FATALS(verify_data("hashkey", "sortkey")); + + // write new data succeed after bulk load + LOG_INFO("Start to SET data..."); + NO_FATALS(operate_data(operation::SET, "valueAfterBulkLoad", 20)); + LOG_INFO("Start to GET data..."); + NO_FATALS(operate_data(operation::GET, "valueAfterBulkLoad", 20)); + + // delete data succeed after bulk load + LOG_INFO("Start to DEL data..."); + NO_FATALS(operate_data(operation::DEL, "", 15)); + LOG_INFO("Start to NO_VALUE data..."); + NO_FATALS(operate_data(operation::NO_VALUE, "", 15)); } diff --git a/src/test/function_test/config.ini b/src/test/function_test/config.ini index 5b76a05a26..01c7bae703 100644 --- a/src/test/function_test/config.ini +++ b/src/test/function_test/config.ini @@ -80,6 +80,9 @@ mycluster = 127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603 onebox = 127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603 single_master_cluster = 127.0.0.1:34601 +[pegasus.server] +encrypt_data_at_rest = false + [function_test.throttle_test] - throttle_test_medium_value_kb = 20 - throttle_test_large_value_kb = 50 +throttle_test_medium_value_kb = 20 +throttle_test_large_value_kb = 50 diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt index 267c825e40..b1e7ac2dff 100644 --- a/src/test_util/CMakeLists.txt +++ b/src/test_util/CMakeLists.txt @@ -22,6 +22,6 @@ set(MY_PROJ_NAME test_utils) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest) +set(MY_PROJ_LIBS gtest rocksdb) dsn_add_static_library() diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index ec7bce6e2e..18e2d38947 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -20,11 +20,22 @@ #pragma once #include +#include +#include "gtest/gtest.h" +#include "utils/flags.h" #include "utils/test_macros.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace pegasus { +class encrypt_data_test_base : public testing::TestWithParam +{ +public: + encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); } +}; + #define ASSERT_EVENTUALLY(expr) \ do { \ AssertEventually(expr); \ diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index e17fef9001..1127a868ef 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -31,7 +31,7 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) -set(MY_PROJ_LIBS dsn_http crypto) +set(MY_PROJ_LIBS dsn_http crypto rocksdb) # Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/utils/alloc.h b/src/utils/alloc.h index 195f740f6b..8541d98e8a 100644 --- a/src/utils/alloc.h +++ b/src/utils/alloc.h @@ -17,18 +17,18 @@ #pragma once +#include +#include +#include +#include +#include + #include "utils/ports.h" // The check for the definition of CACHELINE_SIZE has to be put after including "utils/ports.h", // where CACHELINE_SIZE is defined. #ifdef CACHELINE_SIZE -#include -#include -#include -#include -#include - #ifndef NDEBUG #include "utils/fmt_logging.h" #endif diff --git a/src/utils/command_manager.cpp b/src/utils/command_manager.cpp index 723f8f3447..ab2e22f790 100644 --- a/src/utils/command_manager.cpp +++ b/src/utils/command_manager.cpp @@ -27,7 +27,6 @@ #include "utils/command_manager.h" #include -#include #include #include #include // IWYU pragma: keep @@ -180,7 +179,7 @@ command_manager::~command_manager() { _cmds.clear(); CHECK(_handlers.empty(), - "All commands must be deregistered before command_manager is destroyed, however {} is " + "All commands must be deregistered before command_manager is destroyed, however '{}' is " "still registered", _handlers.begin()->first); } diff --git a/src/utils/config_api.cpp b/src/utils/config_api.cpp index 6319f39dab..537ffed659 100644 --- a/src/utils/config_api.cpp +++ b/src/utils/config_api.cpp @@ -25,6 +25,9 @@ */ #include "utils/config_api.h" + +#include + #include "utils/configuration.h" dsn::configuration g_config; diff --git a/src/utils/configuration.cpp b/src/utils/configuration.cpp index 9aea53448e..c69a556d2c 100644 --- a/src/utils/configuration.cpp +++ b/src/utils/configuration.cpp @@ -33,14 +33,16 @@ * xxxx-xx-xx, author, fix bug about xxx */ -#include +#include +#include +#include #include #include #include #include #include "utils/configuration.h" -#include "utils/filesystem.h" +#include "utils/env.h" #include "utils/strings.h" namespace dsn { @@ -67,33 +69,17 @@ bool configuration::load(const char *file_name, const char *arguments) { _file_name = std::string(file_name); - FILE *fd = ::fopen(file_name, "rb"); - if (fd == nullptr) { - std::string cdir; - dsn::utils::filesystem::get_current_directory(cdir); - printf("ERROR: cannot open file %s in %s, err = %s\n", - file_name, - cdir.c_str(), - strerror(errno)); - return false; - } - ::fseek(fd, 0, SEEK_END); - int len = ftell(fd); - if (len == -1 || len == 0) { - printf("ERROR: cannot get length of %s, err = %s\n", file_name, strerror(errno)); - ::fclose(fd); + auto s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), _file_name, &_file_data); + if (!s.ok()) { + fmt::print(stderr, "ERROR: read file '{}' failed, err = {}\n", _file_name, s.ToString()); return false; } - _file_data.resize(len + 1); - ::fseek(fd, 0, SEEK_SET); - auto sz = ::fread((char *)_file_data.c_str(), len, 1, fd); - ::fclose(fd); - if (sz != 1) { - printf("ERROR: cannot read correct data of %s, err = %s\n", file_name, strerror(errno)); + if (_file_data.empty()) { + fmt::print(stderr, "ERROR: file '{}' is empty\n", _file_name); return false; } - _file_data[len] = '\n'; // replace data with arguments if (arguments != nullptr) { diff --git a/src/utils/env.cpp b/src/utils/env.cpp new file mode 100644 index 0000000000..9380e5db6c --- /dev/null +++ b/src/utils/env.cpp @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "env.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/filesystem.h" +#include "utils/flags.h" +#include "utils/fmt_logging.h" +#include "utils/utils.h" + +DSN_DEFINE_bool(pegasus.server, + encrypt_data_at_rest, + true, + "Whether sensitive files should be encrypted on the file system."); + +DSN_DEFINE_string(pegasus.server, + server_key_for_testing, + "server_key_for_testing", + "The encrypted server key to use in the filesystem. NOTE: only for testing."); + +DSN_DEFINE_string(pegasus.server, + encryption_method, + "AES128CTR", + "The encryption method to use in the filesystem. Now " + "supports AES128CTR, AES192CTR, AES256CTR and SM4CTR."); + +DSN_DEFINE_bool(replication, + enable_direct_io, + false, + "Whether to enable direct I/O when download files"); +DSN_TAG_VARIABLE(enable_direct_io, FT_MUTABLE); + +namespace dsn { +namespace utils { + +rocksdb::Env *NewEncryptedEnv() +{ + std::string provider_id = + fmt::format("AES:{},{}", FLAGS_server_key_for_testing, FLAGS_encryption_method); + std::shared_ptr provider; + auto s = rocksdb::EncryptionProvider::CreateFromString( + rocksdb::ConfigOptions(), provider_id, &provider); + CHECK(s.ok(), "Failed to create encryption provider: {}", s.ToString()); + return NewEncryptedEnv(rocksdb::Env::Default(), provider); +} + +rocksdb::Env *PegasusEnv(FileDataType type) +{ + if (FLAGS_encrypt_data_at_rest && type == FileDataType::kSensitive) { + static rocksdb::Env *env = NewEncryptedEnv(); + return env; + } + + static rocksdb::Env *env = rocksdb::Env::Default(); + return env; +} + +rocksdb::Status do_copy_file(const std::string &src_fname, + dsn::utils::FileDataType src_type, + const std::string &dst_fname, + dsn::utils::FileDataType dst_type, + int64_t remain_size, + uint64_t *total_size) +{ + rocksdb::EnvOptions rd_env_options; + rd_env_options.use_direct_reads = FLAGS_enable_direct_io; + std::unique_ptr sfile; + auto s = dsn::utils::PegasusEnv(src_type)->NewSequentialFile(src_fname, &sfile, rd_env_options); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for reading", src_fname); + + // Limit the size of the file to be copied. + int64_t src_file_size; + CHECK(dsn::utils::filesystem::file_size(src_fname, src_type, src_file_size), ""); + if (remain_size == -1) { + remain_size = src_file_size; + } + remain_size = std::min(remain_size, src_file_size); + + rocksdb::EnvOptions wt_env_options; + wt_env_options.use_direct_writes = FLAGS_enable_direct_io; + std::unique_ptr wfile; + s = dsn::utils::PegasusEnv(dst_type)->NewWritableFile(dst_fname, &wfile, wt_env_options); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, s, "failed to open file {} for writing", dst_fname); + + // Read at most 4MB once. + // TODO(yingchun): make it configurable. + const uint64_t kBlockSize = 4 << 20; + auto buffer = dsn::utils::make_shared_array(kBlockSize); + uint64_t offset = 0; + do { + int bytes_per_copy = std::min(remain_size, static_cast(kBlockSize)); + if (bytes_per_copy <= 0) { + break; + } + + rocksdb::Slice result; + LOG_AND_RETURN_NOT_RDB_OK(WARNING, + sfile->Read(bytes_per_copy, &result, buffer.get()), + "failed to read file {}", + src_fname); + CHECK(!result.empty(), + "read file {} at offset {} with size {} failed", + src_fname, + offset, + bytes_per_copy); + LOG_AND_RETURN_NOT_RDB_OK( + WARNING, wfile->Append(result), "failed to write file {}", dst_fname); + + offset += result.size(); + remain_size -= result.size(); + + // Reach the end of the file. + if (result.size() < bytes_per_copy) { + break; + } + } while (true); + LOG_AND_RETURN_NOT_RDB_OK(WARNING, wfile->Fsync(), "failed to fsync file {}", dst_fname); + + if (total_size != nullptr) { + *total_size = offset; + } + + LOG_INFO("copy file from {} to {}, total size {}", src_fname, dst_fname, offset); + return rocksdb::Status::OK(); +} + +rocksdb::Status +copy_file(const std::string &src_fname, const std::string &dst_fname, uint64_t *total_size) +{ + // TODO(yingchun): Consider to use hard link, LinkFile(). + return do_copy_file( + src_fname, FileDataType::kSensitive, dst_fname, FileDataType::kSensitive, -1, total_size); +} + +rocksdb::Status +encrypt_file(const std::string &src_fname, const std::string &dst_fname, uint64_t *total_size) +{ + return do_copy_file(src_fname, + FileDataType::kNonSensitive, + dst_fname, + FileDataType::kSensitive, + -1, + total_size); +} + +rocksdb::Status encrypt_file(const std::string &fname, uint64_t *total_size) +{ + // TODO(yingchun): add timestamp to the tmp encrypted file name. + std::string tmp_fname = fname + ".encrypted.tmp"; + LOG_AND_RETURN_NOT_RDB_OK( + WARNING, encrypt_file(fname, tmp_fname, total_size), "failed to encrypt file {}", fname); + if (!::dsn::utils::filesystem::rename_path(tmp_fname, fname)) { + LOG_WARNING("rename file from {} to {} failed", tmp_fname, fname); + return rocksdb::Status::IOError("rename file failed"); + } + return rocksdb::Status::OK(); +} + +rocksdb::Status +copy_file_by_size(const std::string &src_fname, const std::string &dst_fname, int64_t limit_size) +{ + return do_copy_file(src_fname, + FileDataType::kSensitive, + dst_fname, + FileDataType::kSensitive, + limit_size, + nullptr); +} + +} // namespace utils +} // namespace dsn diff --git a/src/utils/env.h b/src/utils/env.h new file mode 100644 index 0000000000..7ad6eb6bda --- /dev/null +++ b/src/utils/env.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace rocksdb { +class Env; +} // namespace rocksdb + +namespace dsn { +namespace utils { + +// Indicate whether the file is sensitive or not. +// Only the sensitive file will be encrypted if FLAGS_encrypt_data_at_rest +// is enabled at the same time. +enum class FileDataType +{ + kSensitive = 0, + kNonSensitive = 1 +}; + +rocksdb::Env *PegasusEnv(FileDataType type); + +// The 'total_size' is the total size of the file content, exclude the file encryption header. +// 'src_fname' is not encrypted and 'dst_fname' is encrypted. +rocksdb::Status encrypt_file(const std::string &src_fname, + const std::string &dst_fname, + uint64_t *total_size = nullptr); +// Encrypt the original non-encrypted 'fname'. +rocksdb::Status encrypt_file(const std::string &fname, uint64_t *total_size = nullptr); +// Both 'src_fname' and 'dst_fname' are encrypted files. +rocksdb::Status copy_file(const std::string &src_fname, + const std::string &dst_fname, + uint64_t *total_size = nullptr); +// Both 'src_fname' and 'dst_fname' are encrypted files. 'limit_size' is the max size of the +// size to copy, and -1 means no limit. +rocksdb::Status copy_file_by_size(const std::string &src_fname, + const std::string &dst_fname, + int64_t limit_size = -1); + +} // namespace utils +} // namespace dsn diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index 8bfe4cfa6f..8fc6b46bde 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -33,31 +33,32 @@ * xxxx-xx-xx, author, fix bug about xxx */ -// IWYU pragma: no_include -#include // IWYU pragma: keep #include #include #include -#include #include #include #include #include +#include #include #include #include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include -#include -#include +#include +#include "rocksdb/env.h" +#include "rocksdb/status.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/ports.h" #include "utils/safe_strerror_posix.h" #include "utils/string_view.h" -#include "utils/strings.h" #define getcwd_ getcwd #define rmdir_ rmdir @@ -389,32 +390,16 @@ bool rename_path(const std::string &path1, const std::string &path2) return ret; } -bool file_size(const std::string &path, int64_t &sz) +// TODO(yingchun): refactor to use uint64_t. +bool file_size(const std::string &path, FileDataType type, int64_t &sz) { - struct stat_ st; - std::string npath; - int err; - - if (path.empty()) { - return false; - } - - err = get_normalized_path(path, npath); - if (err != 0) { + uint64_t file_size = 0; + auto s = dsn::utils::PegasusEnv(type)->GetFileSize(path, &file_size); + if (!s.ok()) { + LOG_ERROR("GetFileSize failed, file '{}', err = {}", path, s.ToString()); return false; } - - err = dsn::utils::filesystem::get_stat_internal(npath, st); - if (err != 0) { - return false; - } - - if (!S_ISREG(st.st_mode)) { - return false; - } - - sz = st.st_size; - + sz = file_size; return true; } @@ -497,53 +482,29 @@ bool create_directory(const std::string &path) bool create_file(const std::string &path) { - size_t pos; std::string npath; - int fd; - int mode; - int err; - - if (path.empty()) { - return false; - } - - if (_FS_ISSEP(path.back())) { - return false; - } - - err = get_normalized_path(path, npath); + int err = get_normalized_path(path, npath); if (err != 0) { return false; } - if (dsn::utils::filesystem::path_exists_internal(npath, FTW_F)) { - return true; - } - - if (dsn::utils::filesystem::path_exists_internal(npath, FTW_D)) { - return false; - } - - pos = npath.find_last_of("\\/"); + auto pos = npath.find_last_of("\\/"); if ((pos != std::string::npos) && (pos > 0)) { auto ppath = npath.substr(0, pos); if (!dsn::utils::filesystem::create_directory(ppath)) { + LOG_WARNING("fail to create directory {}", ppath); return false; } } - mode = 0775; - fd = ::creat(npath.c_str(), mode); - if (fd == -1) { - err = errno; - LOG_WARNING("create_file {} failed, err = {}", path, safe_strerror(err)); + std::unique_ptr wfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->ReopenWritableFile(path, &wfile, rocksdb::EnvOptions()); + if (dsn_unlikely(!s.ok())) { + LOG_WARNING("fail to create file {}, err={}", path, s.ToString()); return false; } - if (::close_(fd) != 0) { - LOG_WARNING("create_file {}, failed to close the file handle.", path); - } - return true; } @@ -725,6 +686,55 @@ bool link_file(const std::string &src, const std::string &target) } error_code md5sum(const std::string &file_path, /*out*/ std::string &result) +{ + result.clear(); + if (!::dsn::utils::filesystem::file_exists(file_path)) { + LOG_ERROR("md5sum error: file {} not exist", file_path); + return ERR_OBJECT_NOT_FOUND; + } + + std::unique_ptr sfile; + auto s = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive) + ->NewSequentialFile(file_path, &sfile, rocksdb::EnvOptions()); + if (!sfile) { + LOG_ERROR("md5sum error: open file {} failed, err={}", file_path, s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + + const int64_t kBufferSize = 4096; + char buf[kBufferSize]; + unsigned char out[MD5_DIGEST_LENGTH] = {0}; + MD5_CTX c; + CHECK_EQ(1, MD5_Init(&c)); + while (true) { + rocksdb::Slice res; + s = sfile->Read(kBufferSize, &res, buf); + if (!s.ok()) { + MD5_Final(out, &c); + LOG_ERROR("md5sum error: read file {} failed, err = ", file_path, s.ToString()); + return ERR_FILE_OPERATION_FAILED; + } + if (res.empty()) { + break; + } + CHECK_EQ(1, MD5_Update(&c, buf, res.size())); + if (res.size() < kBufferSize) { + break; + } + } + CHECK_EQ(1, MD5_Final(out, &c)); + + char str[MD5_DIGEST_LENGTH * 2 + 1]; + str[MD5_DIGEST_LENGTH * 2] = 0; + for (int n = 0; n < MD5_DIGEST_LENGTH; n++) { + sprintf(str + n + n, "%02x", out[n]); + } + result.assign(str); + + return ERR_OK; +} + +error_code deprecated_md5sum(const std::string &file_path, /*out*/ std::string &result) { result.clear(); // if file not exist, we return ERR_OBJECT_NOT_FOUND @@ -790,37 +800,8 @@ std::pair is_directory_empty(const std::string &dirname) return res; } -error_code read_file(const std::string &fname, std::string &buf) -{ - if (!file_exists(fname)) { - LOG_ERROR("file({}) doesn't exist", fname); - return ERR_FILE_OPERATION_FAILED; - } - - int64_t file_sz = 0; - if (!file_size(fname, file_sz)) { - LOG_ERROR("get file({}) size failed", fname); - return ERR_FILE_OPERATION_FAILED; - } - - buf.resize(file_sz); - std::ifstream fin(fname, std::ifstream::in); - if (!fin.is_open()) { - LOG_ERROR("open file({}) failed", fname); - return ERR_FILE_OPERATION_FAILED; - } - fin.read(&buf[0], file_sz); - CHECK_EQ_MSG(file_sz, - fin.gcount(), - "read file({}) failed, file_size = {} but read size = {}", - fname, - file_sz, - fin.gcount()); - fin.close(); - return ERR_OK; -} - bool verify_file(const std::string &fname, + FileDataType type, const std::string &expected_md5, const int64_t &expected_fsize) { @@ -829,7 +810,7 @@ bool verify_file(const std::string &fname, return false; } int64_t f_size = 0; - if (!file_size(fname, f_size)) { + if (!file_size(fname, type, f_size)) { LOG_ERROR("verify file({}) failed, becaused failed to get file size", fname); return false; } @@ -850,14 +831,14 @@ bool verify_file(const std::string &fname, return true; } -bool verify_file_size(const std::string &fname, const int64_t &expected_fsize) +bool verify_file_size(const std::string &fname, FileDataType type, const int64_t &expected_fsize) { if (!file_exists(fname)) { LOG_ERROR("file({}) is not existed", fname); return false; } int64_t f_size = 0; - if (!file_size(fname, f_size)) { + if (!file_size(fname, type, f_size)) { LOG_ERROR("verify file({}) size failed, becaused failed to get file size", fname); return false; } @@ -871,22 +852,6 @@ bool verify_file_size(const std::string &fname, const int64_t &expected_fsize) return true; } -bool verify_data_md5(const std::string &fname, - const char *data, - const size_t data_size, - const std::string &expected_md5) -{ - std::string md5 = string_md5(data, data_size); - if (md5 != expected_md5) { - LOG_ERROR("verify data({}) failed, because data damaged, size: md5: {} VS {}", - fname, - md5, - expected_md5); - return false; - } - return true; -} - bool create_directory(const std::string &path, std::string &absolute_path, std::string &err_msg) { FAIL_POINT_INJECT_F("filesystem_create_directory", [path](string_view str) { @@ -908,20 +873,6 @@ bool create_directory(const std::string &path, std::string &absolute_path, std:: return true; } -bool write_file(const std::string &fname, std::string &buf) -{ - if (!file_exists(fname)) { - LOG_ERROR("file({}) doesn't exist", fname); - return false; - } - - std::ofstream fstream; - fstream.open(fname.c_str()); - fstream << buf; - fstream.close(); - return true; -} - bool check_dir_rw(const std::string &path, std::string &err_msg) { FAIL_POINT_INJECT_F("filesystem_check_dir_rw", [path](string_view str) { @@ -932,23 +883,31 @@ bool check_dir_rw(const std::string &path, std::string &err_msg) path.find(broken_disk_dir) == std::string::npos; }); - std::string fname = "read_write_test_file"; - std::string fpath = path_combine(path, fname); - if (!create_file(fpath)) { - err_msg = fmt::format("Fail to create test file {}.", fpath); + static const std::string kTestValue = "test_value"; + static const std::string kFname = "read_write_test_file"; + std::string fpath = path_combine(path, kFname); + auto cleanup = defer([&fpath]() { remove_path(fpath); }); + // Use kSensitive to test encryption functionality as well. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kTestValue), + fpath, + /* should_sync */ true); + if (dsn_unlikely(!s.ok())) { + err_msg = fmt::format("fail to write file {}, err={}", fpath, s.ToString()); return false; } - auto cleanup = defer([&fpath]() { remove_path(fpath); }); - std::string value = "test_value"; - if (!write_file(fpath, value)) { - err_msg = fmt::format("Fail to write file {}.", fpath); + std::string read_data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fpath, &read_data); + if (dsn_unlikely(!s.ok())) { + err_msg = fmt::format("fail to read file {}, err={}", fpath, s.ToString()); return false; } - std::string buf; - if (read_file(fpath, buf) != ERR_OK || buf != value) { - err_msg = fmt::format("Fail to read file {} or get wrong value({}).", fpath, buf); + if (dsn_unlikely(read_data != kTestValue)) { + err_msg = fmt::format("get wrong value '{}' from file", read_data, fpath); return false; } diff --git a/src/utils/filesystem.h b/src/utils/filesystem.h index 4229f551d2..c4b6832049 100644 --- a/src/utils/filesystem.h +++ b/src/utils/filesystem.h @@ -61,8 +61,12 @@ namespace dsn { namespace utils { +enum class FileDataType; + namespace filesystem { +// TODO(yingchun): Consider using rocksdb APIs to rewrite the following functions. + int get_normalized_path(const std::string &path, std::string &npath); bool get_absolute_path(const std::string &path1, std::string &path2); @@ -96,7 +100,7 @@ bool remove_path(const std::string &path); // this will always remove target path if exist bool rename_path(const std::string &path1, const std::string &path2); -bool file_size(const std::string &path, int64_t &sz); +bool file_size(const std::string &path, FileDataType type, int64_t &sz); bool create_directory(const std::string &path); @@ -126,6 +130,7 @@ bool get_disk_space_info(const std::string &path, disk_space_info &info); bool link_file(const std::string &src, const std::string &target); error_code md5sum(const std::string &file_path, /*out*/ std::string &result); +error_code deprecated_md5sum(const std::string &file_path, /*out*/ std::string &result); // return value: // - : @@ -133,27 +138,19 @@ error_code md5sum(const std::string &file_path, /*out*/ std::string &result); // B is represent wheter the directory is empty, true means empty, otherwise false std::pair is_directory_empty(const std::string &dirname); -error_code read_file(const std::string &fname, /*out*/ std::string &buf); - // compare file metadata calculated by fname with expected md5 and file_size bool verify_file(const std::string &fname, + FileDataType type, const std::string &expected_md5, const int64_t &expected_fsize); -bool verify_file_size(const std::string &fname, const int64_t &expected_fsize); - -bool verify_data_md5(const std::string &fname, - const char *data, - const size_t data_size, - const std::string &expected_md5); +bool verify_file_size(const std::string &fname, FileDataType type, const int64_t &expected_fsize); // create driectory and get absolute path bool create_directory(const std::string &path, /*out*/ std::string &absolute_path, /*out*/ std::string &err_msg); -bool write_file(const std::string &fname, std::string &buf); - // check if directory is readable and writable // call `create_directory` before to make `path` exist bool check_dir_rw(const std::string &path, /*out*/ std::string &err_msg); diff --git a/src/utils/flags.h b/src/utils/flags.h index 304238b7bb..a0b9deb46a 100644 --- a/src/utils/flags.h +++ b/src/utils/flags.h @@ -22,6 +22,7 @@ // IWYU pragma: no_include #include #include +#include #include "enum_helper.h" #include "errors.h" @@ -50,7 +51,7 @@ struct hash // Example: // DSN_DEFINE_string(core, filename, "my_file.txt", "The file to read"); // DSN_DEFINE_validator(filename, [](const char *fname){ return is_file(fname); }); -// auto fptr = file::open(FLAGS_filename, O_RDONLY | O_BINARY, 0); +// auto fptr = file::open(FLAGS_filename, file::FileOpenType::kReadOnly); #define DSN_DECLARE_VARIABLE(type, name) extern type FLAGS_##name diff --git a/src/utils/fmt_logging.h b/src/utils/fmt_logging.h index f0c74bd293..35c60aaddf 100644 --- a/src/utils/fmt_logging.h +++ b/src/utils/fmt_logging.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include "utils/api_utilities.h" @@ -272,6 +273,16 @@ inline const char *null_str_printer(const char *s) { return s == nullptr ? "(nul LOG_AND_RETURN_NOT_TRUE(level, _err == ::dsn::ERR_OK, _err, __VA_ARGS__); \ } while (0) +// Return the given rocksdb::Status 's' if it is not OK. +#define LOG_AND_RETURN_NOT_RDB_OK(level, s, ...) \ + do { \ + const auto &_s = (s); \ + if (dsn_unlikely(!_s.ok())) { \ + LOG_##level("{}: {}", _s.ToString(), fmt::format(__VA_ARGS__)); \ + return _s; \ + } \ + } while (0) + #ifndef NDEBUG #define DCHECK CHECK #define DCHECK_NOTNULL CHECK_NOTNULL diff --git a/src/utils/output_utils.h b/src/utils/output_utils.h index c66d55f734..201cee938b 100644 --- a/src/utils/output_utils.h +++ b/src/utils/output_utils.h @@ -20,6 +20,7 @@ // IWYU pragma: no_include #include #include +#include #include // IWYU pragma: keep #include // IWYU pragma: no_include diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index 98b1693d1a..f4c90789a8 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -29,7 +29,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include @@ -183,6 +182,10 @@ simple_logger::simple_logger(const char *log_dir) create_log_file(); + // TODO(yingchun): simple_logger is destroyed after command_manager, so will cause crash like + // "assertion expression: [_handlers.empty()] All commands must be deregistered before + // command_manager is destroyed, however 'flush-log' is still registered". + // We need to fix it. _cmds.emplace_back(::dsn::command_manager::instance().register_command( {"flush-log"}, "flush-log - flush log to stderr or log file", diff --git a/src/utils/strings.h b/src/utils/strings.h index 7584df99bc..1759856f01 100644 --- a/src/utils/strings.h +++ b/src/utils/strings.h @@ -114,6 +114,7 @@ std::string get_last_component(const std::string &input, const char splitters[]) char *trim_string(char *s); +// TODO(yingchun): unify the following functions with the ones in utils::filesystem // calculate the md5 checksum of buffer std::string string_md5(const char *buffer, unsigned int length); diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index fb284b0e86..266b9beeb6 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -33,7 +33,8 @@ set(MY_PROJ_LIBS dsn_http dsn_runtime dsn_utils gtest - ) + rocksdb + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/utils/test/TokenBucketTest.cpp b/src/utils/test/TokenBucketTest.cpp index 74db80227f..8ad5b32a7e 100644 --- a/src/utils/test/TokenBucketTest.cpp +++ b/src/utils/test/TokenBucketTest.cpp @@ -18,7 +18,7 @@ #include // IWYU pragma: no_include -#include +// IWYU pragma: no_include // IWYU pragma: no_include #include #include diff --git a/src/utils/test/autoref_ptr_test.cpp b/src/utils/test/autoref_ptr_test.cpp index 6a8fe8d9d7..4ff6a15294 100644 --- a/src/utils/test/autoref_ptr_test.cpp +++ b/src/utils/test/autoref_ptr_test.cpp @@ -8,6 +8,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include diff --git a/src/utils/test/env.cpp b/src/utils/test/env.cpp index 5a05f1879b..6fad0a975d 100644 --- a/src/utils/test/env.cpp +++ b/src/utils/test/env.cpp @@ -33,17 +33,29 @@ * xxxx-xx-xx, author, fix bug about xxx */ +#include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include #include +#include +#include "test_util/test_util.h" +#include "utils/enum_helper.h" +#include "utils/env.h" +#include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/rand.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + using namespace ::dsn; -TEST(core, env) +TEST(env_test, rand) { uint64_t xs[] = {0, std::numeric_limits::max() - 1, 0xdeadbeef}; @@ -55,3 +67,212 @@ TEST(core, env) EXPECT_TRUE(r == x || r == (x + 1)); } } + +TEST(env_test, get_env) +{ + FLAGS_encrypt_data_at_rest = false; + auto *env_no_enc1 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive); + auto *env_no_enc2 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); + + FLAGS_encrypt_data_at_rest = true; + auto *env_no_enc3 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive); + auto *env_enc1 = dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive); + + ASSERT_EQ(env_no_enc1, env_no_enc2); + ASSERT_EQ(env_no_enc1, env_no_enc3); + ASSERT_NE(env_no_enc1, env_enc1); +} + +class env_file_test : public pegasus::encrypt_data_test_base +{ +public: + env_file_test() : pegasus::encrypt_data_test_base() + { + // The file size should plus 4096 if consider it as kNonSensitive when the if is actually + // encrypted. + if (FLAGS_encrypt_data_at_rest) { + extra_size = 4096; + } + } + uint64_t extra_size = 0; +}; + +INSTANTIATE_TEST_CASE_P(, env_file_test, ::testing::Values(false, true)); + +TEST_P(env_file_test, encrypt_file_2_files) +{ + const std::string kFileName = "encrypt_file_2_files"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + + // Check encrypt_file(src_fname, dst_fname, total_size). + // Loop twice to check overwrite. + for (int i = 0; i < 2; ++i) { + uint64_t encrypt_file_size; + s = dsn::utils::encrypt_file(kFileName, kFileName + ".encrypted", &encrypt_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, encrypt_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName + ".encrypted", dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName + ".encrypted", dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + kFileName + ".encrypted", + &data); + ASSERT_EQ(kFileContent, data); + } +} + +TEST_P(env_file_test, encrypt_file_1_file) +{ + const std::string kFileName = "encrypt_file_1_file"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + + // Check encrypt_file(fname, total_size). + uint64_t encrypt_file_size; + s = dsn::utils::encrypt_file(kFileName, &encrypt_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, encrypt_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), kFileName, &data); + ASSERT_EQ(kFileContent, data); +} + +TEST_P(env_file_test, copy_file) +{ + const std::string kFileName = "copy_file"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + extra_size, wfile_size); + + // Check copy_file(src_fname, dst_fname, total_size). + // Loop twice to check overwrite. + for (int i = 0; i < 2; ++i) { + uint64_t copy_file_size; + s = dsn::utils::copy_file(kFileName, kFileName + ".copy", ©_file_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_EQ(kFileContentSize, copy_file_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName + ".copy", dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName + ".copy", dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + kFileName + ".copy", + &data); + ASSERT_EQ(kFileContent, data); + } +} + +TEST_P(env_file_test, copy_file_by_size) +{ + const std::string kFileName = "copy_file_by_size"; + const uint64_t kFileContentSize = 100; + const std::string kFileContent(kFileContentSize, 'a'); + + // Prepare a non-encrypted test file. + auto s = + rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), + rocksdb::Slice(kFileContent), + kFileName, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // Check file size. + int64_t wfile_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize, wfile_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + kFileName, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(kFileContentSize + extra_size, wfile_size); + + // Check copy_file_by_size(src_fname, dst_fname, limit_size). + struct test_case + { + int64_t limit_size; + int64_t expect_size; + } tests[] = {{-1, kFileContentSize}, + {0, 0}, + {10, 10}, + {kFileContentSize, kFileContentSize}, + {kFileContentSize + 10, kFileContentSize}}; + for (const auto &test : tests) { + std::string copy_file_name = kFileName + ".copy"; + s = dsn::utils::copy_file_by_size(kFileName, copy_file_name, test.limit_size); + ASSERT_TRUE(s.ok()) << s.ToString(); + + int64_t actual_size; + ASSERT_TRUE(dsn::utils::filesystem::file_size( + copy_file_name, dsn::utils::FileDataType::kSensitive, actual_size)); + ASSERT_EQ(test.expect_size, actual_size); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + copy_file_name, dsn::utils::FileDataType::kNonSensitive, wfile_size)); + ASSERT_EQ(test.expect_size + extra_size, wfile_size); + // Check file content. + std::string data; + s = rocksdb::ReadFileToString( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), copy_file_name, &data); + ASSERT_EQ(std::string(test.expect_size, 'a'), data); + } +} diff --git a/src/utils/test/fail_point_test.cpp b/src/utils/test/fail_point_test.cpp index 74d576065a..b7668b03c9 100644 --- a/src/utils/test/fail_point_test.cpp +++ b/src/utils/test/fail_point_test.cpp @@ -31,6 +31,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include "utils/fail_point.h" #include "utils/fail_point_impl.h" diff --git a/src/utils/test/file_system_test.cpp b/src/utils/test/file_system_test.cpp index f6da3e2b38..141d833f85 100644 --- a/src/utils/test/file_system_test.cpp +++ b/src/utils/test/file_system_test.cpp @@ -15,30 +15,83 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include #include +#include "test_util/test_util.h" +#include "utils/env.h" +#include "utils/error_code.h" #include "utils/filesystem.h" namespace dsn { namespace utils { namespace filesystem { -TEST(verify_file, verify_file_test) +// The old filesystem API doesn't support sensitive files, so skip testing +// FLAGS_encrypt_data_at_rest=true. + +TEST(filesystem_test, check_new_md5sum) { + FLAGS_encrypt_data_at_rest = false; + + struct file_info + { + int64_t size; + } tests[]{{4095}, {4096}, {4097}}; + + for (const auto &test : tests) { + std::string fname = "test_file"; + // deprecated_md5sum doesn't support kSensitive files, so use kNonSensitive here. + auto s = rocksdb::WriteStringToFile( + dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), + rocksdb::Slice(std::string(test.size, 'a')), + fname, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + // Check the file size. + int64_t file_fsize; + ASSERT_TRUE(file_size(fname, FileDataType::kNonSensitive, file_fsize)); + ASSERT_EQ(test.size, file_fsize); + + // Get the md5sum. + std::string md5sum1; + ASSERT_EQ(ERR_OK, md5sum(fname, md5sum1)); + ASSERT_FALSE(md5sum1.empty()); + + // Check the md5sum is repeatable. + std::string md5sum2; + ASSERT_EQ(ERR_OK, md5sum(fname, md5sum2)); + ASSERT_EQ(md5sum1, md5sum2); + + // Check the md5sum is the same to deprecated_md5sum. + ASSERT_EQ(ERR_OK, deprecated_md5sum(fname, md5sum2)); + ASSERT_EQ(md5sum1, md5sum2); + + utils::filesystem::remove_path(fname); + } +} + +TEST(filesystem_test, verify_file_test) +{ + FLAGS_encrypt_data_at_rest = false; + const std::string &fname = "test_file"; std::string expected_md5; int64_t expected_fsize; create_file(fname); md5sum(fname, expected_md5); - file_size(fname, expected_fsize); + ASSERT_TRUE(file_size(fname, FileDataType::kNonSensitive, expected_fsize)); - ASSERT_TRUE(verify_file(fname, expected_md5, expected_fsize)); - ASSERT_FALSE(verify_file(fname, "wrong_md5", 10086)); - ASSERT_FALSE(verify_file("file_not_exists", "wrong_md5", 10086)); + ASSERT_TRUE(verify_file(fname, FileDataType::kNonSensitive, expected_md5, expected_fsize)); + ASSERT_FALSE(verify_file(fname, FileDataType::kNonSensitive, "wrong_md5", 10086)); + ASSERT_FALSE(verify_file("file_not_exists", FileDataType::kNonSensitive, "wrong_md5", 10086)); remove_path(fname); } diff --git a/src/utils/test/file_utils.cpp b/src/utils/test/file_utils.cpp index 004e228b4b..e155e27684 100644 --- a/src/utils/test/file_utils.cpp +++ b/src/utils/test/file_utils.cpp @@ -33,6 +33,7 @@ #include #include +#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" @@ -683,12 +684,12 @@ static void file_utils_test_file_size() bool ret; path = "./file_utils_temp.txt"; - ret = dsn::utils::filesystem::file_size(path, sz); - EXPECT_TRUE(ret); - EXPECT_TRUE(sz == 12); + ret = dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kSensitive, sz); + ASSERT_TRUE(ret); + ASSERT_EQ(12, sz); path = "./file_utils_temp2.txt"; - ret = dsn::utils::filesystem::file_size(path, sz); + ret = dsn::utils::filesystem::file_size(path, dsn::utils::FileDataType::kSensitive, sz); EXPECT_FALSE(ret); } diff --git a/src/utils/test/fmt_logging_test.cpp b/src/utils/test/fmt_logging_test.cpp index fc0f18f383..191e31e10d 100644 --- a/src/utils/test/fmt_logging_test.cpp +++ b/src/utils/test/fmt_logging_test.cpp @@ -30,6 +30,7 @@ // IWYU pragma: no_include #include #include +#include #include "common/gpid.h" #include "common/replication.codes.h" diff --git a/src/utils/test/join_point_test.cpp b/src/utils/test/join_point_test.cpp index 6751fe94ae..a23740579a 100644 --- a/src/utils/test/join_point_test.cpp +++ b/src/utils/test/join_point_test.cpp @@ -29,7 +29,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include namespace dsn { diff --git a/src/utils/test/lock.std.cpp b/src/utils/test/lock.std.cpp index c1530e4bd8..c7027a8861 100644 --- a/src/utils/test/lock.std.cpp +++ b/src/utils/test/lock.std.cpp @@ -27,6 +27,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include "runtime/rpc/rpc_address.h" diff --git a/src/utils/test/main.cpp b/src/utils/test/main.cpp index f611528ef4..2be6302e1a 100644 --- a/src/utils/test/main.cpp +++ b/src/utils/test/main.cpp @@ -16,6 +16,7 @@ // under the License. #include +#include #include "utils/flags.h" #include "utils/logging_provider.h" diff --git a/src/utils/test/memutil_test.cpp b/src/utils/test/memutil_test.cpp index 4ddf62eaa1..87c4da8dc3 100644 --- a/src/utils/test/memutil_test.cpp +++ b/src/utils/test/memutil_test.cpp @@ -19,6 +19,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include TEST(MemUtilTest, memmatch) { diff --git a/src/utils/test/sema.cpp b/src/utils/test/sema.cpp index 0982ee8837..40b14824b2 100644 --- a/src/utils/test/sema.cpp +++ b/src/utils/test/sema.cpp @@ -39,6 +39,7 @@ // IWYU pragma: no_include #include #include +#include #include TEST(core, Semaphore) diff --git a/src/utils/test/smart_pointers_test.cpp b/src/utils/test/smart_pointers_test.cpp index a2113b4b36..7a6bba05f0 100644 --- a/src/utils/test/smart_pointers_test.cpp +++ b/src/utils/test/smart_pointers_test.cpp @@ -17,6 +17,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include diff --git a/src/utils/test/utils.cpp b/src/utils/test/utils.cpp index c372b79ff7..756f7f5de0 100644 --- a/src/utils/test/utils.cpp +++ b/src/utils/test/utils.cpp @@ -34,7 +34,7 @@ */ // IWYU pragma: no_include -#include +// IWYU pragma: no_include // IWYU pragma: no_include #include #include diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index a0984a2cad..46faba7277 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -152,59 +152,24 @@ ExternalProject_Add(thrift -DWITH_LIBEVENT=OFF -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DWITH_SHARED_LIB=OFF + -DWITH_SHARED_LIB=ON -DBOOST_ROOT=${TP_OUTPUT} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} DEPENDS boost ) -if (NOT APPLE) - # kerberos - ExternalProject_Add(krb5 - URL ${OSS_URL_PREFIX}/krb5-1.16.1.tar.gz - http://web.mit.edu/kerberos/dist/krb5/1.16/krb5-1.16.1.tar.gz - URL_MD5 848e9b80d6aaaa798e3f3df24b83c407 - CONFIGURE_COMMAND cd src && ./configure --prefix=${TP_OUTPUT} - BUILD_COMMAND cd src && make - INSTALL_COMMAND cd src && make install - BUILD_IN_SOURCE 1 - ) - - # cyrus-sasl - ExternalProject_Add(cyrus-sasl - URL ${OSS_URL_PREFIX}/cyrus-sasl-2.1.27.tar.gz - http://www.cyrusimap.org/releases/cyrus-sasl-2.1.27.tar.gz - URL_MD5 a33820c66e0622222c5aefafa1581083 - CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} - --enable-gssapi=${TP_OUTPUT} - --enable-scram=no - --enable-digest=no - --enable-cram=no - --enable-otp=no - BUILD_COMMAND make - INSTALL_COMMAND make install - BUILD_IN_SOURCE 1 - ) -endif() - check_cxx_compiler_flag(-Wformat-overflow COMPILER_SUPPORTS_FORMAT_OVERFLOW) if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) set(ZOOKEEPER_CFLAGS -Wno-error=format-overflow) endif () -if (NOT APPLE) - set(ZOOKEEPER_WITH_CYRUS_SASL ${TP_OUTPUT}) -else () - set(ZOOKEEPER_WITH_CYRUS_SASL /usr/local/opt/cyrus-sasl/lib) -endif () - ExternalProject_Add(zookeeper URL ${OSS_URL_PREFIX}/apache-zookeeper-3.7.0.tar.gz http://downloads.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz URL_MD5 44c2a33e01931aed94ef7f3d39d0963e PATCH_COMMAND "" - COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && cmake -DCMAKE_BUILD_TYPE=release -DWANT_CPPUNIT=OFF -DWITH_OPENSSL=OFF -DWITH_CYRUS_SASL=${ZOOKEEPER_WITH_CYRUS_SASL} -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && cmake -DCMAKE_BUILD_TYPE=release -DWANT_CPPUNIT=OFF -DWITH_OPENSSL=OFF -DWITH_CYRUS_SASL=ON -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} COMMAND cd zookeeper-client/zookeeper-client-c && make COMMAND cp -R zookeeper-client/zookeeper-client-c/include/. ${TP_OUTPUT}/include/zookeeper && cp zookeeper-client/zookeeper-client-c/generated/zookeeper.jute.h ${TP_OUTPUT}/include/zookeeper && cp zookeeper-client/zookeeper-client-c/libzookeeper.a ${TP_OUTPUT}/lib && cp zookeeper-client/zookeeper-client-c/libhashtable.a ${TP_OUTPUT}/lib CONFIGURE_COMMAND "" @@ -212,9 +177,6 @@ ExternalProject_Add(zookeeper INSTALL_COMMAND "" BUILD_IN_SOURCE 1 ) -if (NOT APPLE) - add_dependencies(zookeeper cyrus-sasl krb5) -endif () ExternalProject_Add(libevent URL ${OSS_URL_PREFIX}/libevent-release-2.1.8-stable.tar.gz @@ -392,15 +354,11 @@ ExternalProject_Add(jemalloc BUILD_IN_SOURCE 1 ) -option(ROCKSDB_PORTABLE "build a portable binary" OFF) +option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) -# The patch name '0879c240' means the patch of rocksdb: -# https://github.com/facebook/rocksdb/commit/0879c240404b00142ba4718f36cd3f2bd537192d ExternalProject_Add(rocksdb - URL ${OSS_URL_PREFIX}/rocksdb-6.6.4.tar.gz - https://github.com/facebook/rocksdb/archive/refs/tags/v6.6.4.tar.gz - URL_MD5 7f7fcca3e96b7d83ef332804c90070c8 - PATCH_COMMAND patch -p1 < ${TP_DIR}/rocksdb_fix_atomic_flush_0879c240.patch + URL https://github.com/acelyc111/rocksdb/archive/ebd8b94b115928193113d8a3175cae048fd1b0a6.zip + URL_MD5 5b2955aa2db375d3f65ba2f5c23369ef DEPENDS jemalloc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DFAIL_ON_WARNINGS=OFF @@ -412,6 +370,7 @@ ExternalProject_Add(rocksdb -DWITH_BZ2=OFF -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF + -DWITH_OPENSSL=ON -DUSE_RTTI=ON -DCMAKE_BUILD_TYPE=Release -DWITH_JEMALLOC=${USE_JEMALLOC}