From de1d4199891fb83e29fc7e677289549d6f00bbe9 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 22 Sep 2023 14:15:44 +0800 Subject: [PATCH] 6 --- src/block_service/test/CMakeLists.txt | 3 +- .../test/block_service_manager_test.cpp | 25 ++---------- src/replica/bulk_load/test/CMakeLists.txt | 3 +- .../test/replica_bulk_loader_test.cpp | 40 ++++++------------- src/replica/test/CMakeLists.txt | 1 + src/test_util/test_util.cpp | 21 ++++++++++ src/test_util/test_util.h | 13 +++++- src/utils/test/env.cpp | 2 +- 8 files changed, 55 insertions(+), 53 deletions(-) diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index 537778caad..f7325f814d 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -36,7 +36,8 @@ set(MY_PROJ_LIBS gtest gtest_main hdfs - rocksdb) + rocksdb + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/block_service/test/block_service_manager_test.cpp b/src/block_service/test/block_service_manager_test.cpp index de088f2d69..ef93ed1962 100644 --- a/src/block_service/test/block_service_manager_test.cpp +++ b/src/block_service/test/block_service_manager_test.cpp @@ -19,9 +19,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include -#include -#include #include #include #include @@ -33,7 +30,6 @@ #include "block_service_mock.h" #include "metadata_types.h" #include "test_util/test_util.h" -#include "utils/env.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/test_macros.h" @@ -60,21 +56,6 @@ class block_service_manager_test : public pegasus::encrypt_data_test_base PROVIDER, LOCAL_DIR, FILE_NAME, _fs.get(), download_size); } - void create_local_file(const std::string &file_name) - { - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice("write some data."), - whole_name, - /* should_sync */ true); - ASSERT_TRUE(s.ok()) << s.ToString(); - - _file_meta.name = whole_name; - ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(whole_name, _file_meta.md5)); - ASSERT_TRUE(utils::filesystem::file_size( - whole_name, dsn::utils::FileDataType::kSensitive, _file_meta.size)); - } - void create_remote_file(const std::string &file_name, int64_t size, const std::string &md5) { std::string whole_file_name = utils::filesystem::path_combine(PROVIDER, file_name); @@ -107,7 +88,8 @@ TEST_P(block_service_manager_test, remote_file_not_exist) TEST_P(block_service_manager_test, local_file_exist) { - NO_FATALS(create_local_file(FILE_NAME)); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), + &_file_meta)); struct remote_file_info { int64_t size; @@ -128,7 +110,8 @@ TEST_P(block_service_manager_test, local_file_exist) TEST_P(block_service_manager_test, do_download_succeed) { - NO_FATALS(create_local_file(FILE_NAME)); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), + &_file_meta)); create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5); // remove local file to mock condition that file not existed std::string file_name = utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME); diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 245a6dc76e..c6d109f7d3 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -27,7 +27,8 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_runtime hashtable gtest - rocksdb) + rocksdb + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp index 6354432c8b..9eb7322a61 100644 --- a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp +++ b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp @@ -17,6 +17,7 @@ #include "replica/bulk_load/replica_bulk_loader.h" +#include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include @@ -36,11 +37,11 @@ #include "replica/test/replica_test_base.h" #include "runtime/rpc/rpc_address.h" #include "runtime/task/task_tracker.h" +#include "test_util/test_util.h" #include "utils/blob.h" -#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" -#include "utils/fmt_logging.h" +#include "utils/test_macros.h" namespace dsn { namespace replication { @@ -252,38 +253,21 @@ class replica_bulk_loader_test : public replica_test_base _replica->set_primary_partition_configuration(config); } - void create_local_file(const std::string &file_name) + void create_local_metadata_file() { - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), - rocksdb::Slice("write some data.\n"), - whole_name, - /* should_sync */ true); - ASSERT_TRUE(s.ok()) << s.ToString(); - _file_meta.name = whole_name; - ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(whole_name, _file_meta.md5)); - ASSERT_TRUE(utils::filesystem::file_size( - whole_name, dsn::utils::FileDataType::kSensitive, _file_meta.size)); - } + NO_FATALS(pegasus::create_local_test_file( + utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), &_file_meta)); - error_code create_local_metadata_file() - { - create_local_file(FILE_NAME); _metadata.files.emplace_back(_file_meta); _metadata.file_total_size = _file_meta.size; - std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); blob bb = json::json_forwarder::encode(_metadata); auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), rocksdb::Slice(bb.data(), bb.length()), whole_name, /* should_sync */ true); - if (!s.ok()) { - LOG_ERROR("write file {} failed", whole_name); - return ERR_FILE_OPERATION_FAILED; - } - - return ERR_OK; + ASSERT_TRUE(s.ok()) << fmt::format( + "write file {} failed, err = {}", whole_name, s.ToString()); } bool validate_metadata() @@ -548,7 +532,8 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt) { // create file can not parse as bulk_load_metadata structure utils::filesystem::create_directory(LOCAL_DIR); - create_local_file(METADATA); + NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, METADATA), + &_file_meta)); std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); error_code ec = test_parse_bulk_load_metadata(metadata_file_name); ASSERT_EQ(ERR_CORRUPTION, ec); @@ -558,11 +543,10 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt) TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) { utils::filesystem::create_directory(LOCAL_DIR); - error_code ec = create_local_metadata_file(); - ASSERT_EQ(ec, ERR_OK); + NO_FATALS(create_local_metadata_file()); std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); - ec = test_parse_bulk_load_metadata(metadata_file_name); + auto ec = test_parse_bulk_load_metadata(metadata_file_name); ASSERT_EQ(ec, ERR_OK); ASSERT_TRUE(validate_metadata()); utils::filesystem::remove_path(LOCAL_DIR); diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 3b82d28e12..cb58e4276c 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -47,6 +47,7 @@ set(MY_PROJ_LIBS dsn_meta_server zookeeper hashtable gtest + rocksdb test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/test_util/test_util.cpp b/src/test_util/test_util.cpp index 5ab185503d..0789c4678b 100644 --- a/src/test_util/test_util.cpp +++ b/src/test_util/test_util.cpp @@ -26,12 +26,33 @@ #include "gtest/gtest-message.h" #include "gtest/gtest-test-part.h" #include "gtest/gtest.h" +#include "metadata_types.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" #include "runtime/api_layer1.h" #include "utils/defer.h" +#include "utils/env.h" +#include "utils/error_code.h" +#include "utils/filesystem.h" #include "utils/fmt_logging.h" namespace pegasus { +void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm) +{ + ASSERT_NE(fm, nullptr); + auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice("write some data."), + full_name, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); + fm->name = full_name; + ASSERT_EQ(dsn::ERR_OK, dsn::utils::filesystem::md5sum(full_name, fm->md5)); + ASSERT_TRUE(dsn::utils::filesystem::file_size( + full_name, dsn::utils::FileDataType::kSensitive, fm->size)); +} + void AssertEventually(const std::function &f, int timeout_sec, WaitBackoff backoff) { // TODO(yingchun): should use mono time diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index 8299601eb1..8e125b9996 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -19,12 +19,21 @@ #pragma once -#include #include +#include +#include +#include +#include "metadata_types.h" #include "utils/flags.h" #include "utils/test_macros.h" +namespace dsn { +namespace replication { +class file_meta; +} // namespace replication +} // namespace dsn + DSN_DECLARE_bool(encrypt_data_at_rest); namespace pegasus { @@ -36,6 +45,8 @@ class encrypt_data_test_base : public testing::TestWithParam encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); } }; +void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm); + #define ASSERT_EVENTUALLY(expr) \ do { \ AssertEventually(expr); \ diff --git a/src/utils/test/env.cpp b/src/utils/test/env.cpp index 813465de26..d7352f1bf2 100644 --- a/src/utils/test/env.cpp +++ b/src/utils/test/env.cpp @@ -47,11 +47,11 @@ #include #include "test_util/test_util.h" -#include "utils/enum_helper.h" #include "utils/env.h" #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/rand.h" +#include "utils/utils.h" DSN_DECLARE_bool(encrypt_data_at_rest);