Skip to content

Commit

Permalink
6
Browse files Browse the repository at this point in the history
  • Loading branch information
@acelyc111
acelyc111 committed Sep 22, 2023
1 parent 13e3694 commit de1d419
Show file tree
Hide file tree
Showing 8 changed files with 55 additions and 53 deletions.
3 changes: 2 additions & 1 deletion src/block_service/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ set(MY_PROJ_LIBS
gtest
gtest_main
hdfs
rocksdb)
rocksdb
test_utils)

set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)

Expand Down
25 changes: 4 additions & 21 deletions src/block_service/test/block_service_manager_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
// IWYU pragma: no_include <gtest/gtest-message.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
#include <gtest/gtest.h>
#include <rocksdb/env.h>
#include <rocksdb/slice.h>
#include <rocksdb/status.h>
#include <cstdint>
#include <map>
#include <memory>
Expand All @@ -33,7 +30,6 @@
#include "block_service_mock.h"
#include "metadata_types.h"
#include "test_util/test_util.h"
#include "utils/env.h"
#include "utils/error_code.h"
#include "utils/filesystem.h"
#include "utils/test_macros.h"
Expand All @@ -60,21 +56,6 @@ class block_service_manager_test : public pegasus::encrypt_data_test_base
PROVIDER, LOCAL_DIR, FILE_NAME, _fs.get(), download_size);
}

void create_local_file(const std::string &file_name)
{
std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name);
auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(),
rocksdb::Slice("write some data."),
whole_name,
/* should_sync */ true);
ASSERT_TRUE(s.ok()) << s.ToString();

_file_meta.name = whole_name;
ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(whole_name, _file_meta.md5));
ASSERT_TRUE(utils::filesystem::file_size(
whole_name, dsn::utils::FileDataType::kSensitive, _file_meta.size));
}

void create_remote_file(const std::string &file_name, int64_t size, const std::string &md5)
{
std::string whole_file_name = utils::filesystem::path_combine(PROVIDER, file_name);
Expand Down Expand Up @@ -107,7 +88,8 @@ TEST_P(block_service_manager_test, remote_file_not_exist)

TEST_P(block_service_manager_test, local_file_exist)
{
NO_FATALS(create_local_file(FILE_NAME));
NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME),
&_file_meta));
struct remote_file_info
{
int64_t size;
Expand All @@ -128,7 +110,8 @@ TEST_P(block_service_manager_test, local_file_exist)

TEST_P(block_service_manager_test, do_download_succeed)
{
NO_FATALS(create_local_file(FILE_NAME));
NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME),
&_file_meta));
create_remote_file(FILE_NAME, _file_meta.size, _file_meta.md5);
// remove local file to mock condition that file not existed
std::string file_name = utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME);
Expand Down
3 changes: 2 additions & 1 deletion src/replica/bulk_load/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ set(MY_PROJ_LIBS dsn_meta_server
dsn_runtime
hashtable
gtest
rocksdb)
rocksdb
test_utils)

set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)

Expand Down
40 changes: 12 additions & 28 deletions src/replica/bulk_load/test/replica_bulk_loader_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "replica/bulk_load/replica_bulk_loader.h"

#include <fmt/core.h>
// IWYU pragma: no_include <gtest/gtest-param-test.h>
// IWYU pragma: no_include <gtest/gtest-message.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
Expand All @@ -36,11 +37,11 @@
#include "replica/test/replica_test_base.h"
#include "runtime/rpc/rpc_address.h"
#include "runtime/task/task_tracker.h"
#include "test_util/test_util.h"
#include "utils/blob.h"
#include "utils/env.h"
#include "utils/fail_point.h"
#include "utils/filesystem.h"
#include "utils/fmt_logging.h"
#include "utils/test_macros.h"

namespace dsn {
namespace replication {
Expand Down Expand Up @@ -252,38 +253,21 @@ class replica_bulk_loader_test : public replica_test_base
_replica->set_primary_partition_configuration(config);
}

void create_local_file(const std::string &file_name)
void create_local_metadata_file()
{
std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name);
auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(),
rocksdb::Slice("write some data.\n"),
whole_name,
/* should_sync */ true);
ASSERT_TRUE(s.ok()) << s.ToString();
_file_meta.name = whole_name;
ASSERT_EQ(ERR_OK, utils::filesystem::md5sum(whole_name, _file_meta.md5));
ASSERT_TRUE(utils::filesystem::file_size(
whole_name, dsn::utils::FileDataType::kSensitive, _file_meta.size));
}
NO_FATALS(pegasus::create_local_test_file(
utils::filesystem::path_combine(LOCAL_DIR, FILE_NAME), &_file_meta));

error_code create_local_metadata_file()
{
create_local_file(FILE_NAME);
_metadata.files.emplace_back(_file_meta);
_metadata.file_total_size = _file_meta.size;

std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA);
blob bb = json::json_forwarder<bulk_load_metadata>::encode(_metadata);
auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(),
rocksdb::Slice(bb.data(), bb.length()),
whole_name,
/* should_sync */ true);
if (!s.ok()) {
LOG_ERROR("write file {} failed", whole_name);
return ERR_FILE_OPERATION_FAILED;
}

return ERR_OK;
ASSERT_TRUE(s.ok()) << fmt::format(
"write file {} failed, err = {}", whole_name, s.ToString());
}

bool validate_metadata()
Expand Down Expand Up @@ -548,7 +532,8 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt)
{
// create file can not parse as bulk_load_metadata structure
utils::filesystem::create_directory(LOCAL_DIR);
create_local_file(METADATA);
NO_FATALS(pegasus::create_local_test_file(utils::filesystem::path_combine(LOCAL_DIR, METADATA),
&_file_meta));
std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA);
error_code ec = test_parse_bulk_load_metadata(metadata_file_name);
ASSERT_EQ(ERR_CORRUPTION, ec);
Expand All @@ -558,11 +543,10 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt)
TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed)
{
utils::filesystem::create_directory(LOCAL_DIR);
error_code ec = create_local_metadata_file();
ASSERT_EQ(ec, ERR_OK);
NO_FATALS(create_local_metadata_file());

std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA);
ec = test_parse_bulk_load_metadata(metadata_file_name);
auto ec = test_parse_bulk_load_metadata(metadata_file_name);
ASSERT_EQ(ec, ERR_OK);
ASSERT_TRUE(validate_metadata());
utils::filesystem::remove_path(LOCAL_DIR);
Expand Down
1 change: 1 addition & 0 deletions src/replica/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ set(MY_PROJ_LIBS dsn_meta_server
zookeeper
hashtable
gtest
rocksdb
test_utils)

set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
Expand Down
21 changes: 21 additions & 0 deletions src/test_util/test_util.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,33 @@
#include "gtest/gtest-message.h"
#include "gtest/gtest-test-part.h"
#include "gtest/gtest.h"
#include "metadata_types.h"
#include "rocksdb/env.h"
#include "rocksdb/slice.h"
#include "rocksdb/status.h"
#include "runtime/api_layer1.h"
#include "utils/defer.h"
#include "utils/env.h"
#include "utils/error_code.h"
#include "utils/filesystem.h"
#include "utils/fmt_logging.h"

namespace pegasus {

void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm)
{
ASSERT_NE(fm, nullptr);
auto s = rocksdb::WriteStringToFile(rocksdb::Env::Default(),
rocksdb::Slice("write some data."),
full_name,
/* should_sync */ true);
ASSERT_TRUE(s.ok()) << s.ToString();
fm->name = full_name;
ASSERT_EQ(dsn::ERR_OK, dsn::utils::filesystem::md5sum(full_name, fm->md5));
ASSERT_TRUE(dsn::utils::filesystem::file_size(
full_name, dsn::utils::FileDataType::kSensitive, fm->size));
}

void AssertEventually(const std::function<void(void)> &f, int timeout_sec, WaitBackoff backoff)
{
// TODO(yingchun): should use mono time
Expand Down
13 changes: 12 additions & 1 deletion src/test_util/test_util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,21 @@

#pragma once

#include <functional>
#include <gtest/gtest.h>
#include <rocksdb/env.h>
#include <functional>
#include <string>

#include "metadata_types.h"
#include "utils/flags.h"
#include "utils/test_macros.h"

namespace dsn {
namespace replication {
class file_meta;
} // namespace replication
} // namespace dsn

DSN_DECLARE_bool(encrypt_data_at_rest);

namespace pegasus {
Expand All @@ -36,6 +45,8 @@ class encrypt_data_test_base : public testing::TestWithParam<bool>
encrypt_data_test_base() { FLAGS_encrypt_data_at_rest = GetParam(); }
};

void create_local_test_file(const std::string &full_name, dsn::replication::file_meta *fm);

#define ASSERT_EVENTUALLY(expr) \
do { \
AssertEventually(expr); \
Expand Down
2 changes: 1 addition & 1 deletion src/utils/test/env.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@
#include <string>

#include "test_util/test_util.h"
#include "utils/enum_helper.h"
#include "utils/env.h"
#include "utils/filesystem.h"
#include "utils/flags.h"
#include "utils/rand.h"
#include "utils/utils.h"

DSN_DECLARE_bool(encrypt_data_at_rest);

Expand Down

0 comments on commit de1d419

Please sign in to comment.