diff --git a/src/block_service/test/block_service_manager_test.cpp b/src/block_service/test/block_service_manager_test.cpp index e8c5a48edf..73b2933fa6 100644 --- a/src/block_service/test/block_service_manager_test.cpp +++ b/src/block_service/test/block_service_manager_test.cpp @@ -91,6 +91,7 @@ class block_service_manager_test : public pegasus::encrypt_data_test_base std::string FILE_NAME = "test_file"; }; +// TODO(yingchun): ENCRYPTION: add enable encryption test. INSTANTIATE_TEST_CASE_P(, block_service_manager_test, ::testing::Values(false)); TEST_P(block_service_manager_test, remote_file_not_exist) diff --git a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp index df123cc0c7..904f6babdb 100644 --- a/src/replica/bulk_load/test/replica_bulk_loader_test.cpp +++ b/src/replica/bulk_load/test/replica_bulk_loader_test.cpp @@ -17,9 +17,13 @@ #include "replica/bulk_load/replica_bulk_loader.h" +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include +#include #include // IWYU pragma: keep #include #include @@ -33,6 +37,7 @@ #include "runtime/rpc/rpc_address.h" #include "runtime/task/task_tracker.h" #include "utils/blob.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" @@ -250,15 +255,16 @@ class replica_bulk_loader_test : public replica_test_base void create_local_file(const std::string &file_name) { std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, file_name); - utils::filesystem::create_file(whole_name); - std::ofstream test_file; - test_file.open(whole_name); - test_file << "write some data.\n"; - test_file.close(); - + auto s = + rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice("write some data.\n"), + whole_name, + /* should_sync */ true); + ASSERT_TRUE(s.ok()) << s.ToString(); _file_meta.name = whole_name; utils::filesystem::md5sum(whole_name, _file_meta.md5); - utils::filesystem::file_size(whole_name, _file_meta.size); + utils::filesystem::file_size( + whole_name, dsn::utils::FileDataType::kSensitive, _file_meta.size); } error_code create_local_metadata_file() @@ -268,21 +274,16 @@ class replica_bulk_loader_test : public replica_test_base _metadata.file_total_size = _file_meta.size; std::string whole_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); - utils::filesystem::create_file(whole_name); - std::ofstream os(whole_name.c_str(), - (std::ofstream::out | std::ios::binary | std::ofstream::trunc)); - if (!os.is_open()) { - LOG_ERROR("open file {} failed", whole_name); - return ERR_FILE_OPERATION_FAILED; - } - blob bb = json::json_forwarder::encode(_metadata); - os.write((const char *)bb.data(), (std::streamsize)bb.length()); - if (os.bad()) { + auto s = + rocksdb::WriteStringToFile(rocksdb::Env::Default(), + rocksdb::Slice(bb.data(), bb.length()), + whole_name, + /* should_sync */ true); + if (!s.ok()) { LOG_ERROR("write file {} failed", whole_name); return ERR_FILE_OPERATION_FAILED; } - os.close(); return ERR_OK; } @@ -451,14 +452,17 @@ class replica_bulk_loader_test : public replica_test_base std::string FILE_NAME = "test_sst_file"; }; +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, replica_bulk_loader_test, ::testing::Values(false)); + // on_bulk_load unit tests -TEST_F(replica_bulk_loader_test, on_bulk_load_not_primary) +TEST_P(replica_bulk_loader_test, on_bulk_load_not_primary) { create_bulk_load_request(bulk_load_status::BLS_DOWNLOADING); ASSERT_EQ(test_on_bulk_load(), ERR_INVALID_STATE); } -TEST_F(replica_bulk_loader_test, on_bulk_load_ballot_change) +TEST_P(replica_bulk_loader_test, on_bulk_load_ballot_change) { create_bulk_load_request(bulk_load_status::BLS_DOWNLOADING, BALLOT + 1); mock_primary_states(); @@ -466,7 +470,7 @@ TEST_F(replica_bulk_loader_test, on_bulk_load_ballot_change) } // on_group_bulk_load unit tests -TEST_F(replica_bulk_loader_test, on_group_bulk_load_test) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_test) { struct test_struct { @@ -493,7 +497,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_test) } // start_downloading unit tests -TEST_F(replica_bulk_loader_test, start_downloading_test) +TEST_P(replica_bulk_loader_test, start_downloading_test) { // Test cases: // - stub concurrent downloading count excceed @@ -520,7 +524,7 @@ TEST_F(replica_bulk_loader_test, start_downloading_test) } // start_downloading unit tests -TEST_F(replica_bulk_loader_test, rollback_to_downloading_test) +TEST_P(replica_bulk_loader_test, rollback_to_downloading_test) { fail::cfg("replica_bulk_loader_download_files", "return()"); struct test_struct @@ -540,23 +544,23 @@ TEST_F(replica_bulk_loader_test, rollback_to_downloading_test) } // parse_bulk_load_metadata unit tests -TEST_F(replica_bulk_loader_test, bulk_load_metadata_not_exist) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_not_exist) { ASSERT_EQ(test_parse_bulk_load_metadata("path_not_exist"), ERR_FILE_OPERATION_FAILED); } -TEST_F(replica_bulk_loader_test, bulk_load_metadata_corrupt) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_corrupt) { // create file can not parse as bulk_load_metadata structure utils::filesystem::create_directory(LOCAL_DIR); create_local_file(METADATA); std::string metadata_file_name = utils::filesystem::path_combine(LOCAL_DIR, METADATA); error_code ec = test_parse_bulk_load_metadata(metadata_file_name); - ASSERT_EQ(ec, ERR_CORRUPTION); + ASSERT_EQ(ERR_CORRUPTION, ec); utils::filesystem::remove_path(LOCAL_DIR); } -TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) +TEST_P(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) { utils::filesystem::create_directory(LOCAL_DIR); error_code ec = create_local_metadata_file(); @@ -570,7 +574,7 @@ TEST_F(replica_bulk_loader_test, bulk_load_metadata_parse_succeed) } // finish download test -TEST_F(replica_bulk_loader_test, finish_download_test) +TEST_P(replica_bulk_loader_test, finish_download_test) { mock_downloading_progress(100, 50, 50); stub->set_bulk_load_downloading_count(3); @@ -581,7 +585,7 @@ TEST_F(replica_bulk_loader_test, finish_download_test) } // start ingestion test -TEST_F(replica_bulk_loader_test, start_ingestion_test) +TEST_P(replica_bulk_loader_test, start_ingestion_test) { mock_group_progress(bulk_load_status::BLS_DOWNLOADED); test_start_ingestion(); @@ -589,7 +593,7 @@ TEST_F(replica_bulk_loader_test, start_ingestion_test) } // handle_bulk_load_finish unit tests -TEST_F(replica_bulk_loader_test, bulk_load_finish_test) +TEST_P(replica_bulk_loader_test, bulk_load_finish_test) { // Test cases // - bulk load succeed @@ -672,7 +676,7 @@ TEST_F(replica_bulk_loader_test, bulk_load_finish_test) } // pause_bulk_load unit tests -TEST_F(replica_bulk_loader_test, pause_bulk_load_test) +TEST_P(replica_bulk_loader_test, pause_bulk_load_test) { const int32_t stub_downloading_count = 3; // Test cases: @@ -703,7 +707,7 @@ TEST_F(replica_bulk_loader_test, pause_bulk_load_test) } // report_group_download_progress unit tests -TEST_F(replica_bulk_loader_test, report_group_download_progress_test) +TEST_P(replica_bulk_loader_test, report_group_download_progress_test) { struct test_struct { @@ -728,7 +732,7 @@ TEST_F(replica_bulk_loader_test, report_group_download_progress_test) } // report_group_ingestion_status unit tests -TEST_F(replica_bulk_loader_test, report_group_ingestion_status_test) +TEST_P(replica_bulk_loader_test, report_group_ingestion_status_test) { struct ingestion_struct @@ -802,27 +806,27 @@ TEST_F(replica_bulk_loader_test, report_group_ingestion_status_test) } // report_group_context_clean_flag unit tests -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_in_unhealthy_state) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_in_unhealthy_state) { // _primary_states.membership.secondaries is empty mock_replica_config(partition_status::PS_PRIMARY); ASSERT_FALSE(test_report_group_cleaned_up()); } -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_not_cleaned_up) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_not_cleaned_up) { mock_group_cleanup_flag(bulk_load_status::BLS_SUCCEED, true, false); ASSERT_FALSE(test_report_group_cleaned_up()); } -TEST_F(replica_bulk_loader_test, report_group_cleanup_flag_all_cleaned_up) +TEST_P(replica_bulk_loader_test, report_group_cleanup_flag_all_cleaned_up) { mock_group_cleanup_flag(bulk_load_status::BLS_INVALID, true, true); ASSERT_TRUE(test_report_group_cleaned_up()); } // report_group_is_paused unit tests -TEST_F(replica_bulk_loader_test, report_group_is_paused_test) +TEST_P(replica_bulk_loader_test, report_group_is_paused_test) { struct test_struct { @@ -836,21 +840,21 @@ TEST_F(replica_bulk_loader_test, report_group_is_paused_test) } // on_group_bulk_load_reply unit tests -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_downloading_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_downloading_error) { mock_group_progress(bulk_load_status::BLS_DOWNLOADING, 30, 30, 60); test_on_group_bulk_load_reply(bulk_load_status::BLS_DOWNLOADING, BALLOT, ERR_BUSY); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_downloaded_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_downloaded_error) { mock_group_progress(bulk_load_status::BLS_DOWNLOADED); test_on_group_bulk_load_reply(bulk_load_status::BLS_DOWNLOADED, BALLOT, ERR_INVALID_STATE); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) { mock_group_ingestion_states(ingestion_status::IS_RUNNING, ingestion_status::IS_SUCCEED); test_on_group_bulk_load_reply( @@ -858,7 +862,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_ingestion_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) { mock_group_cleanup_flag(bulk_load_status::BLS_SUCCEED); test_on_group_bulk_load_reply( @@ -866,14 +870,14 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_succeed_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_failed_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_failed_error) { mock_group_ingestion_states(ingestion_status::IS_RUNNING, ingestion_status::IS_SUCCEED); test_on_group_bulk_load_reply(bulk_load_status::BLS_FAILED, BALLOT, ERR_OK, ERR_TIMEOUT); ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) { mock_group_progress(bulk_load_status::BLS_PAUSED, 100, 50, 10); test_on_group_bulk_load_reply( @@ -881,7 +885,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_pausing_error) ASSERT_TRUE(is_secondary_bulk_load_state_reset()); } -TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) +TEST_P(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) { mock_group_cleanup_flag(bulk_load_status::BLS_INVALID, true, false); test_on_group_bulk_load_reply(bulk_load_status::BLS_CANCELED, BALLOT, ERR_OBJECT_NOT_FOUND); @@ -889,7 +893,7 @@ TEST_F(replica_bulk_loader_test, on_group_bulk_load_reply_rpc_error) } // validate_status unit test -TEST_F(replica_bulk_loader_test, validate_status_test) +TEST_P(replica_bulk_loader_test, validate_status_test) { struct validate_struct { diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index 342669f63c..1d6dbf71fc 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -15,16 +15,18 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include +#include #include +// IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include #include +#include +#include #include -#include +#include "aio/aio_task.h" +#include "aio/file_io.h" #include "common/gpid.h" #include "common/replication.codes.h" #include "common/replication_other_types.h" @@ -44,6 +46,7 @@ #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/chrono_literals.h" +#include "utils/env.h" #include "utils/error_code.h" #include "utils/errors.h" #include "utils/fail_point.h" @@ -58,7 +61,6 @@ #include #include #include -#include #include #include #include @@ -68,6 +70,7 @@ #include "duplication_test_base.h" #include "replica/duplication/load_from_private_log.h" #include "replica/mutation_log_utils.h" +#include "test_util/test_util.h" namespace dsn { namespace replication { @@ -273,78 +276,89 @@ class load_from_private_log_test : public duplication_test_base std::unique_ptr duplicator; }; -TEST_F(load_from_private_log_test, find_log_file_to_start) { test_find_log_file_to_start(); } +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, load_from_private_log_test, ::testing::Values(false)); -TEST_F(load_from_private_log_test, start_duplication_10000_4MB) +TEST_P(load_from_private_log_test, find_log_file_to_start) { test_find_log_file_to_start(); } + +TEST_P(load_from_private_log_test, start_duplication_10000_4MB) { test_start_duplication(10000, 4); } -TEST_F(load_from_private_log_test, start_duplication_50000_4MB) +TEST_P(load_from_private_log_test, start_duplication_50000_4MB) { test_start_duplication(50000, 4); } -TEST_F(load_from_private_log_test, start_duplication_10000_1MB) +TEST_P(load_from_private_log_test, start_duplication_10000_1MB) { test_start_duplication(10000, 1); } -TEST_F(load_from_private_log_test, start_duplication_50000_1MB) +TEST_P(load_from_private_log_test, start_duplication_50000_1MB) { test_start_duplication(50000, 1); } -TEST_F(load_from_private_log_test, start_duplication_100000_4MB) +TEST_P(load_from_private_log_test, start_duplication_100000_4MB) { test_start_duplication(100000, 4); } // Ensure replica_duplicator can correctly handle real-world log file -TEST_F(load_from_private_log_test, handle_real_private_log) +TEST_P(load_from_private_log_test, handle_real_private_log) { + std::vector log_files({"log.1.0.handle_real_private_log", + "log.1.0.handle_real_private_log2", + "log.1.0.all_loaded_are_write_empties"}); + if (FLAGS_encrypt_data_at_rest) { + for (int i = 0; i < log_files.size(); i++) { + auto s = dsn::utils::encrypt_file(log_files[i], log_files[i] + ".encrypted"); + ASSERT_TRUE(s.ok()) << s.ToString(); + log_files[i] += ".encrypted"; + } + } + struct test_data { - std::string fname; int puts; int total; gpid id; } tests[] = { // PUT, PUT, PUT, EMPTY, PUT, EMPTY, EMPTY - {"log.1.0.handle_real_private_log", 4, 6, gpid(1, 4)}, + {4, 6, gpid(1, 4)}, // EMPTY, PUT, EMPTY - {"log.1.0.handle_real_private_log2", 1, 2, gpid(1, 4)}, + {1, 2, gpid(1, 4)}, // EMPTY, EMPTY, EMPTY - {"log.1.0.all_loaded_are_write_empties", 0, 2, gpid(1, 5)}, + {0, 2, gpid(1, 5)}, }; - for (auto tt : tests) { + ASSERT_EQ(log_files.size(), sizeof(tests) / sizeof(test_data)); + for (int i = 0; i < log_files.size(); i++) { // reset replica to specified gpid duplicator.reset(nullptr); - _replica = create_mock_replica(stub.get(), tt.id.get_app_id(), tt.id.get_partition_index()); + _replica = create_mock_replica( + stub.get(), tests[i].id.get_app_id(), tests[i].id.get_partition_index()); // Update '_log_dir' to the corresponding replica created above. _log_dir = _replica->dir(); ASSERT_TRUE(utils::filesystem::path_exists(_log_dir)) << _log_dir; // Copy the log file to '_log_dir' - boost::filesystem::path file(tt.fname); - ASSERT_TRUE(dsn::utils::filesystem::file_exists(tt.fname)) << tt.fname; - boost::system::error_code ec; - boost::filesystem::copy_file( - file, _log_dir + "/log.1.0", boost::filesystem::copy_option::overwrite_if_exists, ec); - ASSERT_TRUE(!ec) << ec.value() << ", " << ec.category().name() << ", " << ec.message(); + auto s = dsn::utils::copy_file(log_files[i], _log_dir + "/log.1.0"); + ASSERT_TRUE(s.ok()) << s.ToString(); // Start to verify. - load_and_wait_all_entries_loaded(tt.puts, tt.total, tt.id, 1, 0); + load_and_wait_all_entries_loaded(tests[i].puts, tests[i].total, tests[i].id, 1, 0); } } -TEST_F(load_from_private_log_test, restart_duplication) { test_restart_duplication(); } +TEST_P(load_from_private_log_test, restart_duplication) { test_restart_duplication(); } -TEST_F(load_from_private_log_test, ignore_useless) +TEST_P(load_from_private_log_test, ignore_useless) { utils::filesystem::remove_path(_log_dir); @@ -410,7 +424,10 @@ class load_fail_mode_test : public load_from_private_log_test std::unique_ptr end_stage; }; -TEST_F(load_fail_mode_test, fail_skip) +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, load_fail_mode_test, ::testing::Values(false)); + +TEST_P(load_fail_mode_test, fail_skip) { duplicator->update_fail_mode(duplication_fail_mode::FAIL_SKIP); ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); @@ -428,7 +445,7 @@ TEST_F(load_fail_mode_test, fail_skip) ASSERT_GT(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); } -TEST_F(load_fail_mode_test, fail_slow) +TEST_P(load_fail_mode_test, fail_slow) { duplicator->update_fail_mode(duplication_fail_mode::FAIL_SLOW); ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); @@ -447,16 +464,31 @@ TEST_F(load_fail_mode_test, fail_slow) ASSERT_EQ(load->_counter_dup_load_skipped_bytes_count->get_integer_value(), 0); } -TEST_F(load_fail_mode_test, fail_skip_real_corrupted_file) +TEST_P(load_fail_mode_test, fail_skip_real_corrupted_file) { { // inject some bad data in the middle of the first file std::string log_path = _log_dir + "/log.1.0"; - auto file_size = boost::filesystem::file_size(log_path); - int fd = open(log_path.c_str(), O_WRONLY); + int64_t file_size; + ASSERT_TRUE(utils::filesystem::file_size( + log_path, dsn::utils::FileDataType::kSensitive, file_size)); + auto wfile = file::open(log_path, O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); + const char buf[] = "xxxxxx"; - auto written_size = pwrite(fd, buf, sizeof(buf), file_size / 2); - ASSERT_EQ(written_size, sizeof(buf)); - close(fd); + auto buff_len = sizeof(buf); + auto t = ::dsn::file::write(wfile, + buf, + buff_len, + file_size / 2, + LPC_AIO_IMMEDIATE_CALLBACK, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(buff_len, n); + }); + t->wait(); + ASSERT_EQ(ERR_OK, ::dsn::file::flush(wfile)); + ASSERT_EQ(ERR_OK, ::dsn::file::close(wfile)); } duplicator->update_fail_mode(duplication_fail_mode::FAIL_SKIP); diff --git a/src/replica/test/mutation_log_test.cpp b/src/replica/test/mutation_log_test.cpp index 527c1e8d98..18e145c44f 100644 --- a/src/replica/test/mutation_log_test.cpp +++ b/src/replica/test/mutation_log_test.cpp @@ -31,7 +31,6 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include #include @@ -39,6 +38,7 @@ #include #include "aio/aio_task.h" +#include "aio/file_io.h" #include "backup_types.h" #include "common/replication.codes.h" #include "consensus_types.h" @@ -49,10 +49,12 @@ #include "replica/test/mock_utils.h" #include "replica_test_base.h" #include "rrdb/rrdb.code.definition.h" +#include "test_util/test_util.h" #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/blob.h" #include "utils/defer.h" +#include "utils/env.h" #include "utils/fail_point.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -66,43 +68,33 @@ class message_ex; using namespace ::dsn; using namespace ::dsn::replication; -static void copy_file(const char *from_file, const char *to_file, int64_t to_size = -1) +static void overwrite_file(const char *file, int offset, const void *buf, int size) { - int64_t from_size; - ASSERT_TRUE(dsn::utils::filesystem::file_size(from_file, from_size)); - ASSERT_LE(to_size, from_size); - FILE *from = fopen(from_file, "rb"); - ASSERT_TRUE(from != nullptr); - FILE *to = fopen(to_file, "wb"); - ASSERT_TRUE(to != nullptr); - if (to_size == -1) - to_size = from_size; - if (to_size > 0) { - std::unique_ptr buf(new char[to_size]); - auto n = fread(buf.get(), 1, to_size, from); - ASSERT_EQ(to_size, n); - n = fwrite(buf.get(), 1, to_size, to); - ASSERT_EQ(to_size, n); - } - int r = fclose(from); - ASSERT_EQ(0, r); - r = fclose(to); - ASSERT_EQ(0, r); + auto wfile = file::open(file, O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_NE(wfile, nullptr); + auto t = ::dsn::file::write(wfile, + (const char *)buf, + size, + offset, + LPC_AIO_IMMEDIATE_CALLBACK, + nullptr, + [=](::dsn::error_code err, size_t n) { + CHECK_EQ(ERR_OK, err); + CHECK_EQ(size, n); + }); + t->wait(); + ASSERT_EQ(ERR_OK, file::flush(wfile)); + ASSERT_EQ(ERR_OK, file::close(wfile)); } -static void overwrite_file(const char *file, int offset, const void *buf, int size) +class replication_test : public pegasus::encrypt_data_test_base { - FILE *f = fopen(file, "r+b"); - ASSERT_TRUE(f != nullptr); - int r = fseek(f, offset, SEEK_SET); - ASSERT_EQ(0, r); - size_t n = fwrite(buf, 1, size, f); - ASSERT_EQ(size, n); - r = fclose(f); - ASSERT_EQ(0, r); -} +}; + +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, replication_test, ::testing::Values(false)); -TEST(replication, log_file) +TEST_P(replication_test, log_file) { replica_log_info_map mdecrees; gpid gpid(1, 0); @@ -198,7 +190,7 @@ TEST(replication, log_file) // bad file data: empty file ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.0")); - copy_file(fpath.c_str(), "log.1.0", 0); + dsn::utils::copy_file_by_size(fpath, "log.1.0", 0); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.0")); lf = log_file::open_read("log.1.0", err); ASSERT_TRUE(lf == nullptr); @@ -208,7 +200,7 @@ TEST(replication, log_file) // bad file data: incomplete log_block_header ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.1")); - copy_file(fpath.c_str(), "log.1.1", sizeof(log_block_header) - 1); + dsn::utils::copy_file_by_size(fpath, "log.1.1", sizeof(log_block_header) - 1); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.1")); lf = log_file::open_read("log.1.1", err); ASSERT_TRUE(lf == nullptr); @@ -218,7 +210,7 @@ TEST(replication, log_file) // bad file data: bad log_block_header (magic = 0xfeadbeef) ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.2")); - copy_file(fpath.c_str(), "log.1.2"); + dsn::utils::copy_file_by_size(fpath, "log.1.2"); int32_t bad_magic = 0xfeadbeef; overwrite_file("log.1.2", FIELD_OFFSET(log_block_header, magic), &bad_magic, sizeof(bad_magic)); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.2")); @@ -230,7 +222,7 @@ TEST(replication, log_file) // bad file data: bad log_block_header (crc check failed) ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.3")); - copy_file(fpath.c_str(), "log.1.3"); + dsn::utils::copy_file_by_size(fpath, "log.1.3"); int32_t bad_crc = 0; overwrite_file("log.1.3", FIELD_OFFSET(log_block_header, body_crc), &bad_crc, sizeof(bad_crc)); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.3")); @@ -242,14 +234,13 @@ TEST(replication, log_file) // bad file data: incomplete block body ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4")); - copy_file(fpath.c_str(), "log.1.4", sizeof(log_block_header) + 1); + dsn::utils::copy_file_by_size(fpath, "log.1.4", sizeof(log_block_header) + 1); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4")); lf = log_file::open_read("log.1.4", err); ASSERT_TRUE(lf == nullptr); ASSERT_EQ(ERR_INCOMPLETE_DATA, err); ASSERT_TRUE(!dsn::utils::filesystem::file_exists("log.1.4")); ASSERT_TRUE(dsn::utils::filesystem::file_exists("log.1.4.removed")); - ASSERT_TRUE(dsn::utils::filesystem::rename_path("log.1.4.removed", "log.1.4")); // read the file for test offset = 100; @@ -259,7 +250,7 @@ TEST(replication, log_file) ASSERT_EQ(1, lf->index()); ASSERT_EQ(100, lf->start_offset()); int64_t sz; - ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, sz)); + ASSERT_TRUE(dsn::utils::filesystem::file_size(fpath, dsn::utils::FileDataType::kSensitive, sz)); ASSERT_EQ(lf->start_offset() + sz, lf->end_offset()); // read data @@ -539,20 +530,23 @@ class mutation_log_test : public replica_test_base } }; -TEST_F(mutation_log_test, replay_single_file_1000) { test_replay_single_file(1000); } +// TODO(yingchun): ENCRYPTION: add enable encryption test. +INSTANTIATE_TEST_CASE_P(, mutation_log_test, ::testing::Values(false)); + +TEST_P(mutation_log_test, replay_single_file_1000) { test_replay_single_file(1000); } -TEST_F(mutation_log_test, replay_single_file_2000) { test_replay_single_file(2000); } +TEST_P(mutation_log_test, replay_single_file_2000) { test_replay_single_file(2000); } -TEST_F(mutation_log_test, replay_single_file_5000) { test_replay_single_file(5000); } +TEST_P(mutation_log_test, replay_single_file_5000) { test_replay_single_file(5000); } -TEST_F(mutation_log_test, replay_single_file_10000) { test_replay_single_file(10000); } +TEST_P(mutation_log_test, replay_single_file_10000) { test_replay_single_file(10000); } -TEST_F(mutation_log_test, replay_single_file_1) { test_replay_single_file(1); } +TEST_P(mutation_log_test, replay_single_file_1) { test_replay_single_file(1); } -TEST_F(mutation_log_test, replay_single_file_10) { test_replay_single_file(10); } +TEST_P(mutation_log_test, replay_single_file_10) { test_replay_single_file(10); } // mutation_log::open -TEST_F(mutation_log_test, open) +TEST_P(mutation_log_test, open) { std::vector mutations; @@ -585,13 +579,13 @@ TEST_F(mutation_log_test, open) } } -TEST_F(mutation_log_test, replay_multiple_files_10000_1mb) { test_replay_multiple_files(10000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_10000_1mb) { test_replay_multiple_files(10000, 1); } -TEST_F(mutation_log_test, replay_multiple_files_20000_1mb) { test_replay_multiple_files(20000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_20000_1mb) { test_replay_multiple_files(20000, 1); } -TEST_F(mutation_log_test, replay_multiple_files_50000_1mb) { test_replay_multiple_files(50000, 1); } +TEST_P(mutation_log_test, replay_multiple_files_50000_1mb) { test_replay_multiple_files(50000, 1); } -TEST_F(mutation_log_test, replay_start_decree) +TEST_P(mutation_log_test, replay_start_decree) { // decree ranges from [1, 30) generate_multiple_log_files(3); @@ -604,7 +598,7 @@ TEST_F(mutation_log_test, replay_start_decree) ASSERT_EQ(mlog->get_log_file_map().size(), 3); } -TEST_F(mutation_log_test, reset_from) +TEST_P(mutation_log_test, reset_from) { std::vector expected; { // writing logs @@ -652,7 +646,7 @@ TEST_F(mutation_log_test, reset_from) // multi-threaded testing. ensure reset_from will wait until // all previous writes complete. -TEST_F(mutation_log_test, reset_from_while_writing) +TEST_P(mutation_log_test, reset_from_while_writing) { std::vector expected; { // writing logs