Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Encrypt demo all #84

Closed
wants to merge 83 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
83 commits
Select commit Hold shift + click to select a range
baec61d
fix mutation_log_test.cpp
acelyc111 Aug 17, 2023
6c14533
encrypt_demo_0
acelyc111 Aug 17, 2023
58ff440
fmt
acelyc111 Aug 23, 2023
7ee766b
app base
acelyc111 Aug 23, 2023
c0f6303
fix dup log test
acelyc111 Aug 24, 2023
a9b23b7
pegasus_unit_test test ok
acelyc111 Aug 24, 2023
ab4bd19
pegasus ut
acelyc111 Aug 24, 2023
7e2c964
fix
acelyc111 Aug 24, 2023
4498c38
pass more tests
acelyc111 Aug 25, 2023
304058c
base api test
acelyc111 Aug 27, 2023
6f12861
function test
acelyc111 Aug 28, 2023
7a6dc4e
enc enc
acelyc111 Aug 28, 2023
0402858
iwyu
acelyc111 Aug 28, 2023
c309480
make
acelyc111 Aug 28, 2023
bee9bd5
revert
acelyc111 Aug 28, 2023
31d214a
cmake
acelyc111 Aug 28, 2023
7c2755f
aio
acelyc111 Aug 28, 2023
ad0b909
iwyu
acelyc111 Aug 28, 2023
ccc2f2e
aio test
acelyc111 Aug 28, 2023
d2d216e
hdfs
acelyc111 Aug 28, 2023
bf71cc5
hdfs fmt
acelyc111 Aug 28, 2023
6837d53
block service
acelyc111 Aug 28, 2023
29cc755
direct io
acelyc111 Aug 28, 2023
a71b2ac
dsn_block_service_test
acelyc111 Aug 28, 2023
3c37234
fmt
acelyc111 Aug 28, 2023
0f6188d
geo
acelyc111 Aug 28, 2023
318e0cc
pegasus_geo_test
acelyc111 Aug 28, 2023
658c147
nfs
acelyc111 Aug 28, 2023
6f36ee8
add copy util
acelyc111 Aug 28, 2023
98cb7a1
fix reopen bug, add new md5sum
acelyc111 Aug 29, 2023
057877a
1
acelyc111 Aug 29, 2023
d019d11
fmt
acelyc111 Aug 29, 2023
c711cd3
add todo
acelyc111 Aug 29, 2023
94c9759
add todo
acelyc111 Aug 29, 2023
3b016d5
dsn_replica_dup_test
acelyc111 Aug 29, 2023
7be7252
src/replica/replica_restore.cpp
acelyc111 Aug 29, 2023
f55d6c8
base info
acelyc111 Aug 29, 2023
f435743
base info
acelyc111 Aug 29, 2023
4325151
/mutation_log_test.cpp
acelyc111 Aug 29, 2023
204e42f
simple_kv.server.impl.cpp
acelyc111 Aug 30, 2023
1db5292
simple kv
acelyc111 Aug 30, 2023
2c55eb0
replica_test.cpp
acelyc111 Aug 30, 2023
1ad91fd
utils
acelyc111 Aug 30, 2023
ab0068d
remove read/write file
acelyc111 Aug 30, 2023
8842d85
fs
acelyc111 Aug 30, 2023
68e5cbf
add ut
acelyc111 Aug 30, 2023
c22740c
src/server/test/hotkey_collector_test.cpp
acelyc111 Aug 30, 2023
46dba17
revert opts
acelyc111 Aug 30, 2023
240a9b3
comment
acelyc111 Aug 30, 2023
46440e1
iwyu
acelyc111 Aug 30, 2023
94a5137
ini
acelyc111 Aug 30, 2023
29b6073
fix cold backup
acelyc111 Aug 30, 2023
69becf1
bulkload test
acelyc111 Aug 31, 2023
8ad6101
1
acelyc111 Aug 31, 2023
0ff0c46
ut
acelyc111 Aug 31, 2023
3479b9a
fix
acelyc111 Aug 31, 2023
670f434
bulk_load_test.bulk_load_test_failed2 ok
acelyc111 Aug 31, 2023
b5654cb
log
acelyc111 Sep 1, 2023
96ac794
fix
acelyc111 Sep 4, 2023
5664262
ut
acelyc111 Sep 4, 2023
51c7497
all ut
acelyc111 Sep 4, 2023
8ebdba7
fix bulkload ut
acelyc111 Sep 5, 2023
d2c54b0
aio fmt
acelyc111 Sep 5, 2023
76fc79f
aio
acelyc111 Sep 5, 2023
f51c939
cmake
acelyc111 Sep 5, 2023
bec0cbc
rename utils
acelyc111 Sep 6, 2023
ccbde3b
copy
acelyc111 Sep 6, 2023
9a76eb4
copy
acelyc111 Sep 6, 2023
f5ec101
copy file test
acelyc111 Sep 6, 2023
924df30
file
acelyc111 Sep 6, 2023
91ef39c
local service
acelyc111 Sep 6, 2023
8d0d191
config
acelyc111 Sep 6, 2023
2717004
copy test
acelyc111 Sep 6, 2023
5dfd6e3
fix
acelyc111 Sep 6, 2023
6edb296
fmt
acelyc111 Sep 6, 2023
90b7aed
close
acelyc111 Sep 6, 2023
e9c9ca8
block service
acelyc111 Sep 6, 2023
99cdf51
log
acelyc111 Sep 6, 2023
5ce23a6
ut
acelyc111 Sep 7, 2023
8fe5e13
hdfs test
acelyc111 Sep 7, 2023
bcc575a
hdfs
acelyc111 Sep 7, 2023
9142ce0
revert
acelyc111 Sep 7, 2023
41b8270
iwyu
acelyc111 Sep 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
dsn_replica_dup_test
acelyc111 committed Aug 29, 2023
commit 3b016d567e6c1f8174b02ee34a71ae2be857b4d0
2 changes: 2 additions & 0 deletions src/replica/duplication/test/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -30,6 +30,8 @@ set(MY_PROJ_LIBS dsn_meta_server
zookeeper
hashtable
gtest
test_utils
rocksdb
)

set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
28 changes: 17 additions & 11 deletions src/replica/duplication/test/load_from_private_log_test.cpp
Original file line number Diff line number Diff line change
@@ -309,47 +309,53 @@ TEST_P(load_from_private_log_test, start_duplication_100000_4MB)
// Ensure replica_duplicator can correctly handle real-world log file
TEST_P(load_from_private_log_test, handle_real_private_log)
{
std::vector<std::string> log_files({"log.1.0.handle_real_private_log",
"log.1.0.handle_real_private_log2",
"log.1.0.all_loaded_are_write_empties"});
if (FLAGS_encrypt_data_at_rest) {
// The testfiles log.1.0.* are not encrypted.
return;
for (int i = 0; i < log_files.size(); i++) {
encrypt_file(log_files[i], log_files[i] + ".encrypted");
log_files[i] += ".encrypted";
}
}

struct test_data
{
std::string fname;
int puts;
int total;
gpid id;
} tests[] = {
// PUT, PUT, PUT, EMPTY, PUT, EMPTY, EMPTY
{"log.1.0.handle_real_private_log", 4, 6, gpid(1, 4)},
{4, 6, gpid(1, 4)},

// EMPTY, PUT, EMPTY
{"log.1.0.handle_real_private_log2", 1, 2, gpid(1, 4)},
{1, 2, gpid(1, 4)},

// EMPTY, EMPTY, EMPTY
{"log.1.0.all_loaded_are_write_empties", 0, 2, gpid(1, 5)},
{0, 2, gpid(1, 5)},
};

for (auto tt : tests) {
ASSERT_EQ(log_files.size(), sizeof(tests) / sizeof(test_data));
for (int i = 0; i < log_files.size(); i++) {
// reset replica to specified gpid
duplicator.reset(nullptr);
_replica = create_mock_replica(stub.get(), tt.id.get_app_id(), tt.id.get_partition_index());
_replica = create_mock_replica(
stub.get(), tests[i].id.get_app_id(), tests[i].id.get_partition_index());

// Update '_log_dir' to the corresponding replica created above.
_log_dir = _replica->dir();
ASSERT_TRUE(utils::filesystem::path_exists(_log_dir)) << _log_dir;

// Copy the log file to '_log_dir'
boost::filesystem::path file(tt.fname);
ASSERT_TRUE(dsn::utils::filesystem::file_exists(tt.fname)) << tt.fname;
boost::filesystem::path file(log_files[i]);
ASSERT_TRUE(dsn::utils::filesystem::file_exists(log_files[i])) << log_files[i];
boost::system::error_code ec;
boost::filesystem::copy_file(
file, _log_dir + "/log.1.0", boost::filesystem::copy_option::overwrite_if_exists, ec);
ASSERT_TRUE(!ec) << ec.value() << ", " << ec.category().name() << ", " << ec.message();

// Start to verify.
load_and_wait_all_entries_loaded(tt.puts, tt.total, tt.id, 1, 0);
load_and_wait_all_entries_loaded(tests[i].puts, tests[i].total, tests[i].id, 1, 0);
}
}

1 change: 1 addition & 0 deletions src/replica/replica_backup.cpp
Original file line number Diff line number Diff line change
@@ -387,6 +387,7 @@ statistic_file_infos_under_dir(const std::string &dir,
total_size = 0;
file_infos.clear();

// TODO(yingchun): check if there are any files that are not sensitive (not encrypted).
for (std::string &file : sub_files) {
std::pair<std::string, int64_t> file_info;