Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Encrypt demo all #84

Closed
wants to merge 83 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
83 commits
Select commit Hold shift + click to select a range
baec61d
fix mutation_log_test.cpp
acelyc111 Aug 17, 2023
6c14533
encrypt_demo_0
acelyc111 Aug 17, 2023
58ff440
fmt
acelyc111 Aug 23, 2023
7ee766b
app base
acelyc111 Aug 23, 2023
c0f6303
fix dup log test
acelyc111 Aug 24, 2023
a9b23b7
pegasus_unit_test test ok
acelyc111 Aug 24, 2023
ab4bd19
pegasus ut
acelyc111 Aug 24, 2023
7e2c964
fix
acelyc111 Aug 24, 2023
4498c38
pass more tests
acelyc111 Aug 25, 2023
304058c
base api test
acelyc111 Aug 27, 2023
6f12861
function test
acelyc111 Aug 28, 2023
7a6dc4e
enc enc
acelyc111 Aug 28, 2023
0402858
iwyu
acelyc111 Aug 28, 2023
c309480
make
acelyc111 Aug 28, 2023
bee9bd5
revert
acelyc111 Aug 28, 2023
31d214a
cmake
acelyc111 Aug 28, 2023
7c2755f
aio
acelyc111 Aug 28, 2023
ad0b909
iwyu
acelyc111 Aug 28, 2023
ccc2f2e
aio test
acelyc111 Aug 28, 2023
d2d216e
hdfs
acelyc111 Aug 28, 2023
bf71cc5
hdfs fmt
acelyc111 Aug 28, 2023
6837d53
block service
acelyc111 Aug 28, 2023
29cc755
direct io
acelyc111 Aug 28, 2023
a71b2ac
dsn_block_service_test
acelyc111 Aug 28, 2023
3c37234
fmt
acelyc111 Aug 28, 2023
0f6188d
geo
acelyc111 Aug 28, 2023
318e0cc
pegasus_geo_test
acelyc111 Aug 28, 2023
658c147
nfs
acelyc111 Aug 28, 2023
6f36ee8
add copy util
acelyc111 Aug 28, 2023
98cb7a1
fix reopen bug, add new md5sum
acelyc111 Aug 29, 2023
057877a
1
acelyc111 Aug 29, 2023
d019d11
fmt
acelyc111 Aug 29, 2023
c711cd3
add todo
acelyc111 Aug 29, 2023
94c9759
add todo
acelyc111 Aug 29, 2023
3b016d5
dsn_replica_dup_test
acelyc111 Aug 29, 2023
7be7252
src/replica/replica_restore.cpp
acelyc111 Aug 29, 2023
f55d6c8
base info
acelyc111 Aug 29, 2023
f435743
base info
acelyc111 Aug 29, 2023
4325151
/mutation_log_test.cpp
acelyc111 Aug 29, 2023
204e42f
simple_kv.server.impl.cpp
acelyc111 Aug 30, 2023
1db5292
simple kv
acelyc111 Aug 30, 2023
2c55eb0
replica_test.cpp
acelyc111 Aug 30, 2023
1ad91fd
utils
acelyc111 Aug 30, 2023
ab0068d
remove read/write file
acelyc111 Aug 30, 2023
8842d85
fs
acelyc111 Aug 30, 2023
68e5cbf
add ut
acelyc111 Aug 30, 2023
c22740c
src/server/test/hotkey_collector_test.cpp
acelyc111 Aug 30, 2023
46dba17
revert opts
acelyc111 Aug 30, 2023
240a9b3
comment
acelyc111 Aug 30, 2023
46440e1
iwyu
acelyc111 Aug 30, 2023
94a5137
ini
acelyc111 Aug 30, 2023
29b6073
fix cold backup
acelyc111 Aug 30, 2023
69becf1
bulkload test
acelyc111 Aug 31, 2023
8ad6101
1
acelyc111 Aug 31, 2023
0ff0c46
ut
acelyc111 Aug 31, 2023
3479b9a
fix
acelyc111 Aug 31, 2023
670f434
bulk_load_test.bulk_load_test_failed2 ok
acelyc111 Aug 31, 2023
b5654cb
log
acelyc111 Sep 1, 2023
96ac794
fix
acelyc111 Sep 4, 2023
5664262
ut
acelyc111 Sep 4, 2023
51c7497
all ut
acelyc111 Sep 4, 2023
8ebdba7
fix bulkload ut
acelyc111 Sep 5, 2023
d2c54b0
aio fmt
acelyc111 Sep 5, 2023
76fc79f
aio
acelyc111 Sep 5, 2023
f51c939
cmake
acelyc111 Sep 5, 2023
bec0cbc
rename utils
acelyc111 Sep 6, 2023
ccbde3b
copy
acelyc111 Sep 6, 2023
9a76eb4
copy
acelyc111 Sep 6, 2023
f5ec101
copy file test
acelyc111 Sep 6, 2023
924df30
file
acelyc111 Sep 6, 2023
91ef39c
local service
acelyc111 Sep 6, 2023
8d0d191
config
acelyc111 Sep 6, 2023
2717004
copy test
acelyc111 Sep 6, 2023
5dfd6e3
fix
acelyc111 Sep 6, 2023
6edb296
fmt
acelyc111 Sep 6, 2023
90b7aed
close
acelyc111 Sep 6, 2023
e9c9ca8
block service
acelyc111 Sep 6, 2023
99cdf51
log
acelyc111 Sep 6, 2023
5ce23a6
ut
acelyc111 Sep 7, 2023
8fe5e13
hdfs test
acelyc111 Sep 7, 2023
bcc575a
hdfs
acelyc111 Sep 7, 2023
9142ce0
revert
acelyc111 Sep 7, 2023
41b8270
iwyu
acelyc111 Sep 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
app base
acelyc111 committed Aug 23, 2023
commit 7ee766be1750acf0f818f7a2fe321af18eae0d0a
1 change: 1 addition & 0 deletions src/aio/file_io.h
Original file line number Diff line number Diff line change
@@ -54,6 +54,7 @@ enum class FileOpenType
kWriteOnly
};

// TODO(yingchun): consider to return a smart pointer
/// open file
///
/// \param file_name filename of the file.
128 changes: 74 additions & 54 deletions src/replica/replication_app_base.cpp
Original file line number Diff line number Diff line change
@@ -75,40 +75,43 @@ namespace replication {
const std::string replica_init_info::kInitInfo = ".init-info";

DEFINE_TASK_CODE_AIO(LPC_AIO_INFO_WRITE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT)
DEFINE_TASK_CODE_AIO(LPC_AIO_INFO_READ, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT)

namespace {
error_code write_blob_to_file(const std::string &fname, const blob &data)
{
// TODO(yingchun): consider not encrypt the meta files.
std::string tmp_fname = fname + ".tmp";
disk_file *dfile = file::open(tmp_fname, file::FileOpenType::kWriteOnly);
LOG_AND_RETURN_NOT_TRUE(
ERROR, dfile, ERR_FILE_OPERATION_FAILED, "open file {} failed", tmp_fname);
auto cleanup = defer([tmp_fname]() {
auto s = dsn::utils::PegasusEnv()->DeleteFile(tmp_fname);
// TODO(yingchun): add macro for rocksdb::Status
LOG_WARNING_IF(!s.ok(), "delete file {} failed, error = {}", tmp_fname, s.ToString());
});

disk_file *wfile = file::open(tmp_fname, file::FileOpenType::kWriteOnly);
LOG_AND_RETURN_NOT_TRUE(
ERROR, wfile != nullptr, ERR_FILE_OPERATION_FAILED, "open file {} failed", tmp_fname);

error_code err;
size_t sz = 0;
size_t write_size = 0;
task_tracker tracker;
aio_task_ptr tsk = file::write(dfile,
data.data(),
data.length(),
0,
LPC_AIO_INFO_WRITE,
&tracker,
[&err, &sz](error_code e, size_t s) {
err = e;
sz = s;
},
0);
auto tsk = file::write(wfile,
data.data(),
data.length(),
0,
LPC_AIO_INFO_WRITE,
&tracker,
[&err, &write_size](error_code e, size_t s) {
err = e;
write_size = s;
},
0);
CHECK_NOTNULL(tsk, "create file::write task failed");
tracker.wait_outstanding_tasks();
LOG_AND_RETURN_NOT_OK(ERROR, file::flush(dfile), "file::flush failed");
LOG_AND_RETURN_NOT_OK(ERROR, file::close(dfile), "file::close failed");
LOG_AND_RETURN_NOT_OK(ERROR, file::flush(wfile), "file::flush failed");
LOG_AND_RETURN_NOT_OK(ERROR, file::close(wfile), "file::close failed");
LOG_AND_RETURN_NOT_OK(ERROR, err, "write file {} failed", tmp_fname);
CHECK_EQ(data.length(), sz);
CHECK_EQ(data.length(), write_size);
LOG_AND_RETURN_NOT_TRUE(ERROR,
utils::filesystem::rename_path(tmp_fname, fname),
ERR_FILE_OPERATION_FAILED,
@@ -152,35 +155,43 @@ error_code replica_init_info::store(const std::string &dir)

error_code replica_init_info::load_json(const std::string &fname)
{
std::unique_ptr<rocksdb::SequentialFile> sfile;
auto s = dsn::utils::PegasusEnv()->NewSequentialFile(fname, &sfile, rocksdb::EnvOptions());
if (!s.ok()) {
LOG_ERROR("open file '{}' failed, err = {}", fname, s.ToString());
return ERR_FILE_OPERATION_FAILED;
}
auto rfile = file::open(fname, file::FileOpenType::kReadOnly);
LOG_AND_RETURN_NOT_TRUE(
ERROR, rfile != nullptr, ERR_FILE_OPERATION_FAILED, "open file {} failed", fname);

int64_t sz = 0;
int64_t file_size = 0;
LOG_AND_RETURN_NOT_TRUE(
ERROR,
utils::filesystem::file_size(fname, utils::filesystem::FileDataType::kSensitive, sz),
utils::filesystem::file_size(fname, utils::filesystem::FileDataType::kSensitive, file_size),
ERR_FILE_OPERATION_FAILED,
"get file size of {} failed",
fname);

error_code err;
size_t read_size = 0;
task_tracker tracker;
rocksdb::Slice result;
char scratch[sz];
s = sfile->Read(sz, &result, scratch);
if (!s.ok()) {
LOG_ERROR("read file '{}' failed, err = {}", fname, s.ToString());
return ERR_FILE_OPERATION_FAILED;
}

LOG_AND_RETURN_NOT_TRUE(ERROR,
json::json_forwarder<replica_init_info>::decode(
blob(result.data(), 0, result.size()), *this),
ERR_FILE_OPERATION_FAILED,
"decode json from file {} failed",
fname);
std::shared_ptr<char> buffer(utils::make_shared_array<char>(file_size));
auto tsk = ::dsn::file::read(rfile,
buffer.get(),
file_size,
0,
LPC_AIO_INFO_READ,
&tracker,
[&err, &read_size](error_code e, size_t s) {
err = e;
read_size = s;
});
CHECK_NOTNULL(tsk, "create file::read task failed");
tracker.wait_outstanding_tasks();
LOG_AND_RETURN_NOT_OK(ERROR, err, "read file {} failed", fname);
CHECK_EQ(file_size, read_size);
LOG_AND_RETURN_NOT_TRUE(
ERROR,
json::json_forwarder<replica_init_info>::decode(blob(buffer, file_size), *this),
ERR_FILE_OPERATION_FAILED,
"decode json from file {} failed",
fname);

return ERR_OK;
}
@@ -202,30 +213,39 @@ std::string replica_init_info::to_string()

error_code replica_app_info::load(const std::string &fname)
{
std::unique_ptr<rocksdb::SequentialFile> sfile;
auto s = dsn::utils::PegasusEnv()->NewSequentialFile(fname, &sfile, rocksdb::EnvOptions());
if (!s.ok()) {
LOG_ERROR("open file '{}' failed, err = {}", fname, s.ToString());
return ERR_FILE_OPERATION_FAILED;
}
auto rfile = file::open(fname, file::FileOpenType::kReadOnly);
LOG_AND_RETURN_NOT_TRUE(
ERROR, rfile != nullptr, ERR_FILE_OPERATION_FAILED, "open file {} failed", fname);

int64_t sz = 0;
int64_t file_size = 0;
LOG_AND_RETURN_NOT_TRUE(
ERROR,
utils::filesystem::file_size(fname, utils::filesystem::FileDataType::kSensitive, sz),
utils::filesystem::file_size(fname, utils::filesystem::FileDataType::kSensitive, file_size),
ERR_FILE_OPERATION_FAILED,
"get file size of {} failed",
fname);

error_code err;
size_t read_size = 0;
task_tracker tracker;
rocksdb::Slice result;
char scratch[sz];
s = sfile->Read(sz, &result, scratch);
if (!s.ok()) {
LOG_ERROR("read file '{}' failed, err = {}", fname, s.ToString());
return ERR_FILE_OPERATION_FAILED;
}
std::shared_ptr<char> buffer(utils::make_shared_array<char>(file_size));
auto tsk = ::dsn::file::read(rfile,
buffer.get(),
file_size,
0,
LPC_AIO_INFO_READ,
&tracker,
[&err, &read_size](error_code e, size_t s) {
err = e;
read_size = s;
});
CHECK_NOTNULL(tsk, "create file::read task failed");
tracker.wait_outstanding_tasks();
LOG_AND_RETURN_NOT_OK(ERROR, err, "read file {} failed", fname);
CHECK_EQ(file_size, read_size);

binary_reader reader(blob(result.data(), 0, result.size()));
binary_reader reader(blob(buffer, file_size));
int magic = 0;
unmarshall(reader, magic, DSF_THRIFT_BINARY);