Skip to content

Commit

Permalink
1
Browse files Browse the repository at this point in the history
  • Loading branch information
@acelyc111
acelyc111 committed Dec 13, 2023
1 parent 56b18b5 commit 92a0c80
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 68 deletions.
72 changes: 31 additions & 41 deletions src/block_service/local/local_service.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,38 @@ namespace block_service {

DEFINE_TASK_CODE(LPC_LOCAL_SERVICE_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE)

bool file_metadata_from_json(const std::string &data, file_metadata &fmeta) noexcept
error_code file_metadata::dump_to_file(const std::string &file_path) const
{
std::string data = nlohmann::json(*this).dump();
auto s =
rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive),
rocksdb::Slice(data),
file_path,
/* should_sync */ true);
if (!s.ok()) {
LOG_WARNING("write to metadata file '{}' failed, err = {}", file_path, s.ToString());
return ERR_FS_INTERNAL;
}

return ERR_OK;
}

error_code file_metadata::load_from_file(const std::string &file_path)
{
std::string data;
auto s = rocksdb::ReadFileToString(
dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), file_path, &data);
if (!s.ok()) {
LOG_WARNING("load from metadata file '{}' failed, err = {}", file_path, s.ToString());
return ERR_FS_INTERNAL;
}

try {
nlohmann::json::parse(data).get_to(fmeta);
return true;
nlohmann::json::parse(data).get_to(*this);
return ERR_OK;
} catch (nlohmann::json::exception &exp) {
LOG_WARNING("decode metadata from json failed: {} [{}]", exp.what(), data);
return false;
LOG_WARNING("decode metadata from json failed: {}, data = [{}]", exp.what(), data);
return ERR_FS_INTERNAL;
}
}

Expand Down Expand Up @@ -281,40 +305,6 @@ error_code local_file_object::load_metadata()
return ERR_OK;
}

error_code file_metadata::write_to_file(std::string filepath) const
{
std::string data = nlohmann::json(*this).dump();
auto s =
rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive),
rocksdb::Slice(data),
filepath,
/* should_sync */ true);
if (!s.ok()) {
LOG_WARNING("write to metadata file '{}' failed, err={}", filepath, s.ToString());
return ERR_FS_INTERNAL;
}

return ERR_OK;
}

error_code file_metadata::load_from_file(std::string filepath)
{
std::string data;
auto s = rocksdb::ReadFileToString(
dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), filepath, &data);
if (!s.ok()) {
LOG_WARNING("load from metadata file '{}' failed, err = {}", filepath, s.ToString());
return ERR_FS_INTERNAL;
}

if (!file_metadata_from_json(data, *this)) {
LOG_WARNING("decode metadata file '{}' failed", filepath);
return ERR_FS_INTERNAL;
}

return ERR_OK;
}

dsn::task_ptr local_file_object::write(const write_request &req,
dsn::task_code code,
const write_callback &cb,
Expand Down Expand Up @@ -367,7 +357,7 @@ dsn::task_ptr local_file_object::write(const write_request &req,
_size = resp.written_size;
_md5_value = utils::string_md5(req.buffer.data(), req.buffer.length());
auto err = file_metadata(_size, _md5_value)
.write_to_file(local_service::get_metafile(file_name()));
.dump_to_file(local_service::get_metafile(file_name()));
if (err != ERR_OK) {
LOG_ERROR("file_metadata write failed");
resp.err = ERR_FS_INTERNAL;
Expand Down Expand Up @@ -509,7 +499,7 @@ dsn::task_ptr local_file_object::upload(const upload_request &req,
}

auto err = file_metadata(_size, _md5_value)
.write_to_file(local_service::get_metafile(file_name()));
.dump_to_file(local_service::get_metafile(file_name()));
if (err != ERR_OK) {
LOG_ERROR("file_metadata write failed");
resp.err = ERR_FS_INTERNAL;
Expand Down
10 changes: 7 additions & 3 deletions src/block_service/local/local_service.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,13 @@ struct file_metadata
int64_t size = 0;
std::string md5;

file_metadata(int64_t s = 0, std::string m = "") : size(s), md5(std::move(m)) {}
error_code write_to_file(std::string filepath) const;
error_code load_from_file(std::string filepath);
file_metadata(int64_t s = 0, const std::string &m = "") : size(s), md5(m) {}

// Dump the object to a file in JSON format.
error_code dump_to_file(const std::string &file_path) const;

// Load the object from a file in JSON format.
error_code load_from_file(const std::string &file_path);
};
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5)

Expand Down
35 changes: 11 additions & 24 deletions src/block_service/test/local_service_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,23 +47,18 @@ class local_service_test : public pegasus::encrypt_data_test_base

INSTANTIATE_TEST_CASE_P(, local_service_test, ::testing::Values(false, true));

TEST_P(local_service_test, store_metadata)
TEST_P(local_service_test, file_metadata)
{
local_file_object file("a.txt");
auto meta_file_path = local_service::get_metafile(file.file_name());

ASSERT_EQ(ERR_OK, file_metadata(0, "").write_to_file(meta_file_path));

const int64_t kSize = 12345;
const std::string kMD5 = "0123456789abcdef0123456789abcdef";
auto meta_file_path = local_service::get_metafile("a.txt");
ASSERT_EQ(ERR_OK, file_metadata(kSize, kMD5).dump_to_file(meta_file_path));
ASSERT_TRUE(boost::filesystem::exists(meta_file_path));

std::string data;
auto s = rocksdb::ReadFileToString(
dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), meta_file_path, &data);
ASSERT_TRUE(s.ok()) << s.ToString();

nlohmann::json j = nlohmann::json::parse(data);
ASSERT_EQ("", j["md5"]);
ASSERT_EQ(0, j["size"]);
file_metadata fm;
fm.load_from_file(meta_file_path);
ASSERT_EQ(kSize, fm.size);
ASSERT_EQ(kMD5, fm.md5);
}

TEST_P(local_service_test, load_metadata)
Expand All @@ -72,15 +67,7 @@ TEST_P(local_service_test, load_metadata)
auto meta_file_path = local_service::get_metafile(file.file_name());

{
nlohmann::json j({{"md5", "abcde"}, {"size", 5}});
std::string data = j.dump();
auto s =
rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive),
rocksdb::Slice(data),
meta_file_path,
/* should_sync */ true);
ASSERT_TRUE(s.ok()) << s.ToString();

ASSERT_EQ(ERR_OK, file_metadata(5, "abcde").dump_to_file(meta_file_path));
ASSERT_EQ(ERR_OK, file.load_metadata());
ASSERT_EQ("abcde", file.get_md5sum());
ASSERT_EQ(5, file.get_size());
Expand All @@ -95,7 +82,7 @@ TEST_P(local_service_test, load_metadata)
ASSERT_TRUE(s.ok()) << s.ToString();

local_file_object file2("a.txt");
ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL);
ASSERT_EQ(ERR_FS_INTERNAL, file2.load_metadata());
}

{
Expand Down

0 comments on commit 92a0c80

Please sign in to comment.