diff --git a/src/block_service/local/local_service.cpp b/src/block_service/local/local_service.cpp index 9120e033fd..a25b882618 100644 --- a/src/block_service/local/local_service.cpp +++ b/src/block_service/local/local_service.cpp @@ -6,9 +6,11 @@ #include #include #include +#include #include #include +#include #include "local_service.h" static const int max_length = 2048; // max data length read from file each time @@ -24,8 +26,21 @@ struct file_metadata { uint64_t size; std::string md5; - DEFINE_JSON_SERIALIZATION(size, md5) }; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5) + +bool file_metadata_from_json(std::ifstream &fin, file_metadata &fmeta) noexcept +{ + std::string data; + fin >> data; + try { + nlohmann::json::parse(data).get_to(fmeta); + return true; + } catch (nlohmann::json::exception &exp) { + dwarn_f("decode meta data from json failed: {} [{}]", exp.what(), data); + return false; + } +} std::string local_service::get_metafile(const std::string &filepath) { @@ -239,14 +254,9 @@ error_code local_file_object::load_metadata() } auto cleanup = dsn::defer([&is]() { is.close(); }); - std::string data; - is >> data; - file_metadata meta; - bool ans = dsn::json::json_forwarder::decode( - dsn::blob(data.c_str(), 0, data.size()), meta); + bool ans = file_metadata_from_json(is, meta); if (!ans) { - dwarn("decode meta data from json(%s) failed", data.c_str()); return ERR_FS_INTERNAL; } _size = meta.size; @@ -270,7 +280,7 @@ error_code local_file_object::store_metadata() return ERR_FS_INTERNAL; } auto cleanup = dsn::defer([&os]() { os.close(); }); - dsn::json::json_forwarder::encode(os, meta); + os << nlohmann::json(meta); return ERR_OK; } diff --git a/src/block_service/test/local_service_test.cpp b/src/block_service/test/local_service_test.cpp new file mode 100644 index 0000000000..9899e3d58e --- /dev/null +++ b/src/block_service/test/local_service_test.cpp @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "block_service/local/local_service.h" + +namespace dsn { +namespace dist { +namespace block_service { + +// Simple tests for nlohmann::json serialization, via NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE. + +TEST(local_service, store_metadata) +{ + local_file_object file("a.txt"); + error_code ec = file.store_metadata(); + ASSERT_EQ(ec, ERR_OK); + + auto meta_file_path = local_service::get_metafile(file.file_name()); + ASSERT_TRUE(boost::filesystem::exists(meta_file_path)); + + std::ifstream ifs(meta_file_path); + nlohmann::json j; + ifs >> j; + ASSERT_EQ(j["md5"], ""); + ASSERT_EQ(j["size"], 0); +} + +TEST(local_service, load_metadata) +{ + local_file_object file("a.txt"); + auto meta_file_path = local_service::get_metafile(file.file_name()); + + { + std::ofstream ofs(meta_file_path); + nlohmann::json j({{"md5", "abcde"}, {"size", 5}}); + ofs << j; + ofs.close(); + + ASSERT_EQ(file.load_metadata(), ERR_OK); + ASSERT_EQ(file.get_md5sum(), "abcde"); + ASSERT_EQ(file.get_size(), 5); + } + + { + std::ofstream ofs(meta_file_path); + ofs << "invalid json string"; + ofs.close(); + + local_file_object file2("a.txt"); + ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL); + } + + { + std::ofstream ofs(meta_file_path); + nlohmann::json j({{"md5", "abcde"}, {"no such key", "illegal"}}); + ofs << j; + ofs.close(); + + local_file_object file2("a.txt"); + ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL); + } +} + +} // namespace block_service +} // namespace dist +} // namespace dsn diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index fa3a2175ff..1341527315 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -1,12 +1,12 @@ project(pegasus_thirdparties) cmake_minimum_required(VERSION 3.0.2) -if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") # require at least gcc 4.8 if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) message(FATAL_ERROR "GCC version must be at least 4.8!") - endif() -endif() + endif () +endif () include(ExternalProject) @@ -182,9 +182,9 @@ ExternalProject_Add(fds ) ExternalProject_Add(fmt - URL ${OSS_URL_PREFIX}/fmt-4.0.0.tar.gz - https://codeload.github.com/fmtlib/fmt/tar.gz/4.0.0 - URL_MD5 c9be9a37bc85493d1116b0af59a25eba + URL ${OSS_URL_PREFIX}/fmt-7.1.2.zip + https://github.com/fmtlib/fmt/releases/download/7.1.2/fmt-7.1.2.zip + URL_MD5 cd838fb47c92b3221986a2f454331178 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_BUILD_TYPE=release -DFMT_TEST=false @@ -256,9 +256,9 @@ ExternalProject_Add(prometheus-cpp # header-only ExternalProject_Add(nlohmann_json - URL ${OSS_URL_PREFIX}/nlohmann_json-3.7.3.zip - https://github.com/nlohmann/json/releases/download/v3.7.3/include.zip - URL_MD5 7249387593792b565dcb30d87bca0de3 + URL ${OSS_URL_PREFIX}/nlohmann_json-3.9.1.zip + https://github.com/nlohmann/json/releases/download/v3.9.1/include.zip + URL_MD5 d2f66c608af689e21d69a33c220e974e CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND cp -R include/nlohmann ${TP_OUTPUT}/include