From 97f357fa3850221ca6ffaf80174832b15548f0f3 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Wed, 29 Apr 2020 14:47:54 +0800 Subject: [PATCH 01/16] init not compile --- include/dsn/tool/abnormal_log.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 include/dsn/tool/abnormal_log.h diff --git a/include/dsn/tool/abnormal_log.h b/include/dsn/tool/abnormal_log.h new file mode 100644 index 0000000000..9f72270a16 --- /dev/null +++ b/include/dsn/tool/abnormal_log.h @@ -0,0 +1,12 @@ +#include + +namespace dsn { +namespace replication { + +class abnormal_log +{ +public: + virtual void print_abnormal_write(const dsn::message_ex **requests) = 0; +}; +} +} \ No newline at end of file From 5ef96224de618fe8d3e08d053186b12bade41651 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Wed, 29 Apr 2020 15:41:10 +0800 Subject: [PATCH 02/16] frame ok --- include/dsn/tool/abnormal_log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dsn/tool/abnormal_log.h b/include/dsn/tool/abnormal_log.h index 9f72270a16..641fc8c59d 100644 --- a/include/dsn/tool/abnormal_log.h +++ b/include/dsn/tool/abnormal_log.h @@ -6,7 +6,7 @@ namespace replication { class abnormal_log { public: - virtual void print_abnormal_write(const dsn::message_ex **requests) = 0; + virtual void print_abnormal_write(dsn::message_ex *request) = 0; }; } } \ No newline at end of file From f5070cec0d37a654352a518cca683705961346a2 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Thu, 30 Apr 2020 16:43:13 +0800 Subject: [PATCH 03/16] add but can't compile --- include/dsn/dist/replication/replica_base.h | 5 ++++- include/dsn/tool/abnormal_log.h | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/dsn/dist/replication/replica_base.h b/include/dsn/dist/replication/replica_base.h index 36fb87ddc5..48446a2b87 100644 --- a/include/dsn/dist/replication/replica_base.h +++ b/include/dsn/dist/replication/replica_base.h @@ -28,6 +28,7 @@ #include #include +#include namespace dsn { namespace replication { @@ -36,7 +37,7 @@ namespace replication { struct replica_base { replica_base(gpid id, string_view name, string_view app_name) - : _gpid(id), _name(name), _app_name(app_name) + : _gpid(id), _name(name), _app_name(app_name), _abnormal_log(nullptr) { } @@ -51,6 +52,8 @@ struct replica_base const char *app_name() const { return _app_name.c_str(); } + std::unique_ptr _abnormal_log; + private: const gpid _gpid; const std::string _name; diff --git a/include/dsn/tool/abnormal_log.h b/include/dsn/tool/abnormal_log.h index 641fc8c59d..864b7a5f27 100644 --- a/include/dsn/tool/abnormal_log.h +++ b/include/dsn/tool/abnormal_log.h @@ -6,7 +6,8 @@ namespace replication { class abnormal_log { public: - virtual void print_abnormal_write(dsn::message_ex *request) = 0; + virtual void dwarn_write_operation(dsn::message_ex *request, + const std::string &message = std::string()) = 0; }; } } \ No newline at end of file From 272697e319a2d05f285af640d28a63efa43b51ba Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Mon, 11 May 2020 18:24:00 +0800 Subject: [PATCH 04/16] refactor --- include/dsn/dist/replication/replica_base.h | 4 +--- include/dsn/dist/replication/replication_app_base.h | 3 +++ include/dsn/tool/abnormal_log.h | 13 ------------- src/dist/replication/lib/replica.h | 2 ++ src/dist/replication/lib/replica_2pc.cpp | 6 +----- src/dist/replication/lib/replication_app_base.cpp | 11 +++++++++++ 6 files changed, 18 insertions(+), 21 deletions(-) delete mode 100644 include/dsn/tool/abnormal_log.h diff --git a/include/dsn/dist/replication/replica_base.h b/include/dsn/dist/replication/replica_base.h index 48446a2b87..9af064bd1d 100644 --- a/include/dsn/dist/replication/replica_base.h +++ b/include/dsn/dist/replication/replica_base.h @@ -37,7 +37,7 @@ namespace replication { struct replica_base { replica_base(gpid id, string_view name, string_view app_name) - : _gpid(id), _name(name), _app_name(app_name), _abnormal_log(nullptr) + : _gpid(id), _name(name), _app_name(app_name) { } @@ -52,8 +52,6 @@ struct replica_base const char *app_name() const { return _app_name.c_str(); } - std::unique_ptr _abnormal_log; - private: const gpid _gpid; const std::string _name; diff --git a/include/dsn/dist/replication/replication_app_base.h b/include/dsn/dist/replication/replication_app_base.h index 628c31d9e9..832bf455cd 100644 --- a/include/dsn/dist/replication/replication_app_base.h +++ b/include/dsn/dist/replication/replication_app_base.h @@ -235,6 +235,9 @@ class replication_app_base : public replica_base // Thread-safe. virtual void set_partition_version(int32_t partition_version){}; + // todo(jiashuo) + virtual void dwarn_write_operation(dsn::message_ex *request, uint64_t max_allowed_write_size); + public: // // utility functions to be used by app diff --git a/include/dsn/tool/abnormal_log.h b/include/dsn/tool/abnormal_log.h deleted file mode 100644 index 864b7a5f27..0000000000 --- a/include/dsn/tool/abnormal_log.h +++ /dev/null @@ -1,13 +0,0 @@ -#include - -namespace dsn { -namespace replication { - -class abnormal_log -{ -public: - virtual void dwarn_write_operation(dsn::message_ex *request, - const std::string &message = std::string()) = 0; -}; -} -} \ No newline at end of file diff --git a/src/dist/replication/lib/replica.h b/src/dist/replication/lib/replica.h index 25c52d4976..2aad3d2b47 100644 --- a/src/dist/replication/lib/replica.h +++ b/src/dist/replication/lib/replica.h @@ -537,6 +537,8 @@ class replica : public serverlet, public ref_counter, public replica_ba dsn::task_tracker _tracker; // the thread access checker dsn::thread_access_checker _checker; + + std::unique_ptr _abnormal_log; }; typedef dsn::ref_ptr replica_ptr; } // namespace replication diff --git a/src/dist/replication/lib/replica_2pc.cpp b/src/dist/replication/lib/replica_2pc.cpp index f9cf2ce4df..146ea52f9d 100644 --- a/src/dist/replication/lib/replica_2pc.cpp +++ b/src/dist/replication/lib/replica_2pc.cpp @@ -47,11 +47,7 @@ void replica::on_client_write(dsn::message_ex *request, bool ignore_throttling) if (dsn_unlikely(_stub->_max_allowed_write_size && request->body_size() > _stub->_max_allowed_write_size)) { - dwarn_replica("client from {} write request body size exceed threshold, request_body_size " - "= {}, max_allowed_write_size = {}, it will be rejected!", - request->header->from_address.to_string(), - request->body_size(), - _stub->_max_allowed_write_size); + _app->dwarn_write_operation(request, _stub->_max_allowed_write_size); _stub->_counter_recent_write_size_exceed_threshold_count->increment(); response_client_write(request, ERR_INVALID_DATA); return; diff --git a/src/dist/replication/lib/replication_app_base.cpp b/src/dist/replication/lib/replication_app_base.cpp index d524b840fd..62d56bd875 100644 --- a/src/dist/replication/lib/replication_app_base.cpp +++ b/src/dist/replication/lib/replication_app_base.cpp @@ -581,5 +581,16 @@ ::dsn::error_code replication_app_base::update_init_info_ballot_and_decree(repli _info.init_offset_in_private_log, r->last_durable_decree()); } + +void replication_app_base::dwarn_write_operation(dsn::message_ex *request, + uint64_t max_allowed_write_size) +{ + dwarn_replica("client from {} write request body size exceed threshold, request_body_size " + "= {}, max_allowed_write_size = {}, it will be rejected!", + request->header->from_address.to_string(), + request->body_size(), + max_allowed_write_size); +} + } // namespace replication } // namespace dsn From 4e0c5997e19418d24b706589c47e5c1328ff020c Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Mon, 11 May 2020 18:30:26 +0800 Subject: [PATCH 05/16] del include --- include/dsn/dist/replication/replica_base.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/dsn/dist/replication/replica_base.h b/include/dsn/dist/replication/replica_base.h index 9af064bd1d..36fb87ddc5 100644 --- a/include/dsn/dist/replication/replica_base.h +++ b/include/dsn/dist/replication/replica_base.h @@ -28,7 +28,6 @@ #include #include -#include namespace dsn { namespace replication { From e0c5c20a742e48e901f9383ffe2185f994b27bd2 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Mon, 11 May 2020 18:34:08 +0800 Subject: [PATCH 06/16] del replica --- src/dist/replication/lib/replica.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/dist/replication/lib/replica.h b/src/dist/replication/lib/replica.h index 2aad3d2b47..25c52d4976 100644 --- a/src/dist/replication/lib/replica.h +++ b/src/dist/replication/lib/replica.h @@ -537,8 +537,6 @@ class replica : public serverlet, public ref_counter, public replica_ba dsn::task_tracker _tracker; // the thread access checker dsn::thread_access_checker _checker; - - std::unique_ptr _abnormal_log; }; typedef dsn::ref_ptr replica_ptr; } // namespace replication From 41e07ddf70b9f784db33b48029c34374b5e649b7 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Mon, 11 May 2020 18:37:35 +0800 Subject: [PATCH 07/16] add dwarn_replica include --- src/dist/replication/lib/replication_app_base.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dist/replication/lib/replication_app_base.cpp b/src/dist/replication/lib/replication_app_base.cpp index 62d56bd875..57c3cbcee1 100644 --- a/src/dist/replication/lib/replication_app_base.cpp +++ b/src/dist/replication/lib/replication_app_base.cpp @@ -36,6 +36,7 @@ #include #include #include +#include namespace dsn { namespace replication { From ecf075e6139d253b071bf3cc0fd5aef23339fab8 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 12 May 2020 10:26:59 +0800 Subject: [PATCH 08/16] add comment --- include/dsn/dist/replication/replication_app_base.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dsn/dist/replication/replication_app_base.h b/include/dsn/dist/replication/replication_app_base.h index 832bf455cd..b4f30db59e 100644 --- a/include/dsn/dist/replication/replication_app_base.h +++ b/include/dsn/dist/replication/replication_app_base.h @@ -235,7 +235,7 @@ class replication_app_base : public replica_base // Thread-safe. virtual void set_partition_version(int32_t partition_version){}; - // todo(jiashuo) + // log for the large write size operation virtual void dwarn_write_operation(dsn::message_ex *request, uint64_t max_allowed_write_size); public: From 0273d78788739c70923c28591f84786d82a794f8 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Tue, 12 May 2020 10:34:12 +0800 Subject: [PATCH 09/16] rename --- include/dsn/dist/replication/replication_app_base.h | 4 ++-- src/dist/replication/lib/replica_2pc.cpp | 2 +- src/dist/replication/lib/replication_app_base.cpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/dsn/dist/replication/replication_app_base.h b/include/dsn/dist/replication/replication_app_base.h index b4f30db59e..d3f02f3748 100644 --- a/include/dsn/dist/replication/replication_app_base.h +++ b/include/dsn/dist/replication/replication_app_base.h @@ -235,8 +235,8 @@ class replication_app_base : public replica_base // Thread-safe. virtual void set_partition_version(int32_t partition_version){}; - // log for the large write size operation - virtual void dwarn_write_operation(dsn::message_ex *request, uint64_t max_allowed_write_size); + // log for the large write size request + virtual void dwarn_write_request(dsn::message_ex *request, uint64_t max_allowed_write_size); public: // diff --git a/src/dist/replication/lib/replica_2pc.cpp b/src/dist/replication/lib/replica_2pc.cpp index 146ea52f9d..162ee98429 100644 --- a/src/dist/replication/lib/replica_2pc.cpp +++ b/src/dist/replication/lib/replica_2pc.cpp @@ -47,7 +47,7 @@ void replica::on_client_write(dsn::message_ex *request, bool ignore_throttling) if (dsn_unlikely(_stub->_max_allowed_write_size && request->body_size() > _stub->_max_allowed_write_size)) { - _app->dwarn_write_operation(request, _stub->_max_allowed_write_size); + _app->dwarn_write_request(request, _stub->_max_allowed_write_size); _stub->_counter_recent_write_size_exceed_threshold_count->increment(); response_client_write(request, ERR_INVALID_DATA); return; diff --git a/src/dist/replication/lib/replication_app_base.cpp b/src/dist/replication/lib/replication_app_base.cpp index 57c3cbcee1..b5e27b9d52 100644 --- a/src/dist/replication/lib/replication_app_base.cpp +++ b/src/dist/replication/lib/replication_app_base.cpp @@ -583,8 +583,8 @@ ::dsn::error_code replication_app_base::update_init_info_ballot_and_decree(repli r->last_durable_decree()); } -void replication_app_base::dwarn_write_operation(dsn::message_ex *request, - uint64_t max_allowed_write_size) +void replication_app_base::dwarn_write_request(dsn::message_ex *request, + uint64_t max_allowed_write_size) { dwarn_replica("client from {} write request body size exceed threshold, request_body_size " "= {}, max_allowed_write_size = {}, it will be rejected!", From 695be32a4d1dd71226e3bbad245409b7a875c9b8 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Thu, 14 May 2020 12:11:13 +0800 Subject: [PATCH 10/16] restore request --- include/dsn/dist/replication/replication_app_base.h | 4 ++-- src/dist/replication/lib/replica_2pc.cpp | 9 ++++++++- src/dist/replication/lib/replication_app_base.cpp | 10 ---------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/include/dsn/dist/replication/replication_app_base.h b/include/dsn/dist/replication/replication_app_base.h index d3f02f3748..a2c1fa7f2a 100644 --- a/include/dsn/dist/replication/replication_app_base.h +++ b/include/dsn/dist/replication/replication_app_base.h @@ -235,8 +235,8 @@ class replication_app_base : public replica_base // Thread-safe. virtual void set_partition_version(int32_t partition_version){}; - // log for the large write size request - virtual void dwarn_write_request(dsn::message_ex *request, uint64_t max_allowed_write_size); + // restore the write request some info to string, it overload + virtual std::string restore_write_request(dsn::message_ex *request){}; public: // diff --git a/src/dist/replication/lib/replica_2pc.cpp b/src/dist/replication/lib/replica_2pc.cpp index 162ee98429..e25e8d9b0f 100644 --- a/src/dist/replication/lib/replica_2pc.cpp +++ b/src/dist/replication/lib/replica_2pc.cpp @@ -47,7 +47,14 @@ void replica::on_client_write(dsn::message_ex *request, bool ignore_throttling) if (dsn_unlikely(_stub->_max_allowed_write_size && request->body_size() > _stub->_max_allowed_write_size)) { - _app->dwarn_write_request(request, _stub->_max_allowed_write_size); + std::string request_info = _app->restore_write_request(request); + dwarn_replica("client from {} write request body size exceed threshold, request=[{}], " + "request_body_size " + "= {}, max_allowed_write_size = {}, it will be rejected!", + request->header->from_address.to_string(), + request_info, + request->body_size(), + max_allowed_write_size); _stub->_counter_recent_write_size_exceed_threshold_count->increment(); response_client_write(request, ERR_INVALID_DATA); return; diff --git a/src/dist/replication/lib/replication_app_base.cpp b/src/dist/replication/lib/replication_app_base.cpp index b5e27b9d52..0744b31c27 100644 --- a/src/dist/replication/lib/replication_app_base.cpp +++ b/src/dist/replication/lib/replication_app_base.cpp @@ -583,15 +583,5 @@ ::dsn::error_code replication_app_base::update_init_info_ballot_and_decree(repli r->last_durable_decree()); } -void replication_app_base::dwarn_write_request(dsn::message_ex *request, - uint64_t max_allowed_write_size) -{ - dwarn_replica("client from {} write request body size exceed threshold, request_body_size " - "= {}, max_allowed_write_size = {}, it will be rejected!", - request->header->from_address.to_string(), - request->body_size(), - max_allowed_write_size); -} - } // namespace replication } // namespace dsn From 8b75e69135d713743270ab40ced39da7eb76c44b Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Thu, 14 May 2020 12:17:34 +0800 Subject: [PATCH 11/16] restore request --- include/dsn/dist/replication/replication_app_base.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dsn/dist/replication/replication_app_base.h b/include/dsn/dist/replication/replication_app_base.h index a2c1fa7f2a..9871a3cd82 100644 --- a/include/dsn/dist/replication/replication_app_base.h +++ b/include/dsn/dist/replication/replication_app_base.h @@ -236,7 +236,7 @@ class replication_app_base : public replica_base virtual void set_partition_version(int32_t partition_version){}; // restore the write request some info to string, it overload - virtual std::string restore_write_request(dsn::message_ex *request){}; + virtual std::string restore_write_request(dsn::message_ex *request); public: // From 1d617f256dd7363224b06a7a0b22bc364f0ab7d3 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Thu, 14 May 2020 14:55:10 +0800 Subject: [PATCH 12/16] restore request --- include/dsn/dist/replication/replication_app_base.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/dsn/dist/replication/replication_app_base.h b/include/dsn/dist/replication/replication_app_base.h index 9871a3cd82..cebaccfbd2 100644 --- a/include/dsn/dist/replication/replication_app_base.h +++ b/include/dsn/dist/replication/replication_app_base.h @@ -235,8 +235,8 @@ class replication_app_base : public replica_base // Thread-safe. virtual void set_partition_version(int32_t partition_version){}; - // restore the write request some info to string, it overload - virtual std::string restore_write_request(dsn::message_ex *request); + // restore the write request some info to string, it may need overload + virtual std::string restore_write_request(dsn::message_ex *request) { return "write request"; }; public: // From ee8ede0c3ba4290d0235ff4bffab56119aceb5da Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Thu, 14 May 2020 15:08:06 +0800 Subject: [PATCH 13/16] rename --- src/dist/replication/lib/replica_2pc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dist/replication/lib/replica_2pc.cpp b/src/dist/replication/lib/replica_2pc.cpp index e25e8d9b0f..182f18ce20 100644 --- a/src/dist/replication/lib/replica_2pc.cpp +++ b/src/dist/replication/lib/replica_2pc.cpp @@ -54,7 +54,7 @@ void replica::on_client_write(dsn::message_ex *request, bool ignore_throttling) request->header->from_address.to_string(), request_info, request->body_size(), - max_allowed_write_size); + _stub->_max_allowed_write_size); _stub->_counter_recent_write_size_exceed_threshold_count->increment(); response_client_write(request, ERR_INVALID_DATA); return; From 04336eebfbf095eb0525b2965eb653ed2e42862b Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Thu, 14 May 2020 16:03:59 +0800 Subject: [PATCH 14/16] format --- src/dist/replication/lib/replica_2pc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dist/replication/lib/replica_2pc.cpp b/src/dist/replication/lib/replica_2pc.cpp index 182f18ce20..bd885daa60 100644 --- a/src/dist/replication/lib/replica_2pc.cpp +++ b/src/dist/replication/lib/replica_2pc.cpp @@ -48,7 +48,7 @@ void replica::on_client_write(dsn::message_ex *request, bool ignore_throttling) if (dsn_unlikely(_stub->_max_allowed_write_size && request->body_size() > _stub->_max_allowed_write_size)) { std::string request_info = _app->restore_write_request(request); - dwarn_replica("client from {} write request body size exceed threshold, request=[{}], " + dwarn_replica("client from {} write request body size exceed threshold, request = [{}], " "request_body_size " "= {}, max_allowed_write_size = {}, it will be rejected!", request->header->from_address.to_string(), From 67c3a53b377c9cde2ffd6b5d97cd4fbf6e6bc7f6 Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Thu, 14 May 2020 16:09:39 +0800 Subject: [PATCH 15/16] format --- src/dist/replication/lib/replication_app_base.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/dist/replication/lib/replication_app_base.cpp b/src/dist/replication/lib/replication_app_base.cpp index 0744b31c27..d524b840fd 100644 --- a/src/dist/replication/lib/replication_app_base.cpp +++ b/src/dist/replication/lib/replication_app_base.cpp @@ -36,7 +36,6 @@ #include #include #include -#include namespace dsn { namespace replication { @@ -582,6 +581,5 @@ ::dsn::error_code replication_app_base::update_init_info_ballot_and_decree(repli _info.init_offset_in_private_log, r->last_durable_decree()); } - } // namespace replication } // namespace dsn From 27cfaf37d4c852431e7e4e344502659afc3cafec Mon Sep 17 00:00:00 2001 From: JiaShuo Date: Fri, 15 May 2020 19:13:05 +0800 Subject: [PATCH 16/16] rename --- include/dsn/dist/replication/replication_app_base.h | 4 ++-- src/dist/replication/lib/replica_2pc.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/dsn/dist/replication/replication_app_base.h b/include/dsn/dist/replication/replication_app_base.h index cebaccfbd2..1d55c4850e 100644 --- a/include/dsn/dist/replication/replication_app_base.h +++ b/include/dsn/dist/replication/replication_app_base.h @@ -235,8 +235,8 @@ class replication_app_base : public replica_base // Thread-safe. virtual void set_partition_version(int32_t partition_version){}; - // restore the write request some info to string, it may need overload - virtual std::string restore_write_request(dsn::message_ex *request) { return "write request"; }; + // dump the write request some info to string, it may need overload + virtual std::string dump_write_request(dsn::message_ex *request) { return "write request"; }; public: // diff --git a/src/dist/replication/lib/replica_2pc.cpp b/src/dist/replication/lib/replica_2pc.cpp index bd885daa60..6b5c8a9fb1 100644 --- a/src/dist/replication/lib/replica_2pc.cpp +++ b/src/dist/replication/lib/replica_2pc.cpp @@ -47,7 +47,7 @@ void replica::on_client_write(dsn::message_ex *request, bool ignore_throttling) if (dsn_unlikely(_stub->_max_allowed_write_size && request->body_size() > _stub->_max_allowed_write_size)) { - std::string request_info = _app->restore_write_request(request); + std::string request_info = _app->dump_write_request(request); dwarn_replica("client from {} write request body size exceed threshold, request = [{}], " "request_body_size " "= {}, max_allowed_write_size = {}, it will be rejected!",