From b66756ca3d424a2e816c6dafef8716984b9dcb0d Mon Sep 17 00:00:00 2001 From: Weijie Sun Date: Thu, 12 Apr 2018 10:49:33 +0800 Subject: [PATCH 1/2] replica_server: reimplement uniq_timestamp generator Reimplement this for 2 reasons: 1. All threads shared a lock in the old implementation, which was not friendly to performance. As a matter of fact, it's not necessary for different replicas to keep an global increasing timestamp, so we can try this optimization 2. Although the timestamp was replicated to secondaries from primary, timestamp value of secondaries never updated accordingly, for which reason we were exposed to the risks that a newer mutation may had smaller timestamp if primary switched. --- include/dsn/tool-api/uniq_timestamp_us.h | 58 +++++++++++++++++++++++ src/dist/replication/lib/replica.h | 14 ++++++ src/dist/replication/lib/replica_2pc.cpp | 14 +----- src/dist/replication/lib/replica_init.cpp | 1 + 4 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 include/dsn/tool-api/uniq_timestamp_us.h diff --git a/include/dsn/tool-api/uniq_timestamp_us.h b/include/dsn/tool-api/uniq_timestamp_us.h new file mode 100644 index 0000000000..25ca2e835b --- /dev/null +++ b/include/dsn/tool-api/uniq_timestamp_us.h @@ -0,0 +1,58 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Microsoft Corporation + * + * -=- Robust Distributed System Nucleus (rDSN) -=- + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#pragma once + +#include +#include +#include + +namespace dsn { +// +// uniq_timestamp_us is used to generate an increasing unique microsecond timestamp +// in rdsn, it's mainly used for replica to set mutation's timestamp +// +// Notice: this module is not thread-safe, +// please ensure that it is accessed only by one thread +// +class uniq_timestamp_us { +private: + uint64_t last_ts; +public: + uniq_timestamp_us() { last_ts = dsn_now_us(); } + + void try_update(uint64_t new_ts) + { + if (new_ts > last_ts) + last_ts = new_ts; + } + + uint64_t next() + { + last_ts = std::max(dsn_now_us(), last_ts+1); + return last_ts; + } +}; +} diff --git a/src/dist/replication/lib/replica.h b/src/dist/replication/lib/replica.h index 881252e65a..cc2d2b195a 100644 --- a/src/dist/replication/lib/replica.h +++ b/src/dist/replication/lib/replica.h @@ -42,6 +42,7 @@ // which is binded to this replication partition // +#include #include #include #include "dist/replication/client_lib/replication_common.h" @@ -329,6 +330,19 @@ class replica : public serverlet, public ref_counter const app_info _app_info; std::map _extra_envs; + // uniq timestamp generator for this replica. + // + // we use it to generate an increasing timestamp for current replica + // and replicate it to secondary in preparing mutations, and secodaries' + // timestamp value will also updated if value from primary is larger + // + // as the timestamp is recorded in mutation log with mutations, we also update the value + // when do replaying + // + // in addition, as a replica can only be accessed by one thread, + // so the "thread-unsafe" generator works fine + uniq_timestamp_us _uniq_timestamp_us; + // replica status specific states primary_context _primary_states; secondary_context _secondary_states; diff --git a/src/dist/replication/lib/replica_2pc.cpp b/src/dist/replication/lib/replica_2pc.cpp index 1106be6101..57dee35819 100644 --- a/src/dist/replication/lib/replica_2pc.cpp +++ b/src/dist/replication/lib/replica_2pc.cpp @@ -42,16 +42,6 @@ namespace dsn { namespace replication { -static int64_t get_uniq_timestamp() -{ - static int64_t last = 0; - static ::dsn::utils::ex_lock_nr_spin _lock; - int64_t time = dsn_now_ns() / 1000; - ::dsn::utils::auto_lock<::dsn::utils::ex_lock_nr_spin> l(_lock); - last = std::max(time, last + 1); - return last; -} - void replica::on_client_write(task_code code, dsn_message_t request) { check_hashed_access(); @@ -93,7 +83,7 @@ void replica::init_prepare(mutation_ptr &mu, bool reconciliation) if (_options->prepare_decree_gap_for_debug_logging > 0 && mu->get_decree() % _options->prepare_decree_gap_for_debug_logging == 0) level = LOG_LEVEL_DEBUG; - mu->set_timestamp(get_uniq_timestamp()); + mu->set_timestamp(_uniq_timestamp_us.next()); } else { mu->set_id(get_ballot(), mu->data.header.decree); } @@ -335,7 +325,7 @@ void replica::on_prepare(dsn_message_t request) } // real prepare start - + _uniq_timestamp_us.try_update(mu->data.header.timestamp); auto mu2 = _prepare_list->get_mutation_by_decree(decree); if (mu2 != nullptr && mu2->data.header.ballot == mu->data.header.ballot) { if (mu2->is_logged()) { diff --git a/src/dist/replication/lib/replica_init.cpp b/src/dist/replication/lib/replica_init.cpp index 3e2f9b40b2..84ba9a6f9e 100644 --- a/src/dist/replication/lib/replica_init.cpp +++ b/src/dist/replication/lib/replica_init.cpp @@ -437,6 +437,7 @@ bool replica::replay_mutation(mutation_ptr &mu, bool is_private) mu->data.header.last_committed_decree); // prepare + _uniq_timestamp_us.try_update(mu->data.header.timestamp); error_code err = _prepare_list->prepare(mu, partition_status::PS_INACTIVE); dassert(err == ERR_OK, "prepare failed, err = %s", err.to_string()); From 5d921a6d0b809e91a3987c0dc43dad6e175a309c Mon Sep 17 00:00:00 2001 From: Weijie Sun Date: Thu, 12 Apr 2018 14:29:31 +0800 Subject: [PATCH 2/2] uniq_timestamp_us: fix variable name --- include/dsn/tool-api/uniq_timestamp_us.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/include/dsn/tool-api/uniq_timestamp_us.h b/include/dsn/tool-api/uniq_timestamp_us.h index 25ca2e835b..406644f8ef 100644 --- a/include/dsn/tool-api/uniq_timestamp_us.h +++ b/include/dsn/tool-api/uniq_timestamp_us.h @@ -27,6 +27,7 @@ #include #include +#include #include namespace dsn { @@ -39,20 +40,20 @@ namespace dsn { // class uniq_timestamp_us { private: - uint64_t last_ts; + uint64_t _last_ts; public: - uniq_timestamp_us() { last_ts = dsn_now_us(); } + uniq_timestamp_us() { _last_ts = dsn_now_us(); } void try_update(uint64_t new_ts) { - if (new_ts > last_ts) - last_ts = new_ts; + if ( dsn_likely(new_ts > _last_ts) ) + _last_ts = new_ts; } uint64_t next() { - last_ts = std::max(dsn_now_us(), last_ts+1); - return last_ts; + _last_ts = std::max(dsn_now_us(), _last_ts+1); + return _last_ts; } }; }