From 1cce1611820a5ceb595523c276f8f92020ecbc0c Mon Sep 17 00:00:00 2001 From: VishnuK007 <60417098+VishnuK007@users.noreply.github.com> Date: Thu, 21 Jan 2021 10:09:41 -0800 Subject: [PATCH] [#6772] Display tserver clock information in yb-master UI Display tserver clock information in yb-master UI PR #6776 --- src/yb/client/snapshot-txn-test.cc | 6 +- .../master_path_handlers-itest.cc | 2 +- src/yb/integration-tests/mini_cluster.cc | 3 +- src/yb/master/master-path-handlers.cc | 117 ++++++++++++------ src/yb/master/master-path-handlers.h | 11 +- src/yb/master/master.cc | 1 + src/yb/master/master.proto | 6 + src/yb/master/master_service.cc | 2 + src/yb/master/ts_descriptor.h | 26 ++++ src/yb/server/hybrid_clock.h | 2 +- src/yb/tserver/heartbeater.cc | 17 +++ src/yb/util/date_time.cc | 4 + src/yb/util/date_time.h | 3 + src/yb/util/timestamp.cc | 5 +- src/yb/util/timestamp.h | 4 + 15 files changed, 164 insertions(+), 45 deletions(-) diff --git a/src/yb/client/snapshot-txn-test.cc b/src/yb/client/snapshot-txn-test.cc index 3fb2310ee48b..94fbba3aff90 100644 --- a/src/yb/client/snapshot-txn-test.cc +++ b/src/yb/client/snapshot-txn-test.cc @@ -155,7 +155,7 @@ std::thread RandomClockSkewWalkThread(MiniCluster* cluster, std::atomic* s auto* tserver = cluster->mini_tablet_server(i)->server(); auto* hybrid_clock = down_cast(tserver->clock()); auto skewed_clock = - std::static_pointer_cast(hybrid_clock->TEST_clock()); + std::static_pointer_cast(hybrid_clock->physical_clock()); auto shift = RandomUniformInt(-10, 10); std::chrono::milliseconds change(1 << std::abs(shift)); if (shift < 0) { @@ -183,7 +183,7 @@ std::thread StrobeThread(MiniCluster* cluster, std::atomic* stop) { auto* tserver = cluster->mini_tablet_server(i)->server(); auto* hybrid_clock = down_cast(tserver->clock()); auto skewed_clock = - std::static_pointer_cast(hybrid_clock->TEST_clock()); + std::static_pointer_cast(hybrid_clock->physical_clock()); server::SkewedClock::DeltaTime time_delta; if (iteration & 1) { time_delta = server::SkewedClock::DeltaTime(); @@ -254,7 +254,7 @@ void SnapshotTxnTest::TestBankAccounts(BankAccountsOptions options, CoarseDurati auto* tserver = cluster_->mini_tablet_server(0)->server(); auto* hybrid_clock = down_cast(tserver->clock()); auto skewed_clock = - std::static_pointer_cast(hybrid_clock->TEST_clock()); + std::static_pointer_cast(hybrid_clock->physical_clock()); auto old_delta = skewed_clock->SetDelta(duration); std::this_thread::sleep_for(1s); skewed_clock->SetDelta(old_delta); diff --git a/src/yb/integration-tests/master_path_handlers-itest.cc b/src/yb/integration-tests/master_path_handlers-itest.cc index c214f1e02f0b..7394534a30dd 100644 --- a/src/yb/integration-tests/master_path_handlers-itest.cc +++ b/src/yb/integration-tests/master_path_handlers-itest.cc @@ -54,7 +54,7 @@ class MasterPathHandlersItest : public YBMiniClusterTestBase { ASSERT_OK(cluster_->Start()); Endpoint master_http_endpoint = cluster_->leader_mini_master()->bound_http_addr(); - master_http_url_ = "http://" + ToString(master_http_endpoint); + master_http_url_ = "http://" + AsString(master_http_endpoint); } void DoTearDown() override { diff --git a/src/yb/integration-tests/mini_cluster.cc b/src/yb/integration-tests/mini_cluster.cc index 2b83cd643cfc..186a69e13c4d 100644 --- a/src/yb/integration-tests/mini_cluster.cc +++ b/src/yb/integration-tests/mini_cluster.cc @@ -567,7 +567,8 @@ std::vector SkewClocks( auto* tserver = cluster->mini_tablet_server(i)->server(); auto* hybrid_clock = down_cast(tserver->clock()); delta_changers.emplace_back( - i * clock_skew, std::static_pointer_cast(hybrid_clock->TEST_clock())); + i * clock_skew, std::static_pointer_cast( + hybrid_clock->physical_clock())); } return delta_changers; } diff --git a/src/yb/master/master-path-handlers.cc b/src/yb/master/master-path-handlers.cc index 6cee58b4ed28..b15b53f50c23 100644 --- a/src/yb/master/master-path-handlers.cc +++ b/src/yb/master/master-path-handlers.cc @@ -40,6 +40,7 @@ #include #include +#include "yb/common/hybrid_time.h" #include "yb/common/partition.h" #include "yb/common/schema.h" #include "yb/consensus/consensus.pb.h" @@ -60,6 +61,7 @@ #include "yb/server/webui_util.h" #include "yb/util/curl_util.h" #include "yb/util/string_case.h" +#include "yb/util/timestamp.h" #include "yb/util/url-coding.h" #include "yb/util/version_info.h" #include "yb/util/version_info.pb.h" @@ -213,25 +215,38 @@ void MasterPathHandlers::CallIfLeaderOrPrintRedirect( } } -inline void MasterPathHandlers::TServerTable(std::stringstream* output) { +inline void MasterPathHandlers::TServerTable(std::stringstream* output, + TServersViewType viewType) { *output << "\n"; *output << " \n" << " \n" << " \n" - << " \n" - << " \n" - << " \n" - << " \n" - << " \n" - << " \n" - << " \n" - << " \n" - << " \n" + << " \n"; + + if (viewType == TServersViewType::kTServersClocksView) { + *output << " \n" + << " \n"; + } else { + DCHECK_EQ(viewType, TServersViewType::kTServersDefaultView); + *output << " \n" + << " \n" + << " \n" + << " \n" + << " \n" + << " \n" + << " \n"; + } + + *output << " \n" << " \n" - << " \n" - << " \n" - << " \n" - << " \n"; + << " \n"; + + if (viewType == TServersViewType::kTServersDefaultView) { + *output << " \n" + << " \n"; + } + + *output << " \n"; } namespace { @@ -311,7 +326,8 @@ void MasterPathHandlers::TServerDisplay(const std::string& current_uuid, std::vector>* descs, TabletCountMap* tablet_map, std::stringstream* output, - const int hide_dead_node_threshold_mins) { + const int hide_dead_node_threshold_mins, + TServersViewType viewType) { // Copy vector to avoid changes to the reference descs passed std::vector> local_descs(*descs); @@ -339,22 +355,44 @@ void MasterPathHandlers::TServerDisplay(const std::string& current_uuid, auto tserver = tablet_map->find(desc->permanent_uuid()); bool no_tablets = tserver == tablet_map->end(); - *output << " "; - *output << " "; - *output << " "; - *output << " "; - *output << " "; - *output << " "; - *output << " "; + + if (viewType == TServersViewType::kTServersClocksView) { + // Render physical time. + const Timestamp p_ts(desc->physical_time()); + *output << " "; + + // Render the physical and logical components of the hybrid time. + const HybridTime ht = desc->hybrid_time(); + const Timestamp h_ts(ht.GetPhysicalValueMicros()); + *output << " "; + } else { + DCHECK_EQ(viewType, TServersViewType::kTServersDefaultView); + *output << " "; + *output << " "; + *output << " "; + *output << " "; + *output << " "; + *output << " "; + *output << " "; + } + *output << " "; *output << " "; *output << " "; - *output << " "; - *output << " "; + + if (viewType == TServersViewType::kTServersDefaultView) { + *output << " "; + *output << " "; + } + *output << " \n"; } } @@ -479,7 +517,8 @@ MasterPathHandlers::ZoneTabletCounts::CloudTree MasterPathHandlers::CalculateTab } void MasterPathHandlers::HandleTabletServers(const Webserver::WebRequest& req, - Webserver::WebResponse* resp) { + Webserver::WebResponse* resp, + TServersViewType viewType) { std::stringstream *output = &resp->output; master_->catalog_manager()->AssertLeaderLockAcquiredForReading(); @@ -521,15 +560,16 @@ void MasterPathHandlers::HandleTabletServers(const Webserver::WebRequest& req, << live_id << "\n"; } - TServerTable(output); - TServerDisplay(live_id, &descs, &tablet_map, output, hide_dead_node_threshold_override); + TServerTable(output, viewType); + TServerDisplay(live_id, &descs, &tablet_map, output, hide_dead_node_threshold_override, + viewType); for (const auto& read_replica_uuid : read_replica_uuids) { *output << "

Read Replica UUID: " << (read_replica_uuid.empty() ? kNoPlacementUUID : read_replica_uuid) << "

\n"; - TServerTable(output); - TServerDisplay( - read_replica_uuid, &descs, &tablet_map, output, hide_dead_node_threshold_override); + TServerTable(output, viewType); + TServerDisplay(read_replica_uuid, &descs, &tablet_map, output, + hide_dead_node_threshold_override, viewType); } ZoneTabletCounts::CloudTree counts_tree = CalculateTabletCountsTree(descs, tablet_map); @@ -1782,11 +1822,18 @@ Status MasterPathHandlers::Register(Webserver* server) { "/", "Home", std::bind(&MasterPathHandlers::RootHandler, this, _1, _2), is_styled, is_on_nav_bar, "fa fa-home"); Webserver::PathHandlerCallback cb = - std::bind(&MasterPathHandlers::HandleTabletServers, this, _1, _2); + std::bind(&MasterPathHandlers::HandleTabletServers, this, _1, _2, + TServersViewType::kTServersDefaultView); server->RegisterPathHandler( "/tablet-servers", "Tablet Servers", std::bind(&MasterPathHandlers::CallIfLeaderOrPrintRedirect, this, _1, _2, cb), is_styled, is_on_nav_bar, "fa fa-server"); + cb = std::bind(&MasterPathHandlers::HandleTabletServers, this, _1, _2, + TServersViewType::kTServersClocksView); + server->RegisterPathHandler( + "/tablet-server-clocks", "Tablet Server Clocks", + std::bind(&MasterPathHandlers::CallIfLeaderOrPrintRedirect, this, _1, _2, cb), is_styled, + false /* is_on_nav_bar */); cb = std::bind(&MasterPathHandlers::HandleCatalogManager, this, _1, _2, false /* only_user_tables */); server->RegisterPathHandler( diff --git a/src/yb/master/master-path-handlers.h b/src/yb/master/master-path-handlers.h index dcdebf70808b..77e331f8f881 100644 --- a/src/yb/master/master-path-handlers.h +++ b/src/yb/master/master-path-handlers.h @@ -41,6 +41,7 @@ #include "yb/master/catalog_entity_info.h" #include "yb/master/catalog_manager.h" #include "yb/server/webserver.h" +#include "yb/util/enums.h" namespace yb { @@ -56,6 +57,8 @@ struct TabletReplica; class TSDescriptor; class TSRegistrationPB; +YB_DEFINE_ENUM(TServersViewType, (kTServersDefaultView)(kTServersClocksView)); + // Web page support for the master. class MasterPathHandlers { public: @@ -120,13 +123,14 @@ class MasterPathHandlers { const string kNoPlacementUUID = "NONE"; - static inline void TServerTable(std::stringstream* output); + static inline void TServerTable(std::stringstream* output, TServersViewType viewType); void TServerDisplay(const std::string& current_uuid, std::vector>* descs, TabletCountMap* tmap, std::stringstream* output, - const int hide_dead_node_threshold_override); + const int hide_dead_node_threshold_override, + TServersViewType viewType); // Outputs a ZoneTabletCounts::CloudTree as an html table with a heading. static void DisplayTabletZonesTable( @@ -148,7 +152,8 @@ class MasterPathHandlers { void RootHandler(const Webserver::WebRequest& req, Webserver::WebResponse* resp); void HandleTabletServers(const Webserver::WebRequest& req, - Webserver::WebResponse* resp); + Webserver::WebResponse* resp, + TServersViewType viewType); void HandleCatalogManager(const Webserver::WebRequest& req, Webserver::WebResponse* resp, bool only_user_tables = false); diff --git a/src/yb/master/master.cc b/src/yb/master/master.cc index 8f4fd1966278..4506d846efa9 100644 --- a/src/yb/master/master.cc +++ b/src/yb/master/master.cc @@ -249,6 +249,7 @@ void Master::DisplayGeneralInfoIcons(std::stringstream* output) { // Tasks. DisplayIconTile(output, "fa-check", "Tasks", "/tasks"); DisplayIconTile(output, "fa-clone", "Replica Info", "/tablet-replication"); + DisplayIconTile(output, "fa-check", "TServer Clocks", "/tablet-server-clocks"); } Status Master::StartAsync() { diff --git a/src/yb/master/master.proto b/src/yb/master/master.proto index 18eb7aedc156..06157a5834a3 100644 --- a/src/yb/master/master.proto +++ b/src/yb/master/master.proto @@ -749,6 +749,12 @@ message TSHeartbeatRequestPB { // List of candidate tablets for split based on tablet splitting strategy and settings. repeated TabletForSplitPB tablets_for_split = 9; + + // Physical time on tablet server + optional fixed64 ts_physical_time = 10; + + // Hybrid time on tablet server + optional fixed64 ts_hybrid_time = 11; } message TSHeartbeatResponsePB { diff --git a/src/yb/master/master_service.cc b/src/yb/master/master_service.cc index b80decfff726..7dbb3b132935 100644 --- a/src/yb/master/master_service.cc +++ b/src/yb/master/master_service.cc @@ -185,6 +185,8 @@ void MasterServiceImpl::TSHeartbeat(const TSHeartbeatRequestPB* req, ts_desc->UpdateHeartbeatTime(); ts_desc->set_num_live_replicas(req->num_live_tablets()); ts_desc->set_leader_count(req->leader_count()); + ts_desc->set_physical_time(req->ts_physical_time()); + ts_desc->set_hybrid_time(HybridTime::FromPB(req->ts_hybrid_time())); // Adjust the table report limit per heartbeat so this can be dynamically changed. if (ts_desc->HasCapability(CAPABILITY_TabletReportLimit)) { diff --git a/src/yb/master/ts_descriptor.h b/src/yb/master/ts_descriptor.h index c303b68f9566..1495e095c50f 100644 --- a/src/yb/master/ts_descriptor.h +++ b/src/yb/master/ts_descriptor.h @@ -39,6 +39,7 @@ #include #include +#include "yb/common/hybrid_time.h" #include "yb/gutil/gscoped_ptr.h" #include "yb/master/master_fwd.h" @@ -49,6 +50,7 @@ #include "yb/util/capabilities.h" #include "yb/util/locks.h" #include "yb/util/monotime.h" +#include "yb/util/physical_time.h" #include "yb/util/status.h" #include "yb/util/shared_ptr_tuple.h" #include "yb/util/shared_lock.h" @@ -177,6 +179,26 @@ class TSDescriptor { return leader_count_; } + void set_physical_time(MicrosTime physical_time) { + std::lock_guard l(lock_); + physical_time_ = physical_time; + } + + MicrosTime physical_time() const { + SharedLock l(lock_); + return physical_time_; + } + + void set_hybrid_time(HybridTime hybrid_time) { + std::lock_guard l(lock_); + hybrid_time_ = hybrid_time; + } + + HybridTime hybrid_time() const { + SharedLock l(lock_); + return hybrid_time_; + } + void set_total_memory_usage(uint64_t total_memory_usage) { std::lock_guard l(lock_); ts_metrics_.total_memory_usage = total_memory_usage; @@ -337,6 +359,10 @@ class TSDescriptor { // The last time a heartbeat was received for this node. MonoTime last_heartbeat_; + // The physical and hybrid times on this node at the time of heartbeat + MicrosTime physical_time_; + HybridTime hybrid_time_; + // Set to true once this instance has reported all of its tablets. bool has_tablet_report_; diff --git a/src/yb/server/hybrid_clock.h b/src/yb/server/hybrid_clock.h index 97c97eaf5bf3..e27f341af2d5 100644 --- a/src/yb/server/hybrid_clock.h +++ b/src/yb/server/hybrid_clock.h @@ -142,7 +142,7 @@ class HybridClock : public Clock { // Enables check whether clock skew within configured bounds. static void EnableClockSkewControl(); - const PhysicalClockPtr& TEST_clock() { return clock_; } + const PhysicalClockPtr& physical_clock() { return clock_; } private: enum State { diff --git a/src/yb/tserver/heartbeater.cc b/src/yb/tserver/heartbeater.cc index 537f6cf96cf2..a3ba97eca2a2 100644 --- a/src/yb/tserver/heartbeater.cc +++ b/src/yb/tserver/heartbeater.cc @@ -382,6 +382,23 @@ Status Heartbeater::Thread::TryHeartbeat() { req.set_config_index(server_->GetCurrentMasterIndex()); req.set_cluster_config_version(server_->cluster_config_version()); + // Include the hybrid time of this tablet server in the heartbeat. + auto* hybrid_clock = dynamic_cast(server_->Clock()); + if (hybrid_clock) { + req.set_ts_hybrid_time(hybrid_clock->Now().ToUint64()); + // Also include the physical clock time of this tablet server in the heartbeat. + Result now = hybrid_clock->physical_clock()->Now(); + if (!now.ok()) { + YB_LOG_EVERY_N_SECS(WARNING, 10) << "Failed to read clock: " << now.status(); + req.set_ts_physical_time(0); + } else { + req.set_ts_physical_time(now->time_point); + } + } else { + req.set_ts_hybrid_time(0); + req.set_ts_physical_time(0); + } + { VLOG_WITH_PREFIX(2) << "Sending heartbeat:\n" << req.DebugString(); master::TSHeartbeatResponsePB resp; diff --git a/src/yb/util/date_time.cc b/src/yb/util/date_time.cc index 23d852c55328..38903744423b 100644 --- a/src/yb/util/date_time.cc +++ b/src/yb/util/date_time.cc @@ -447,4 +447,8 @@ const DateTime::OutputFormat DateTime::CqlOutputFormat = OutputFormat( locale(locale::classic(), new local_time_facet("%Y-%m-%dT%H:%M:%S.%f%q")) ); +const DateTime::OutputFormat DateTime::HumanReadableOutputFormat = OutputFormat( + locale(locale::classic(), new local_time_facet("%Y-%m-%d %H:%M:%S.%f")) +); + } // namespace yb diff --git a/src/yb/util/date_time.h b/src/yb/util/date_time.h index aa4085dfe7e8..1547948b1999 100644 --- a/src/yb/util/date_time.h +++ b/src/yb/util/date_time.h @@ -53,6 +53,9 @@ class DateTime { static const InputFormat CqlInputFormat; static const OutputFormat CqlOutputFormat; + // Human readable format. + static const OutputFormat HumanReadableOutputFormat; + //---------------------------------------------------------------------------------------------- static Result TimestampFromString(const std::string& str, const InputFormat& input_format = CqlInputFormat); diff --git a/src/yb/util/timestamp.cc b/src/yb/util/timestamp.cc index d0fefdebcb29..c61659222f86 100644 --- a/src/yb/util/timestamp.cc +++ b/src/yb/util/timestamp.cc @@ -11,7 +11,6 @@ // under the License. // - #include "yb/util/timestamp.h" #include "yb/util/date_time.h" #include "yb/gutil/strings/substitute.h" @@ -38,4 +37,8 @@ string Timestamp::ToFormattedString() const { return DateTime::TimestampToString(*this, DateTime::CqlOutputFormat); } +string Timestamp::ToHumanReadableTime() const { + return DateTime::TimestampToString(*this, DateTime::HumanReadableOutputFormat); +} + } // namespace yb diff --git a/src/yb/util/timestamp.h b/src/yb/util/timestamp.h index 3c1983b28aa5..825e17badbab 100644 --- a/src/yb/util/timestamp.h +++ b/src/yb/util/timestamp.h @@ -32,6 +32,10 @@ class Timestamp { std::string ToFormattedString() const; + // Return date in human readable format (in local time zone). + // For example, 2021-Jan-10 22:29:35.776000. + std::string ToHumanReadableTime() const; + val_type value() const { return value_; } void set_value(int64_t value) {value_ = value;}
ServerTime since
heartbeat		Status & UptimeUser Tablet-Peers / LeadersRAM UsedNum SST FilesTotal SST Files SizeUncompressed SST
Files Size		Read ops/secWrite ops/secCloudStatus & UptimePhysical Time (UTC)Hybrid Time (UTC)User Tablet-Peers / LeadersRAM UsedNum SST FilesTotal SST Files SizeUncompressed SST
Files Size		Read ops/secWrite ops/secCloudRegionZoneSystem Tablet-Peers / LeadersActive Tablet-Peers
ZoneSystem Tablet-Peers / LeadersActive Tablet-Peers
" << (no_tablets ? 0 - : tserver->second.user_tablet_leaders + tserver->second.user_tablet_followers) - << " / " << (no_tablets ? 0 : tserver->second.user_tablet_leaders) << "" << HumanizeBytes(desc->total_memory_usage()) << "" << desc->num_sst_files() << "" << HumanizeBytes(desc->total_sst_file_size()) << "" << HumanizeBytes(desc->uncompressed_sst_file_size()) << "" << desc->read_ops_per_sec() << "" << desc->write_ops_per_sec() << "" << p_ts.ToHumanReadableTime() << "" << h_ts.ToHumanReadableTime(); + if (ht.GetLogicalValue()) { + *output << " / Logical: " << ht.GetLogicalValue(); + } + *output << "" << (no_tablets ? 0 + : tserver->second.user_tablet_leaders + tserver->second.user_tablet_followers) + << " / " << (no_tablets ? 0 : tserver->second.user_tablet_leaders) << "" << HumanizeBytes(desc->total_memory_usage()) << "" << desc->num_sst_files() << "" << HumanizeBytes(desc->total_sst_file_size()) << "" << HumanizeBytes(desc->uncompressed_sst_file_size()) << "" << desc->read_ops_per_sec() << "" << desc->write_ops_per_sec() << "" << reg.common().cloud_info().placement_cloud() << "" << reg.common().cloud_info().placement_region() << "" << reg.common().cloud_info().placement_zone() << "" << (no_tablets ? 0 - : tserver->second.system_tablet_leaders + tserver->second.system_tablet_followers) - << " / " << (no_tablets ? 0 : tserver->second.system_tablet_leaders) << "" << (no_tablets ? 0 : desc->num_live_replicas()) << "" << (no_tablets ? 0 + : tserver->second.system_tablet_leaders + tserver->second.system_tablet_followers) + << " / " << (no_tablets ? 0 : tserver->second.system_tablet_leaders) << "" << (no_tablets ? 0 : desc->num_live_replicas()) << "