Cherry pick v3.0.0 0215 (#3898)

vesoft-inc · Feb 15, 2022 · 25fd187 · 25fd187
1 parent f37129a
commit 25fd187
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 30 deletions.
diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp
@@ -402,7 +402,6 @@ bool MetaClient::loadData() {
 
 TagSchemas MetaClient::buildTagSchemas(std::vector<cpp2::TagItem> tagItemVec) {
   TagSchemas tagSchemas;
-  TagID lastTagId = -1;
   for (auto& tagIt : tagItemVec) {
     // meta will return the different version from new to old
     auto schema = std::make_shared<NebulaSchemaProvider>(tagIt.get_version());
@@ -411,20 +410,21 @@ TagSchemas MetaClient::buildTagSchemas(std::vector<cpp2::TagItem> tagItemVec) {
     }
     // handle schema property
     schema->setProp(tagIt.get_schema().get_schema_prop());
-    if (tagIt.get_tag_id() != lastTagId) {
-      // init schema vector, since schema version is zero-based, need to add one
-      tagSchemas[tagIt.get_tag_id()].resize(schema->getVersion() + 1);
-      lastTagId = tagIt.get_tag_id();
+    auto& schemas = tagSchemas[tagIt.get_tag_id()];
+    // Because of the byte order of schema version in meta is not same as numerical order, we have
+    // to check schema version
+    if (schemas.size() <= static_cast<size_t>(schema->getVersion())) {
+      // since schema version is zero-based, need to add one
+      schemas.resize(schema->getVersion() + 1);
     }
-    tagSchemas[tagIt.get_tag_id()][schema->getVersion()] = std::move(schema);
+    schemas[schema->getVersion()] = std::move(schema);
   }
   return tagSchemas;
 }
 
 EdgeSchemas MetaClient::buildEdgeSchemas(std::vector<cpp2::EdgeItem> edgeItemVec) {
   EdgeSchemas edgeSchemas;
   std::unordered_set<std::pair<GraphSpaceID, EdgeType>> edges;
-  EdgeType lastEdgeType = -1;
   for (auto& edgeIt : edgeItemVec) {
     // meta will return the different version from new to old
     auto schema = std::make_shared<NebulaSchemaProvider>(edgeIt.get_version());
@@ -433,12 +433,14 @@ EdgeSchemas MetaClient::buildEdgeSchemas(std::vector<cpp2::EdgeItem> edgeItemVec
     }
     // handle shcem property
     schema->setProp(edgeIt.get_schema().get_schema_prop());
-    if (edgeIt.get_edge_type() != lastEdgeType) {
-      // init schema vector, since schema version is zero-based, need to add one
-      edgeSchemas[edgeIt.get_edge_type()].resize(schema->getVersion() + 1);
-      lastEdgeType = edgeIt.get_edge_type();
+    auto& schemas = edgeSchemas[edgeIt.get_edge_type()];
+    // Because of the byte order of schema version in meta is not same as numerical order, we have
+    // to check schema version
+    if (schemas.size() <= static_cast<size_t>(schema->getVersion())) {
+      // since schema version is zero-based, need to add one
+      schemas.resize(schema->getVersion() + 1);
     }
-    edgeSchemas[edgeIt.get_edge_type()][schema->getVersion()] = std::move(schema);
+    schemas[schema->getVersion()] = std::move(schema);
   }
   return edgeSchemas;
 }

diff --git a/src/common/stats/StatsManager.cpp b/src/common/stats/StatsManager.cpp
@@ -279,14 +279,18 @@ void StatsManager::addValue(const CounterId& id, VT value) {
   bool isHisto = id.isHisto();
   if (!isHisto) {
     // Stats
-    DCHECK(sm.stats_.find(index) != sm.stats_.end());
-    std::lock_guard<std::mutex> g(*(sm.stats_[index].first));
-    sm.stats_[index].second->addValue(seconds(time::WallClock::fastNowInSec()), value);
+    auto iter = sm.stats_.find(index);
+    if (iter != sm.stats_.end()) {
+      std::lock_guard<std::mutex> g(*(iter->second.first));
+      iter->second.second->addValue(seconds(time::WallClock::fastNowInSec()), value);
+    }
   } else {
     // Histogram
-    DCHECK(sm.histograms_.find(index) != sm.histograms_.end());
-    std::lock_guard<std::mutex> g(*(sm.histograms_[index].first));
-    sm.histograms_[index].second->addValue(seconds(time::WallClock::fastNowInSec()), value);
+    auto iter = sm.histograms_.find(index);
+    if (iter != sm.histograms_.end()) {
+      std::lock_guard<std::mutex> g(*(iter->second.first));
+      iter->second.second->addValue(seconds(time::WallClock::fastNowInSec()), value);
+    }
   }
 }
 
@@ -464,16 +468,18 @@ StatusOr<StatsManager::VT> StatsManager::readStats(const CounterId& id,
 
   if (!id.isHisto()) {
     // stats
-    DCHECK(sm.stats_.find(index) != sm.stats_.end());
-    std::lock_guard<std::mutex> g(*(sm.stats_[index].first));
-    sm.stats_[index].second->update(seconds(time::WallClock::fastNowInSec()));
-    return readValue(*(sm.stats_[index].second), range, method);
+    auto iter = sm.stats_.find(index);
+    DCHECK(iter != sm.stats_.end());
+    std::lock_guard<std::mutex> g(*(iter->second.first));
+    iter->second.second->update(seconds(time::WallClock::fastNowInSec()));
+    return readValue(*(iter->second.second), range, method);
   } else {
     // histograms_
-    DCHECK(sm.histograms_.find(index) != sm.histograms_.end());
-    std::lock_guard<std::mutex> g(*(sm.histograms_[index].first));
-    sm.histograms_[index].second->update(seconds(time::WallClock::fastNowInSec()));
-    return readValue(*(sm.histograms_[index].second), range, method);
+    auto iter = sm.histograms_.find(index);
+    DCHECK(iter != sm.histograms_.end());
+    std::lock_guard<std::mutex> g(*(iter->second.first));
+    iter->second.second->update(seconds(time::WallClock::fastNowInSec()));
+    return readValue(*(iter->second.second), range, method);
   }
 }
 

diff --git a/src/common/stats/StatsManager.h b/src/common/stats/StatsManager.h
@@ -7,6 +7,7 @@
 #define COMMON_STATS_STATSMANAGER_H_
 
 #include <folly/RWSpinLock.h>
+#include <folly/concurrency/ConcurrentHashMap.h>
 #include <folly/stats/MultiLevelTimeSeries.h>
 #include <folly/stats/TimeseriesHistogram.h>
 
@@ -206,8 +207,8 @@ class StatsManager final {
   std::unordered_map<std::string, CounterInfo> nameMap_;
 
   // All time series stats
-  std::unordered_map<std::string,
-                     std::pair<std::unique_ptr<std::mutex>, std::unique_ptr<StatsType>>>
+  folly::ConcurrentHashMap<std::string,
+                           std::pair<std::unique_ptr<std::mutex>, std::unique_ptr<StatsType>>>
       stats_;
 
   // All histogram stats

diff --git a/tests/Makefile b/tests/Makefile
@@ -32,8 +32,8 @@ test_j_sa = $(test_without_skip_sa) -n$(J)
 
 
 install-deps:
-	pip3 install --user -U setuptools wheel -i $(PYPI_MIRROR)
-	pip3 install --user -r $(CURR_DIR)/requirements.txt -i $(PYPI_MIRROR)
+	pip3 install --user -Ur $(CURR_DIR)/requirements.txt -i $(PYPI_MIRROR)
+
 install-nebula-py: install-deps
 	git clone --branch master https://github.com/vesoft-inc/nebula-python $(CURR_DIR)/nebula-python
 	cd $(CURR_DIR)/nebula-python \

diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -1,3 +1,5 @@
+setuptools==59.6.0
+wheel==0.37.1
 pytest==5.3.2
 pytest-html==2.0.1
 pytest-metadata==1.8.0