Support min commit ts filter & Fix compute FsStats (#713) (#714)

Signed-off-by: Tong Zhigao <[email protected]>

dbms/src/Storages/PathCapacityMetrics.cpp

@@ -144,7 +144,7 @@ FsStats PathCapacityMetrics::CapacityInfo::getStats(Poco::Logger * log) const
     // capacity
     uint64_t capacity = 0;
     const uint64_t disk_capacity_size = vfs.f_blocks * vfs.f_frsize;
-    if (capacity_bytes == 0 || disk_capacity_size < capacity)
+    if (capacity_bytes == 0 || disk_capacity_size < capacity_bytes)
         capacity = disk_capacity_size;
     else
         capacity = capacity_bytes;

dbms/src/Storages/PathCapacityMetrics.h

@@ -1,6 +1,5 @@
 #pragma once
 #include <Core/Types.h>
-#include <Storages/Transaction/ProxyFFIType.h>
 #include <common/logger_useful.h>
 #include <sys/statvfs.h>
 
@@ -13,6 +12,8 @@ namespace DB
 class PathCapacityMetrics;
 using PathCapacityMetricsPtr = std::shared_ptr<PathCapacityMetrics>;
 
+struct FsStats;
+
 class PathCapacityMetrics : private boost::noncopyable
 {
 public:

dbms/src/Storages/Transaction/ProxyFFIType.h

@@ -83,7 +83,9 @@ struct FsStats
     uint64_t avail_size;
     uint64_t capacity_size;
 
-    uint8_t ok = 0;
+    uint8_t ok;
+
+    FsStats() { memset(this, 0, sizeof(*this)); }
 };
 
 struct TiFlashServerHelper

dbms/src/Storages/Transaction/RegionData.cpp

@@ -125,13 +125,14 @@ LockInfoPtr RegionData::getLockInfo(const RegionLockReadQuery & query) const
         std::ignore = handle;
 
         const auto & [tikv_key, tikv_val, decoded_val, decoded_key] = value;
-        const auto & [lock_type, primary, ts, ttl] = decoded_val;
+        const auto & [lock_type, primary, ts, ttl, min_commit_ts] = decoded_val;
         std::ignore = tikv_key;
         std::ignore = tikv_val;
 
         if (ts > query.read_tso || lock_type == Lock || lock_type == Pessimistic)
             continue;
-
+        if (min_commit_ts > query.read_tso)
+            continue;
         if (query.bypass_lock_ts && query.bypass_lock_ts->count(ts))
             continue;
 

dbms/src/Storages/Transaction/TiKVRecordFormat.h

@@ -174,14 +174,15 @@ inline TiKVValue encodeLockCfValue(UInt8 lock_type, const String & primary, Time
     return internalEncodeLockCfValue(lock_type, primary, ts, ttl, nullptr);
 }
 
-using DecodedLockCFValue = std::tuple<UInt8, String, Timestamp, UInt64>;
+using DecodedLockCFValue = std::tuple<UInt8, String, Timestamp, UInt64, Timestamp>;
 
 inline DecodedLockCFValue decodeLockCfValue(const TiKVValue & value)
 {
     UInt8 lock_type;
     String primary;
-    UInt64 ts;
+    Timestamp ts;
     UInt64 ttl = 0;
+    Timestamp min_commit_ts = 0;
 
     const char * data = value.data();
     size_t len = value.dataSize();
@@ -216,7 +217,7 @@ inline DecodedLockCFValue decodeLockCfValue(const TiKVValue & value)
                 };
                 case MIN_COMMIT_TS_PREFIX:
                 {
-                    readBigEndian<UInt64>(data);
+                    min_commit_ts = readBigEndian<UInt64>(data);
                     data += sizeof(UInt64);
                     len -= sizeof(UInt64);
                     break;
@@ -246,7 +247,7 @@ inline DecodedLockCFValue decodeLockCfValue(const TiKVValue & value)
     if (len != 0)
         throw Exception("invalid lock value " + value.toHex(), ErrorCodes::LOGICAL_ERROR);
 
-    return std::make_tuple(lock_type, primary, ts, ttl);
+    return std::make_tuple(lock_type, primary, ts, ttl, min_commit_ts);
 }
 
 using DecodedWriteCFValue = std::tuple<UInt8, Timestamp, std::shared_ptr<const TiKVValue>>;

dbms/src/Storages/Transaction/tests/gtest_tikv_keyvalue.cpp

@@ -58,20 +58,22 @@ TEST(TiKVKeyValue_test, DISABLED_PortedTests)
     {
         auto lock_value
             = RecordKVFormat::encodeLockCfValue(Region::PutFlag, "primary key", 421321, std::numeric_limits<UInt64>::max(), "value");
-        auto [lock_type, primary, ts, ttl] = RecordKVFormat::decodeLockCfValue(lock_value);
+        auto [lock_type, primary, ts, ttl, min_commit_ts] = RecordKVFormat::decodeLockCfValue(lock_value);
         ASSERT_TRUE(Region::PutFlag == lock_type);
         ASSERT_TRUE("primary key" == primary);
         ASSERT_TRUE(421321 == ts);
         ASSERT_TRUE(std::numeric_limits<UInt64>::max() == ttl);
+        ASSERT_TRUE(0 == min_commit_ts);
     }
 
     {
         auto lock_value = RecordKVFormat::encodeLockCfValue(Region::PutFlag, "primary key", 421321, std::numeric_limits<UInt64>::max());
-        auto [lock_type, primary, ts, ttl] = RecordKVFormat::decodeLockCfValue(lock_value);
+        auto [lock_type, primary, ts, ttl, min_commit_ts] = RecordKVFormat::decodeLockCfValue(lock_value);
         ASSERT_TRUE(Region::PutFlag == lock_type);
         ASSERT_TRUE("primary key" == primary);
         ASSERT_TRUE(421321 == ts);
         ASSERT_TRUE(std::numeric_limits<UInt64>::max() == ttl);
+        ASSERT_TRUE(0 == min_commit_ts);
     }
 
     {

tests/docker/cluster.yaml

@@ -51,6 +51,7 @@ services:
       - --path=pd0:2379
       - --config=/tidb.toml
       - --log-file=/log/tidb.log
+      - --advertise-address=tidb0
     depends_on:
       - "tikv0"
     restart: on-failure

tests/docker/config/tiflash_dt.toml

@@ -4,7 +4,7 @@ default_profile = "default"
 users_config = "users.toml"
 # specify paths used for store data, multiple path should be seperated by comma
 path = "/tmp/tiflash/data/db"
-capacity = "107374182400" 
+capacity = "10737418240"
 # multi-paths example
 # path = "/tmp/tiflash/data/db,/tmp/tiflash1,/tmp/tiflash2"
 # capacity = "0,0,0"

tests/docker/config/tiflash_tmt.toml

@@ -4,7 +4,7 @@ default_profile = "default"
 users_config = "users.toml"
 # specify paths used for store data, multiple path should be seperated by comma
 path = "/tmp/tiflash/data/db"
-capacity = "107374182400" 
+capacity = "10737418240"
 # multi-paths example
 # path = "/tmp/tiflash/data/db,/tmp/tiflash1,/tmp/tiflash2"
 # capacity = "0,0,0"