*: bump client-go with some regression tests (#57282) (#57563)

ref #55287, close #56837, ref #57338, close #57425

DEPS.bzl

@@ -6933,13 +6933,13 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sha256 = "537e3204b8178e2ce0ce43c744fc2699883bb33e718e267da2f1dd6c389968c2",
-        strip_prefix = "github.com/tikv/client-go/[email protected].20241111090227-70049ae310bf",
+        sha256 = "cdcad188042c4d716dd9d4a304a2e36bc9d4edccaf86a19b85b1682f01df193c",
+        strip_prefix = "github.com/tikv/client-go/[email protected].20241121061241-006dfb024c26",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241111090227-70049ae310bf.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241111090227-70049ae310bf.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241111090227-70049ae310bf.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241111090227-70049ae310bf.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241121061241-006dfb024c26.zip",
+            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241121061241-006dfb024c26.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241121061241-006dfb024c26.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241121061241-006dfb024c26.zip",
         ],
     )
     go_repository(

errors.toml

@@ -2171,6 +2171,11 @@ error = '''
 not implemented
 '''
 
+["kv:8178"]
+error = '''
+key is too large, the size of given key is %d
+'''
+
 ["kv:9007"]
 error = '''
 Write conflict, txnStartTS=%d, conflictStartTS=%d, conflictCommitTS=%d, key=%s%s%s%s, reason=%s [try again later]

go.mod

@@ -109,7 +109,7 @@ require (
 	github.com/tdakkota/asciicheck v0.2.0
 	github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
 	github.com/tidwall/btree v1.7.0
-	github.com/tikv/client-go/v2 v2.0.8-0.20241111090227-70049ae310bf
+	github.com/tikv/client-go/v2 v2.0.8-0.20241121061241-006dfb024c26
 	github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31
 	github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a
 	github.com/twmb/murmur3 v1.1.6

go.sum

@@ -824,8 +824,8 @@ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW
 github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
 github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/tikv/client-go/v2 v2.0.8-0.20241111090227-70049ae310bf h1:qCi6BiBUPk3Ky4f2CCgBxgUmi3ZpuQLYDLgxw1ilXPA=
-github.com/tikv/client-go/v2 v2.0.8-0.20241111090227-70049ae310bf/go.mod h1:p9zPFlKBrxhp3b/cBmKBWL9M0X4HtJjgi1ThUtQYF7o=
+github.com/tikv/client-go/v2 v2.0.8-0.20241121061241-006dfb024c26 h1:CwiOzQZl7qCJi4QhNbzptX0hJoG10Q/gyLc5QULNW7I=
+github.com/tikv/client-go/v2 v2.0.8-0.20241121061241-006dfb024c26/go.mod h1:p9zPFlKBrxhp3b/cBmKBWL9M0X4HtJjgi1ThUtQYF7o=
 github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 h1:oAYc4m5Eu1OY9ogJ103VO47AYPHvhtzbUPD8L8B67Qk=
 github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31/go.mod h1:W5a0sDadwUpI9k8p7M77d3jo253ZHdmua+u4Ho4Xw8U=
 github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo=

pkg/errno/errcode.go

@@ -1085,6 +1085,7 @@ const (
 	ErrMemoryExceedForQuery                = 8175
 	ErrMemoryExceedForInstance             = 8176
 	ErrDeleteNotFoundColumn                = 8177
+	ErrKeyTooLarge                         = 8178
 
 	// Error codes used by TiDB ddl package
 	ErrUnsupportedDDLOperation            = 8200

pkg/errno/errname.go

@@ -1078,7 +1078,9 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
 	ErrMemoryExceedForQuery:             mysql.Message("Your query has been cancelled due to exceeding the allowed memory limit for a single SQL query. Please try narrowing your query scope or increase the tidb_mem_quota_query limit and try again.[conn=%d]", nil),
 	ErrMemoryExceedForInstance:          mysql.Message("Your query has been cancelled due to exceeding the allowed memory limit for the tidb-server instance and this query is currently using the most memory. Please try narrowing your query scope or increase the tidb_server_memory_limit and try again.[conn=%d]", nil),
 	ErrDeleteNotFoundColumn:             mysql.Message("Delete can not find column %s for table %s", nil),
-	ErrHTTPServiceError:                 mysql.Message("HTTP request failed with status %s", nil),
+	ErrKeyTooLarge:                      mysql.Message("key is too large, the size of given key is %d", nil),
+
+	ErrHTTPServiceError: mysql.Message("HTTP request failed with status %s", nil),
 
 	ErrWarnOptimizerHintInvalidInteger:  mysql.Message("integer value is out of range in '%s'", nil),
 	ErrWarnOptimizerHintUnsupportedHint: mysql.Message("Optimizer hint %s is not supported by TiDB and is ignored", nil),

pkg/executor/adapter.go

@@ -659,7 +659,7 @@ func (a *ExecStmt) handleStmtForeignKeyTrigger(ctx context.Context, e exec.Execu
 	if stmtCtx.ForeignKeyTriggerCtx.HasFKCascades {
 		// If the ExecStmt has foreign key cascade to be executed, we need call `StmtCommit` to commit the ExecStmt itself
 		// change first.
-		// Since `UnionScanExec` use `SnapshotIter` and `SnapshotGetter` to read txn mem-buffer, if we don't  do `StmtCommit`,
+		// Since `UnionScanExec` use `SnapshotIter` and `SnapshotGetter` to read txn mem-buffer, if we don't do `StmtCommit`,
 		// then the fk cascade executor can't read the mem-buffer changed by the ExecStmt.
 		a.Ctx.StmtCommit(ctx)
 	}

pkg/executor/mem_reader.go

@@ -345,7 +345,6 @@ func (iter *txnMemBufferIter) Valid() bool {
 		if iter.curr.Valid() {
 			return true
 		}
-		iter.curr = nil
 		iter.idx++
 	}
 	for iter.idx < len(iter.kvRanges) {
@@ -372,7 +371,6 @@ func (iter *txnMemBufferIter) Valid() bool {
 		if iter.curr.Valid() {
 			return true
 		}
-		iter.curr = nil
 		iter.idx++
 	}
 	return false
@@ -398,7 +396,10 @@ func (iter *txnMemBufferIter) Value() []byte {
 	return iter.curr.Value()
 }
 
-func (*txnMemBufferIter) Close() {
+func (iter *txnMemBufferIter) Close() {
+	if iter.curr != nil {
+		iter.curr.Close()
+	}
 }
 
 func (m *memTableReader) getMemRowsIter(ctx context.Context) (memRowsIter, error) {
@@ -882,6 +883,8 @@ func buildMemIndexMergeReader(ctx context.Context, us *UnionScanExec, indexMerge
 
 type memRowsIter interface {
 	Next() ([]types.Datum, error)
+	// Close will release the snapshot it holds, so be sure to call Close.
+	Close()
 }
 
 type defaultRowsIter struct {
@@ -898,6 +901,8 @@ func (iter *defaultRowsIter) Next() ([]types.Datum, error) {
 	return nil, nil
 }
 
+func (*defaultRowsIter) Close() {}
+
 // memRowsIterForTable combine a kv.Iterator and a kv decoder to get a memRowsIter.
 type memRowsIterForTable struct {
 	kvIter   *txnMemBufferIter // txnMemBufferIter is the kv.Iterator
@@ -966,6 +971,12 @@ func (iter *memRowsIterForTable) Next() ([]types.Datum, error) {
 	return ret, nil
 }
 
+func (iter *memRowsIterForTable) Close() {
+	if iter.kvIter != nil {
+		iter.kvIter.Close()
+	}
+}
+
 type memRowsIterForIndex struct {
 	kvIter     *txnMemBufferIter
 	tps        []*types.FieldType
@@ -1018,6 +1029,12 @@ func (iter *memRowsIterForIndex) Next() ([]types.Datum, error) {
 	return ret, nil
 }
 
+func (iter *memRowsIterForIndex) Close() {
+	if iter.kvIter != nil {
+		iter.kvIter.Close()
+	}
+}
+
 func (m *memIndexMergeReader) getMemRowsIter(ctx context.Context) (memRowsIter, error) {
 	data, err := m.getMemRows(ctx)
 	if err != nil {

pkg/executor/union_scan.go

@@ -191,6 +191,9 @@ func (us *UnionScanExec) Close() error {
 	us.cursor4AddRows = nil
 	us.cursor4SnapshotRows = 0
 	us.snapshotRows = us.snapshotRows[:0]
+	if us.addedRowsIter != nil {
+		us.addedRowsIter.Close()
+	}
 	return exec.Close(us.Children(0))
 }
 

pkg/executor/union_scan_test.go

@@ -360,6 +360,21 @@ func TestIssue32422(t *testing.T) {
 	tk.MustExec("rollback")
 }
 
+func TestSnapshotWithConcurrentWrite(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("create table t1 (id int auto_increment key, b int, index(b));")
+
+	tk.MustExec("begin")
+	tk.MustExec("insert into t1 (b) values (1),(2),(3),(4),(5),(6),(7),(8);")
+	for j := 0; j < 16; j++ {
+		tk.MustExec("insert into t1 (b) select /*+ use_index(t1, b) */ id from t1;")
+	}
+	tk.MustQuery("select count(1) from t1").Check(testkit.Rows("524288")) // 8 * 2^16 rows
+	tk.MustExec("rollback")
+}
+
 func BenchmarkUnionScanRead(b *testing.B) {
 	store := testkit.CreateMockStore(b)
 

pkg/kv/error.go

@@ -47,6 +47,8 @@ var (
 	ErrTxnTooLarge = dbterror.ClassKV.NewStd(mysql.ErrTxnTooLarge)
 	// ErrEntryTooLarge is the error when a key value entry is too large.
 	ErrEntryTooLarge = dbterror.ClassKV.NewStd(mysql.ErrEntryTooLarge)
+	// ErrKeyTooLarge is the error when a key is too large to be handled by MemBuffer.
+	ErrKeyTooLarge = dbterror.ClassKV.NewStd(mysql.ErrKeyTooLarge)
 	// ErrKeyExists returns when key is already exist. Caller should try to use
 	// GenKeyExistsErr to generate this error for correct format.
 	ErrKeyExists = dbterror.ClassKV.NewStd(mysql.ErrDupEntry)

pkg/session/test/txn/BUILD.bazel

@@ -9,7 +9,7 @@ go_test(
     ],
     flaky = True,
     race = "on",
-    shard_count = 9,
+    shard_count = 10,
     deps = [
         "//pkg/config",
         "//pkg/kv",

pkg/session/test/txn/txn_test.go

@@ -553,3 +553,29 @@ func TestMemBufferSnapshotRead(t *testing.T) {
 	tk.MustExec("set session tidb_max_chunk_size=default;")
 	tk.MustExec("set session tidb_index_join_batch_size = default")
 }
+
+func TestMemBufferCleanupMemoryLeak(t *testing.T) {
+	// Test if cleanup memory will cause a memory leak.
+	// When an in-txn statement fails, TiDB cleans up the mutations from this statement.
+	// If there's a memory leak, the memory usage could increase uncontrollably with retries.
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("create table t(a varchar(255) primary key)")
+	key1 := strings.Repeat("a", 255)
+	key2 := strings.Repeat("b", 255)
+	tk.MustExec(`set global tidb_mem_oom_action='cancel'`)
+	tk.MustExec("set session tidb_mem_quota_query=10240")
+	tk.MustExec("begin")
+	tk.MustExec("insert into t values(?)", key2)
+	for i := 0; i < 100; i++ {
+		// The insert statement will fail because of the duplicate key error.
+		err := tk.ExecToErr("insert into t values(?), (?)", key1, key2)
+		require.Error(t, err)
+		if strings.Contains(err.Error(), "Duplicate") {
+			continue
+		}
+		require.NoError(t, err)
+	}
+	tk.MustExec("commit")
+}

pkg/store/driver/error/error.go

@@ -101,6 +101,11 @@ func ToTiDBErr(err error) error {
 		return kv.ErrEntryTooLarge.GenWithStackByArgs(entryTooLarge.Limit, entryTooLarge.Size)
 	}
 
+	var keyTooLarge *tikverr.ErrKeyTooLarge
+	if stderrs.As(err, &keyTooLarge) {
+		return kv.ErrKeyTooLarge.GenWithStackByArgs(keyTooLarge.KeySize)
+	}
+
 	if stderrs.Is(err, tikverr.ErrInvalidTxn) {
 		return kv.ErrInvalidTxn
 	}

pkg/store/driver/error/error_test.go

@@ -15,6 +15,7 @@
 package error //nolint: predeclared
 
 import (
+	"math"
 	"testing"
 
 	"github.com/pingcap/errors"
@@ -52,3 +53,16 @@ func TestConvertError(t *testing.T) {
 		assert.True(t, errors.ErrorEqual(tidbErr, terror.ErrResultUndetermined))
 	}
 }
+
+func TestMemBufferOversizeError(t *testing.T) {
+	err2str := map[error]string{
+		&tikverr.ErrTxnTooLarge{Size: 100}:                   "Transaction is too large, size: 100",
+		&tikverr.ErrEntryTooLarge{Limit: 10, Size: 20}:       "entry too large, the max entry size is 10, the size of data is 20",
+		&tikverr.ErrKeyTooLarge{KeySize: math.MaxUint16 + 1}: "key is too large, the size of given key is 65536",
+	}
+	for err, errString := range err2str {
+		tidbErr := ToTiDBErr(err)
+		assert.NotNil(t, tidbErr)
+		assert.Contains(t, tidbErr.Error(), errString)
+	}
+}

tests/integrationtest/r/executor/ddl.result

@@ -379,7 +379,10 @@ create or replace definer='root'@'localhost' view v_nested as select * from v_ne
 Error 1146 (42S02): Table 'executor__ddl.v_nested' doesn't exist
 drop table test_v_nested;
 drop view v_nested, v_nested2;
-create view v_stale as select * from source_table as of timestamp current_timestamp(3);
+select sleep(1);
+sleep(1)
+0
+create view v_stale as select * from source_table as of timestamp date_sub(current_timestamp(3), interval 1 second);
 Error 1356 (HY000): View 'executor__ddl.v_stale' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them
 drop view if exists v1,v2;
 drop table if exists t1;

tests/integrationtest/r/globalindex/insert.result

@@ -6,7 +6,7 @@ a	b
 1	3
 alter table t add unique index idx1(b) global;
 insert into t values (2, 4), (3, 4) on duplicate key update a=2, b=5;
-select * from t use index (idx1);
+select * from t use index (idx1) order by a desc;
 a	b
 2	5
 1	3

tests/integrationtest/t/executor/ddl.test

@@ -313,8 +313,9 @@ create or replace definer='root'@'localhost' view v_nested as select * from v_ne
 drop table test_v_nested;
 drop view v_nested, v_nested2;
 ## Refer https://github.com/pingcap/tidb/issues/25876
+select sleep(1);
 -- error 1356
-create view v_stale as select * from source_table as of timestamp current_timestamp(3);
+create view v_stale as select * from source_table as of timestamp date_sub(current_timestamp(3), interval 1 second);
 ## Refer https://github.com/pingcap/tidb/issues/32682
 drop view if exists v1,v2;
 drop table if exists t1;

tests/integrationtest/t/globalindex/insert.test

@@ -5,6 +5,6 @@ select * from t use index (idx);
 
 alter table t add unique index idx1(b) global;
 insert into t values (2, 4), (3, 4) on duplicate key update a=2, b=5;
-select * from t use index (idx1);
+select * from t use index (idx1) order by a desc;