From 6722c39cdebcc158180816cc3e3e25d6a0cb7ea9 Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Mon, 16 Dec 2024 15:12:24 +0800 Subject: [PATCH] *: bump client-go with some regression tests (#57282) (#57764) ref pingcap/tidb#55287, close pingcap/tidb#56837, ref pingcap/tidb#57338, close pingcap/tidb#57425 --- DEPS.bzl | 4 +- executor/adapter.go | 2 +- executor/union_scan_test.go | 15 ++++++ go.mod | 2 +- go.sum | 4 +- pkg/sessionctx/BUILD.bazel | 0 session/sessiontest/session_test.go | 71 +++++++++++++++++++++++++++++ 7 files changed, 92 insertions(+), 6 deletions(-) delete mode 100644 pkg/sessionctx/BUILD.bazel diff --git a/DEPS.bzl b/DEPS.bzl index 4d9b0f44834f3..a50b3d1a4a604 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -4179,8 +4179,8 @@ def go_deps(): name = "com_github_tikv_client_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sum = "h1:DeZMstuDPx80CNsJdmN47biH+8PL5qK4TmTyNcDRDdI=", - version = "v2.0.8-0.20241111142004-e10335846244", + sum = "h1:E+JsLmxpa4rsT0rS9m52+FhiiRbQxZYn079T4JD12jU=", + version = "v2.0.8-0.20241204085508-80a6b021f0f6", ) go_repository( name = "com_github_tikv_pd", diff --git a/executor/adapter.go b/executor/adapter.go index 94301d4467e6a..5886f9355708f 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -590,7 +590,7 @@ func (a *ExecStmt) handleStmtForeignKeyTrigger(ctx context.Context, e Executor) if stmtCtx.ForeignKeyTriggerCtx.HasFKCascades { // If the ExecStmt has foreign key cascade to be executed, we need call `StmtCommit` to commit the ExecStmt itself // change first. - // Since `UnionScanExec` use `SnapshotIter` and `SnapshotGetter` to read txn mem-buffer, if we don't do `StmtCommit`, + // Since `UnionScanExec` use `SnapshotIter` and `SnapshotGetter` to read txn mem-buffer, if we don't do `StmtCommit`, // then the fk cascade executor can't read the mem-buffer changed by the ExecStmt. a.Ctx.StmtCommit(ctx) } diff --git a/executor/union_scan_test.go b/executor/union_scan_test.go index 26db22e22c897..2b2062bcb4530 100644 --- a/executor/union_scan_test.go +++ b/executor/union_scan_test.go @@ -595,6 +595,21 @@ func TestIssue36903(t *testing.T) { tk.MustQuery("select pkey from t_vwvgdc where 0 <> 0 union select pkey from t_vwvgdc;").Sort().Check(testkit.Rows("15000", "228000")) } +func TestSnapshotWithConcurrentWrite(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t1 (id int auto_increment key, b int, index(b));") + + tk.MustExec("begin") + tk.MustExec("insert into t1 (b) values (1),(2),(3),(4),(5),(6),(7),(8);") + for j := 0; j < 16; j++ { + tk.MustExec("insert into t1 (b) select /*+ use_index(t1, b) */ id from t1;") + } + tk.MustQuery("select count(1) from t1").Check(testkit.Rows("524288")) // 8 * 2^16 rows + tk.MustExec("rollback") +} + func BenchmarkUnionScanRead(b *testing.B) { store := testkit.CreateMockStore(b) diff --git a/go.mod b/go.mod index 3f5f0517fbe83..09cf4365e3f43 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/stretchr/testify v1.8.2 github.com/tdakkota/asciicheck v0.2.0 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/client-go/v2 v2.0.8-0.20241111142004-e10335846244 + github.com/tikv/client-go/v2 v2.0.8-0.20241204085508-80a6b021f0f6 github.com/tikv/pd/client v0.0.0-20240725070735-fb162bf0aa3f github.com/timakin/bodyclose v0.0.0-20221125081123-e39cf3fc478e github.com/twmb/murmur3 v1.1.6 diff --git a/go.sum b/go.sum index d3d085b284161..a581bb88152cb 100644 --- a/go.sum +++ b/go.sum @@ -968,8 +968,8 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= -github.com/tikv/client-go/v2 v2.0.8-0.20241111142004-e10335846244 h1:DeZMstuDPx80CNsJdmN47biH+8PL5qK4TmTyNcDRDdI= -github.com/tikv/client-go/v2 v2.0.8-0.20241111142004-e10335846244/go.mod h1:45NuHB8x+VAoztMIjF6hEgXvPQXhXWPfMxDg0N8CoRY= +github.com/tikv/client-go/v2 v2.0.8-0.20241204085508-80a6b021f0f6 h1:E+JsLmxpa4rsT0rS9m52+FhiiRbQxZYn079T4JD12jU= +github.com/tikv/client-go/v2 v2.0.8-0.20241204085508-80a6b021f0f6/go.mod h1:45NuHB8x+VAoztMIjF6hEgXvPQXhXWPfMxDg0N8CoRY= github.com/tikv/pd/client v0.0.0-20240725070735-fb162bf0aa3f h1:Szw9YxqGGEneSniBd4ep09jgB77cKUy+AuhKOmdGPdE= github.com/tikv/pd/client v0.0.0-20240725070735-fb162bf0aa3f/go.mod h1:QCBn54O5lhfkYfxj8Tyiqaxue/mthHEMyi7AqJP/+n4= github.com/timakin/bodyclose v0.0.0-20221125081123-e39cf3fc478e h1:MV6KaVu/hzByHP0UvJ4HcMGE/8a6A4Rggc/0wx2AvJo= diff --git a/pkg/sessionctx/BUILD.bazel b/pkg/sessionctx/BUILD.bazel deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/session/sessiontest/session_test.go b/session/sessiontest/session_test.go index 65bfd2e7de160..25368e5c5136a 100644 --- a/session/sessiontest/session_test.go +++ b/session/sessiontest/session_test.go @@ -3685,3 +3685,74 @@ func TestPrepareExecuteWithSQLHints(t *testing.T) { } } } + +func TestMemBufferSnapshotRead(t *testing.T) { + store := testkit.CreateMockStore(t) + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a int primary key, b int, index i(b));") + + tk.MustExec("set session tidb_distsql_scan_concurrency = 1;") + tk.MustExec("set session tidb_index_lookup_join_concurrency = 1;") + tk.MustExec("set session tidb_projection_concurrency=1;") + tk.MustExec("set session tidb_init_chunk_size=1;") + tk.MustExec("set session tidb_max_chunk_size=40;") + tk.MustExec("set session tidb_index_join_batch_size = 10") + + tk.MustExec("begin;") + // write (0, 0), (1, 1), ... ,(100, 100) into membuffer + var sb strings.Builder + sb.WriteString("insert into t values ") + for i := 0; i <= 100; i++ { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("(%d, %d)", i, i)) + } + tk.MustExec(sb.String()) + + // insert on duplicate key statement should update the table to (0, 100), (1, 99), ... (100, 0) + // This statement will create UnionScan dynamically during execution, and some UnionScan will see staging data(should be bypassed), + // so it relies on correct snapshot read to get the expected result. + tk.MustExec("insert into t (select /*+ INL_JOIN(t1) */ 100 - t1.a as a, t1.b from t t1, (select a, b from t) t2 where t1.b = t2.b) on duplicate key update b = values(b)") + + require.Empty(t, tk.MustQuery("select a, b from t where a + b != 100;").Rows()) + tk.MustExec("commit;") + require.Empty(t, tk.MustQuery("select a, b from t where a + b != 100;").Rows()) + + tk.MustExec("set session tidb_distsql_scan_concurrency = default;") + tk.MustExec("set session tidb_index_lookup_join_concurrency = default;") + tk.MustExec("set session tidb_projection_concurrency=default;") + tk.MustExec("set session tidb_init_chunk_size=default;") + tk.MustExec("set session tidb_max_chunk_size=default;") + tk.MustExec("set session tidb_index_join_batch_size = default") +} + +func TestMemBufferCleanupMemoryLeak(t *testing.T) { + // Test if cleanup memory will cause a memory leak. + // When an in-txn statement fails, TiDB cleans up the mutations from this statement. + // If there's a memory leak, the memory usage could increase uncontrollably with retries. + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t(a varchar(255) primary key)") + key1 := strings.Repeat("a", 255) + key2 := strings.Repeat("b", 255) + tk.MustExec(`set global tidb_mem_oom_action='cancel'`) + tk.MustExec("set session tidb_mem_quota_query=10240") + tk.MustExec("begin") + tk.MustExec("insert into t values(?)", key2) + for i := 0; i < 100; i++ { + // The insert statement will fail because of the duplicate key error. + err := tk.ExecToErr("insert into t values(?), (?)", key1, key2) + require.Error(t, err) + if strings.Contains(err.Error(), "Duplicate") { + continue + } + require.NoError(t, err) + } + tk.MustExec("commit") +}