diff --git a/bindinfo/main_test.go b/bindinfo/main_test.go index 2d358809e8059..ede7172be10a6 100644 --- a/bindinfo/main_test.go +++ b/bindinfo/main_test.go @@ -26,6 +26,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/ddl/index.go b/ddl/index.go index 931580e586af0..4404ea4f551bd 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -1293,7 +1293,10 @@ func (w *baseIndexWorker) getNextKey(taskRange reorgBackfillTask, taskDone bool) recordKey := tablecodec.EncodeRecordKey(w.table.RecordPrefix(), lastHandle) return recordKey.Next() } - return taskRange.endKey.Next() + if taskRange.endInclude { + return taskRange.endKey.Next() + } + return taskRange.endKey } func (w *baseIndexWorker) updateRowDecoder(handle kv.Handle, rawRecord []byte) error { diff --git a/ddl/index_modify_test.go b/ddl/index_modify_test.go index 18ff753aee618..00695b3da8f6e 100644 --- a/ddl/index_modify_test.go +++ b/ddl/index_modify_test.go @@ -1067,3 +1067,17 @@ func TestAddIndexWithDupIndex(t *testing.T) { err = tk.ExecToErr("alter table test_add_index_with_dup add index idx (a)") require.ErrorIs(t, err, errors.Cause(err2)) } + +func TestAddIndexUniqueFailOnDuplicate(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t (a bigint primary key clustered, b int);") + tk.MustExec("set @@global.tidb_ddl_reorg_worker_cnt = 2;") + for i := 1; i <= 12; i++ { + tk.MustExec("insert into t values (?, ?)", i, i) + } + tk.MustExec("insert into t values (0, 1);") // Insert a duplicate key. + tk.MustQuery("split table t by (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11), (12);").Check(testkit.Rows("13 1")) + tk.MustGetErrCode("alter table t add unique index idx (b);", errno.ErrDupEntry) +} diff --git a/ddl/main_test.go b/ddl/main_test.go index 91558d1e44e27..9341be34dfb96 100644 --- a/ddl/main_test.go +++ b/ddl/main_test.go @@ -62,6 +62,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) diff --git a/docs/design/2022-09-22-global-memory-control.md b/docs/design/2022-09-22-global-memory-control.md new file mode 100644 index 0000000000000..7eb3e04b307a2 --- /dev/null +++ b/docs/design/2022-09-22-global-memory-control.md @@ -0,0 +1,66 @@ +# Proposal: Global Memory Control + +* Authors: [wshwsh12](https://github.com/wshwsh12), [Xuhuaiyu](https://github.com/Xuhuaiyu) +* Tracking issue: [#37816](https://github.com/pingcap/tidb/issues/37816) + +## Abstract + +This proposes a design of how to control global memory of TiDB instance. + +## Background + +Currently, TiDB has a query-level memory control strategy `mem-quota-query`, which triggers Cancel when the memory usage of a single SQL exceeds `mem-quota-query`. However, there is currently no global memory control strategy. + +When TiDB has multiple SQLs whose memory usage does not exceed `mem-quota-query` or memory tracking inaccurate, it will lead to high memory usage or even OOM. + +Therefore, we need an observer to check whether the memory usage of the current system is normal. When there are some problems, try to control TiDB's memory no longer continue to grow, to reduce the risk of process crashes. + +## Goal + +- Control the TiDB execution memory within the system variable `tidb_server_memory_limit`. + +## Design + +New system variables: +- `tidb_server_memory_limit`: TiDB maintains the overall memory usage within `tidb_server_memory_limit` +- `tidb_server_memory_gc_trigger`: When TiDB memory usage reaches a certain percentage of `tidb_server_memory_limit`, try to take the initiative to trigger golang GC to release memory +- `tidb_server_memory_limit_sess_min_size`: The minimum memory of a session that can be killed by TiDB + +We need to implement the following three functions to control the memory usage of TiDB: +1. Kill the SQL with the most memory usage in the current system, when `HeapInuse` is larger than `tidb_server_memory_limit`. +2. Take the initiative to trigger `runtime.GC()`, when `HeapInuse` is large than `tidb_server_memory_limit`*`tidb_server_memory_limit_gc_trigger`. +3. Introduce some memory tables to observe the memory status of the current system. + +### Kill the SQL with the max memory usage + +New variables: + +1. Global variable `MemUsageTop1Tracker atomic.Pointer[Tracker]`: Indicates the Tracker with the largest memory usage. +2. The flag `NeedKill atomic.Bool` in the structure `Tracker`: Indicates whether the SQL for the current Tracker needs to be Killed. +3. `SessionID int64` in Structure Tracker: Indicates the Session ID corresponding to the current Tracker. + +Implements: + +#### How to get the current TiDB memory usage Top 1 +When `Tracker.Consume()` calling, check the following logic. If all are satisfied, update the `MemUsageTop1Tracker`. +1. Is it a Session-level Tracker? +2. Whether the flag `NeedKill` is false, to avoid cancel the current SQL twice +3. Whether the memory usage exceeds the threshold `tidb_server_memory_limit_sess_min_size`(default 128MB, can be dynamically adjusted), can be candidate of the `MemUsageTop1Tracker` +4. Is the memory usage of the current Tracker greater than the current `MemUsageTop1Tracker` + +#### How to Cancel the current top 1 memory usage and recycle memory in time +1. Create a goroutine that calls Golang's `ReadMemStat` interface in a 100 ms cycle. (Get the memory usage of the current TiDB instance) +2. If the `heapInuse` of the current instance is greater than `tidb_server_memory_limit`, set `MemUsageTop1Tracker`'s `NeedKill` flag. (Sends a Kill signal) +3. When the SQL call to `Tracker.Consume()`, check its own `NeedKill` flag. If it is true, trigger Panic and exit. (terminates the execution of SQL) +4. Get the `SessionID` from the tracker and continuously query its status, waiting for it to complete exited. When SQL successfully exited, explicitly trigger Golang GC to release memory. (Wait for SQL exited completely and release memory) + +### Take the initiative to trigger GC + +The inspiration for this design comes from uber-go-gc-tuner: +1. Use the Go1.19 `SetMemoryLimit` feature to set the soft limit to `tidb_server_memory_limit` * `tidb_server_memory_limit_gc_trigger` to ensure that GC can be triggered when reaching the certain threshold. +2. After each GC, check whether this GC is caused by memory limit. If it is caused by this, temporarily set memory limit to infinite, and then set it back to the specified threshold after 1 minute. In this way, the problem of frequent GC caused by `heapInUse` being larger than the soft limit can be avoided. + +### Introduce some memory tables + +Introduce `performance_schema.memory_usage` and `performance_schema.memory_usage_ops_history` to display the current system memory usage and historical operations. +This can be implemented by maintaining a set of global data, and reading and outputting directly from the global data when querying. diff --git a/executor/fktest/foreign_key_test.go b/executor/fktest/foreign_key_test.go index 79b36ff7f586f..64908a77b271a 100644 --- a/executor/fktest/foreign_key_test.go +++ b/executor/fktest/foreign_key_test.go @@ -486,14 +486,22 @@ func TestForeignKeyOnInsertIgnore(t *testing.T) { tk.MustExec("set @@global.tidb_enable_foreign_key=1") tk.MustExec("set @@foreign_key_checks=1") tk.MustExec("use test") - + // Test for foreign key index is primary key. tk.MustExec("CREATE TABLE t1 (i INT PRIMARY KEY);") tk.MustExec("CREATE TABLE t2 (i INT, FOREIGN KEY (i) REFERENCES t1 (i));") tk.MustExec("INSERT INTO t1 VALUES (1),(3);") - tk.MustExec("INSERT IGNORE INTO t2 VALUES (1),(2),(3),(4);") + tk.MustExec("INSERT IGNORE INTO t2 VALUES (1), (null), (1), (2),(3),(4);") warning := "Warning 1452 Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `fk_1` FOREIGN KEY (`i`) REFERENCES `t1` (`i`))" tk.MustQuery("show warnings;").Check(testkit.Rows(warning, warning)) - tk.MustQuery("select * from t2").Check(testkit.Rows("1", "3")) + tk.MustQuery("select * from t2 order by i").Check(testkit.Rows("", "1", "1", "3")) + // Test for foreign key index is non-unique key. + tk.MustExec("drop table t1,t2") + tk.MustExec("CREATE TABLE t1 (i INT, index(i));") + tk.MustExec("CREATE TABLE t2 (i INT, FOREIGN KEY (i) REFERENCES t1 (i));") + tk.MustExec("INSERT INTO t1 VALUES (1),(3);") + tk.MustExec("INSERT IGNORE INTO t2 VALUES (1), (null), (1), (2), (3), (2);") + tk.MustQuery("show warnings;").Check(testkit.Rows(warning, warning)) + tk.MustQuery("select * from t2 order by i").Check(testkit.Rows("", "1", "1", "3")) } func TestForeignKeyOnInsertOnDuplicateParentTableCheck(t *testing.T) { diff --git a/executor/foreign_key.go b/executor/foreign_key.go index f8bc4569ee3e7..32c27961023d8 100644 --- a/executor/foreign_key.go +++ b/executor/foreign_key.go @@ -533,8 +533,9 @@ func (fkc FKCheckExec) checkRows(ctx context.Context, sc *stmtctx.StatementConte rows[i].ignored = true sc.AppendWarning(fkc.FailedErr) fkc.checkRowsCache[string(k)] = true + } else { + fkc.checkRowsCache[string(k)] = false } - fkc.checkRowsCache[string(k)] = false if fkc.stats != nil { fkc.stats.Keys++ } diff --git a/executor/main_test.go b/executor/main_test.go index f418387a5ed25..0c951915222cf 100644 --- a/executor/main_test.go +++ b/executor/main_test.go @@ -59,6 +59,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun"), goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } callback := func(i int) int { testDataMap.GenerateOutputIfNeeded() diff --git a/expression/integration_test.go b/expression/integration_test.go index 7f70b994b4f52..5a9c0cf3454b5 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -7768,6 +7768,23 @@ func TestJSONStorageFree(t *testing.T) { require.Error(t, err, "[json:3140]Invalid JSON text: The document root must not be followed by other values.") } +func TestIssue38736(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("CREATE TABLE t0(c0 BOOL, c1 INT);") + tk.MustExec("CREATE TABLE t1 LIKE t0;") + tk.MustExec("CREATE definer='root'@'localhost' VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;") + tk.MustExec("INSERT INTO t0(c0, c1) VALUES (true, 0);") + tk.MustExec("INSERT INTO t1(c0, c1) VALUES (true, 2);") + + // The filter is evaled as false. + tk.MustQuery("SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);").Check(testkit.Rows()) + + // Also the filter is evaled as false. + tk.MustQuery("SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0) or v0.c0 > 0").Check(testkit.Rows()) +} + func TestJSONExtractFromLast(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/expression/main_test.go b/expression/main_test.go index 55af0163dc0fd..16fdf0574eb75 100644 --- a/expression/main_test.go +++ b/expression/main_test.go @@ -54,6 +54,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } callback := func(i int) int { diff --git a/expression/util.go b/expression/util.go index 72f512038f162..22637d1b8617f 100644 --- a/expression/util.go +++ b/expression/util.go @@ -58,7 +58,7 @@ func (c *cowExprRef) Set(i int, changed bool, val Expression) { return } c.new = make([]Expression, len(c.ref)) - copy(c.new, c.ref[:i]) + copy(c.new, c.ref) c.new[i] = val } @@ -382,17 +382,6 @@ func SetExprColumnInOperand(expr Expression) Expression { return expr } -// ColumnSubstitute4PPD substitutes the columns in filter to expressions in select fields. -// Only used for predicate push down to projection. Some columns can not be substituted for some reasons. -// So we should return the bool value to indicate some expressions can not be pushed down. -// e.g. CREATE TABLE t3(c0 INT, primary key(c0)); -// SELECT v2.c0 FROM (select 1 as c0 from t3) v2 WHERE (v2.c0)like(True); -// The cond `(v2.c0)like(True)` can not be substituted when the new collation enable. So we shouldn't push the cond down to the projection. -func ColumnSubstitute4PPD(expr Expression, schema *Schema, newExprs []Expression) (bool, Expression) { - substituted, _, resExpr := ColumnSubstituteImpl(expr, schema, newExprs, false) - return substituted, resExpr -} - // ColumnSubstitute substitutes the columns in filter to expressions in select fields. // e.g. select * from (select b as a from t) k where a < 10 => select * from (select b as a from t where b < 10) k. func ColumnSubstitute(expr Expression, schema *Schema, newExprs []Expression) Expression { @@ -432,41 +421,43 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression substituted := false hasFail := false if v.FuncName.L == ast.Cast { - newFunc := v.Clone().(*ScalarFunction) - substituted, hasFail, newFunc.GetArgs()[0] = ColumnSubstituteImpl(newFunc.GetArgs()[0], schema, newExprs, fail1Return) + var newArg Expression + substituted, hasFail, newArg = ColumnSubstituteImpl(v.GetArgs()[0], schema, newExprs, fail1Return) if fail1Return && hasFail { - return substituted, hasFail, newFunc + return substituted, hasFail, v } if substituted { - // Workaround for issue https://github.com/pingcap/tidb/issues/28804 - e := NewFunctionInternal(v.GetCtx(), v.FuncName.L, v.RetType, newFunc.GetArgs()...) + e := BuildCastFunction(v.GetCtx(), newArg, v.RetType) e.SetCoercibility(v.Coercibility()) return true, false, e } - return false, false, newFunc + return false, false, v } // cowExprRef is a copy-on-write util, args array allocation happens only // when expr in args is changed refExprArr := cowExprRef{v.GetArgs(), nil} _, coll := DeriveCollationFromExprs(v.GetCtx(), v.GetArgs()...) + var tmpArgForCollCheck []Expression + if collate.NewCollationEnabled() { + tmpArgForCollCheck = make([]Expression, len(v.GetArgs())) + } for idx, arg := range v.GetArgs() { - changed, hasFail, newFuncExpr := ColumnSubstituteImpl(arg, schema, newExprs, fail1Return) - if fail1Return && hasFail { - return changed, hasFail, v + changed, failed, newFuncExpr := ColumnSubstituteImpl(arg, schema, newExprs, fail1Return) + if fail1Return && failed { + return changed, failed, v } oldChanged := changed - if collate.NewCollationEnabled() { + if collate.NewCollationEnabled() && changed { // Make sure the collation used by the ScalarFunction isn't changed and its result collation is not weaker than the collation used by the ScalarFunction. - if changed { - changed = false - tmpArgs := make([]Expression, 0, len(v.GetArgs())) - _ = append(append(append(tmpArgs, refExprArr.Result()[0:idx]...), refExprArr.Result()[idx+1:]...), newFuncExpr) - _, newColl := DeriveCollationFromExprs(v.GetCtx(), append(v.GetArgs(), newFuncExpr)...) - if coll == newColl { - changed = checkCollationStrictness(coll, newFuncExpr.GetType().GetCollate()) - } + changed = false + copy(tmpArgForCollCheck, refExprArr.Result()) + tmpArgForCollCheck[idx] = newFuncExpr + _, newColl := DeriveCollationFromExprs(v.GetCtx(), tmpArgForCollCheck...) + if coll == newColl { + changed = checkCollationStrictness(coll, newFuncExpr.GetType().GetCollate()) } } + hasFail = hasFail || failed || oldChanged != changed if fail1Return && oldChanged != changed { // Only when the oldChanged is true and changed is false, we will get here. // And this means there some dependency in this arg can be substituted with @@ -481,7 +472,7 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression } } if substituted { - return true, false, NewFunctionInternal(v.GetCtx(), v.FuncName.L, v.RetType, refExprArr.Result()...) + return true, hasFail, NewFunctionInternal(v.GetCtx(), v.FuncName.L, v.RetType, refExprArr.Result()...) } } return false, false, expr diff --git a/extension/main_test.go b/extension/main_test.go index 25fa00793b5ac..82f6f903456ef 100644 --- a/extension/main_test.go +++ b/extension/main_test.go @@ -26,6 +26,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/planner/cascades/main_test.go b/planner/cascades/main_test.go index 5ebbbc50eebfb..c135838a2a1fb 100644 --- a/planner/cascades/main_test.go +++ b/planner/cascades/main_test.go @@ -49,6 +49,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } if err := goleak.Find(opts...); err != nil { diff --git a/planner/cascades/transformation_rules.go b/planner/cascades/transformation_rules.go index 0991379df468b..3dd85fbaa5120 100644 --- a/planner/cascades/transformation_rules.go +++ b/planner/cascades/transformation_rules.go @@ -550,8 +550,8 @@ func (*PushSelDownProjection) OnTransform(old *memo.ExprIter) (newExprs []*memo. canBePushed := make([]expression.Expression, 0, len(sel.Conditions)) canNotBePushed := make([]expression.Expression, 0, len(sel.Conditions)) for _, cond := range sel.Conditions { - substituted, newFilter := expression.ColumnSubstitute4PPD(cond, projSchema, proj.Exprs) - if substituted && !expression.HasGetSetVarFunc(newFilter) { + substituted, hasFailed, newFilter := expression.ColumnSubstituteImpl(cond, projSchema, proj.Exprs, true) + if substituted && !hasFailed && !expression.HasGetSetVarFunc(newFilter) { canBePushed = append(canBePushed, newFilter) } else { canNotBePushed = append(canNotBePushed, cond) diff --git a/planner/core/main_test.go b/planner/core/main_test.go index 5818baba86199..a6c646923cb30 100644 --- a/planner/core/main_test.go +++ b/planner/core/main_test.go @@ -59,6 +59,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun"), goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } callback := func(i int) int { diff --git a/planner/core/rule_predicate_push_down.go b/planner/core/rule_predicate_push_down.go index bebed1cab141a..94f92f780202a 100644 --- a/planner/core/rule_predicate_push_down.go +++ b/planner/core/rule_predicate_push_down.go @@ -474,8 +474,8 @@ func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression } } for _, cond := range predicates { - substituted, newFilter := expression.ColumnSubstitute4PPD(cond, p.Schema(), p.Exprs) - if substituted && !expression.HasGetSetVarFunc(newFilter) { + substituted, hasFailed, newFilter := expression.ColumnSubstituteImpl(cond, p.Schema(), p.Exprs, true) + if substituted && !hasFailed && !expression.HasGetSetVarFunc(newFilter) { canBePushed = append(canBePushed, newFilter) } else { canNotBePushed = append(canNotBePushed, cond) diff --git a/planner/funcdep/main_test.go b/planner/funcdep/main_test.go index e1e8cd745f387..4387fad57e5b8 100644 --- a/planner/funcdep/main_test.go +++ b/planner/funcdep/main_test.go @@ -26,6 +26,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/planner/memo/main_test.go b/planner/memo/main_test.go index 0fcbd9568d818..bb84aa0af9800 100644 --- a/planner/memo/main_test.go +++ b/planner/memo/main_test.go @@ -26,6 +26,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/session/advisory_locks.go b/session/advisory_locks.go index f51bb061a119c..aff3d80ed7e88 100644 --- a/session/advisory_locks.go +++ b/session/advisory_locks.go @@ -50,7 +50,7 @@ func (a *advisoryLock) DecrReferences() { a.referenceCount-- } -// References returns the current reference count for the advisory lock. +// ReferenceCount returns the current reference count for the advisory lock. func (a *advisoryLock) ReferenceCount() int { return a.referenceCount } diff --git a/session/session.go b/session/session.go index 1d3e8c17753e6..a0b19481c77c2 100644 --- a/session/session.go +++ b/session/session.go @@ -46,6 +46,7 @@ import ( "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/domain/infosync" "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/expression" @@ -1815,10 +1816,11 @@ func (s *session) GetAdvisoryLock(lockName string, timeout int64) error { lock.IncrReferences() return nil } - sess, err := createSession(s.GetStore()) + sess, err := createSession(s.store) if err != nil { return err } + infosync.StoreInternalSession(sess) lock := &advisoryLock{session: sess, ctx: context.TODO()} err = lock.GetLock(lockName, timeout) if err != nil { @@ -1840,6 +1842,7 @@ func (s *session) ReleaseAdvisoryLock(lockName string) (released bool) { if lock.ReferenceCount() <= 0 { lock.Close() delete(s.advisoryLocks, lockName) + infosync.DeleteInternalSession(lock.session) } return true } @@ -1856,6 +1859,7 @@ func (s *session) ReleaseAllAdvisoryLocks() int { lock.Close() count += lock.ReferenceCount() delete(s.advisoryLocks, lockName) + infosync.DeleteInternalSession(lock.session) } return count } @@ -3024,6 +3028,10 @@ func createSessions(store kv.Storage, cnt int) ([]*session, error) { return ses, nil } +// createSession creates a new session. +// Please note that such a session is not tracked by the internal session list. +// This means the min ts reporter is not aware of it and may report a wrong min start ts. +// In most cases you should use a session pool in domain instead. func createSession(store kv.Storage) (*session, error) { return createSessionWithOpt(store, nil) } diff --git a/sessionctx/variable/main_test.go b/sessionctx/variable/main_test.go index f3edec39107f0..977294bdc70d9 100644 --- a/sessionctx/variable/main_test.go +++ b/sessionctx/variable/main_test.go @@ -26,6 +26,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/statistics/handle/main_test.go b/statistics/handle/main_test.go index b346b095e13cf..b6bf59c2e27c3 100644 --- a/statistics/handle/main_test.go +++ b/statistics/handle/main_test.go @@ -25,6 +25,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } testsetup.SetupForCommonTest() goleak.VerifyTestMain(m, opts...) diff --git a/store/helper/main_test.go b/store/helper/main_test.go index e459f147efedb..9c95a1fdbd9cf 100644 --- a/store/helper/main_test.go +++ b/store/helper/main_test.go @@ -26,6 +26,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/table/tables/main_test.go b/table/tables/main_test.go index f707c7babbf74..63a616eeb69b0 100644 --- a/table/tables/main_test.go +++ b/table/tables/main_test.go @@ -26,6 +26,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/telemetry/main_test.go b/telemetry/main_test.go index 6d4b5d2c5aaf8..698dd58e98635 100644 --- a/telemetry/main_test.go +++ b/telemetry/main_test.go @@ -39,6 +39,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) diff --git a/tidb-server/main_test.go b/tidb-server/main_test.go index a6a0003aedc65..5242f804f4378 100644 --- a/tidb-server/main_test.go +++ b/tidb-server/main_test.go @@ -34,6 +34,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/tools/check/ut.go b/tools/check/ut.go index f957f3ea75e94..08b31d243a157 100644 --- a/tools/check/ut.go +++ b/tools/check/ut.go @@ -906,9 +906,12 @@ func listNewTestCases(pkg string) ([]string, error) { // session.test -test.list Test cmd := exec.Command(exe, "-test.list", "Test") cmd.Dir = path.Join(workDir, pkg) - res, err := cmdToLines(cmd) - if err != nil { - return nil, withTrace(err) + var buf bytes.Buffer + cmd.Stdout = &buf + err := cmd.Run() + res := strings.Split(buf.String(), "\n") + if err != nil && len(res) == 0 { + fmt.Println("err ==", err) } return filter(res, func(s string) bool { return strings.HasPrefix(s, "Test") && s != "TestT" && s != "TestBenchDaily" diff --git a/types/main_test.go b/types/main_test.go index 81069e118ea61..af337f93a762d 100644 --- a/types/main_test.go +++ b/types/main_test.go @@ -26,6 +26,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/util/chunk/iterator.go b/util/chunk/iterator.go index 63aabde256c5d..74d78b9cb62ea 100644 --- a/util/chunk/iterator.go +++ b/util/chunk/iterator.go @@ -14,6 +14,8 @@ package chunk +import "golang.org/x/sys/cpu" + var ( _ Iterator = (*Iterator4Chunk)(nil) _ Iterator = (*iterator4RowPtr)(nil) @@ -58,8 +60,10 @@ func NewIterator4Slice(rows []Row) Iterator { // Iterator4Slice is used to iterate rows inside a slice. type Iterator4Slice struct { + _ cpu.CacheLinePad rows []Row cursor int + _ cpu.CacheLinePad } // Begin implements the Iterator interface. diff --git a/util/ranger/main_test.go b/util/ranger/main_test.go index 92ffa6f422fc1..7e4131603ab90 100644 --- a/util/ranger/main_test.go +++ b/util/ranger/main_test.go @@ -45,6 +45,7 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } if err := goleak.Find(opts...); err != nil { diff --git a/util/servermemorylimit/servermemorylimit.go b/util/servermemorylimit/servermemorylimit.go index 4a67f1fa2a70e..a13e2e8b0e081 100644 --- a/util/servermemorylimit/servermemorylimit.go +++ b/util/servermemorylimit/servermemorylimit.go @@ -77,9 +77,10 @@ func (smqh *Handle) Run() { } type sessionToBeKilled struct { - isKilling bool - sqlStartTime time.Time - sessionID uint64 + isKilling bool + sqlStartTime time.Time + sessionID uint64 + sessionTracker *memory.Tracker } func killSessIfNeeded(s *sessionToBeKilled, bt uint64, sm util.SessionManager) { @@ -91,6 +92,7 @@ func killSessIfNeeded(s *sessionToBeKilled, bt uint64, sm util.SessionManager) { } s.isKilling = false IsKilling.Store(false) + memory.MemUsageTop1Tracker.CompareAndSwap(s.sessionTracker, nil) //nolint: all_revive,revive runtime.GC() } @@ -109,6 +111,7 @@ func killSessIfNeeded(s *sessionToBeKilled, bt uint64, sm util.SessionManager) { s.sessionID = t.SessionID s.sqlStartTime = info.Time s.isKilling = true + s.sessionTracker = t t.NeedKill.Store(true) killTime := time.Now()