infoschema, util: add table events_statements_summary_by_digest_history (#13813)

config/config.go

@@ -323,10 +323,16 @@ type PessimisticTxn struct {
 
 // StmtSummary is the config for statement summary.
 type StmtSummary struct {
+	// Enable statement summary or not.
+	Enable bool `toml:"enable" json:"enable"`
 	// The maximum number of statements kept in memory.
 	MaxStmtCount uint `toml:"max-stmt-count" json:"max-stmt-count"`
 	// The maximum length of displayed normalized SQL and sample SQL.
 	MaxSQLLength uint `toml:"max-sql-length" json:"max-sql-length"`
+	// The refresh interval of statement summary.
+	RefreshInterval int `toml:"refresh-interval" json:"refresh-interval"`
+	// The maximum history size of statement summary.
+	HistorySize int `toml:"history-size" json:"history-size"`
 }
 
 var defaultConf = Config{
@@ -428,8 +434,11 @@ var defaultConf = Config{
 		MaxRetryCount: 256,
 	},
 	StmtSummary: StmtSummary{
-		MaxStmtCount: 100,
-		MaxSQLLength: 4096,
+		Enable:          false,
+		MaxStmtCount:    100,
+		MaxSQLLength:    4096,
+		RefreshInterval: 1800,
+		HistorySize:     24,
 	},
 }
 
@@ -603,6 +612,13 @@ func (c *Config) Valid() error {
 	if c.TiKVClient.MaxTxnTimeUse == 0 {
 		return fmt.Errorf("max-txn-time-use should be greater than 0")
 	}
+
+	if c.StmtSummary.HistorySize < 0 {
+		return fmt.Errorf("history-size in [stmt-summary] should be greater than or equal to 0")
+	}
+	if c.StmtSummary.RefreshInterval <= 0 {
+		return fmt.Errorf("refresh-interval in [stmt-summary] should be greater than 0")
+	}
 	return nil
 }
 

config/config.toml.example

@@ -339,8 +339,17 @@ enable = true
 max-retry-count = 256
 
 [stmt-summary]
+# enable statement summary.
+enable = false
+
 # max number of statements kept in memory.
 max-stmt-count = 100
 
 # max length of displayed normalized sql and sample sql.
 max-sql-length = 4096
+
+# the refresh interval of statement summary, it's counted in seconds.
+refresh-interval = 1800
+
+# the maximum history size of statement summary.
+history-size = 24

config/config_test.go

@@ -76,8 +76,11 @@ max-batch-size=128
 region-cache-ttl=6000
 store-limit=0
 [stmt-summary]
+enable=true
 max-stmt-count=1000
 max-sql-length=1024
+refresh-interval=100
+history-size=100
 `)
 
 	c.Assert(err, IsNil)
@@ -104,8 +107,11 @@ max-sql-length=1024
 	c.Assert(conf.TiKVClient.StoreLimit, Equals, int64(0))
 	c.Assert(conf.TokenLimit, Equals, uint(1000))
 	c.Assert(conf.SplitRegionMaxNum, Equals, uint64(10000))
+	c.Assert(conf.StmtSummary.Enable, Equals, true)
 	c.Assert(conf.StmtSummary.MaxStmtCount, Equals, uint(1000))
 	c.Assert(conf.StmtSummary.MaxSQLLength, Equals, uint(1024))
+	c.Assert(conf.StmtSummary.RefreshInterval, Equals, 100)
+	c.Assert(conf.StmtSummary.HistorySize, Equals, 100)
 	c.Assert(f.Close(), IsNil)
 	c.Assert(os.Remove(configFile), IsNil)
 

domain/global_vars_cache.go

@@ -70,19 +70,17 @@ func (gvc *GlobalVariableCache) Disable() {
 // checkEnableServerGlobalVar processes variables that acts in server and global level.
 func checkEnableServerGlobalVar(rows []chunk.Row) {
 	for _, row := range rows {
+		sVal := ""
+		if !row.IsNull(1) {
+			sVal = row.GetString(1)
+		}
 		switch row.GetString(0) {
 		case variable.TiDBEnableStmtSummary:
-			sVal := ""
-			if !row.IsNull(1) {
-				sVal = row.GetString(1)
-			}
 			stmtsummary.StmtSummaryByDigestMap.SetEnabled(sVal, false)
 		case variable.TiDBStmtSummaryRefreshInterval:
-			sVal := ""
-			if !row.IsNull(1) {
-				sVal = row.GetString(1)
-			}
 			stmtsummary.StmtSummaryByDigestMap.SetRefreshInterval(sVal, false)
+		case variable.TiDBStmtSummaryHistorySize:
+			stmtsummary.StmtSummaryByDigestMap.SetHistorySize(sVal, false)
 		}
 	}
 }

executor/set.go

@@ -202,6 +202,8 @@ func (e *SetExecutor) setSysVariable(name string, v *expression.VarAssignment) e
 		stmtsummary.StmtSummaryByDigestMap.SetEnabled(valStr, !v.IsGlobal)
 	case variable.TiDBStmtSummaryRefreshInterval:
 		stmtsummary.StmtSummaryByDigestMap.SetRefreshInterval(valStr, !v.IsGlobal)
+	case variable.TiDBStmtSummaryHistorySize:
+		stmtsummary.StmtSummaryByDigestMap.SetHistorySize(valStr, !v.IsGlobal)
 	}
 
 	return nil

infoschema/perfschema/const.go

@@ -37,6 +37,7 @@ var perfSchemaTables = []string{
 	tableStagesHistory,
 	tableStagesHistoryLong,
 	tableEventsStatementsSummaryByDigest,
+	tableEventsStatementsSummaryByDigestHistory,
 	tableTiDBProfileCPU,
 	tableTiDBProfileMemory,
 	tableTiDBProfileMutex,
@@ -444,7 +445,72 @@ const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists events_
 	"LAST_SEEN TIMESTAMP(6) NOT NULL," +
 	"QUERY_SAMPLE_TEXT LONGTEXT DEFAULT NULL);"
 
-// tableTiDBProfileCPU contains the columns name definitions for table events_cpu_profile_graph
+// tableEventsStatementsSummaryByDigestHistory contains the column name definitions for table
+// events_statements_summary_by_digest_history.
+const tableEventsStatementsSummaryByDigestHistory = "CREATE TABLE if not exists events_statements_summary_by_digest_history (" +
+	"SUMMARY_BEGIN_TIME TIMESTAMP(6) NOT NULL," +
+	"STMT_TYPE VARCHAR(64) NOT NULL," +
+	"SCHEMA_NAME VARCHAR(64) DEFAULT NULL," +
+	"DIGEST VARCHAR(64) NOT NULL," +
+	"DIGEST_TEXT LONGTEXT NOT NULL," +
+	"TABLE_NAMES TEXT DEFAULT NULL," +
+	"INDEX_NAMES TEXT DEFAULT NULL," +
+	"SAMPLE_USER VARCHAR(64) DEFAULT NULL," +
+	"EXEC_COUNT BIGINT(20) UNSIGNED NOT NULL," +
+	"SUM_LATENCY BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_LATENCY BIGINT(20) UNSIGNED NOT NULL," +
+	"MIN_LATENCY BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_LATENCY BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_PARSE_LATENCY BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_PARSE_LATENCY BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_COMPILE_LATENCY BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_COMPILE_LATENCY BIGINT(20) UNSIGNED NOT NULL," +
+	"COP_TASK_NUM BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_COP_PROCESS_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_COP_PROCESS_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_COP_PROCESS_ADDRESS VARCHAR(256) DEFAULT NULL," +
+	"AVG_COP_WAIT_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_COP_WAIT_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_COP_WAIT_ADDRESS VARCHAR(256) DEFAULT NULL," +
+	"AVG_PROCESS_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_PROCESS_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_WAIT_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_WAIT_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_BACKOFF_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_BACKOFF_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_TOTAL_KEYS BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_TOTAL_KEYS BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_PROCESSED_KEYS BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_PROCESSED_KEYS BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_PREWRITE_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_PREWRITE_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_COMMIT_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_COMMIT_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_GET_COMMIT_TS_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_GET_COMMIT_TS_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_COMMIT_BACKOFF_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_COMMIT_BACKOFF_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_RESOLVE_LOCK_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_RESOLVE_LOCK_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_LOCAL_LATCH_WAIT_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_LOCAL_LATCH_WAIT_TIME BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_WRITE_KEYS DOUBLE UNSIGNED NOT NULL," +
+	"MAX_WRITE_KEYS BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_WRITE_SIZE DOUBLE NOT NULL," +
+	"MAX_WRITE_SIZE BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_PREWRITE_REGIONS DOUBLE NOT NULL," +
+	"MAX_PREWRITE_REGIONS INT(11) UNSIGNED NOT NULL," +
+	"AVG_TXN_RETRY DOUBLE NOT NULL," +
+	"MAX_TXN_RETRY INT(11) UNSIGNED NOT NULL," +
+	"BACKOFF_TYPES VARCHAR(1024) DEFAULT NULL," +
+	"AVG_MEM BIGINT(20) UNSIGNED NOT NULL," +
+	"MAX_MEM BIGINT(20) UNSIGNED NOT NULL," +
+	"AVG_AFFECTED_ROWS DOUBLE UNSIGNED NOT NULL," +
+	"FIRST_SEEN TIMESTAMP(6) NOT NULL," +
+	"LAST_SEEN TIMESTAMP(6) NOT NULL," +
+	"QUERY_SAMPLE_TEXT LONGTEXT DEFAULT NULL);"
+
+// tableTiDBProfileCPU contains the columns name definitions for table tidb_profile_cpu
 const tableTiDBProfileCPU = "CREATE TABLE IF NOT EXISTS " + tableNameTiDBProfileCPU + " (" +
 	"FUNCTION VARCHAR(512) NOT NULL," +
 	"PERCENT_ABS VARCHAR(8) NOT NULL," +

infoschema/perfschema/tables.go

@@ -26,13 +26,14 @@ import (
 )
 
 const (
-	tableNameEventsStatementsSummaryByDigest = "events_statements_summary_by_digest"
-	tableNameTiDBProfileCPU                  = "tidb_profile_cpu"
-	tableNameTiDBProfileMemory               = "tidb_profile_memory"
-	tableNameTiDBProfileMutex                = "tidb_profile_mutex"
-	tableNameTiDBProfileAllocs               = "tidb_profile_allocs"
-	tableNameTiDBProfileBlock                = "tidb_profile_block"
-	tableNameTiDBProfileGoroutines           = "tidb_profile_goroutines"
+	tableNameEventsStatementsSummaryByDigest        = "events_statements_summary_by_digest"
+	tableNameEventsStatementsSummaryByDigestHistory = "events_statements_summary_by_digest_history"
+	tableNameTiDBProfileCPU                         = "tidb_profile_cpu"
+	tableNameTiDBProfileMemory                      = "tidb_profile_memory"
+	tableNameTiDBProfileMutex                       = "tidb_profile_mutex"
+	tableNameTiDBProfileAllocs                      = "tidb_profile_allocs"
+	tableNameTiDBProfileBlock                       = "tidb_profile_block"
+	tableNameTiDBProfileGoroutines                  = "tidb_profile_goroutines"
 )
 
 // perfSchemaTable stands for the fake table all its data is in the memory.
@@ -96,7 +97,9 @@ func (vt *perfSchemaTable) Meta() *model.TableInfo {
 func (vt *perfSchemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) (fullRows [][]types.Datum, err error) {
 	switch vt.meta.Name.O {
 	case tableNameEventsStatementsSummaryByDigest:
-		fullRows = stmtsummary.StmtSummaryByDigestMap.ToDatum()
+		fullRows = stmtsummary.StmtSummaryByDigestMap.ToCurrentDatum()
+	case tableNameEventsStatementsSummaryByDigestHistory:
+		fullRows = stmtsummary.StmtSummaryByDigestMap.ToHistoryDatum()
 	case tableNameTiDBProfileCPU:
 		fullRows, err = (&profile.Collector{}).ProfileGraph("cpu")
 	case tableNameTiDBProfileMemory:

infoschema/perfschema/tables_test.go

@@ -188,3 +188,48 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) {
 		from performance_schema.events_statements_summary_by_digest`,
 	).Check(testkit.Rows())
 }
+
+// Test events_statements_summary_by_digest_history.
+func (s *testTableSuite) TestStmtSummaryHistoryTable(c *C) {
+	tk := testkit.NewTestKitWithInit(c, s.store)
+
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b varchar(10), key k(a))")
+
+	// Statement summary is disabled by default.
+	tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0"))
+	tk.MustExec("insert into t values(1, 'a')")
+	tk.MustQuery("select * from performance_schema.events_statements_summary_by_digest_history").Check(testkit.Rows())
+
+	tk.MustExec("set global tidb_enable_stmt_summary = 1")
+	tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1"))
+	defer tk.MustExec("set global tidb_enable_stmt_summary = false")
+
+	// Invalidate the cache manually so that tidb_enable_stmt_summary works immediately.
+	s.dom.GetGlobalVarsCache().Disable()
+	// Disable refreshing summary.
+	tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999")
+	tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Check(testkit.Rows("999999999"))
+
+	// Create a new session to test.
+	tk = testkit.NewTestKitWithInit(c, s.store)
+
+	// Test INSERT
+	tk.MustExec("insert into t values(1, 'a')")
+	tk.MustExec("insert into t    values(2, 'b')")
+	tk.MustExec("insert into t VALUES(3, 'c')")
+	tk.MustExec("/**/insert into t values(4, 'd')")
+	tk.MustQuery(`select stmt_type, schema_name, table_names, index_names, exec_count, cop_task_num, avg_total_keys, 
+		max_total_keys, avg_processed_keys, max_processed_keys, avg_write_keys, max_write_keys, avg_prewrite_regions, 
+		max_prewrite_regions, avg_affected_rows, query_sample_text 
+		from performance_schema.events_statements_summary_by_digest_history
+		where digest_text like 'insert into t%'`,
+	).Check(testkit.Rows("insert test test.t <nil> 4 0 0 0 0 0 2 2 1 1 1 /**/insert into t values(4, 'd')"))
+
+	tk.MustExec("set global tidb_stmt_summary_history_size = 0")
+	tk.MustQuery(`select stmt_type, schema_name, table_names, index_names, exec_count, cop_task_num, avg_total_keys, 
+		max_total_keys, avg_processed_keys, max_processed_keys, avg_write_keys, max_write_keys, avg_prewrite_regions, 
+		max_prewrite_regions, avg_affected_rows, query_sample_text 
+		from performance_schema.events_statements_summary_by_digest_history`,
+	).Check(testkit.Rows())
+}

session/session.go

@@ -1721,6 +1721,7 @@ var builtinGlobalVariable = []string{
 	variable.TiDBTxnMode,
 	variable.TiDBEnableStmtSummary,
 	variable.TiDBStmtSummaryRefreshInterval,
+	variable.TiDBStmtSummaryHistorySize,
 	variable.TiDBMaxDeltaSchemaCount,
 	variable.TiDBStoreLimit,
 }

sessionctx/variable/sysvar.go

@@ -714,8 +714,9 @@ var defaultSysVars = []*SysVar{
 	{ScopeSession, TiDBLowResolutionTSO, "0"},
 	{ScopeSession, TiDBExpensiveQueryTimeThreshold, strconv.Itoa(DefTiDBExpensiveQueryTimeThreshold)},
 	{ScopeSession, TiDBAllowRemoveAutoInc, BoolToIntStr(DefTiDBAllowRemoveAutoInc)},
-	{ScopeGlobal | ScopeSession, TiDBEnableStmtSummary, "0"},
-	{ScopeGlobal | ScopeSession, TiDBStmtSummaryRefreshInterval, strconv.Itoa(DefTiDBStmtSummaryRefreshInterval)},
+	{ScopeGlobal | ScopeSession, TiDBEnableStmtSummary, BoolToIntStr(config.GetGlobalConfig().StmtSummary.Enable)},
+	{ScopeGlobal | ScopeSession, TiDBStmtSummaryRefreshInterval, strconv.Itoa(config.GetGlobalConfig().StmtSummary.RefreshInterval)},
+	{ScopeGlobal | ScopeSession, TiDBStmtSummaryHistorySize, strconv.Itoa(config.GetGlobalConfig().StmtSummary.HistorySize)},
 	{ScopeGlobal | ScopeSession, TiDBStoreLimit, strconv.FormatInt(atomic.LoadInt64(&config.GetGlobalConfig().TiKVClient.StoreLimit), 10)},
 }
 

sessionctx/variable/tidb_vars.go

@@ -298,6 +298,9 @@ const (
 	// TiDBStmtSummaryRefreshInterval indicates the refresh interval in seconds for each statement summary.
 	TiDBStmtSummaryRefreshInterval = "tidb_stmt_summary_refresh_interval"
 
+	// TiDBStmtSummaryHistorySize indicates the history size of each statement summary.
+	TiDBStmtSummaryHistorySize = "tidb_stmt_summary_history_size"
+
 	// TiDBStoreLimit indicates the limit of sending request to a store, 0 means without limit.
 	TiDBStoreLimit = "tidb_store_limit"
 )
@@ -365,8 +368,7 @@ const (
 	DefTiDBExpensiveQueryTimeThreshold = 60  // 60s
 	DefWaitSplitRegionTimeout          = 300 // 300s
 	DefTiDBAllowRemoveAutoInc          = false
-	DefTiDBStmtSummaryRefreshInterval  = 1800 // 1800s
-	DefInnodbLockWaitTimeout           = 50   // 50s
+	DefInnodbLockWaitTimeout           = 50 // 50s
 	DefTiDBStoreLimit                  = 0
 )
 

sessionctx/variable/varsutil.go

@@ -616,6 +616,11 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string,
 			return "", nil
 		}
 		return checkUInt64SystemVar(name, value, 1, math.MaxUint32, vars)
+	case TiDBStmtSummaryHistorySize:
+		if value == "" {
+			return "", nil
+		}
+		return checkUInt64SystemVar(name, value, 0, math.MaxUint8, vars)
 	}
 	return value, nil
 }

sessionctx/variable/varsutil_test.go

@@ -304,6 +304,11 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) {
 	val, err = GetSessionSystemVar(v, TiDBStmtSummaryRefreshInterval)
 	c.Assert(err, IsNil)
 	c.Assert(val, Equals, "10")
+
+	SetSessionSystemVar(v, TiDBStmtSummaryHistorySize, types.NewStringDatum("10"))
+	val, err = GetSessionSystemVar(v, TiDBStmtSummaryHistorySize)
+	c.Assert(err, IsNil)
+	c.Assert(val, Equals, "10")
 }
 
 func (s *testVarsutilSuite) TestValidate(c *C) {
@@ -363,6 +368,8 @@ func (s *testVarsutilSuite) TestValidate(c *C) {
 		{TiDBEnableStmtSummary, "", false},
 		{TiDBStmtSummaryRefreshInterval, "a", true},
 		{TiDBStmtSummaryRefreshInterval, "", false},
+		{TiDBStmtSummaryHistorySize, "a", true},
+		{TiDBStmtSummaryHistorySize, "", false},
 	}
 
 	for _, t := range tests {