From 1664104e8d4551632a2d9e012f4815eefa163b95 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 30 Apr 2019 15:50:23 +0800 Subject: [PATCH 1/9] infoschema/slow_query: fix token too long --- config/config.go | 2 ++ executor/set_test.go | 9 +++++++++ infoschema/slow_log.go | 12 +++++++++++- infoschema/slow_log_test.go | 21 +++++++++++++++++++++ sessionctx/variable/session.go | 3 +++ 5 files changed, 46 insertions(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index cada69fe3edbd..6bacf4247c773 100644 --- a/config/config.go +++ b/config/config.go @@ -48,6 +48,8 @@ var ( CheckTableBeforeDrop = false // checkBeforeDropLDFlag is a go build flag. checkBeforeDropLDFlag = "None" + // QueryLogMaxLenRecord use to record the max value of QueryLogMaxLen. + QueryLogMaxLenRecord = uint64(logutil.DefaultQueryLogMaxLen) ) // Config contains configuration options. diff --git a/executor/set_test.go b/executor/set_test.go index 0a0796c1eb61f..4e8a004e3611d 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -15,12 +15,15 @@ package executor_test import ( "context" + "fmt" + "sync/atomic" . "github.com/pingcap/check" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testutil" ) @@ -272,8 +275,14 @@ func (s *testSuite2) TestSetVar(c *C) { tk.MustExec("set tidb_query_log_max_len = 0") tk.MustQuery("select @@session.tidb_query_log_max_len;").Check(testkit.Rows("0")) + c.Assert(atomic.LoadUint64(&config.QueryLogMaxLenRecord), Equals, uint64(logutil.DefaultQueryLogMaxLen)) + tk.MustExec(fmt.Sprintf("set tidb_query_log_max_len = %v", logutil.DefaultQueryLogMaxLen+1)) + tk.MustQuery("select @@session.tidb_query_log_max_len;").Check(testkit.Rows(fmt.Sprintf("%v", logutil.DefaultQueryLogMaxLen+1))) + c.Assert(atomic.LoadUint64(&config.QueryLogMaxLenRecord), Equals, uint64(logutil.DefaultQueryLogMaxLen+1)) tk.MustExec("set tidb_query_log_max_len = 20") tk.MustQuery("select @@session.tidb_query_log_max_len;").Check(testkit.Rows("20")) + // QueryLogMaxLenRecord record the max len. + c.Assert(atomic.LoadUint64(&config.QueryLogMaxLenRecord), Equals, uint64(logutil.DefaultQueryLogMaxLen+1)) _, err = tk.Exec("set global tidb_query_log_max_len = 20") c.Assert(err, NotNil) diff --git a/infoschema/slow_log.go b/infoschema/slow_log.go index 6afa6dab3ea47..dac1445788f3e 100644 --- a/infoschema/slow_log.go +++ b/infoschema/slow_log.go @@ -19,10 +19,13 @@ import ( "os" "strconv" "strings" + "sync/atomic" "time" "github.com/pingcap/errors" "github.com/pingcap/parser/mysql" + "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" @@ -77,7 +80,6 @@ func parseSlowLogFile(tz *time.Location, filePath string) ([][]types.Datum, erro logutil.Logger(context.Background()).Error("close slow log file failed.", zap.String("file", filePath), zap.Error(err)) } }() - return ParseSlowLog(tz, bufio.NewScanner(file)) } @@ -88,6 +90,11 @@ func ParseSlowLog(tz *time.Location, scanner *bufio.Scanner) ([][]types.Datum, e startFlag := false var st *slowQueryTuple var err error + // Adjust buffer size. + if atomic.LoadUint64(&config.QueryLogMaxLenRecord) > (bufio.MaxScanTokenSize - 100) { + maxBuf := int(atomic.LoadUint64(&config.QueryLogMaxLenRecord) + 100) + scanner.Buffer([]byte{}, maxBuf) + } for scanner.Scan() { line := scanner.Text() // Check slow log entry start flag. @@ -128,6 +135,9 @@ func ParseSlowLog(tz *time.Location, scanner *bufio.Scanner) ([][]types.Datum, e } } if err := scanner.Err(); err != nil { + if terror.ErrorEqual(err, bufio.ErrTooLong) { + err = errors.Errorf("read file buffer overflow, please try to enlarge the variable 'tidb_query_log_max_len'") + } return nil, errors.AddStack(err) } return rows, nil diff --git a/infoschema/slow_log_test.go b/infoschema/slow_log_test.go index 3c75daecd5c2c..7dcd6d02a7eb2 100644 --- a/infoschema/slow_log_test.go +++ b/infoschema/slow_log_test.go @@ -16,9 +16,12 @@ package infoschema_test import ( "bufio" "bytes" + "strings" + "sync/atomic" "time" . "github.com/pingcap/check" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/util/logutil" ) @@ -88,6 +91,24 @@ select * from t; t1Str, err := rows[1][0].ToString() c.Assert(err, IsNil) c.Assert(t1Str, Equals, "2019-04-24 19:41:21.716221") + + // test for bufio.Scanner: token too long. + slowLog = bytes.NewBufferString( + `# Time: 2019-04-28T15:24:04.309074+08:00 +select * from t; +# Time: 2019-04-24-19:41:21.716221 +0800 +`) + sql := strings.Repeat("x", bufio.MaxScanTokenSize) + slowLog.WriteString(sql) + scanner = bufio.NewScanner(slowLog) + rows, err = infoschema.ParseSlowLog(loc, scanner) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "read file buffer overflow, please try to enlarge the variable 'tidb_query_log_max_len'") + + atomic.StoreUint64(&config.QueryLogMaxLenRecord, bufio.MaxScanTokenSize) + scanner = bufio.NewScanner(slowLog) + rows, err = infoschema.ParseSlowLog(loc, scanner) + c.Assert(err, IsNil) } func (s *testSuite) TestSlowLogParseTime(c *C) { diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 81a58c6243489..f7bebd286e7f0 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -739,6 +739,9 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { atomic.StoreUint32(&DDLSlowOprThreshold, uint32(tidbOptPositiveInt32(val, DefTiDBDDLSlowOprThreshold))) case TiDBQueryLogMaxLen: atomic.StoreUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen, uint64(tidbOptInt64(val, logutil.DefaultQueryLogMaxLen))) + if atomic.LoadUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen) > atomic.LoadUint64(&config.QueryLogMaxLenRecord) { + atomic.StoreUint64(&config.QueryLogMaxLenRecord, atomic.LoadUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen)) + } case TiDBRetryLimit: s.RetryLimit = tidbOptInt64(val, DefTiDBRetryLimit) case TiDBDisableTxnAutoRetry: From c9f0daf927872d88242cf7a143b5087bcaaa0e1e Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 30 Apr 2019 16:13:27 +0800 Subject: [PATCH 2/9] fix ci --- infoschema/slow_log_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infoschema/slow_log_test.go b/infoschema/slow_log_test.go index 7dcd6d02a7eb2..b27d2729fd4c5 100644 --- a/infoschema/slow_log_test.go +++ b/infoschema/slow_log_test.go @@ -101,13 +101,13 @@ select * from t; sql := strings.Repeat("x", bufio.MaxScanTokenSize) slowLog.WriteString(sql) scanner = bufio.NewScanner(slowLog) - rows, err = infoschema.ParseSlowLog(loc, scanner) + _, err = infoschema.ParseSlowLog(loc, scanner) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "read file buffer overflow, please try to enlarge the variable 'tidb_query_log_max_len'") atomic.StoreUint64(&config.QueryLogMaxLenRecord, bufio.MaxScanTokenSize) scanner = bufio.NewScanner(slowLog) - rows, err = infoschema.ParseSlowLog(loc, scanner) + _, err = infoschema.ParseSlowLog(loc, scanner) c.Assert(err, IsNil) } From 13919dc1313bf8ca0e17cd9e51d2f2784eb9cde4 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 30 Apr 2019 17:22:11 +0800 Subject: [PATCH 3/9] use reader.ReadLine --- infoschema/slow_log.go | 52 +++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/infoschema/slow_log.go b/infoschema/slow_log.go index dac1445788f3e..4909b85f04e52 100644 --- a/infoschema/slow_log.go +++ b/infoschema/slow_log.go @@ -16,16 +16,14 @@ package infoschema import ( "bufio" "context" + "io" "os" "strconv" "strings" - "sync/atomic" "time" "github.com/pingcap/errors" "github.com/pingcap/parser/mysql" - "github.com/pingcap/parser/terror" - "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" @@ -80,23 +78,45 @@ func parseSlowLogFile(tz *time.Location, filePath string) ([][]types.Datum, erro logutil.Logger(context.Background()).Error("close slow log file failed.", zap.String("file", filePath), zap.Error(err)) } }() - return ParseSlowLog(tz, bufio.NewScanner(file)) + return ParseSlowLog(tz, bufio.NewReader(file)) } +const maxSingleLineLength = 1024 * 1024 * 1024 + // ParseSlowLog exports for testing. // TODO: optimize for parse huge log-file. -func ParseSlowLog(tz *time.Location, scanner *bufio.Scanner) ([][]types.Datum, error) { +func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, error) { var rows [][]types.Datum startFlag := false var st *slowQueryTuple - var err error // Adjust buffer size. - if atomic.LoadUint64(&config.QueryLogMaxLenRecord) > (bufio.MaxScanTokenSize - 100) { - maxBuf := int(atomic.LoadUint64(&config.QueryLogMaxLenRecord) + 100) - scanner.Buffer([]byte{}, maxBuf) - } - for scanner.Scan() { - line := scanner.Text() + //if atomic.LoadUint64(&config.QueryLogMaxLenRecord) > (bufio.MaxScanTokenSize - 100) { + // maxBuf := int(atomic.LoadUint64(&config.QueryLogMaxLenRecord) + 100) + // scanner.Buffer([]byte{}, maxBuf) + //} + var tempLine []byte + for { + lineByte, isPrefix, err := reader.ReadLine() + if err != nil { + if err == io.EOF { + err = nil + } + return rows, err + } + for isPrefix { + tempLine, isPrefix, err = reader.ReadLine() + if err != nil { + if err == io.EOF { + err = nil + } + return rows, err + } + lineByte = append(lineByte, tempLine...) + if len(lineByte) > maxSingleLineLength { + return rows, errors.Errorf("single line length exceeds limit: %v", maxSingleLineLength) + } + } + line := string(hack.String(lineByte)) // Check slow log entry start flag. if !startFlag && strings.HasPrefix(line, variable.SlowLogStartPrefixStr) { st = &slowQueryTuple{} @@ -134,13 +154,7 @@ func ParseSlowLog(tz *time.Location, scanner *bufio.Scanner) ([][]types.Datum, e } } } - if err := scanner.Err(); err != nil { - if terror.ErrorEqual(err, bufio.ErrTooLong) { - err = errors.Errorf("read file buffer overflow, please try to enlarge the variable 'tidb_query_log_max_len'") - } - return nil, errors.AddStack(err) - } - return rows, nil + //return rows, nil } type slowQueryTuple struct { From c68b4daf5421ee61efc7ecc6080985701e1242da Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 5 May 2019 16:41:27 +0800 Subject: [PATCH 4/9] use MaxOfMaxAllowedPacket to check single line limit --- config/config.go | 2 -- executor/set_test.go | 12 ------------ infoschema/slow_log.go | 11 ++--------- infoschema/slow_log_test.go | 31 ++++++++++++++++--------------- sessionctx/variable/session.go | 3 --- sessionctx/variable/tidb_vars.go | 7 ++++--- sessionctx/variable/varsutil.go | 2 +- 7 files changed, 23 insertions(+), 45 deletions(-) diff --git a/config/config.go b/config/config.go index 6bacf4247c773..cada69fe3edbd 100644 --- a/config/config.go +++ b/config/config.go @@ -48,8 +48,6 @@ var ( CheckTableBeforeDrop = false // checkBeforeDropLDFlag is a go build flag. checkBeforeDropLDFlag = "None" - // QueryLogMaxLenRecord use to record the max value of QueryLogMaxLen. - QueryLogMaxLenRecord = uint64(logutil.DefaultQueryLogMaxLen) ) // Config contains configuration options. diff --git a/executor/set_test.go b/executor/set_test.go index 4e8a004e3611d..2326702a583c2 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -15,15 +15,11 @@ package executor_test import ( "context" - "fmt" - "sync/atomic" - . "github.com/pingcap/check" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" - "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testutil" ) @@ -275,14 +271,6 @@ func (s *testSuite2) TestSetVar(c *C) { tk.MustExec("set tidb_query_log_max_len = 0") tk.MustQuery("select @@session.tidb_query_log_max_len;").Check(testkit.Rows("0")) - c.Assert(atomic.LoadUint64(&config.QueryLogMaxLenRecord), Equals, uint64(logutil.DefaultQueryLogMaxLen)) - tk.MustExec(fmt.Sprintf("set tidb_query_log_max_len = %v", logutil.DefaultQueryLogMaxLen+1)) - tk.MustQuery("select @@session.tidb_query_log_max_len;").Check(testkit.Rows(fmt.Sprintf("%v", logutil.DefaultQueryLogMaxLen+1))) - c.Assert(atomic.LoadUint64(&config.QueryLogMaxLenRecord), Equals, uint64(logutil.DefaultQueryLogMaxLen+1)) - tk.MustExec("set tidb_query_log_max_len = 20") - tk.MustQuery("select @@session.tidb_query_log_max_len;").Check(testkit.Rows("20")) - // QueryLogMaxLenRecord record the max len. - c.Assert(atomic.LoadUint64(&config.QueryLogMaxLenRecord), Equals, uint64(logutil.DefaultQueryLogMaxLen+1)) _, err = tk.Exec("set global tidb_query_log_max_len = 20") c.Assert(err, NotNil) diff --git a/infoschema/slow_log.go b/infoschema/slow_log.go index 4909b85f04e52..420f9dd3f18ef 100644 --- a/infoschema/slow_log.go +++ b/infoschema/slow_log.go @@ -81,19 +81,12 @@ func parseSlowLogFile(tz *time.Location, filePath string) ([][]types.Datum, erro return ParseSlowLog(tz, bufio.NewReader(file)) } -const maxSingleLineLength = 1024 * 1024 * 1024 - // ParseSlowLog exports for testing. // TODO: optimize for parse huge log-file. func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, error) { var rows [][]types.Datum startFlag := false var st *slowQueryTuple - // Adjust buffer size. - //if atomic.LoadUint64(&config.QueryLogMaxLenRecord) > (bufio.MaxScanTokenSize - 100) { - // maxBuf := int(atomic.LoadUint64(&config.QueryLogMaxLenRecord) + 100) - // scanner.Buffer([]byte{}, maxBuf) - //} var tempLine []byte for { lineByte, isPrefix, err := reader.ReadLine() @@ -112,8 +105,8 @@ func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, err return rows, err } lineByte = append(lineByte, tempLine...) - if len(lineByte) > maxSingleLineLength { - return rows, errors.Errorf("single line length exceeds limit: %v", maxSingleLineLength) + if len(lineByte) > int(variable.MaxOfMaxAllowedPacket) { + return rows, errors.Errorf("single line length exceeds limit: %v", variable.MaxOfMaxAllowedPacket) } } line := string(hack.String(lineByte)) diff --git a/infoschema/slow_log_test.go b/infoschema/slow_log_test.go index b27d2729fd4c5..d21aed2230244 100644 --- a/infoschema/slow_log_test.go +++ b/infoschema/slow_log_test.go @@ -16,12 +16,11 @@ package infoschema_test import ( "bufio" "bytes" + "github.com/pingcap/tidb/sessionctx/variable" "strings" - "sync/atomic" "time" . "github.com/pingcap/check" - "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/util/logutil" ) @@ -39,10 +38,10 @@ func (s *testSuite) TestParseSlowLogFile(c *C) { # Cop_wait_avg: 0.05 Cop_wait_p90: 0.6 Cop_wait_max: 0.8 Cop_wait_addr: 0.0.0.0:20160 # Mem_max: 70724 select * from t;`) - scanner := bufio.NewScanner(slowLog) + reader := bufio.NewReader(slowLog) loc, err := time.LoadLocation("Asia/Shanghai") c.Assert(err, IsNil) - rows, err := infoschema.ParseSlowLog(loc, scanner) + rows, err := infoschema.ParseSlowLog(loc, reader) c.Assert(err, IsNil) c.Assert(len(rows), Equals, 1) recordString := "" @@ -70,8 +69,8 @@ select a# from t; # Stats: t1:1,t2:2 select * from t; `) - scanner = bufio.NewScanner(slowLog) - _, err = infoschema.ParseSlowLog(loc, scanner) + reader = bufio.NewReader(slowLog) + _, err = infoschema.ParseSlowLog(loc, reader) c.Assert(err, IsNil) // test for time format compatibility. @@ -81,8 +80,8 @@ select * from t; # Time: 2019-04-24-19:41:21.716221 +0800 select * from t; `) - scanner = bufio.NewScanner(slowLog) - rows, err = infoschema.ParseSlowLog(loc, scanner) + reader = bufio.NewReader(slowLog) + rows, err = infoschema.ParseSlowLog(loc, reader) c.Assert(err, IsNil) c.Assert(len(rows) == 2, IsTrue) t0Str, err := rows[0][0].ToString() @@ -98,16 +97,18 @@ select * from t; select * from t; # Time: 2019-04-24-19:41:21.716221 +0800 `) - sql := strings.Repeat("x", bufio.MaxScanTokenSize) + originValue := variable.MaxOfMaxAllowedPacket + variable.MaxOfMaxAllowedPacket = 65536 + sql := strings.Repeat("x", int(variable.MaxOfMaxAllowedPacket+1)) slowLog.WriteString(sql) - scanner = bufio.NewScanner(slowLog) - _, err = infoschema.ParseSlowLog(loc, scanner) + reader = bufio.NewReader(slowLog) + _, err = infoschema.ParseSlowLog(loc, reader) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "read file buffer overflow, please try to enlarge the variable 'tidb_query_log_max_len'") + c.Assert(err.Error(), Equals, "single line length exceeds limit: 65536") - atomic.StoreUint64(&config.QueryLogMaxLenRecord, bufio.MaxScanTokenSize) - scanner = bufio.NewScanner(slowLog) - _, err = infoschema.ParseSlowLog(loc, scanner) + variable.MaxOfMaxAllowedPacket = originValue + reader = bufio.NewReader(slowLog) + _, err = infoschema.ParseSlowLog(loc, reader) c.Assert(err, IsNil) } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index f7bebd286e7f0..81a58c6243489 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -739,9 +739,6 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { atomic.StoreUint32(&DDLSlowOprThreshold, uint32(tidbOptPositiveInt32(val, DefTiDBDDLSlowOprThreshold))) case TiDBQueryLogMaxLen: atomic.StoreUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen, uint64(tidbOptInt64(val, logutil.DefaultQueryLogMaxLen))) - if atomic.LoadUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen) > atomic.LoadUint64(&config.QueryLogMaxLenRecord) { - atomic.StoreUint64(&config.QueryLogMaxLenRecord, atomic.LoadUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen)) - } case TiDBRetryLimit: s.RetryLimit = tidbOptInt64(val, DefTiDBRetryLimit) case TiDBDisableTxnAutoRetry: diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index f16aec3d5cc4e..7dd5ff0af54c5 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -331,7 +331,8 @@ var ( MaxDDLReorgBatchSize int32 = 10240 MinDDLReorgBatchSize int32 = 32 // DDLSlowOprThreshold is the threshold for ddl slow operations, uint is millisecond. - DDLSlowOprThreshold uint32 = DefTiDBDDLSlowOprThreshold - ForcePriority = int32(DefTiDBForcePriority) - ServerHostname, _ = os.Hostname() + DDLSlowOprThreshold uint32 = DefTiDBDDLSlowOprThreshold + ForcePriority = int32(DefTiDBForcePriority) + ServerHostname, _ = os.Hostname() + MaxOfMaxAllowedPacket uint64 = 1073741824 ) diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index a9e49e323daae..5cbb2e77e14d2 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -305,7 +305,7 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string, return checkUInt64SystemVar(name, value, 1, 1073741824, vars) // See "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_allowed_packet" case MaxAllowedPacket: - return checkUInt64SystemVar(name, value, 1024, 1073741824, vars) + return checkUInt64SystemVar(name, value, 1024, MaxOfMaxAllowedPacket, vars) case MaxConnections: return checkUInt64SystemVar(name, value, 1, 100000, vars) case MaxConnectErrors: From 775be133bdda94a770744edd303f2927e4bde61c Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 5 May 2019 16:44:29 +0800 Subject: [PATCH 5/9] refine code --- infoschema/slow_log.go | 2 +- infoschema/slow_log_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infoschema/slow_log.go b/infoschema/slow_log.go index 420f9dd3f18ef..f2073423faa5d 100644 --- a/infoschema/slow_log.go +++ b/infoschema/slow_log.go @@ -147,7 +147,7 @@ func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, err } } } - //return rows, nil + return rows, nil } type slowQueryTuple struct { diff --git a/infoschema/slow_log_test.go b/infoschema/slow_log_test.go index d21aed2230244..cd5abc659e222 100644 --- a/infoschema/slow_log_test.go +++ b/infoschema/slow_log_test.go @@ -16,12 +16,12 @@ package infoschema_test import ( "bufio" "bytes" - "github.com/pingcap/tidb/sessionctx/variable" "strings" "time" . "github.com/pingcap/check" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util/logutil" ) From be791f41481a1d3765eb572e153a22f0a1cdbe28 Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 5 May 2019 17:20:14 +0800 Subject: [PATCH 6/9] fix bug --- infoschema/slow_log.go | 48 ++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/infoschema/slow_log.go b/infoschema/slow_log.go index f2073423faa5d..6d514c2682e2c 100644 --- a/infoschema/slow_log.go +++ b/infoschema/slow_log.go @@ -87,28 +87,19 @@ func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, err var rows [][]types.Datum startFlag := false var st *slowQueryTuple - var tempLine []byte + finish := false for { - lineByte, isPrefix, err := reader.ReadLine() + if finish { + break + } + lineByte, err := getOneLine(reader) + if err == io.EOF { + finish = true + err = nil + } if err != nil { - if err == io.EOF { - err = nil - } return rows, err } - for isPrefix { - tempLine, isPrefix, err = reader.ReadLine() - if err != nil { - if err == io.EOF { - err = nil - } - return rows, err - } - lineByte = append(lineByte, tempLine...) - if len(lineByte) > int(variable.MaxOfMaxAllowedPacket) { - return rows, errors.Errorf("single line length exceeds limit: %v", variable.MaxOfMaxAllowedPacket) - } - } line := string(hack.String(lineByte)) // Check slow log entry start flag. if !startFlag && strings.HasPrefix(line, variable.SlowLogStartPrefixStr) { @@ -150,6 +141,27 @@ func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, err return rows, nil } +func getOneLine(reader *bufio.Reader) ([]byte, error) { + lineByte, isPrefix, err := reader.ReadLine() + var tempLine []byte + if err != nil { + return lineByte, err + } + for isPrefix { + tempLine, isPrefix, err = reader.ReadLine() + lineByte = append(lineByte, tempLine...) + + // Use the max value of max_allowed_packet to check the single line length. + if len(lineByte) > int(variable.MaxOfMaxAllowedPacket) { + return lineByte, errors.Errorf("single line length exceeds limit: %v", variable.MaxOfMaxAllowedPacket) + } + if err != nil { + return lineByte, err + } + } + return lineByte, err +} + type slowQueryTuple struct { time time.Time txnStartTs uint64 From c0a67fa9b1f3f5ed3a311b9fa1adeec304727a67 Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 5 May 2019 17:24:07 +0800 Subject: [PATCH 7/9] fix ci --- executor/set_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/executor/set_test.go b/executor/set_test.go index 2326702a583c2..0a0796c1eb61f 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -15,6 +15,7 @@ package executor_test import ( "context" + . "github.com/pingcap/check" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" @@ -271,6 +272,8 @@ func (s *testSuite2) TestSetVar(c *C) { tk.MustExec("set tidb_query_log_max_len = 0") tk.MustQuery("select @@session.tidb_query_log_max_len;").Check(testkit.Rows("0")) + tk.MustExec("set tidb_query_log_max_len = 20") + tk.MustQuery("select @@session.tidb_query_log_max_len;").Check(testkit.Rows("20")) _, err = tk.Exec("set global tidb_query_log_max_len = 20") c.Assert(err, NotNil) From b6a6b6b96a5249669c954b347e4f404cc6b1511b Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 8 May 2019 12:10:28 +0800 Subject: [PATCH 8/9] refine code --- infoschema/slow_log.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/infoschema/slow_log.go b/infoschema/slow_log.go index 6d514c2682e2c..a47370a9b4491 100644 --- a/infoschema/slow_log.go +++ b/infoschema/slow_log.go @@ -87,17 +87,12 @@ func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, err var rows [][]types.Datum startFlag := false var st *slowQueryTuple - finish := false for { - if finish { - break - } lineByte, err := getOneLine(reader) - if err == io.EOF { - finish = true - err = nil - } if err != nil { + if err == io.EOF { + return rows, nil + } return rows, err } line := string(hack.String(lineByte)) @@ -143,10 +138,10 @@ func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, err func getOneLine(reader *bufio.Reader) ([]byte, error) { lineByte, isPrefix, err := reader.ReadLine() - var tempLine []byte if err != nil { return lineByte, err } + var tempLine []byte for isPrefix { tempLine, isPrefix, err = reader.ReadLine() lineByte = append(lineByte, tempLine...) From 0f827baf42cccebc1e2c0072f64237b3e51cd9dd Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 8 May 2019 13:30:47 +0800 Subject: [PATCH 9/9] remove useless code --- infoschema/slow_log.go | 1 - 1 file changed, 1 deletion(-) diff --git a/infoschema/slow_log.go b/infoschema/slow_log.go index a47370a9b4491..bcc864a0a77f4 100644 --- a/infoschema/slow_log.go +++ b/infoschema/slow_log.go @@ -133,7 +133,6 @@ func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, err } } } - return rows, nil } func getOneLine(reader *bufio.Reader) ([]byte, error) {