From 270cd47967c709ca37e53c62a41ed976f4d75ef3 Mon Sep 17 00:00:00 2001 From: D3Hunter Date: Thu, 11 May 2023 20:08:56 +0800 Subject: [PATCH 1/2] change --- br/pkg/lightning/backend/kv/sql2kv.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/br/pkg/lightning/backend/kv/sql2kv.go b/br/pkg/lightning/backend/kv/sql2kv.go index 5be0717a442e1..771fa42f25be9 100644 --- a/br/pkg/lightning/backend/kv/sql2kv.go +++ b/br/pkg/lightning/backend/kv/sql2kv.go @@ -181,6 +181,11 @@ func Row2KvPairs(row encode.Row) []common.KvPair { // `columnPermutation` parameter. func (kvcodec *tableKVEncoder) Encode(row []types.Datum, rowID int64, columnPermutation []int, _ int64) (encode.Row, error) { + // we ignore warnings when encoding rows now, but warnings uses the same memory as parser, since the input + // row []types.Datum share the same underlying buf, and when doing CastValue, we're using hack.String/hack.Slice. + // when generating error such as mysql.ErrDataOutOfRange, the data will be part of the error, causing the buf + // unable to release. So we truncate the warnings here. + defer kvcodec.SessionCtx.Vars.StmtCtx.TruncateWarnings(0) var value types.Datum var err error From 8586481cb4dc8487bf10063fa316053402549893 Mon Sep 17 00:00:00 2001 From: D3Hunter Date: Thu, 11 May 2023 20:29:55 +0800 Subject: [PATCH 2/2] fix comments --- br/pkg/lightning/backend/kv/base.go | 5 +++++ br/pkg/lightning/backend/kv/sql2kv.go | 2 +- executor/importer/kv_encode.go | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index 5020d38753fbc..4d288bd3db03c 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -322,6 +322,11 @@ func (e *BaseKVEncoder) LogEvalGenExprFailed(row []types.Datum, colInfo *model.C ) } +// TruncateWarns resets the warnings in session context. +func (e *BaseKVEncoder) TruncateWarns() { + e.SessionCtx.Vars.StmtCtx.TruncateWarnings(0) +} + func evalGeneratedColumns(se *Session, record []types.Datum, cols []*table.Column, genCols []GeneratedCol) (errCol *model.ColumnInfo, err error) { mutRow := chunk.MutRowFromDatums(record) diff --git a/br/pkg/lightning/backend/kv/sql2kv.go b/br/pkg/lightning/backend/kv/sql2kv.go index 771fa42f25be9..5f4050b0444b2 100644 --- a/br/pkg/lightning/backend/kv/sql2kv.go +++ b/br/pkg/lightning/backend/kv/sql2kv.go @@ -185,7 +185,7 @@ func (kvcodec *tableKVEncoder) Encode(row []types.Datum, // row []types.Datum share the same underlying buf, and when doing CastValue, we're using hack.String/hack.Slice. // when generating error such as mysql.ErrDataOutOfRange, the data will be part of the error, causing the buf // unable to release. So we truncate the warnings here. - defer kvcodec.SessionCtx.Vars.StmtCtx.TruncateWarnings(0) + defer kvcodec.TruncateWarns() var value types.Datum var err error diff --git a/executor/importer/kv_encode.go b/executor/importer/kv_encode.go index ce499360c9e02..5e6ca4353dfc7 100644 --- a/executor/importer/kv_encode.go +++ b/executor/importer/kv_encode.go @@ -79,6 +79,11 @@ func newTableKVEncoder( // Encode implements the kvEncoder interface. func (en *tableKVEncoder) Encode(row []types.Datum, rowID int64) (*kv.Pairs, error) { + // we ignore warnings when encoding rows now, but warnings uses the same memory as parser, since the input + // row []types.Datum share the same underlying buf, and when doing CastValue, we're using hack.String/hack.Slice. + // when generating error such as mysql.ErrDataOutOfRange, the data will be part of the error, causing the buf + // unable to release. So we truncate the warnings here. + defer en.TruncateWarns() record, err := en.parserData2TableData(row, rowID) if err != nil { return nil, err