lightning: fix oom when mem/cpu ratio is low

br/pkg/lightning/backend/kv/base.go

@@ -322,6 +322,11 @@ func (e *BaseKVEncoder) LogEvalGenExprFailed(row []types.Datum, colInfo *model.C
 	)
 }
 
+// TruncateWarns resets the warnings in session context.
+func (e *BaseKVEncoder) TruncateWarns() {
+	e.SessionCtx.Vars.StmtCtx.TruncateWarnings(0)
+}
+
 func evalGeneratedColumns(se *Session, record []types.Datum, cols []*table.Column,
 	genCols []GeneratedCol) (errCol *model.ColumnInfo, err error) {
 	mutRow := chunk.MutRowFromDatums(record)

br/pkg/lightning/backend/kv/sql2kv.go

@@ -181,6 +181,11 @@ func Row2KvPairs(row encode.Row) []common.KvPair {
 // `columnPermutation` parameter.
 func (kvcodec *tableKVEncoder) Encode(row []types.Datum,
 	rowID int64, columnPermutation []int, _ int64) (encode.Row, error) {
+	// we ignore warnings when encoding rows now, but warnings uses the same memory as parser, since the input
+	// row []types.Datum share the same underlying buf, and when doing CastValue, we're using hack.String/hack.Slice.
+	// when generating error such as mysql.ErrDataOutOfRange, the data will be part of the error, causing the buf
+	// unable to release. So we truncate the warnings here.
+	defer kvcodec.TruncateWarns()
 	var value types.Datum
 	var err error
 

executor/importer/kv_encode.go

@@ -79,6 +79,11 @@ func newTableKVEncoder(
 
 // Encode implements the kvEncoder interface.
 func (en *tableKVEncoder) Encode(row []types.Datum, rowID int64) (*kv.Pairs, error) {
+	// we ignore warnings when encoding rows now, but warnings uses the same memory as parser, since the input
+	// row []types.Datum share the same underlying buf, and when doing CastValue, we're using hack.String/hack.Slice.
+	// when generating error such as mysql.ErrDataOutOfRange, the data will be part of the error, causing the buf
+	// unable to release. So we truncate the warnings here.
+	defer en.TruncateWarns()
 	record, err := en.parserData2TableData(row, rowID)
 	if err != nil {
 		return nil, err