skip encode some kv to speed up test

D3Hunter · Oct 26, 2023 · bec9359 · bec9359
1 parent 51e1fc0
commit bec9359
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 6 deletions.
diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go
@@ -189,6 +189,7 @@ func (e *BaseKVEncoder) GetOrCreateRecord() []types.Datum {
 
 // Record2KV converts a row into a KV pair.
 func (e *BaseKVEncoder) Record2KV(record, originalRow []types.Datum, rowID int64) (*Pairs, error) {
+	e.Table.(*tables.TableCommon).SkipEncodeDataKV = true
 	_, err := e.Table.AddRecord(e.SessionCtx, record)
 	if err != nil {
 		e.logger.Error("kv encode failed",

diff --git a/br/pkg/lightning/backend/kv/session.go b/br/pkg/lightning/backend/kv/session.go
@@ -33,6 +33,7 @@ import (
 	"github.com/pingcap/tidb/pkg/parser/model"
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"
+	"github.com/pingcap/tidb/pkg/tablecodec"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util/topsql/stmtstats"
 	"go.uber.org/zap"
@@ -134,6 +135,16 @@ func (mb *MemBuf) AllocateBuf(size int) {
 
 // Set sets the key-value pair.
 func (mb *MemBuf) Set(k kv.Key, v []byte) error {
+	if tablecodec.IsRecordKey(k) {
+		return nil
+	}
+	indexID, err := tablecodec.DecodeIndexID(k)
+	if err != nil {
+		return err
+	}
+	if indexID != 7 && indexID != 4 {
+		return nil
+	}
 	kvPairs := mb.kvPairs
 	size := len(k) + len(v)
 	if mb.buf == nil || mb.buf.cap-mb.buf.idx < size {

diff --git a/pkg/table/tables/tables.go b/pkg/table/tables/tables.go
@@ -79,8 +79,9 @@ type TableCommon struct {
 	WritableConstraints             []*table.Constraint
 
 	// recordPrefix and indexPrefix are generated using physicalTableID.
-	recordPrefix kv.Key
-	indexPrefix  kv.Key
+	recordPrefix     kv.Key
+	indexPrefix      kv.Key
+	SkipEncodeDataKV bool
 }
 
 // MockTableFromMeta only serves for test.
@@ -986,10 +987,12 @@ func (t *TableCommon) AddRecord(sctx sessionctx.Context, r []types.Datum, opts .
 	logutil.BgLogger().Debug("addRecord",
 		zap.Stringer("key", key))
 	sc, rd := sessVars.StmtCtx, &sessVars.RowEncoder
-	checksums, writeBufs.RowValBuf = t.calcChecksums(sctx, recordID, checksumData, writeBufs.RowValBuf)
-	writeBufs.RowValBuf, err = tablecodec.EncodeRow(sc, row, colIDs, writeBufs.RowValBuf, writeBufs.AddRowValues, rd, checksums...)
-	if err != nil {
-		return nil, err
+	if !t.SkipEncodeDataKV {
+		checksums, writeBufs.RowValBuf = t.calcChecksums(sctx, recordID, checksumData, writeBufs.RowValBuf)
+		writeBufs.RowValBuf, err = tablecodec.EncodeRow(sc, row, colIDs, writeBufs.RowValBuf, writeBufs.AddRowValues, rd, checksums...)
+		if err != nil {
+			return nil, err
+		}
 	}
 	value := writeBufs.RowValBuf