diff --git a/pkg/catalog/types.go b/pkg/catalog/types.go index 4b15301ef7442..acb70c45fe21d 100644 --- a/pkg/catalog/types.go +++ b/pkg/catalog/types.go @@ -833,3 +833,13 @@ const ( SAVED_ROW_COUNT_IDX = 14 QUERY_ROW_COUNT_IDX = 15 ) + +var SystemDatabases = []string{ + "information_schema", + "mo_catalog", + "mo_debug", + "mo_task", + "mysql", + "system", + "system_metrics", +} diff --git a/pkg/cdc/reader.go b/pkg/cdc/reader.go index e6b9d5fc5ee8c..b6a8bbcf4bdd7 100644 --- a/pkg/cdc/reader.go +++ b/pkg/cdc/reader.go @@ -16,6 +16,7 @@ package cdc import ( "context" + "sync" "time" "github.com/matrixorigin/matrixone/pkg/common/moerr" @@ -39,27 +40,29 @@ type tableReader struct { packerPool *fileservice.Pool[*types.Packer] info *DbTableInfo sinker Sinker - wMarkUpdater *WatermarkUpdater + wMarkUpdater IWatermarkUpdater tick *time.Ticker resetWatermarkFunc func(*DbTableInfo) error initSnapshotSplitTxn bool + runningReaders *sync.Map tableDef *plan.TableDef insTsColIdx, insCompositedPkColIdx int delTsColIdx, delCompositedPkColIdx int } -func NewTableReader( +var NewTableReader = func( cnTxnClient client.TxnClient, cnEngine engine.Engine, mp *mpool.MPool, packerPool *fileservice.Pool[*types.Packer], info *DbTableInfo, sinker Sinker, - wMarkUpdater *WatermarkUpdater, + wMarkUpdater IWatermarkUpdater, tableDef *plan.TableDef, resetWatermarkFunc func(*DbTableInfo) error, initSnapshotSplitTxn bool, + runningReaders *sync.Map, ) Reader { reader := &tableReader{ cnTxnClient: cnTxnClient, @@ -72,6 +75,7 @@ func NewTableReader( tick: time.NewTicker(200 * time.Millisecond), resetWatermarkFunc: resetWatermarkFunc, initSnapshotSplitTxn: initSnapshotSplitTxn, + runningReaders: runningReaders, tableDef: tableDef, } @@ -98,14 +102,16 @@ func (reader *tableReader) Run( logutil.Infof("cdc tableReader(%v).Run: start", reader.info) defer func() { if err != nil { - if err = reader.wMarkUpdater.SaveErrMsg(reader.info.SourceTblIdStr, err.Error()); err != nil { + if err = reader.wMarkUpdater.SaveErrMsg(reader.info.SourceDbName, reader.info.SourceTblName, err.Error()); err != nil { logutil.Infof("cdc tableReader(%v).Run: save err msg failed, err: %v", reader.info, err) } } reader.Close() + reader.runningReaders.Delete(GenDbTblKey(reader.info.SourceDbName, reader.info.SourceTblName)) logutil.Infof("cdc tableReader(%v).Run: end", reader.info) }() + reader.runningReaders.Store(GenDbTblKey(reader.info.SourceDbName, reader.info.SourceTblName), reader) for { select { case <-ctx.Done(): @@ -192,7 +198,7 @@ func (reader *tableReader) readTableWithTxn( //step2 : define time range // from = last wmark // to = txn operator snapshot ts - fromTs := reader.wMarkUpdater.GetFromMem(reader.info.SourceTblIdStr) + fromTs := reader.wMarkUpdater.GetFromMem(reader.info.SourceDbName, reader.info.SourceTblName) toTs := types.TimestampToTS(GetSnapshotTS(txnOp)) start := time.Now() changes, err = CollectChanges(ctx, rel, fromTs, toTs, reader.mp) @@ -264,7 +270,7 @@ func (reader *tableReader) readTableWithTxn( } if err == nil { - reader.wMarkUpdater.UpdateMem(reader.info.SourceTblIdStr, toTs) + reader.wMarkUpdater.UpdateMem(reader.info.SourceDbName, reader.info.SourceTblName, toTs) } } else { // has error already if hasBegin { diff --git a/pkg/cdc/reader_test.go b/pkg/cdc/reader_test.go index 773f6b5a07751..2df4c9864c60a 100644 --- a/pkg/cdc/reader_test.go +++ b/pkg/cdc/reader_test.go @@ -38,15 +38,16 @@ import ( func TestNewTableReader(t *testing.T) { type args struct { - cnTxnClient client.TxnClient - cnEngine engine.Engine - mp *mpool.MPool - packerPool *fileservice.Pool[*types.Packer] - info *DbTableInfo - sinker Sinker - wMarkUpdater *WatermarkUpdater - tableDef *plan.TableDef - restartFunc func(*DbTableInfo) error + cnTxnClient client.TxnClient + cnEngine engine.Engine + mp *mpool.MPool + packerPool *fileservice.Pool[*types.Packer] + info *DbTableInfo + sinker Sinker + wMarkUpdater *WatermarkUpdater + tableDef *plan.TableDef + restartFunc func(*DbTableInfo) error + runningReaders *sync.Map } tableDef := &plan.TableDef{ @@ -76,7 +77,29 @@ func TestNewTableReader(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.NotNilf(t, NewTableReader(tt.args.cnTxnClient, tt.args.cnEngine, tt.args.mp, tt.args.packerPool, tt.args.info, tt.args.sinker, tt.args.wMarkUpdater, tt.args.tableDef, tt.args.restartFunc, true), "NewTableReader(%v, %v, %v, %v, %v, %v, %v, %v, %v)", tt.args.cnTxnClient, tt.args.cnEngine, tt.args.mp, tt.args.packerPool, tt.args.info, tt.args.sinker, tt.args.wMarkUpdater, tt.args.tableDef, tt.args.restartFunc) + assert.NotNilf(t, NewTableReader( + tt.args.cnTxnClient, + tt.args.cnEngine, + tt.args.mp, + tt.args.packerPool, + tt.args.info, + tt.args.sinker, + tt.args.wMarkUpdater, + tt.args.tableDef, + tt.args.restartFunc, + true, + tt.args.runningReaders, + ), + "NewTableReader(%v,%v,%v,%v,%v,%v,%v,%v,%v)", + tt.args.cnTxnClient, + tt.args.cnEngine, + tt.args.mp, + tt.args.packerPool, + tt.args.info, + tt.args.sinker, + tt.args.wMarkUpdater, + tt.args.tableDef, + tt.args.restartFunc) }) } } @@ -218,6 +241,7 @@ func Test_tableReader_Run(t *testing.T) { insCompositedPkColIdx: tt.fields.insCompositedPkColIdx, delTsColIdx: tt.fields.delTsColIdx, delCompositedPkColIdx: tt.fields.delCompositedPkColIdx, + runningReaders: &sync.Map{}, } reader.Run(tt.args.ctx, tt.args.ar) }) @@ -238,6 +262,11 @@ func Test_tableReader_Run_StaleRead(t *testing.T) { tick: time.NewTicker(time.Millisecond * 300), sinker: NewConsoleSinker(nil, nil), resetWatermarkFunc: func(*DbTableInfo) error { return nil }, + runningReaders: &sync.Map{}, + info: &DbTableInfo{ + SourceDbName: "db1", + SourceTblName: "t1", + }, } reader.Run(ctx, NewCdcActiveRoutine()) cancel() @@ -254,10 +283,12 @@ func Test_tableReader_Run_StaleRead(t *testing.T) { tick: time.NewTicker(time.Millisecond * 300), sinker: NewConsoleSinker(nil, nil), info: &DbTableInfo{ - SourceTblIdStr: "1_0", + SourceDbName: "db1", + SourceTblName: "t1", }, wMarkUpdater: u, resetWatermarkFunc: func(*DbTableInfo) error { return moerr.NewInternalErrorNoCtx("") }, + runningReaders: &sync.Map{}, } reader.Run(ctx, NewCdcActiveRoutine()) cancel() @@ -283,10 +314,12 @@ func Test_tableReader_Run_NonStaleReadErr(t *testing.T) { tick: time.NewTicker(time.Millisecond * 300), sinker: NewConsoleSinker(nil, nil), info: &DbTableInfo{ - SourceTblIdStr: "1_0", + SourceDbName: "db1", + SourceTblName: "t1", }, wMarkUpdater: u, resetWatermarkFunc: func(*DbTableInfo) error { return nil }, + runningReaders: &sync.Map{}, } reader.Run(ctx, NewCdcActiveRoutine()) } @@ -319,8 +352,8 @@ func Test_tableReader_readTableWithTxn(t *testing.T) { reader := &tableReader{ info: &DbTableInfo{ - SourceTblName: "t1", - SourceTblIdStr: "123", + SourceDbName: "db1", + SourceTblName: "t1", }, packerPool: pool, wMarkUpdater: watermarkUpdater, @@ -328,6 +361,7 @@ func Test_tableReader_readTableWithTxn(t *testing.T) { insTsColIdx: 0, insCompositedPkColIdx: 3, sinker: NewConsoleSinker(nil, nil), + runningReaders: &sync.Map{}, } getRelationByIdStub := gostub.Stub(&GetRelationById, func(_ context.Context, _ engine.Engine, _ client.TxnOperator, _ uint64) (string, string, engine.Relation, error) { diff --git a/pkg/cdc/sinker.go b/pkg/cdc/sinker.go index 0ef171aa5109a..d55a1dc5d8f85 100644 --- a/pkg/cdc/sinker.go +++ b/pkg/cdc/sinker.go @@ -35,9 +35,11 @@ import ( const ( // sqlBufReserved leave 5 bytes for mysql driver - sqlBufReserved = 5 - sqlPrintLen = 200 - fakeSql = "fakeSql" + sqlBufReserved = 5 + sqlPrintLen = 200 + fakeSql = "fakeSql" + createTable = "create table" + createTableIfNotExists = "create table if not exists" ) var ( @@ -47,10 +49,10 @@ var ( dummy = []byte("") ) -func NewSinker( +var NewSinker = func( sinkUri UriInfo, dbTblInfo *DbTableInfo, - watermarkUpdater *WatermarkUpdater, + watermarkUpdater IWatermarkUpdater, tableDef *plan.TableDef, retryTimes int, retryDuration time.Duration, @@ -68,6 +70,21 @@ func NewSinker( return nil, err } + ctx := context.Background() + padding := strings.Repeat(" ", sqlBufReserved) + // create db + _ = sink.Send(ctx, ar, []byte(padding+fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", dbTblInfo.SinkDbName))) + // use db + _ = sink.Send(ctx, ar, []byte(padding+fmt.Sprintf("use `%s`", dbTblInfo.SinkDbName))) + // create table + createSql := strings.TrimSpace(dbTblInfo.SourceCreateSql) + if len(createSql) < len(createTableIfNotExists) || !strings.EqualFold(createSql[:len(createTableIfNotExists)], createTableIfNotExists) { + createSql = createTableIfNotExists + createSql[len(createTable):] + } + createSql = strings.ReplaceAll(createSql, dbTblInfo.SourceDbName, dbTblInfo.SinkDbName) + createSql = strings.ReplaceAll(createSql, dbTblInfo.SourceTblName, dbTblInfo.SinkTblName) + _ = sink.Send(ctx, ar, []byte(padding+createSql)) + return NewMysqlSinker(sink, dbTblInfo, watermarkUpdater, tableDef, ar, maxSqlLength), nil } @@ -75,12 +92,12 @@ var _ Sinker = new(consoleSinker) type consoleSinker struct { dbTblInfo *DbTableInfo - watermarkUpdater *WatermarkUpdater + watermarkUpdater IWatermarkUpdater } func NewConsoleSinker( dbTblInfo *DbTableInfo, - watermarkUpdater *WatermarkUpdater, + watermarkUpdater IWatermarkUpdater, ) Sinker { return &consoleSinker{ dbTblInfo: dbTblInfo, @@ -142,7 +159,7 @@ var _ Sinker = new(mysqlSinker) type mysqlSinker struct { mysql Sink dbTblInfo *DbTableInfo - watermarkUpdater *WatermarkUpdater + watermarkUpdater IWatermarkUpdater ar *ActiveRoutine // buf of sql statement @@ -179,7 +196,7 @@ type mysqlSinker struct { var NewMysqlSinker = func( mysql Sink, dbTblInfo *DbTableInfo, - watermarkUpdater *WatermarkUpdater, + watermarkUpdater IWatermarkUpdater, tableDef *plan.TableDef, ar *ActiveRoutine, maxSqlLength uint64, @@ -288,7 +305,7 @@ func (s *mysqlSinker) Run(ctx context.Context, ar *ActiveRoutine) { } func (s *mysqlSinker) Sink(ctx context.Context, data *DecoderOutput) { - watermark := s.watermarkUpdater.GetFromMem(s.dbTblInfo.SourceTblIdStr) + watermark := s.watermarkUpdater.GetFromMem(s.dbTblInfo.SourceDbName, s.dbTblInfo.SourceTblName) if data.toTs.LE(&watermark) { logutil.Errorf("cdc mysqlSinker(%v): unexpected watermark: %s, current watermark: %s", s.dbTblInfo, data.toTs.ToString(), watermark.ToString()) @@ -417,7 +434,7 @@ func (s *mysqlSinker) sinkSnapshot(ctx context.Context, bat *batch.Batch) { } // step3: append to sqlBuf, send sql if sqlBuf is full - if err = s.appendSqlBuf(ctx, InsertRow); err != nil { + if err = s.appendSqlBuf(InsertRow); err != nil { s.err.Store(err) return } @@ -501,7 +518,7 @@ func (s *mysqlSinker) sinkInsert(ctx context.Context, insertIter *atomicBatchRow } // step3: append to sqlBuf - if err = s.appendSqlBuf(ctx, InsertRow); err != nil { + if err = s.appendSqlBuf(InsertRow); err != nil { return } @@ -530,7 +547,7 @@ func (s *mysqlSinker) sinkDelete(ctx context.Context, deleteIter *atomicBatchRow } // step3: append to sqlBuf - if err = s.appendSqlBuf(ctx, DeleteRow); err != nil { + if err = s.appendSqlBuf(DeleteRow); err != nil { return } @@ -539,7 +556,7 @@ func (s *mysqlSinker) sinkDelete(ctx context.Context, deleteIter *atomicBatchRow // appendSqlBuf appends rowBuf to sqlBuf if not exceed its cap // otherwise, send sql to downstream first, then reset sqlBuf and append -func (s *mysqlSinker) appendSqlBuf(ctx context.Context, rowType RowType) (err error) { +func (s *mysqlSinker) appendSqlBuf(rowType RowType) (err error) { // insert suffix: `;`, delete suffix: `);` suffixLen := 1 if rowType == DeleteRow { @@ -668,8 +685,8 @@ var NewMysqlSink = func( return ret, err } +// Send must leave 5 bytes at the head of sqlBuf func (s *mysqlSink) Send(ctx context.Context, ar *ActiveRoutine, sqlBuf []byte) error { - // must leave 5 bytes at the head of sqlBuf reuseQueryArg := sql.NamedArg{ Name: mysql.ReuseQueryBuf, Value: sqlBuf, diff --git a/pkg/cdc/sinker_test.go b/pkg/cdc/sinker_test.go index 3a0b40eb2677f..682d7964f06e3 100644 --- a/pkg/cdc/sinker_test.go +++ b/pkg/cdc/sinker_test.go @@ -40,7 +40,7 @@ func TestNewSinker(t *testing.T) { type args struct { sinkUri UriInfo dbTblInfo *DbTableInfo - watermarkUpdater *WatermarkUpdater + watermarkUpdater IWatermarkUpdater tableDef *plan.TableDef retryTimes int retryDuration time.Duration @@ -75,23 +75,42 @@ func TestNewSinker(t *testing.T) { sinkUri: UriInfo{ SinkTyp: MysqlSink, }, - dbTblInfo: &DbTableInfo{}, + dbTblInfo: &DbTableInfo{ + SourceCreateSql: "create table t1 (a int, b int, c int)", + }, watermarkUpdater: nil, tableDef: nil, retryTimes: 0, retryDuration: 0, + ar: NewCdcActiveRoutine(), }, want: nil, wantErr: assert.NoError, }, } + db, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectExec(fakeSql).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(fakeSql).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(fakeSql).WillReturnResult(sqlmock.NewResult(1, 1)) + + sink := &mysqlSink{ + user: "root", + password: "123456", + ip: "127.0.0.1", + port: 3306, + retryTimes: DefaultRetryTimes, + retryDuration: DefaultRetryDuration, + conn: db, + } + sinkStub := gostub.Stub(&NewMysqlSink, func(_, _, _ string, _, _ int, _ time.Duration, _ string) (Sink, error) { - return nil, nil + return sink, nil }) defer sinkStub.Reset() - sinkerStub := gostub.Stub(&NewMysqlSinker, func(_ Sink, _ *DbTableInfo, _ *WatermarkUpdater, _ *plan.TableDef, _ *ActiveRoutine, _ uint64) Sinker { + sinkerStub := gostub.Stub(&NewMysqlSinker, func(Sink, *DbTableInfo, IWatermarkUpdater, *plan.TableDef, *ActiveRoutine, uint64) Sinker { return nil }) defer sinkerStub.Reset() @@ -110,7 +129,7 @@ func TestNewSinker(t *testing.T) { func TestNewConsoleSinker(t *testing.T) { type args struct { dbTblInfo *DbTableInfo - watermarkUpdater *WatermarkUpdater + watermarkUpdater IWatermarkUpdater } tests := []struct { name string @@ -370,11 +389,11 @@ func Test_mysqlSinker_appendSqlBuf(t *testing.T) { s.sqlBuf = append(s.sqlBuf[:sqlBufReserved], s.tsInsertPrefix...) s.rowBuf = []byte("insert") // not exceed cap - err = s.appendSqlBuf(ctx, InsertRow) + err = s.appendSqlBuf(InsertRow) assert.NoError(t, err) assert.Equal(t, []byte(prefix+tsInsertPrefix+"insert"), s.sqlBuf) // exceed cap - err = s.appendSqlBuf(ctx, InsertRow) + err = s.appendSqlBuf(InsertRow) assert.NoError(t, err) assert.Equal(t, []byte(prefix+tsInsertPrefix+"insert"), s.sqlBuf) @@ -382,11 +401,11 @@ func Test_mysqlSinker_appendSqlBuf(t *testing.T) { s.sqlBuf = append(s.sqlBuf[:sqlBufReserved], s.tsDeletePrefix...) s.rowBuf = []byte("delete") // not exceed cap - err = s.appendSqlBuf(ctx, DeleteRow) + err = s.appendSqlBuf(DeleteRow) assert.NoError(t, err) assert.Equal(t, []byte(prefix+tsDeletePrefix+"delete"), s.sqlBuf) // exceed cap - err = s.appendSqlBuf(ctx, DeleteRow) + err = s.appendSqlBuf(DeleteRow) assert.NoError(t, err) assert.Equal(t, []byte(prefix+tsDeletePrefix+"delete"), s.sqlBuf) } @@ -473,7 +492,7 @@ func Test_mysqlSinker_Sink(t *testing.T) { watermarkUpdater := &WatermarkUpdater{ watermarkMap: &sync.Map{}, } - watermarkUpdater.UpdateMem("1_0", t0) + watermarkUpdater.UpdateMem("db1", "t1", t0) tableDef := &plan.TableDef{ Cols: []*plan.ColDef{ @@ -592,13 +611,14 @@ func Test_mysqlSinker_Sink_NoMoreData(t *testing.T) { mock.ExpectExec(".*").WillReturnError(moerr.NewInternalErrorNoCtx("")) dbTblInfo := &DbTableInfo{ - SourceTblIdStr: "1_0", + SourceDbName: "db1", + SourceTblName: "t1", } watermarkUpdater := &WatermarkUpdater{ watermarkMap: &sync.Map{}, } - watermarkUpdater.UpdateMem("1_0", types.BuildTS(0, 1)) + watermarkUpdater.UpdateMem("db1", "t1", types.BuildTS(0, 1)) ar := NewCdcActiveRoutine() @@ -773,14 +793,16 @@ func Test_mysqlsink(t *testing.T) { sink := &mysqlSinker{ watermarkUpdater: wmark, dbTblInfo: &DbTableInfo{ - SourceTblId: 0, + SourceTblId: 0, + SourceTblName: "t1", + SourceDbName: "db1", }, } tts := timestamp.Timestamp{ PhysicalTime: 100, LogicalTime: 100, } - sink.watermarkUpdater.watermarkMap.Store(uint64(0), types.TimestampToTS(tts)) + sink.watermarkUpdater.UpdateMem("db1", "t1", types.TimestampToTS(tts)) sink.Sink(context.Background(), &DecoderOutput{}) } diff --git a/pkg/cdc/table_scanner.go b/pkg/cdc/table_scanner.go new file mode 100644 index 0000000000000..af65bcb0a88c4 --- /dev/null +++ b/pkg/cdc/table_scanner.go @@ -0,0 +1,173 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cdc + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/util/executor" +) + +var ( + scanner *TableScanner + once sync.Once + + scanSql = fmt.Sprintf("select "+ + " rel_id, "+ + " relname, "+ + " reldatabase_id, "+ + " reldatabase, "+ + " rel_createsql, "+ + " account_id "+ + "from "+ + " mo_catalog.mo_tables "+ + "where "+ + " relkind = '%s'"+ + " and reldatabase not in (%s)", + catalog.SystemOrdinaryRel, // only scan ordinary tables + AddSingleQuotesJoin(catalog.SystemDatabases), // skip system databases + ) +) + +var getSqlExecutor = func(cnUUID string) executor.SQLExecutor { + v, _ := runtime.ServiceRuntime(cnUUID).GetGlobalVariables(runtime.InternalSQLExecutor) + return v.(executor.SQLExecutor) +} + +var GetTableScanner = func(cnUUID string) *TableScanner { + once.Do(func() { + scanner = &TableScanner{ + Mutex: sync.Mutex{}, + Mp: make(map[uint32]TblMap), + Callbacks: make(map[string]func(map[uint32]TblMap)), + } + scanner.exec = getSqlExecutor(cnUUID) + }) + return scanner +} + +// TblMap key is dbName.tableName, e.g. db1.t1 +type TblMap map[string]*DbTableInfo + +type TableScanner struct { + sync.Mutex + + Mp map[uint32]TblMap + Callbacks map[string]func(map[uint32]TblMap) + exec executor.SQLExecutor + cancel context.CancelFunc +} + +func (s *TableScanner) Register(id string, cb func(map[uint32]TblMap)) { + s.Lock() + defer s.Unlock() + + if len(s.Callbacks) == 0 { + ctx, cancel := context.WithCancel(defines.AttachAccountId(context.Background(), catalog.System_Account)) + s.cancel = cancel + go s.scanTableLoop(ctx) + } + s.Callbacks[id] = cb +} + +func (s *TableScanner) UnRegister(id string) { + s.Lock() + defer s.Unlock() + + delete(s.Callbacks, id) + if len(s.Callbacks) == 0 && s.cancel != nil { + s.cancel() + s.cancel = nil + } +} + +func (s *TableScanner) scanTableLoop(ctx context.Context) { + logutil.Infof("cdc TableScanner.scanTableLoop: start") + defer func() { + logutil.Infof("cdc TableScanner.scanTableLoop: end") + }() + + timeTick := time.Tick(10 * time.Second) + for { + select { + case <-ctx.Done(): + return + case <-timeTick: + s.scanTable() + // do callbacks + s.Lock() + for _, cb := range s.Callbacks { + go cb(s.Mp) + } + s.Unlock() + } + } +} + +func (s *TableScanner) scanTable() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + result, err := s.exec.Exec(ctx, scanSql, executor.Options{}) + if err != nil { + return + } + defer result.Close() + + mp := make(map[uint32]TblMap) + result.ReadRows(func(rows int, cols []*vector.Vector) bool { + for i := 0; i < rows; i++ { + tblId := vector.MustFixedColWithTypeCheck[uint64](cols[0])[i] + tblName := cols[1].UnsafeGetStringAt(i) + dbId := vector.MustFixedColWithTypeCheck[uint64](cols[2])[i] + dbName := cols[3].UnsafeGetStringAt(i) + createSql := cols[4].UnsafeGetStringAt(i) + accountId := vector.MustFixedColWithTypeCheck[uint32](cols[5])[i] + + // skip table with foreign key + if strings.Contains(strings.ToLower("createSql"), "foreign key") { + continue + } + + if _, ok := mp[accountId]; !ok { + mp[accountId] = make(TblMap) + } + + key := GenDbTblKey(dbName, tblName) + mp[accountId][key] = &DbTableInfo{ + SourceDbId: dbId, + SourceDbName: dbName, + SourceTblId: tblId, + SourceTblName: tblName, + SourceCreateSql: createSql, + } + } + return true + }) + + // replace the old table map + s.Lock() + s.Mp = mp + s.Unlock() +} diff --git a/pkg/cdc/table_scanner_test.go b/pkg/cdc/table_scanner_test.go new file mode 100644 index 0000000000000..0eb7b50beb900 --- /dev/null +++ b/pkg/cdc/table_scanner_test.go @@ -0,0 +1,73 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cdc + +import ( + "sync" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/util/executor" + mock_executor "github.com/matrixorigin/matrixone/pkg/util/executor/test" + "github.com/prashantv/gostub" + "github.com/stretchr/testify/assert" +) + +func TestGetTableScanner(t *testing.T) { + gostub.Stub(&getSqlExecutor, func(cnUUID string) executor.SQLExecutor { + return &mock_executor.MockSQLExecutor{} + }) + assert.NotNil(t, GetTableScanner("cnUUID")) +} + +func TestTableScanner(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + bat := batch.New([]string{"tblId", "tblName", "dbId", "dbName", "createSql", "accountId"}) + bat.Vecs[0] = testutil.MakeUint64Vector([]uint64{1}, nil) + bat.Vecs[1] = testutil.MakeVarcharVector([]string{"tblName"}, nil) + bat.Vecs[2] = testutil.MakeUint64Vector([]uint64{1}, nil) + bat.Vecs[3] = testutil.MakeVarcharVector([]string{"dbName"}, nil) + bat.Vecs[4] = testutil.MakeVarcharVector([]string{"createSql"}, nil) + bat.Vecs[5] = testutil.MakeUint32Vector([]uint32{1}, nil) + bat.SetRowCount(1) + res := executor.Result{ + Mp: testutil.TestUtilMp, + Batches: []*batch.Batch{bat}, + } + + mockSqlExecutor := mock_executor.NewMockSQLExecutor(ctrl) + mockSqlExecutor.EXPECT().Exec(gomock.Any(), gomock.Any(), gomock.Any()).Return(res, nil).AnyTimes() + + scanner = &TableScanner{ + Mutex: sync.Mutex{}, + Mp: make(map[uint32]TblMap), + Callbacks: make(map[string]func(map[uint32]TblMap)), + exec: mockSqlExecutor, + } + + scanner.Register("id", func(mp map[uint32]TblMap) {}) + assert.Equal(t, 1, len(scanner.Callbacks)) + + // one round of scanTable + time.Sleep(11 * time.Second) + + scanner.UnRegister("id") + assert.Equal(t, 0, len(scanner.Callbacks)) +} diff --git a/pkg/cdc/types.go b/pkg/cdc/types.go index 643b4672e2262..2b0663bfe08f8 100644 --- a/pkg/cdc/types.go +++ b/pkg/cdc/types.go @@ -35,8 +35,11 @@ import ( ) const ( - AccountLevel = "account" - ClusterLevel = "cluster" + TableLevel = "table" + DbLevel = "database" + AccountLevel = "account" + MatchAll = "*" + MysqlSink = "mysql" MatrixoneSink = "matrixone" ConsoleSink = "console" @@ -92,6 +95,18 @@ type Sink interface { Close() } +type IWatermarkUpdater interface { + Run(ctx context.Context, ar *ActiveRoutine) + InsertIntoDb(dbTableInfo *DbTableInfo, watermark types.TS) error + GetFromMem(dbName, tblName string) types.TS + GetFromDb(dbName, tblName string) (watermark types.TS, err error) + UpdateMem(dbName, tblName string, watermark types.TS) + DeleteFromMem(dbName, tblName string) + DeleteFromDb(dbName, tblName string) error + DeleteAllFromDb() error + SaveErrMsg(dbName, tblName string, errMsg string) error +} + type ActiveRoutine struct { sync.Mutex Pause chan struct{} @@ -161,17 +176,14 @@ type RowIterator interface { } type DbTableInfo struct { - SourceAccountName string - SourceDbName string - SourceTblName string - SourceAccountId uint64 - SourceDbId uint64 - SourceTblId uint64 - SourceTblIdStr string + SourceDbId uint64 + SourceDbName string + SourceTblId uint64 + SourceTblName string + SourceCreateSql string - SinkAccountName string - SinkDbName string - SinkTblName string + SinkDbName string + SinkTblName string } func (info DbTableInfo) String() string { @@ -365,16 +377,12 @@ func (info *UriInfo) String() string { } type PatternTable struct { - AccountId uint64 `json:"account_id"` - Account string `json:"account"` - Database string `json:"database"` - Table string `json:"table"` - TableIsRegexp bool `json:"table_is_regexp"` - Reserved bool `json:"reserved"` + Database string `json:"database"` + Table string `json:"table"` } func (table PatternTable) String() string { - return fmt.Sprintf("(%s,%d,%s,%s)", table.Account, table.AccountId, table.Database, table.Table) + return fmt.Sprintf("%s.%s", table.Database, table.Table) } type PatternTuple struct { diff --git a/pkg/cdc/types_test.go b/pkg/cdc/types_test.go index 6a08ce82052ce..b4780896ae15d 100644 --- a/pkg/cdc/types_test.go +++ b/pkg/cdc/types_test.go @@ -205,15 +205,12 @@ func Test_atomicBatchRowIter(t *testing.T) { func TestDbTableInfo_String(t *testing.T) { type fields struct { - SourceAccountName string - SourceDbName string - SourceTblName string - SourceAccountId uint64 - SourceDbId uint64 - SourceTblId uint64 - SinkAccountName string - SinkDbName string - SinkTblName string + SourceDbName string + SourceTblName string + SourceDbId uint64 + SourceTblId uint64 + SinkDbName string + SinkTblName string } tests := []struct { name string @@ -235,15 +232,12 @@ func TestDbTableInfo_String(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { info := DbTableInfo{ - SourceAccountName: tt.fields.SourceAccountName, - SourceDbName: tt.fields.SourceDbName, - SourceTblName: tt.fields.SourceTblName, - SourceAccountId: tt.fields.SourceAccountId, - SourceDbId: tt.fields.SourceDbId, - SourceTblId: tt.fields.SourceTblId, - SinkAccountName: tt.fields.SinkAccountName, - SinkDbName: tt.fields.SinkDbName, - SinkTblName: tt.fields.SinkTblName, + SourceDbName: tt.fields.SourceDbName, + SourceTblName: tt.fields.SourceTblName, + SourceDbId: tt.fields.SourceDbId, + SourceTblId: tt.fields.SourceTblId, + SinkDbName: tt.fields.SinkDbName, + SinkTblName: tt.fields.SinkTblName, } assert.Equalf(t, tt.want, info.String(), "String()") }) @@ -331,12 +325,9 @@ func TestOutputType_String(t *testing.T) { func TestPatternTable_String(t *testing.T) { type fields struct { - AccountId uint64 - Account string - Database string - Table string - TableIsRegexp bool - Reserved bool + Database string + Table string + Reserved bool } tests := []struct { name string @@ -345,23 +336,17 @@ func TestPatternTable_String(t *testing.T) { }{ { fields: fields{ - AccountId: 123, - Account: "account", - Database: "database", - Table: "table", + Database: "database", + Table: "table", }, - want: "(account,123,database,table)", + want: "database.table", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { table := PatternTable{ - AccountId: tt.fields.AccountId, - Account: tt.fields.Account, - Database: tt.fields.Database, - Table: tt.fields.Table, - TableIsRegexp: tt.fields.TableIsRegexp, - Reserved: tt.fields.Reserved, + Database: tt.fields.Database, + Table: tt.fields.Table, } assert.Equalf(t, tt.want, table.String(), "String()") }) @@ -383,19 +368,15 @@ func TestPatternTuple_String(t *testing.T) { { fields: fields{ Source: PatternTable{ - AccountId: 123, - Account: "account1", - Database: "database1", - Table: "table1", + Database: "database1", + Table: "table1", }, Sink: PatternTable{ - AccountId: 456, - Account: "account2", - Database: "database2", - Table: "table2", + Database: "database2", + Table: "table2", }, }, - want: "(account1,123,database1,table1),(account2,456,database2,table2)", + want: "database1.table1,database2.table2", }, } for _, tt := range tests { @@ -434,10 +415,8 @@ func TestPatternTuples_Append(t *testing.T) { args: args{ pt: &PatternTuple{ Source: PatternTable{ - AccountId: 123, - Account: "account1", - Database: "database1", - Table: "table1", + Database: "database1", + Table: "table1", }, }, }, @@ -475,21 +454,17 @@ func TestPatternTuples_String(t *testing.T) { Pts: []*PatternTuple{ { Source: PatternTable{ - AccountId: 123, - Account: "account1", - Database: "database1", - Table: "table1", + Database: "database1", + Table: "table1", }, Sink: PatternTable{ - AccountId: 456, - Account: "account2", - Database: "database2", - Table: "table2", + Database: "database2", + Table: "table2", }, }, }, }, - want: "(account1,123,database1,table1),(account2,456,database2,table2)", + want: "database1.table1,database2.table2", }, } for _, tt := range tests { diff --git a/pkg/cdc/util.go b/pkg/cdc/util.go index a67411db18a24..72dd13d9ce0d2 100644 --- a/pkg/cdc/util.go +++ b/pkg/cdc/util.go @@ -26,6 +26,7 @@ import ( "math/rand" "slices" "strconv" + "strings" "time" "go.uber.org/zap" @@ -623,7 +624,7 @@ var ExitRunSql = func(txnOp client.TxnOperator) { txnOp.ExitRunSql() } -func GetTableDef( +var GetTableDef = func( ctx context.Context, txnOp client.TxnOperator, cnEngine engine.Engine, @@ -732,3 +733,23 @@ func batchRowCount(bat *batch.Batch) int { } return bat.Vecs[0].Length() } + +// AddSingleQuotesJoin [a, b, c] -> 'a','b','c' +func AddSingleQuotesJoin(s []string) string { + if len(s) == 0 { + return "" + } + return "'" + strings.Join(s, "','") + "'" +} + +func GenDbTblKey(dbName, tblName string) string { + return dbName + "." + tblName +} + +func SplitDbTblKey(dbTblKey string) (dbName, tblName string) { + s := strings.Split(dbTblKey, ".") + if len(s) != 2 { + return + } + return s[0], s[1] +} diff --git a/pkg/cdc/watermark_updater.go b/pkg/cdc/watermark_updater.go index 90d2f8fd36ce8..f04308bc51822 100644 --- a/pkg/cdc/watermark_updater.go +++ b/pkg/cdc/watermark_updater.go @@ -35,21 +35,21 @@ const ( maxErrMsgLen = 256 - insertWatermarkFormat = "insert into mo_catalog.mo_cdc_watermark values (%d, '%s', '%s', '%s', '%s', '%s', '%s')" + insertWatermarkFormat = "insert into mo_catalog.mo_cdc_watermark values (%d, '%s', '%s', '%s', '%s', '%s')" - getWatermarkFormat = "select watermark from mo_catalog.mo_cdc_watermark where account_id = %d and task_id = '%s' and table_id = '%s'" + getWatermarkFormat = "select watermark from mo_catalog.mo_cdc_watermark where account_id = %d and task_id = '%s' and db_name = '%s' and table_name = '%s'" - getAllWatermarkFormat = "select table_id, watermark from mo_catalog.mo_cdc_watermark where account_id = %d and task_id = '%s'" - - updateWatermarkFormat = "update mo_catalog.mo_cdc_watermark set watermark='%s' where account_id = %d and task_id = '%s' and table_id = '%s'" + updateWatermarkFormat = "update mo_catalog.mo_cdc_watermark set watermark='%s' where account_id = %d and task_id = '%s' and db_name = '%s' and table_name = '%s'" deleteWatermarkFormat = "delete from mo_catalog.mo_cdc_watermark where account_id = %d and task_id = '%s'" - deleteWatermarkByTableFormat = "delete from mo_catalog.mo_cdc_watermark where account_id = %d and task_id = '%s' and table_id = '%s'" + deleteWatermarkByTableFormat = "delete from mo_catalog.mo_cdc_watermark where account_id = %d and task_id = '%s' and db_name = '%s' and table_name = '%s'" - updateErrMsgFormat = "update mo_catalog.mo_cdc_watermark set err_msg='%s' where account_id = %d and task_id = '%s' and table_id = '%s'" + updateErrMsgFormat = "update mo_catalog.mo_cdc_watermark set err_msg='%s' where account_id = %d and task_id = '%s' and db_name = '%s' and table_name = '%s'" ) +var _ IWatermarkUpdater = new(WatermarkUpdater) + type WatermarkUpdater struct { accountId uint32 taskId uuid.UUID @@ -59,9 +59,9 @@ type WatermarkUpdater struct { watermarkMap *sync.Map } -func NewWatermarkUpdater(accountId uint64, taskId string, ie ie.InternalExecutor) *WatermarkUpdater { +func NewWatermarkUpdater(accountId uint32, taskId string, ie ie.InternalExecutor) *WatermarkUpdater { u := &WatermarkUpdater{ - accountId: uint32(accountId), + accountId: accountId, ie: ie, watermarkMap: &sync.Map{}, } @@ -93,29 +93,29 @@ func (u *WatermarkUpdater) Run(ctx context.Context, ar *ActiveRoutine) { func (u *WatermarkUpdater) InsertIntoDb(dbTableInfo *DbTableInfo, watermark types.TS) error { sql := fmt.Sprintf(insertWatermarkFormat, u.accountId, u.taskId, - dbTableInfo.SourceTblIdStr, dbTableInfo.SourceDbName, dbTableInfo.SourceTblName, + dbTableInfo.SourceDbName, dbTableInfo.SourceTblName, watermark.ToString(), "") ctx := defines.AttachAccountId(context.Background(), catalog.System_Account) return u.ie.Exec(ctx, sql, ie.SessionOverrideOptions{}) } -func (u *WatermarkUpdater) GetFromMem(tableIdStr string) types.TS { - if value, ok := u.watermarkMap.Load(tableIdStr); ok { +func (u *WatermarkUpdater) GetFromMem(dbName, tblName string) types.TS { + if value, ok := u.watermarkMap.Load(GenDbTblKey(dbName, tblName)); ok { return value.(types.TS) } return types.TS{} } -func (u *WatermarkUpdater) GetFromDb(tableIdStr string) (watermark types.TS, err error) { - sql := fmt.Sprintf(getWatermarkFormat, u.accountId, u.taskId, tableIdStr) +func (u *WatermarkUpdater) GetFromDb(dbName, tblName string) (watermark types.TS, err error) { + sql := fmt.Sprintf(getWatermarkFormat, u.accountId, u.taskId, dbName, tblName) ctx := defines.AttachAccountId(context.Background(), catalog.System_Account) res := u.ie.Query(ctx, sql, ie.SessionOverrideOptions{}) if res.Error() != nil { err = res.Error() } else if res.RowCount() < 1 { - err = moerr.NewInternalErrorf(ctx, "no watermark found for task: %s, tableIdStr: %v\n", u.taskId, tableIdStr) + err = moerr.NewErrNoWatermarkFoundNoCtx(dbName, tblName) } else if res.RowCount() > 1 { - err = moerr.NewInternalErrorf(ctx, "duplicate watermark found for task: %s, tableIdStr: %v\n", u.taskId, tableIdStr) + err = moerr.NewInternalErrorf(ctx, "duplicate watermark found for task: %s, table: %s.%s", u.taskId, dbName, tblName) } if err != nil { return @@ -128,40 +128,16 @@ func (u *WatermarkUpdater) GetFromDb(tableIdStr string) (watermark types.TS, err return types.StringToTS(watermarkStr), nil } -func (u *WatermarkUpdater) GetAllFromDb() (mp map[string]types.TS, err error) { - sql := fmt.Sprintf(getAllWatermarkFormat, u.accountId, u.taskId) - ctx := defines.AttachAccountId(context.Background(), catalog.System_Account) - res := u.ie.Query(ctx, sql, ie.SessionOverrideOptions{}) - if res.Error() != nil { - err = res.Error() - return - } - - var tableIdStr string - var watermarkStr string - mp = make(map[string]types.TS) - for i := uint64(0); i < res.RowCount(); i++ { - if tableIdStr, err = res.GetString(ctx, i, 0); err != nil { - return - } - if watermarkStr, err = res.GetString(ctx, i, 1); err != nil { - return - } - mp[tableIdStr] = types.StringToTS(watermarkStr) - } - return -} - -func (u *WatermarkUpdater) UpdateMem(tableIdStr string, watermark types.TS) { - u.watermarkMap.Store(tableIdStr, watermark) +func (u *WatermarkUpdater) UpdateMem(dbName, tblName string, watermark types.TS) { + u.watermarkMap.Store(GenDbTblKey(dbName, tblName), watermark) } -func (u *WatermarkUpdater) DeleteFromMem(tableIdStr string) { - u.watermarkMap.Delete(tableIdStr) +func (u *WatermarkUpdater) DeleteFromMem(dbName, tblName string) { + u.watermarkMap.Delete(GenDbTblKey(dbName, tblName)) } -func (u *WatermarkUpdater) DeleteFromDb(tableIdStr string) error { - sql := fmt.Sprintf(deleteWatermarkByTableFormat, u.accountId, u.taskId, tableIdStr) +func (u *WatermarkUpdater) DeleteFromDb(dbName, tblName string) error { + sql := fmt.Sprintf(deleteWatermarkByTableFormat, u.accountId, u.taskId, dbName, tblName) ctx := defines.AttachAccountId(context.Background(), catalog.System_Account) return u.ie.Exec(ctx, sql, ie.SessionOverrideOptions{}) } @@ -172,28 +148,29 @@ func (u *WatermarkUpdater) DeleteAllFromDb() error { return u.ie.Exec(ctx, sql, ie.SessionOverrideOptions{}) } -func (u *WatermarkUpdater) SaveErrMsg(tableIdStr string, errMsg string) error { +func (u *WatermarkUpdater) SaveErrMsg(dbName, tblName string, errMsg string) error { if len(errMsg) > maxErrMsgLen { errMsg = errMsg[:maxErrMsgLen] } - sql := fmt.Sprintf(updateErrMsgFormat, errMsg, u.accountId, u.taskId, tableIdStr) + sql := fmt.Sprintf(updateErrMsgFormat, errMsg, u.accountId, u.taskId, dbName, tblName) ctx := defines.AttachAccountId(context.Background(), catalog.System_Account) return u.ie.Exec(ctx, sql, ie.SessionOverrideOptions{}) } func (u *WatermarkUpdater) flushAll() { u.watermarkMap.Range(func(k, v any) bool { - tableIdStr := k.(string) + key := k.(string) ts := v.(types.TS) - if err := u.flush(tableIdStr, ts); err != nil { - logutil.Errorf("flush table %s failed, current watermark: %s err: %v\n", tableIdStr, ts.ToString(), err) + if err := u.flush(key, ts); err != nil { + logutil.Errorf("flush table %s failed, current watermark: %s err: %v", key, ts.ToString(), err) } return true }) } -func (u *WatermarkUpdater) flush(tableIdStr string, watermark types.TS) error { - sql := fmt.Sprintf(updateWatermarkFormat, watermark.ToString(), u.accountId, u.taskId, tableIdStr) +func (u *WatermarkUpdater) flush(key string, watermark types.TS) error { + dbName, tblName := SplitDbTblKey(key) + sql := fmt.Sprintf(updateWatermarkFormat, watermark.ToString(), u.accountId, u.taskId, dbName, tblName) ctx := defines.AttachAccountId(context.Background(), catalog.System_Account) return u.ie.Exec(ctx, sql, ie.SessionOverrideOptions{}) } diff --git a/pkg/cdc/watermark_updater_test.go b/pkg/cdc/watermark_updater_test.go index 55917c0c0ccb1..842c3c4195c90 100644 --- a/pkg/cdc/watermark_updater_test.go +++ b/pkg/cdc/watermark_updater_test.go @@ -17,14 +17,12 @@ package cdc import ( "context" "regexp" - "strconv" "strings" "sync" "testing" "time" "github.com/google/uuid" - "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" ie "github.com/matrixorigin/matrixone/pkg/util/internalExecutor" "github.com/stretchr/testify/assert" @@ -32,35 +30,34 @@ import ( ) type wmMockSQLExecutor struct { - mp map[string]string - insertRe *regexp.Regexp - updateRe *regexp.Regexp - selectRe *regexp.Regexp - selectAllRe *regexp.Regexp + mp map[string]string + insertRe *regexp.Regexp + updateRe *regexp.Regexp + selectRe *regexp.Regexp } func newWmMockSQLExecutor() *wmMockSQLExecutor { return &wmMockSQLExecutor{ - mp: make(map[string]string), - insertRe: regexp.MustCompile(`^insert .* values \(.*\, .*\, \'(.*)\'\, .*\, .*\, \'(.*)\'\, \'\'\)$`), - updateRe: regexp.MustCompile(`^update .* set watermark\=\'(.*)\' where .* and table_id \= '(.*)'$`), - selectRe: regexp.MustCompile(`^select .* and table_id \= '(.*)'$`), - selectAllRe: regexp.MustCompile(`^select .* where account_id \= (.*) and task_id .*`), + mp: make(map[string]string), + // matches[1] = db_name, matches[2] = table_name, matches[3] = watermark + insertRe: regexp.MustCompile(`^insert .* values \(.*\, .*\, \'(.*)\'\, \'(.*)\'\, \'(.*)\'\, \'\'\)$`), + updateRe: regexp.MustCompile(`^update .* set watermark\=\'(.*)\' where .* and db_name \= '(.*)' and table_name \= '(.*)'$`), + selectRe: regexp.MustCompile(`^select .* and db_name \= '(.*)' and table_name \= '(.*)'$`), } } func (m *wmMockSQLExecutor) Exec(_ context.Context, sql string, _ ie.SessionOverrideOptions) error { if strings.HasPrefix(sql, "insert") { matches := m.insertRe.FindStringSubmatch(sql) - m.mp[matches[1]] = matches[2] + m.mp[GenDbTblKey(matches[1], matches[2])] = matches[3] } else if strings.HasPrefix(sql, "update mo_catalog.mo_cdc_watermark set err_msg") { // do nothing } else if strings.HasPrefix(sql, "update") { matches := m.updateRe.FindStringSubmatch(sql) - m.mp[matches[2]] = matches[1] + m.mp[GenDbTblKey(matches[2], matches[3])] = matches[1] } else if strings.HasPrefix(sql, "delete") { if strings.Contains(sql, "table_id") { - delete(m.mp, "1_0") + delete(m.mp, "db1.t1") } else { m.mp = make(map[string]string) } @@ -120,27 +117,7 @@ func (res *internalExecResult) GetString(ctx context.Context, i uint64, j uint64 } func (m *wmMockSQLExecutor) Query(ctx context.Context, sql string, pts ie.SessionOverrideOptions) ie.InternalExecResult { - if strings.HasPrefix(sql, "select table_id") { - matches := m.selectAllRe.FindStringSubmatch(sql) - accountId, _ := strconv.Atoi(matches[1]) - if accountId == 1 { // normal path - var data [][]interface{} - for k, v := range m.mp { - data = append(data, []interface{}{k, v}) - } - return &internalExecResult{ - affectedRows: 1, - resultSet: &MysqlResultSet{ - Columns: nil, - Name2Index: nil, - Data: data, - }, - err: nil, - } - } else { // error path - return &internalExecResult{err: moerr.NewInternalErrorNoCtx("error")} - } - } else if strings.HasPrefix(sql, "select") { + if strings.HasPrefix(sql, "select") { matches := m.selectRe.FindStringSubmatch(sql) return &internalExecResult{ affectedRows: 1, @@ -148,13 +125,12 @@ func (m *wmMockSQLExecutor) Query(ctx context.Context, sql string, pts ie.Sessio Columns: nil, Name2Index: nil, Data: [][]interface{}{ - {m.mp[matches[1]]}, + {m.mp[GenDbTblKey(matches[1], matches[2])]}, }, }, err: nil, } } - return nil } @@ -165,7 +141,7 @@ func TestNewWatermarkUpdater(t *testing.T) { require.NoError(t, err) type args struct { - accountId uint64 + accountId uint32 taskId string ie ie.InternalExecutor } @@ -205,12 +181,12 @@ func TestWatermarkUpdater_MemOps(t *testing.T) { } t1 := types.BuildTS(1, 1) - u.UpdateMem("1_0", t1) - actual := u.GetFromMem("1_0") + u.UpdateMem("db1", "t1", t1) + actual := u.GetFromMem("db1", "t1") assert.Equal(t, t1, actual) - u.DeleteFromMem("1_0") - actual = u.GetFromMem("1_0") + u.DeleteFromMem("db1", "t1") + actual = u.GetFromMem("db1", "t1") assert.Equal(t, types.TS{}, actual) } @@ -222,69 +198,49 @@ func TestWatermarkUpdater_DbOps(t *testing.T) { watermarkMap: &sync.Map{}, } - // ---------- init count is 0 - mp, err := u.GetAllFromDb() - assert.NoError(t, err) - assert.Equal(t, 0, len(mp)) - // ---------- insert into a record t1 := types.BuildTS(1, 1) info1 := &DbTableInfo{ - SourceTblIdStr: "1_0", - SourceDbName: "dbName", - SourceTblName: "tblName", + SourceDbName: "db1", + SourceTblName: "t1", } - err = u.InsertIntoDb(info1, t1) - assert.NoError(t, err) - // count is 1 - mp, err = u.GetAllFromDb() + err := u.InsertIntoDb(info1, t1) assert.NoError(t, err) - assert.Equal(t, 1, len(mp)) // get value of tableId 1 - actual, err := u.GetFromDb("1_0") + actual, err := u.GetFromDb("db1", "t1") assert.NoError(t, err) assert.Equal(t, t1, actual) // ---------- update t1 -> t2 t2 := types.BuildTS(2, 1) - err = u.flush("1_0", t2) + err = u.flush("db1.t1", t2) assert.NoError(t, err) // value is t2 - actual, err = u.GetFromDb("1_0") + actual, err = u.GetFromDb("db1", "t1") assert.NoError(t, err) assert.Equal(t, t2, actual) + // ---------- delete tableId 1 + err = u.DeleteFromDb("db1", "t1") + assert.NoError(t, err) + // ---------- insert more records info2 := &DbTableInfo{ - SourceTblIdStr: "2_0", - SourceDbName: "dbName", - SourceTblName: "tblName", + SourceDbName: "db2", + SourceTblName: "t2", } err = u.InsertIntoDb(info2, t1) assert.NoError(t, err) info3 := &DbTableInfo{ - SourceTblIdStr: "3_0", - SourceDbName: "dbName", - SourceTblName: "tblName", + SourceDbName: "db3", + SourceTblName: "t3", } err = u.InsertIntoDb(info3, t1) assert.NoError(t, err) - // ---------- delete tableId 1 - err = u.DeleteFromDb("1_0") - assert.NoError(t, err) - // count is 2 - mp, err = u.GetAllFromDb() - assert.NoError(t, err) - assert.Equal(t, 2, len(mp)) - // ---------- delete all err = u.DeleteAllFromDb() assert.NoError(t, err) - // count is 0 - mp, err = u.GetAllFromDb() - assert.NoError(t, err) - assert.Equal(t, 0, len(mp)) } func TestWatermarkUpdater_Run(t *testing.T) { @@ -311,50 +267,37 @@ func TestWatermarkUpdater_flushAll(t *testing.T) { t1 := types.BuildTS(1, 1) info1 := &DbTableInfo{ - SourceTblIdStr: "1_0", - SourceDbName: "dbName", - SourceTblName: "tblName", + SourceDbName: "db1", + SourceTblName: "t1", } err := u.InsertIntoDb(info1, t1) assert.NoError(t, err) info2 := &DbTableInfo{ - SourceTblIdStr: "1_0", - SourceDbName: "dbName", - SourceTblName: "tblName", + SourceDbName: "db2", + SourceTblName: "t2", } err = u.InsertIntoDb(info2, t1) assert.NoError(t, err) info3 := &DbTableInfo{ - SourceTblIdStr: "1_0", - SourceDbName: "dbName", - SourceTblName: "tblName", + SourceDbName: "db3", + SourceTblName: "t3", } err = u.InsertIntoDb(info3, t1) assert.NoError(t, err) t2 := types.BuildTS(2, 1) - u.UpdateMem("1_0", t2) - u.UpdateMem("2_0", t2) - u.UpdateMem("3_0", t2) + u.UpdateMem("db1", "t1", t2) + u.UpdateMem("db2", "t2", t2) + u.UpdateMem("db3", "t3", t2) u.flushAll() - actual, err := u.GetFromDb("1_0") + actual, err := u.GetFromDb("db1", "t1") assert.NoError(t, err) assert.Equal(t, t2, actual) - actual, err = u.GetFromDb("2_0") + actual, err = u.GetFromDb("db2", "t2") assert.NoError(t, err) assert.Equal(t, t2, actual) - actual, err = u.GetFromDb("3_0") + actual, err = u.GetFromDb("db3", "t3") assert.NoError(t, err) assert.Equal(t, t2, actual) } - -func TestWatermarkUpdater_GetAllFromDb(t *testing.T) { - u := &WatermarkUpdater{ - accountId: 2, - taskId: uuid.New(), - ie: newWmMockSQLExecutor(), - } - _, err := u.GetAllFromDb() - assert.Error(t, err) -} diff --git a/pkg/common/moerr/cause.go b/pkg/common/moerr/cause.go index e0f25e599bf86..b1ebb38a9726f 100644 --- a/pkg/common/moerr/cause.go +++ b/pkg/common/moerr/cause.go @@ -249,6 +249,7 @@ var ( //pkg/vm/engine/tae/db/merge CauseCleanUpUselessFiles = NewInternalError(context.Background(), "CleanUpUselessFiles") CauseOnObject = NewInternalError(context.Background(), "OnObject") + CauseCreateCNMerge = NewInternalError(context.Background(), "CreateCNMergeTask") //pkg/vm/engine/tae/logstore/driver/logservicedriver CauseDriverAppender1 = NewInternalError(context.Background(), "DriverAppender append 1") CauseDriverAppender2 = NewInternalError(context.Background(), "DriverAppender append 2") diff --git a/pkg/common/moerr/error.go b/pkg/common/moerr/error.go index 38b41e3222781..5309b3ebd5f83 100644 --- a/pkg/common/moerr/error.go +++ b/pkg/common/moerr/error.go @@ -290,7 +290,8 @@ const ( ErrTooLargeObjectSize uint16 = 22001 // Group 13: CDC - ErrStaleRead uint16 = 22101 + ErrStaleRead uint16 = 22101 + ErrNoWatermarkFound uint16 = 22102 // ErrEnd, the max value of MOErrorCode ErrEnd uint16 = 65535 @@ -525,7 +526,9 @@ var errorMsgRefer = map[uint16]moErrorMsgItem{ ErrTooLargeObjectSize: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "objectio: too large object size %d"}, // Group 13: CDC - ErrStaleRead: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "CDC handle: stale read, min TS is %v, receive %v"}, + ErrStaleRead: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "CDC handle: stale read, min TS is %v, receive %v"}, + ErrNoWatermarkFound: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "CDC task: no watermark found of table %s.%s"}, + // Group End: max value of MOErrorCode ErrEnd: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "internal error: end of errcode code"}, } diff --git a/pkg/common/moerr/error_no_ctx.go b/pkg/common/moerr/error_no_ctx.go index 2ab51c92068c5..547e361e6c29f 100644 --- a/pkg/common/moerr/error_no_ctx.go +++ b/pkg/common/moerr/error_no_ctx.go @@ -438,6 +438,10 @@ func NewErrStaleReadNoCtx(minTS, start string) *Error { return newError(Context(), ErrStaleRead, minTS, start) } +func NewErrNoWatermarkFoundNoCtx(dbName, tblName string) *Error { + return newError(Context(), ErrNoWatermarkFound, dbName, tblName) +} + func NewArenaFullNoCtx() *Error { return newError(Context(), ErrArenaFull) } diff --git a/pkg/container/batch/compact_batchs.go b/pkg/container/batch/compact_batchs.go index 822a68f380890..baf5799fea250 100644 --- a/pkg/container/batch/compact_batchs.go +++ b/pkg/container/batch/compact_batchs.go @@ -18,7 +18,6 @@ import ( "context" "github.com/matrixorigin/matrixone/pkg/common/mpool" - "github.com/matrixorigin/matrixone/pkg/container/vector" ) const ( @@ -31,7 +30,6 @@ const ( // until bats.Batchs[lastIdx].rowCount to DefaultBatchMaxRow type CompactBatchs struct { batchs []*Batch - ufs []func(*vector.Vector, *vector.Vector) error // functions for vector union } func NewCompactBatchs() *CompactBatchs { @@ -160,13 +158,6 @@ func (bats *CompactBatchs) fillData(mpool *mpool.MPool, inBatch *Batch) error { var tmpBat *Batch var err error - if len(bats.ufs) == 0 { - for i := 0; i < inBatch.VectorCount(); i++ { - typ := *inBatch.GetVector(int32(i)).GetType() - bats.ufs = append(bats.ufs, vector.GetUnionAllFunction(typ, mpool)) - } - } - //fill data start, end := 0, inBatch.RowCount() isNewBat := false @@ -193,21 +184,15 @@ func (bats *CompactBatchs) fillData(mpool *mpool.MPool, inBatch *Batch) error { return err } } + tmpBat.AddRowCount(addRowCount) } else { - for i := range tmpBat.Vecs { - srcVec, err := inBatch.Vecs[i].Window(start, start+addRowCount) - if err != nil { - return err - } - err = bats.ufs[i](tmpBat.Vecs[i], srcVec) - if err != nil { - return err - } + err := tmpBat.UnionWindow(inBatch, start, addRowCount, mpool) + if err != nil { + return err } } start = start + addRowCount - tmpBat.AddRowCount(addRowCount) } return nil diff --git a/pkg/frontend/cdc.go b/pkg/frontend/cdc.go index 02085c7e3a448..0247877300450 100644 --- a/pkg/frontend/cdc.go +++ b/pkg/frontend/cdc.go @@ -19,9 +19,10 @@ import ( "database/sql" "encoding/json" "fmt" - "math" + "regexp" "strconv" "strings" + "sync" "time" "github.com/google/uuid" @@ -287,67 +288,33 @@ func doCreateCdc(ctx context.Context, ses *Session, create *tree.CreateCDC) (err cdcTaskOptionsMap[create.Option[i]] = create.Option[i+1] } - cdcLevel := cdcTaskOptionsMap["Level"] - cdcAccount := cdcTaskOptionsMap["Account"] - - if cdcLevel != cdc2.AccountLevel { - return moerr.NewInternalError(ctx, "invalid level. only support account level in 1.3") - } - - if cdcAccount != ses.GetTenantInfo().GetTenant() { - return moerr.NewInternalErrorf(ctx, "invalid account. account must be %s", ses.GetTenantInfo().GetTenant()) - } - - //////////// - //!!!NOTE!!! - //1.3 - // level: account level - // account: must be designated. - /////////// - - //step 1 : handle tables - tablePts, err := preprocessTables( - ctx, - ses, - cdcLevel, - cdcAccount, - create.Tables, - ) - if err != nil { - return err + // step 1: handle tables + level := cdcTaskOptionsMap["Level"] + if level != cdc2.AccountLevel && level != cdc2.DbLevel && level != cdc2.TableLevel { + return moerr.NewInternalErrorf(ctx, "invalid level: %s", level) } - - //step 2: handle filters - //There must be no special characters (',' '.' ':' '`') in the single rule. - filters := cdcTaskOptionsMap["Rules"] - - jsonFilters, filterPts, err := preprocessRules(ctx, filters) + tablePts, err := getPatternTuples(ctx, level, create.Tables) if err != nil { return err } - - err = attachAccountToFilters(ctx, ses, cdcLevel, cdcAccount, filterPts) + jsonTables, err := cdc2.JsonEncode(tablePts) if err != nil { - return err + return } - //TODO: refine it after 1.3 - //check table be filtered or not - if filterTable(tablePts, filterPts) == 0 { - return moerr.NewInternalError(ctx, "all tables has been excluded by filters. create cdc failed.") + // step 2: handle exclude (regular expression) + exclude := cdcTaskOptionsMap["Exclude"] + if _, err = regexp.Compile(exclude); err != nil { + return moerr.NewInternalErrorf(ctx, "invalid exclude expression: %s, err: %v", exclude, err) } - dat := time.Now().UTC() - - creatorAccInfo := ses.GetTenantInfo() - cdcId, _ := uuid.NewV7() - - //step 4: check uri format and strip password + //step 4: check source uri format and strip password jsonSrcUri, _, err := extractUriInfo(ctx, create.SourceUri, cdc2.SourceUriPrefix) if err != nil { return err } + //step 5: check sink uri format, strip password and check connection sinkType := strings.ToLower(create.SinkType) useConsole := false if cdc2.EnableConsoleSink && sinkType == cdc2.ConsoleSink { @@ -358,7 +325,6 @@ func doCreateCdc(ctx context.Context, ses *Session, create *tree.CreateCDC) (err return moerr.NewInternalErrorf(ctx, "unsupported sink type: %s", create.SinkType) } - //step 5: check downstream connectivity jsonSinkUri, sinkUriInfo, err := extractUriInfo(ctx, create.SinkUri, cdc2.SinkUriPrefix) if err != nil { return @@ -368,22 +334,27 @@ func doCreateCdc(ctx context.Context, ses *Session, create *tree.CreateCDC) (err return } - noFull := false - if cdcTaskOptionsMap["NoFull"] == "true" { - noFull = true - } + //step 6: no full + noFull, _ := strconv.ParseBool(cdcTaskOptionsMap["NoFull"]) + //step 7: additionalConfig additionalConfig := make(map[string]any) + + // InitSnapshotSplitTxn additionalConfig[cdc2.InitSnapshotSplitTxn] = cdc2.DefaultInitSnapshotSplitTxn if cdcTaskOptionsMap[cdc2.InitSnapshotSplitTxn] == "false" { additionalConfig[cdc2.InitSnapshotSplitTxn] = false } + + // MaxSqlLength additionalConfig[cdc2.MaxSqlLength] = cdc2.DefaultMaxSqlLength if val, ok := cdcTaskOptionsMap[cdc2.MaxSqlLength]; ok { if additionalConfig[cdc2.MaxSqlLength], err = strconv.ParseUint(val, 10, 64); err != nil { return } } + + // SendSqlTimeout additionalConfig[cdc2.SendSqlTimeout] = cdc2.DefaultSendSqlTimeout if val, ok := cdcTaskOptionsMap[cdc2.SendSqlTimeout]; ok { // check duration format @@ -393,24 +364,30 @@ func doCreateCdc(ctx context.Context, ses *Session, create *tree.CreateCDC) (err additionalConfig[cdc2.SendSqlTimeout] = val } + // marshal additionalConfigBytes, err := json.Marshal(additionalConfig) if err != nil { return err } + //step 8: details + accountInfo := ses.GetTenantInfo() + accountId := accountInfo.GetTenantID() + accountName := accountInfo.GetTenant() + cdcId, _ := uuid.NewV7() details := &task.Details{ //account info that create cdc - AccountID: creatorAccInfo.GetTenantID(), - Account: creatorAccInfo.GetTenant(), - Username: creatorAccInfo.GetUser(), + AccountID: accountId, + Account: accountName, + Username: accountInfo.GetUser(), Details: &task.Details_CreateCdc{ CreateCdc: &task.CreateCdcDetails{ TaskName: create.TaskName.String(), TaskId: cdcId.String(), Accounts: []*task.Account{ { - Id: uint64(creatorAccInfo.GetTenantID()), - Name: cdcAccount, + Id: uint64(accountId), + Name: accountName, }, }, }, @@ -418,21 +395,6 @@ func doCreateCdc(ctx context.Context, ses *Session, create *tree.CreateCDC) (err } addCdcTaskCallback := func(ctx context.Context, tx taskservice.SqlExecutor) (ret int, err error) { - err = checkAccounts(ctx, tx, tablePts, filterPts) - if err != nil { - return 0, err - } - - ret, err = checkTables(ctx, tx, tablePts, filterPts) - if err != nil { - return - } - - jsonTables, err := cdc2.JsonEncode(tablePts) - if err != nil { - return 0, err - } - var encodedSinkPwd string if !useConsole { // TODO replace with creatorAccountId @@ -445,9 +407,11 @@ func doCreateCdc(ctx context.Context, ses *Session, create *tree.CreateCDC) (err } } + exclude = strings.ReplaceAll(exclude, "\\", "\\\\") + //step 5: create daemon task insertSql := getSqlForNewCdcTask( - uint64(creatorAccInfo.GetTenantID()), //the account_id of cdc creator + uint64(accountId), //the account_id of cdc creator cdcId, create.TaskName.String(), jsonSrcUri, //json bytes @@ -459,14 +423,14 @@ func doCreateCdc(ctx context.Context, ses *Session, create *tree.CreateCDC) (err "", "", jsonTables, - jsonFilters, + exclude, "", cdc2.SASCommon, cdc2.SASCommon, "", //1.3 does not support startTs "", //1.3 does not support endTs cdcTaskOptionsMap["ConfigFile"], - dat, + time.Now().UTC(), CdcRunning, 0, noFull, @@ -506,114 +470,6 @@ func cdcTaskMetadata(cdcId string) task.TaskMetadata { } } -// checkAccounts checks the accounts exists or not -func checkAccounts(ctx context.Context, tx taskservice.SqlExecutor, tablePts, filterPts *cdc2.PatternTuples) error { - //step1 : collect accounts - accounts := make(map[string]uint64) - for _, pt := range tablePts.Pts { - if pt == nil || pt.Source.Account == "" { - continue - } - accounts[pt.Source.Account] = math.MaxUint64 - } - - for _, pt := range filterPts.Pts { - if pt == nil || pt.Source.Account == "" { - continue - } - accounts[pt.Source.Account] = math.MaxUint64 - } - - //step2 : collect account id - //after this step, all account has accountid - res := make(map[string]uint64) - for acc := range accounts { - exists, accId, err := checkAccountExists(ctx, tx, acc) - if err != nil { - return err - } - if !exists { - return moerr.NewInternalErrorf(ctx, "account %s does not exist", acc) - } - res[acc] = accId - } - - //step3: attach accountId - for _, pt := range tablePts.Pts { - if pt == nil || pt.Source.Account == "" { - continue - } - pt.Source.AccountId = res[pt.Source.Account] - } - - for _, pt := range filterPts.Pts { - if pt == nil || pt.Source.Account == "" { - continue - } - pt.Source.AccountId = res[pt.Source.Account] - } - - return nil -} - -// checkTables -// checks the table existed or not -// checks the table having the primary key -// filters the table -func checkTables(ctx context.Context, tx taskservice.SqlExecutor, tablePts, filterPts *cdc2.PatternTuples) (int, error) { - var err error - var found bool - var hasPrimaryKey bool - for _, pt := range tablePts.Pts { - if pt == nil { - continue - } - - //skip tables that is filtered - if needSkipThisTable(pt.Source.Account, pt.Source.Database, pt.Source.Table, filterPts) { - continue - } - - //check tables exists or not and filter the table - found, err = checkTableExists(ctx, tx, pt.Source.AccountId, pt.Source.Database, pt.Source.Table) - if err != nil { - return 0, err - } - if !found { - return 0, moerr.NewInternalErrorf(ctx, "no table %s:%s", pt.Source.Database, pt.Source.Table) - } - - //check table has primary key - hasPrimaryKey, err = checkPrimaryKey(ctx, tx, pt.Source.AccountId, pt.Source.Database, pt.Source.Table) - if err != nil { - return 0, err - } - if !hasPrimaryKey { - return 0, moerr.NewInternalErrorf(ctx, "table %s:%s does not have primary key", pt.Source.Database, pt.Source.Table) - } - } - return 0, err -} - -// filterTable checks the table filtered or not -// returns the count of tables that not be filtered -func filterTable(tablePts, filterPts *cdc2.PatternTuples) int { - //check table be filtered or not - leftCount := 0 - for _, pt := range tablePts.Pts { - if pt == nil { - continue - } - - //skip tables that is filtered - if needSkipThisTable(pt.Source.Account, pt.Source.Database, pt.Source.Table, filterPts) { - continue - } - leftCount++ - } - return leftCount -} - func queryTable( ctx context.Context, tx taskservice.SqlExecutor, @@ -645,299 +501,105 @@ func queryTable( return false, nil } -func checkAccountExists(ctx context.Context, tx taskservice.SqlExecutor, account string) (bool, uint64, error) { - checkSql := getSqlForCheckAccount(account) - var err error - var ret bool - var accountId uint64 - ret, err = queryTable(ctx, tx, checkSql, func(ctx context.Context, rows *sql.Rows) (bool, error) { - accountId = 0 - if err = rows.Scan(&accountId); err != nil { - return false, err - } - return true, nil - }) - - return ret, accountId, err -} - -func checkTableExists(ctx context.Context, tx taskservice.SqlExecutor, accountId uint64, db, table string) (bool, error) { - //select from mo_tables - checkSql := getSqlForGetTable(accountId, db, table) - var err error - var ret bool - var tableId uint64 - ret, err = queryTable(ctx, tx, checkSql, func(ctx context.Context, rows *sql.Rows) (bool, error) { - tableId = 0 - if err = rows.Scan(&tableId); err != nil { - return false, err - } - return true, nil - }) - - return ret, err -} - -func checkPrimaryKey(ctx context.Context, tx taskservice.SqlExecutor, accountId uint64, db, table string) (bool, error) { - checkSql := getSqlForGetPkCount(accountId, db, table) - var ret bool - var err error - var pkCount uint64 - - ret, err = queryTable(ctx, tx, checkSql, func(ctx context.Context, rows *sql.Rows) (bool, error) { - pkCount = 0 - - if err = rows.Scan( - &pkCount, - ); err != nil { - return false, err - } - if pkCount > 0 { - return true, nil - } - return false, nil - }) - - return ret, err -} - -// extractTablePair -// extract source:sink pair from the pattern +// getPatternTuple pattern example: // -// There must be no special characters (',' '.' ':' '`') in account name & database name & table name. -func extractTablePair(ctx context.Context, pattern string, defaultAcc string) (*cdc2.PatternTuple, error) { - var err error - pattern = strings.TrimSpace(pattern) - //step1 : split table pair by ':' => table0 table1 - //step2 : split table0/1 by '.' => account database table - //step3 : check table accord with regular expression - pt := &cdc2.PatternTuple{OriginString: pattern} - if strings.Contains(pattern, ":") { - //Format: account.db.table:db.table - splitRes := strings.Split(pattern, ":") - if len(splitRes) != 2 { - return nil, moerr.NewInternalErrorf(ctx, "invalid table format: %s, must be `source:sink`.", pattern) - } - - //handle source part - pt.Source.Account, pt.Source.Database, pt.Source.Table, pt.Source.TableIsRegexp, err = extractTableInfo(ctx, splitRes[0], false) - if err != nil { - return nil, err - } - if pt.Source.Account == "" { - pt.Source.Account = defaultAcc - } - - //handle sink part - pt.Sink.Account, pt.Sink.Database, pt.Sink.Table, pt.Sink.TableIsRegexp, err = extractTableInfo(ctx, splitRes[1], false) - if err != nil { - return nil, err - } - if pt.Sink.Account == "" { - pt.Sink.Account = defaultAcc - } - return pt, nil +// db1 +// db1:db2 +// db1.t1 +// db1.t1:db2.t2 +// +// There must be no special characters (',' '.' ':' '`') in database name & table name. +func getPatternTuple(ctx context.Context, level string, pattern string, dup map[string]struct{}) (pt *cdc2.PatternTuple, err error) { + splitRes := strings.Split(strings.TrimSpace(pattern), ":") + if len(splitRes) > 2 { + err = moerr.NewInternalErrorf(ctx, "invalid pattern format: %s, must be `source` or `source:sink`.", pattern) + return } - //Format: account.db.table - //handle source part only - pt.Source.Account, pt.Source.Database, pt.Source.Table, pt.Source.TableIsRegexp, err = extractTableInfo(ctx, pattern, false) - if err != nil { - return nil, err + pt = &cdc2.PatternTuple{OriginString: pattern} + + // handle source part + if pt.Source.Database, pt.Source.Table, err = extractTableInfo(ctx, splitRes[0], level); err != nil { + return } - if pt.Source.Account == "" { - pt.Source.Account = defaultAcc + key := cdc2.GenDbTblKey(pt.Source.Database, pt.Source.Table) + if _, ok := dup[key]; ok { + err = moerr.NewInternalErrorf(ctx, "one db/table: %s can't be used as multi sources in a cdc task", key) + return } + dup[key] = struct{}{} - pt.Sink.AccountId = pt.Source.AccountId - pt.Sink.Account = pt.Source.Account - pt.Sink.Database = pt.Source.Database - pt.Sink.Table = pt.Source.Table - pt.Sink.TableIsRegexp = pt.Source.TableIsRegexp - return pt, nil + // handle sink part + if len(splitRes) > 1 { + if pt.Sink.Database, pt.Sink.Table, err = extractTableInfo(ctx, splitRes[1], level); err != nil { + return + } + } else { + // if not specify sink, then sink = source + pt.Sink.Database = pt.Source.Database + pt.Sink.Table = pt.Source.Table + } + return } -// extractTableInfo -// get account,database,table info from string +// extractTableInfo get account,database,table info from string +// +// input format: // -// account: may be empty -// database: must be concrete name instead of pattern. -// table: must be concrete name or pattern in the source part. must be concrete name in the destination part -// isRegexpTable: table name is regular expression -// !!!NOTE!!! +// DbLevel: database +// TableLevel: database.table // -// There must be no special characters (',' '.' ':' '`') in account name & database name & table name. -func extractTableInfo(ctx context.Context, input string, mustBeConcreteTable bool) (account string, db string, table string, isRegexpTable bool, err error) { +// There must be no special characters (',' '.' ':' '`') in database name & table name. +func extractTableInfo(ctx context.Context, input string, level string) (db string, table string, err error) { parts := strings.Split(strings.TrimSpace(input), ".") - if len(parts) != 2 && len(parts) != 3 { - err = moerr.NewInternalErrorf(ctx, "invalid table format: %s, needs account.database.table or database.table.", input) + if level == cdc2.DbLevel && len(parts) != 1 { + err = moerr.NewInternalErrorf(ctx, "invalid databases format: %s", input) + return + } else if level == cdc2.TableLevel && len(parts) != 2 { + err = moerr.NewInternalErrorf(ctx, "invalid tables format: %s", input) return } - if len(parts) == 2 { - db, table = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) - } else { - account, db, table = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), strings.TrimSpace(parts[2]) - - if !accountNameIsLegal(account) { - err = moerr.NewInternalErrorf(ctx, "invalid account name: %s", account) - return - } - } - + db = strings.TrimSpace(parts[0]) if !dbNameIsLegal(db) { err = moerr.NewInternalErrorf(ctx, "invalid database name: %s", db) return } - if !tableNameIsLegal(table) { - err = moerr.NewInternalErrorf(ctx, "invalid table name: %s", table) - return - } - - return -} - -// preprocessTables extract tables and serialize them -func preprocessTables( - ctx context.Context, - ses *Session, - level string, - account string, - tables string, -) (*cdc2.PatternTuples, error) { - tablesPts, err := extractTablePairs(ctx, tables, account) - if err != nil { - return nil, err - } - - //step 2: check privilege - if err = canCreateCdcTask(ctx, ses, level, account, tablesPts); err != nil { - return nil, err - } - - return tablesPts, nil -} - -/* -extractTablePairs extracts all source:sink pairs from the pattern -There must be no special characters (',' '.' ':' '`') in account name & database name & table name. -*/ -func extractTablePairs(ctx context.Context, pattern string, defaultAcc string) (*cdc2.PatternTuples, error) { - pattern = strings.TrimSpace(pattern) - pts := &cdc2.PatternTuples{} - - tablePairs := strings.Split(pattern, ",") - - //step1 : split pattern by ',' => table pair - for _, pair := range tablePairs { - pt, err := extractTablePair(ctx, pair, defaultAcc) - if err != nil { - return nil, err + if level == cdc2.TableLevel { + table = strings.TrimSpace(parts[1]) + if !tableNameIsLegal(table) { + err = moerr.NewInternalErrorf(ctx, "invalid table name: %s", table) + return } - pts.Append(pt) + } else { + table = cdc2.MatchAll } - - return pts, nil + return } -func preprocessRules(ctx context.Context, rules string) (string, *cdc2.PatternTuples, error) { - pts, err := extractRules(ctx, rules) - if err != nil { - return "", nil, err - } - - jsonPts, err := cdc2.JsonEncode(pts) - if err != nil { - return "", nil, err - } - return jsonPts, pts, nil -} +func getPatternTuples(ctx context.Context, level string, tables string) (pts *cdc2.PatternTuples, err error) { + pts = &cdc2.PatternTuples{} -/* -extractRules extracts filters -pattern maybe empty string. then, it returns empty PatternTuples -There must be no special characters (',' '.' ':' '`') in account name & database name & table name. -*/ -func extractRules(ctx context.Context, pattern string) (*cdc2.PatternTuples, error) { - pattern = strings.TrimSpace(pattern) - pts := &cdc2.PatternTuples{} - if len(pattern) == 0 { - return pts, nil + if level == cdc2.AccountLevel { + pts.Append(&cdc2.PatternTuple{ + Source: cdc2.PatternTable{Database: cdc2.MatchAll, Table: cdc2.MatchAll}, + Sink: cdc2.PatternTable{Database: cdc2.MatchAll, Table: cdc2.MatchAll}, + }) + return } - tablePairs := strings.Split(pattern, ",") - if len(tablePairs) == 0 { - return nil, moerr.NewInternalErrorf(ctx, "invalid pattern format: %s", pattern) - } - var err error - //step1 : split pattern by ',' => table pair + // split tables by ',' => table pair + var pt *cdc2.PatternTuple + tablePairs := strings.Split(strings.TrimSpace(tables), ",") + dup := make(map[string]struct{}) for _, pair := range tablePairs { - pt := &cdc2.PatternTuple{} - pt.Source.Account, pt.Source.Database, pt.Source.Table, pt.Source.TableIsRegexp, err = extractTableInfo(ctx, pair, false) - if err != nil { - return nil, err + if pt, err = getPatternTuple(ctx, level, pair, dup); err != nil { + return } pts.Append(pt) } - - return pts, nil -} - -func canCreateCdcTask(ctx context.Context, ses *Session, level string, account string, pts *cdc2.PatternTuples) error { - if strings.EqualFold(level, cdc2.ClusterLevel) { - if !ses.tenant.IsMoAdminRole() { - return moerr.NewInternalError(ctx, "Only sys account administrator are allowed to create cluster level task") - } - for _, pt := range pts.Pts { - if isBannedDatabase(pt.Source.Database) { - return moerr.NewInternalError(ctx, "The system database cannot be subscribed to") - } - } - } else if strings.EqualFold(level, cdc2.AccountLevel) { - if !ses.tenant.IsMoAdminRole() && ses.GetTenantName() != account { - return moerr.NewInternalErrorf(ctx, "No privilege to create task on %s", account) - } - for _, pt := range pts.Pts { - if account != pt.Source.Account { - return moerr.NewInternalErrorf(ctx, "No privilege to create task on table %s", pt.OriginString) - } - if isBannedDatabase(pt.Source.Database) { - return moerr.NewInternalError(ctx, "The system database cannot be subscribed to") - } - } - } else { - return moerr.NewInternalErrorf(ctx, "Incorrect level %s", level) - } - return nil -} - -func attachAccountToFilters(ctx context.Context, ses *Session, level string, account string, pts *cdc2.PatternTuples) error { - if strings.EqualFold(level, cdc2.ClusterLevel) { - if !ses.tenant.IsMoAdminRole() { - return moerr.NewInternalError(ctx, "Only sys account administrator are allowed to create cluster level task") - } - for _, pt := range pts.Pts { - if pt.Source.Account == "" { - pt.Source.Account = ses.GetTenantName() - } - } - } else if strings.EqualFold(level, cdc2.AccountLevel) { - if !ses.tenant.IsMoAdminRole() && ses.GetTenantName() != account { - return moerr.NewInternalErrorf(ctx, "No privilege to create task on %s", account) - } - for _, pt := range pts.Pts { - if pt.Source.Account == "" { - pt.Source.Account = account - } - if account != pt.Source.Account { - return moerr.NewInternalErrorf(ctx, "No privilege to create task on table %s", pt.OriginString) - } - } - } else { - return moerr.NewInternalErrorf(ctx, "Incorrect level %s", level) - } - return nil + return } func RegisterCdcExecutor( @@ -1001,16 +663,19 @@ type CdcTask struct { sinkUri cdc2.UriInfo tables cdc2.PatternTuples - filters cdc2.PatternTuples + exclude *regexp.Regexp startTs types.TS - noFull string + noFull bool additionalConfig map[string]interface{} activeRoutine *cdc2.ActiveRoutine - // sunkWatermarkUpdater update the watermark of the items that has been sunk to downstream - sunkWatermarkUpdater *cdc2.WatermarkUpdater - isRunning bool - holdCh chan int + // watermarkUpdater update the watermark of the items that has been sunk to downstream + watermarkUpdater cdc2.IWatermarkUpdater + // runningReaders store the running execute pipelines, map key pattern: db.table + runningReaders *sync.Map + + isRunning bool + holdCh chan int // start wrapper, for ut startFunc func(ctx context.Context) error @@ -1053,7 +718,11 @@ func NewCdcTask( } func (cdc *CdcTask) Start(rootCtx context.Context) (err error) { - logutil.Infof("cdc task %s start on cn %s", cdc.cdcTask.TaskName, cdc.cnUUID) + taskId := cdc.cdcTask.TaskId + taskName := cdc.cdcTask.TaskName + cnUUID := cdc.cnUUID + accountId := uint32(cdc.cdcTask.Accounts[0].GetId()) + logutil.Infof("cdc task %s start on cn %s", taskName, cnUUID) defer func() { if err != nil { @@ -1063,56 +732,35 @@ func (cdc *CdcTask) Start(rootCtx context.Context) (err error) { cdc.activeRoutine.CloseCancel() if updateErrMsgErr := cdc.updateErrMsg(rootCtx, err.Error()); updateErrMsgErr != nil { - logutil.Errorf("cdc task %s update err msg failed, err: %v", cdc.cdcTask.TaskName, updateErrMsgErr) + logutil.Errorf("cdc task %s update err msg failed, err: %v", taskName, updateErrMsgErr) } } - // if Resume/Restart successfully will reach here, do nothing + + cdc2.GetTableScanner(cnUUID).UnRegister(taskId) }() - ctx := defines.AttachAccountId(rootCtx, uint32(cdc.cdcTask.Accounts[0].GetId())) + ctx := defines.AttachAccountId(rootCtx, accountId) // get cdc task definition if err = cdc.retrieveCdcTask(ctx); err != nil { return err } - // check table be filtered or not - if filterTable(&cdc.tables, &cdc.filters) == 0 { - return moerr.NewInternalError(ctx, "all tables has been excluded by filters. start cdc failed.") - } - - // get source table id - var info *cdc2.DbTableInfo - dbTableInfos := make([]*cdc2.DbTableInfo, 0, len(cdc.tables.Pts)) - tblIdStrMap := make(map[string]bool) - for _, tuple := range cdc.tables.Pts { - accId, accName, dbName, tblName := tuple.Source.AccountId, tuple.Source.Account, tuple.Source.Database, tuple.Source.Table - if needSkipThisTable(accName, dbName, tblName, &cdc.filters) { - logutil.Infof("cdc skip table %s:%s by filter", dbName, tblName) - continue - } - // get db id, table id for the source table - info, err = cdc.retrieveTable(ctx, accId, accName, dbName, tblName, tblIdStrMap) - if err != nil { - return err - } + // reset runningReaders + cdc.runningReaders = &sync.Map{} - info.SinkAccountName = tuple.Sink.Account - info.SinkDbName = tuple.Sink.Database - info.SinkTblName = tuple.Sink.Table + // start watermarkUpdater + cdc.watermarkUpdater = cdc2.NewWatermarkUpdater(accountId, taskId, cdc.ie) + go cdc.watermarkUpdater.Run(ctx, cdc.activeRoutine) - dbTableInfos = append(dbTableInfos, info) - } - - if err = cdc.startWatermarkAndPipeline(ctx, dbTableInfos); err != nil { - return err - } + // register to table scanner + cdc2.GetTableScanner(cnUUID).Register(taskId, cdc.handleNewTables) cdc.isRunning = true - logutil.Infof("cdc task %s start on cn %s success", cdc.cdcTask.TaskName, cdc.cnUUID) + logutil.Infof("cdc task %s start on cn %s success", taskName, cnUUID) // start success, clear err msg if err = cdc.updateErrMsg(ctx, ""); err != nil { - logutil.Errorf("cdc task %s update err msg failed, err: %v", cdc.cdcTask.TaskName, err) + logutil.Errorf("cdc task %s update err msg failed, err: %v", taskName, err) } // hold @@ -1157,7 +805,7 @@ func (cdc *CdcTask) Restart() (err error) { }() // delete previous records - if err = cdc.sunkWatermarkUpdater.DeleteAllFromDb(); err != nil { + if err = cdc.watermarkUpdater.DeleteAllFromDb(); err != nil { return } go func() { @@ -1199,7 +847,7 @@ func (cdc *CdcTask) Cancel() (err error) { cdc.activeRoutine.CloseCancel() cdc.isRunning = false } - if err = cdc.sunkWatermarkUpdater.DeleteAllFromDb(); err != nil { + if err = cdc.watermarkUpdater.DeleteAllFromDb(); err != nil { return err } // let Start() go @@ -1207,70 +855,20 @@ func (cdc *CdcTask) Cancel() (err error) { return } -func (cdc *CdcTask) addExecPipelineForTable(info *cdc2.DbTableInfo, txnOp client.TxnOperator) error { - // reader --call--> sinker ----> remote db - ctx := defines.AttachAccountId(context.Background(), uint32(cdc.cdcTask.Accounts[0].GetId())) - - // add watermark to updater - watermark, err := cdc.sunkWatermarkUpdater.GetFromDb(info.SourceTblIdStr) - if err != nil { - return err - } - cdc.sunkWatermarkUpdater.UpdateMem(info.SourceTblIdStr, watermark) - - tableDef, err := cdc2.GetTableDef(ctx, txnOp, cdc.cnEngine, info.SourceTblId) - if err != nil { - return err - } - - // make sinker for table - sinker, err := cdc2.NewSinker( - cdc.sinkUri, - info, - cdc.sunkWatermarkUpdater, - tableDef, - cdc2.DefaultRetryTimes, - cdc2.DefaultRetryDuration, - cdc.activeRoutine, - uint64(cdc.additionalConfig[cdc2.MaxSqlLength].(float64)), - cdc.additionalConfig[cdc2.SendSqlTimeout].(string), - ) - if err != nil { - return err - } - go sinker.Run(ctx, cdc.activeRoutine) - - // make reader - reader := cdc2.NewTableReader( - cdc.cnTxnClient, - cdc.cnEngine, - cdc.mp, - cdc.packerPool, - info, - sinker, - cdc.sunkWatermarkUpdater, - tableDef, - cdc.resetWatermarkForTable, - cdc.additionalConfig[cdc2.InitSnapshotSplitTxn].(bool), - ) - go reader.Run(ctx, cdc.activeRoutine) - - return nil -} - func (cdc *CdcTask) resetWatermarkForTable(info *cdc2.DbTableInfo) (err error) { - tblIdStr := info.SourceTblIdStr + dbName, tblName := info.SourceDbName, info.SourceTblName // delete old watermark of table - cdc.sunkWatermarkUpdater.DeleteFromMem(tblIdStr) - if err = cdc.sunkWatermarkUpdater.DeleteFromDb(tblIdStr); err != nil { + cdc.watermarkUpdater.DeleteFromMem(dbName, tblName) + if err = cdc.watermarkUpdater.DeleteFromDb(dbName, tblName); err != nil { return } // use start_ts as init watermark - if err = cdc.sunkWatermarkUpdater.InsertIntoDb(info, cdc.startTs); err != nil { + // TODO handle no_full + if err = cdc.watermarkUpdater.InsertIntoDb(info, cdc.startTs); err != nil { return } - cdc.sunkWatermarkUpdater.UpdateMem(tblIdStr, cdc.startTs) + cdc.watermarkUpdater.UpdateMem(dbName, tblName, cdc.startTs) return } @@ -1311,54 +909,133 @@ func (cdc *CdcTask) updateErrMsg(ctx context.Context, errMsg string) (err error) return cdc.ie.Exec(defines.AttachAccountId(ctx, catalog.System_Account), sql, ie.SessionOverrideOptions{}) } -func (cdc *CdcTask) startWatermarkAndPipeline(ctx context.Context, dbTableInfos []*cdc2.DbTableInfo) (err error) { - var info *cdc2.DbTableInfo - txnOp, err := cdc2.GetTxnOp(ctx, cdc.cnEngine, cdc.cnTxnClient, "cdc-startWatermarkAndPipeline") +func (cdc *CdcTask) handleNewTables(allAccountTbls map[uint32]cdc2.TblMap) { + accountId := uint32(cdc.cdcTask.Accounts[0].GetId()) + ctx := defines.AttachAccountId(context.Background(), accountId) + + txnOp, err := cdc2.GetTxnOp(ctx, cdc.cnEngine, cdc.cnTxnClient, "cdc-handleNewTables") if err != nil { - return err + logutil.Errorf("cdc task %s get txn op failed, err: %v", cdc.cdcTask.TaskName, err) } defer func() { cdc2.FinishTxnOp(ctx, err, txnOp, cdc.cnEngine) }() if err = cdc.cnEngine.New(ctx, txnOp); err != nil { - return err + logutil.Errorf("cdc task %s new engine failed, err: %v", cdc.cdcTask.TaskName, err) } - if cdc.noFull == "true" { - cdc.startTs = types.TimestampToTS(txnOp.SnapshotTS()) - } + for key, info := range allAccountTbls[accountId] { + // already running + if _, ok := cdc.runningReaders.Load(key); ok { + continue + } - // start watermark updater - cdc.sunkWatermarkUpdater = cdc2.NewWatermarkUpdater(cdc.cdcTask.Accounts[0].GetId(), cdc.cdcTask.TaskId, cdc.ie) + if !cdc.matchAnyPattern(key, info) { + continue + } - mp, err := cdc.sunkWatermarkUpdater.GetAllFromDb() - if err != nil { - return err + if cdc.exclude != nil && cdc.exclude.MatchString(key) { + continue + } + + logutil.Infof("cdc task find new table: %s", info) + if err = cdc.addExecPipelineForTable(ctx, info, txnOp); err != nil { + logutil.Errorf("cdc task %s add exec pipeline for table %s failed, err: %v", cdc.cdcTask.TaskName, key, err) + } else { + logutil.Infof("cdc task %s add exec pipeline for table %s successfully", cdc.cdcTask.TaskName, key) + } } - for _, info = range dbTableInfos { - // insert if not exists - if _, ok := mp[info.SourceTblIdStr]; !ok { - // use startTs as watermark - if err = cdc.sunkWatermarkUpdater.InsertIntoDb(info, cdc.startTs); err != nil { - return - } +} + +func (cdc *CdcTask) matchAnyPattern(key string, info *cdc2.DbTableInfo) bool { + match := func(s, p string) bool { + if p == cdc2.MatchAll { + return true } - delete(mp, info.SourceTblIdStr) + return s == p } - // delete outdated watermark - for tableIdStr := range mp { - if err = cdc.sunkWatermarkUpdater.DeleteFromDb(tableIdStr); err != nil { - return err + + db, table := cdc2.SplitDbTblKey(key) + for _, pt := range cdc.tables.Pts { + if match(db, pt.Source.Database) && match(table, pt.Source.Table) { + // complete sink info + info.SinkDbName = pt.Sink.Database + if info.SinkDbName == cdc2.MatchAll { + info.SinkDbName = db + } + info.SinkTblName = pt.Sink.Table + if info.SinkTblName == cdc2.MatchAll { + info.SinkTblName = table + } + return true } } - go cdc.sunkWatermarkUpdater.Run(ctx, cdc.activeRoutine) + return false +} - // create exec pipelines - for _, info = range dbTableInfos { - if err = cdc.addExecPipelineForTable(info, txnOp); err != nil { +// reader ----> sinker ----> remote db +func (cdc *CdcTask) addExecPipelineForTable(ctx context.Context, info *cdc2.DbTableInfo, txnOp client.TxnOperator) (err error) { + // step 1. init watermarkUpdater + // get watermark from db + watermark, err := cdc.watermarkUpdater.GetFromDb(info.SourceDbName, info.SourceTblName) + if moerr.IsMoErrCode(err, moerr.ErrNoWatermarkFound) { + // add watermark into db if not exists + watermark = cdc.startTs + if cdc.noFull { + watermark = types.TimestampToTS(txnOp.SnapshotTS()) + } + + if err = cdc.watermarkUpdater.InsertIntoDb(info, watermark); err != nil { return } + } else if err != nil { + return + } + // clear err msg + if err = cdc.watermarkUpdater.SaveErrMsg(info.SourceDbName, info.SourceTblName, ""); err != nil { + return + } + // add watermark into memory + cdc.watermarkUpdater.UpdateMem(info.SourceDbName, info.SourceTblName, watermark) + + tableDef, err := cdc2.GetTableDef(ctx, txnOp, cdc.cnEngine, info.SourceTblId) + if err != nil { + return } + + // step 2. new sinker + sinker, err := cdc2.NewSinker( + cdc.sinkUri, + info, + cdc.watermarkUpdater, + tableDef, + cdc2.DefaultRetryTimes, + cdc2.DefaultRetryDuration, + cdc.activeRoutine, + uint64(cdc.additionalConfig[cdc2.MaxSqlLength].(float64)), + cdc.additionalConfig[cdc2.SendSqlTimeout].(string), + ) + if err != nil { + return err + } + go sinker.Run(ctx, cdc.activeRoutine) + + // step 3. new reader + reader := cdc2.NewTableReader( + cdc.cnTxnClient, + cdc.cnEngine, + cdc.mp, + cdc.packerPool, + info, + sinker, + cdc.watermarkUpdater, + tableDef, + cdc.resetWatermarkForTable, + cdc.additionalConfig[cdc2.InitSnapshotSplitTxn].(bool), + cdc.runningReaders, + ) + go reader.Run(ctx, cdc.activeRoutine) + return } @@ -1385,7 +1062,6 @@ func (cdc *CdcTask) retrieveCdcTask(ctx context.Context) error { return err } - var sinkPwd string if sinkTyp != cdc2.ConsoleSink { //sink uri jsonSinkUri, err := res.GetString(ctx, 0, 0) @@ -1393,13 +1069,12 @@ func (cdc *CdcTask) retrieveCdcTask(ctx context.Context) error { return err } - err = cdc2.JsonDecode(jsonSinkUri, &cdc.sinkUri) - if err != nil { + if err = cdc2.JsonDecode(jsonSinkUri, &cdc.sinkUri); err != nil { return err } //sink_password - sinkPwd, err = res.GetString(ctx, 0, 2) + sinkPwd, err := res.GetString(ctx, 0, 2) if err != nil { return err } @@ -1409,8 +1084,7 @@ func (cdc *CdcTask) retrieveCdcTask(ctx context.Context) error { return err } - cdc.sinkUri.Password, err = cdc2.AesCFBDecode(ctx, sinkPwd) - if err != nil { + if cdc.sinkUri.Password, err = cdc2.AesCFBDecode(ctx, sinkPwd); err != nil { return err } } @@ -1418,26 +1092,25 @@ func (cdc *CdcTask) retrieveCdcTask(ctx context.Context) error { //update sink type after deserialize cdc.sinkUri.SinkTyp = sinkTyp - //tables + // tables jsonTables, err := res.GetString(ctx, 0, 3) if err != nil { return err } - err = cdc2.JsonDecode(jsonTables, &cdc.tables) - if err != nil { + if err = cdc2.JsonDecode(jsonTables, &cdc.tables); err != nil { return err } - //filters - jsonFilters, err := res.GetString(ctx, 0, 4) + // exclude + exclude, err := res.GetString(ctx, 0, 4) if err != nil { return err } - - err = cdc2.JsonDecode(jsonFilters, &cdc.filters) - if err != nil { - return err + if exclude != "" { + if cdc.exclude, err = regexp.Compile(exclude); err != nil { + return err + } } // noFull @@ -1445,9 +1118,10 @@ func (cdc *CdcTask) retrieveCdcTask(ctx context.Context) error { if err != nil { return err } + cdc.noFull, _ = strconv.ParseBool(noFull) + // startTs cdc.startTs = types.TS{} - cdc.noFull = noFull // additionalConfig additionalConfigStr, err := res.GetString(ctx, 0, 6) @@ -1457,54 +1131,6 @@ func (cdc *CdcTask) retrieveCdcTask(ctx context.Context) error { return json.Unmarshal([]byte(additionalConfigStr), &cdc.additionalConfig) } -func (cdc *CdcTask) retrieveTable(ctx context.Context, accId uint64, accName, dbName, tblName string, tblIdStrMap map[string]bool) (*cdc2.DbTableInfo, error) { - var dbId, tblId uint64 - var err error - ctx = defines.AttachAccountId(ctx, catalog.System_Account) - sql := getSqlForDbIdAndTableId(accId, dbName, tblName) - res := cdc.ie.Query(ctx, sql, ie.SessionOverrideOptions{}) - if res.Error() != nil { - return nil, res.Error() - } - - getTblIdStr := func(tblId uint64) string { - for i := 0; ; i++ { - tblIdStr := fmt.Sprintf("%d_%d", tblId, i) - if _, ok := tblIdStrMap[tblIdStr]; !ok { - tblIdStrMap[tblIdStr] = true - return tblIdStr - } - } - } - - /* - missing table will be handled in the future. - */ - if res.RowCount() < 1 { - return nil, moerr.NewInternalErrorf(ctx, "no table %s:%s", dbName, tblName) - } else if res.RowCount() > 1 { - return nil, moerr.NewInternalErrorf(ctx, "duplicate table %s:%s", dbName, tblName) - } - - if dbId, err = res.GetUint64(ctx, 0, 0); err != nil { - return nil, err - } - - if tblId, err = res.GetUint64(ctx, 0, 1); err != nil { - return nil, err - } - - return &cdc2.DbTableInfo{ - SourceAccountName: accName, - SourceDbName: dbName, - SourceTblName: tblName, - SourceAccountId: accId, - SourceDbId: dbId, - SourceTblId: tblId, - SourceTblIdStr: getTblIdStr(tblId), - }, err -} - var initAesKeyByInternalExecutor = func(ctx context.Context, cdc *CdcTask, accountId uint32) error { return cdc.initAesKeyByInternalExecutor(ctx, accountId) } @@ -1958,20 +1584,3 @@ func initAesKeyBySqlExecutor(ctx context.Context, executor taskservice.SqlExecut cdc2.AesKey, err = decrypt(ctx, encryptedKey, []byte(getGlobalPuWrapper(service).SV.KeyEncryptionKey)) return } - -func needSkipThisTable(accountName, dbName, tblName string, filters *cdc2.PatternTuples) bool { - if len(filters.Pts) == 0 { - return false - } - for _, filter := range filters.Pts { - if filter == nil { - continue - } - if filter.Source.Account == accountName && - filter.Source.Database == dbName && - filter.Source.Table == tblName { - return true - } - } - return false -} diff --git a/pkg/frontend/cdc_test.go b/pkg/frontend/cdc_test.go index ab9c934f35dda..ea0d35d64323b 100644 --- a/pkg/frontend/cdc_test.go +++ b/pkg/frontend/cdc_test.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" "regexp" + "sync" "testing" "time" @@ -106,527 +107,254 @@ func Test_newCdcSqlFormat(t *testing.T) { assert.Equal(t, wantsql7, sql7) } -func Test_parseTables(t *testing.T) { +func Test_getPatternTuples(t *testing.T) { //tables := []string{ - // "acc1.users.t1:acc1.users.t1", - // "acc1.users.t*:acc1.users.t*", - // "acc*.users.t?:acc*.users.t?", - // "acc*.users|items.*[12]/:acc*.users|items.*[12]/", - // "acc*.*./table./", - // //"acc*.*.table*", - // //"/sys|acc.*/.*.t*", - // //"/sys|acc.*/.*./t.$/", - // //"/sys|acc.*/.test*./t1{1,3}$/,/acc[23]/.items./.*/", + // - table level + // "db1.t1:db2.t2,db3.t3:db4.t4", + // "db1.t1,db3.t3:db4.t4", + // "db1.t1,db3.t3", + // "db1.t1:db2.t2,db1.t1:db4.t4", // error + // - db level + // "db1:db2,db3:db4", + // "db1,db3:db4", + // "db1,db3", + // - account level //} - type tableInfo struct { - account, db, table string - tableIsRegexp bool - } - - isSame := func(info tableInfo, account, db, table string, isRegexp bool, tip string) { - assert.Equalf(t, info.account, account, tip) - assert.Equalf(t, info.db, db, tip) - assert.Equalf(t, info.table, table, tip) - assert.Equalf(t, info.tableIsRegexp, isRegexp, tip) - } - type kase struct { - input string + tables string + level string wantErr bool - src tableInfo - dst tableInfo + expect *cdc2.PatternTuples } kases := []kase{ + // table level { - input: "acc1.users.t1:acc1.users.t1", + tables: "db1.t1:db2.t2,db3.t3:db4.t4", + level: cdc2.TableLevel, wantErr: false, - src: tableInfo{ - account: "acc1", - db: "users", - table: "t1", - }, - dst: tableInfo{ - account: "acc1", - db: "users", - table: "t1", + expect: &cdc2.PatternTuples{ + Pts: []*cdc2.PatternTuple{ + { + Source: cdc2.PatternTable{ + Database: "db1", + Table: "t1", + }, + Sink: cdc2.PatternTable{ + Database: "db2", + Table: "t2", + }, + }, + { + Source: cdc2.PatternTable{ + Database: "db3", + Table: "t3", + }, + Sink: cdc2.PatternTable{ + Database: "db4", + Table: "t4", + }, + }, + }, }, }, { - input: "acc1.users.t*:acc1.users.t*", + tables: "db1.t1,db3.t3:db4.t4", + level: cdc2.TableLevel, wantErr: false, - src: tableInfo{ - account: "acc1", - db: "users", - table: "t*", - }, - dst: tableInfo{ - account: "acc1", - db: "users", - table: "t*", + expect: &cdc2.PatternTuples{ + Pts: []*cdc2.PatternTuple{ + { + Source: cdc2.PatternTable{ + Database: "db1", + Table: "t1", + }, + Sink: cdc2.PatternTable{ + Database: "db1", + Table: "t1", + }, + }, + { + Source: cdc2.PatternTable{ + Database: "db3", + Table: "t3", + }, + Sink: cdc2.PatternTable{ + Database: "db4", + Table: "t4", + }, + }, + }, }, }, { - input: "acc*.users.t?:acc*.users.t?", + tables: "db1.t1,db3.t3", + level: cdc2.TableLevel, wantErr: false, - src: tableInfo{ - account: "acc*", - db: "users", - table: "t?", - }, - dst: tableInfo{ - account: "acc*", - db: "users", - table: "t?", + expect: &cdc2.PatternTuples{ + Pts: []*cdc2.PatternTuple{ + { + Source: cdc2.PatternTable{ + Database: "db1", + Table: "t1", + }, + Sink: cdc2.PatternTable{ + Database: "db1", + Table: "t1", + }, + }, + { + Source: cdc2.PatternTable{ + Database: "db3", + Table: "t3", + }, + Sink: cdc2.PatternTable{ + Database: "db3", + Table: "t3", + }, + }, + }, }, }, { - input: "acc*.users|items.*[12]/:acc*.users|items.*[12]/", + tables: "db1.t1:db2.t2,db1.t1:db4.t4", + level: cdc2.TableLevel, + wantErr: true, + }, + + // db level + { + tables: "db1:db2,db3:db4", + level: cdc2.DbLevel, wantErr: false, - src: tableInfo{ - account: "acc*", - db: "users|items", - table: "*[12]/", - }, - dst: tableInfo{ - account: "acc*", - db: "users|items", - table: "*[12]/", + expect: &cdc2.PatternTuples{ + Pts: []*cdc2.PatternTuple{ + { + Source: cdc2.PatternTable{ + Database: "db1", + Table: cdc2.MatchAll, + }, + Sink: cdc2.PatternTable{ + Database: "db2", + Table: cdc2.MatchAll, + }, + }, + { + Source: cdc2.PatternTable{ + Database: "db3", + Table: cdc2.MatchAll, + }, + Sink: cdc2.PatternTable{ + Database: "db4", + Table: cdc2.MatchAll, + }, + }, + }, }, }, { - input: "acc*.*./table./", - wantErr: true, - src: tableInfo{}, - dst: tableInfo{}, + tables: "db1,db3:db4", + level: cdc2.DbLevel, + wantErr: false, + expect: &cdc2.PatternTuples{ + Pts: []*cdc2.PatternTuple{ + { + Source: cdc2.PatternTable{ + Database: "db1", + Table: cdc2.MatchAll, + }, + Sink: cdc2.PatternTable{ + Database: "db1", + Table: cdc2.MatchAll, + }, + }, + { + Source: cdc2.PatternTable{ + Database: "db3", + Table: cdc2.MatchAll, + }, + Sink: cdc2.PatternTable{ + Database: "db4", + Table: cdc2.MatchAll, + }, + }, + }, + }, }, { - input: "acc*.*.table*:acc*.*.table*", + tables: "db1,db3", + level: cdc2.DbLevel, wantErr: false, - src: tableInfo{ - account: "acc*", - db: "*", - table: "table*", - }, - dst: tableInfo{ - account: "acc*", - db: "*", - table: "table*", + expect: &cdc2.PatternTuples{ + Pts: []*cdc2.PatternTuple{ + { + Source: cdc2.PatternTable{ + Database: "db1", + Table: cdc2.MatchAll, + }, + Sink: cdc2.PatternTable{ + Database: "db1", + Table: cdc2.MatchAll, + }, + }, + { + Source: cdc2.PatternTable{ + Database: "db3", + Table: cdc2.MatchAll, + }, + Sink: cdc2.PatternTable{ + Database: "db3", + Table: cdc2.MatchAll, + }, + }, + }, }, }, + + // account level { - input: "", - wantErr: true, + tables: "", + level: cdc2.AccountLevel, + wantErr: false, + expect: &cdc2.PatternTuples{ + Pts: []*cdc2.PatternTuple{ + { + Source: cdc2.PatternTable{ + Database: cdc2.MatchAll, + Table: cdc2.MatchAll, + }, + Sink: cdc2.PatternTable{ + Database: cdc2.MatchAll, + Table: cdc2.MatchAll, + }, + }, + }, + }, }, } + isSame := func(pt0, pt1 *cdc2.PatternTuples) { + assert.Equal(t, len(pt0.Pts), len(pt1.Pts)) + for i := 0; i < len(pt0.Pts); i++ { + assert.Equal(t, pt0.Pts[i].Source.Database, pt1.Pts[i].Source.Database) + assert.Equal(t, pt0.Pts[i].Source.Table, pt1.Pts[i].Source.Table) + assert.Equal(t, pt0.Pts[i].Sink.Database, pt1.Pts[i].Sink.Database) + assert.Equal(t, pt0.Pts[i].Sink.Table, pt1.Pts[i].Sink.Table) + } + } + for _, tkase := range kases { - pirs, err := extractTablePairs(context.Background(), tkase.input, "") + pts, err := getPatternTuples(context.Background(), tkase.level, tkase.tables) if tkase.wantErr { - assert.Errorf(t, err, tkase.input) + assert.Errorf(t, err, tkase.tables) } else { - assert.NoErrorf(t, err, tkase.input) - assert.Equal(t, len(pirs.Pts), 1, tkase.input) - pir := pirs.Pts[0] - isSame(tkase.src, pir.Source.Account, pir.Source.Database, pir.Source.Table, pir.Source.TableIsRegexp, tkase.input) - isSame(tkase.dst, pir.Sink.Account, pir.Sink.Database, pir.Sink.Table, pir.Sink.TableIsRegexp, tkase.input) + assert.NoErrorf(t, err, tkase.tables) + isSame(pts, tkase.expect) } } } -func Test_privilegeCheck(t *testing.T) { - var tenantInfo *TenantInfo - var err error - var pts []*cdc2.PatternTuple - ctx := context.Background() - ses := &Session{} - - gen := func(pts []*cdc2.PatternTuple) *cdc2.PatternTuples { - return &cdc2.PatternTuples{Pts: pts} - } - - tenantInfo = &TenantInfo{ - Tenant: sysAccountName, - DefaultRole: moAdminRoleName, - } - ses.tenant = tenantInfo - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc1"}}, - {Source: cdc2.PatternTable{Account: sysAccountName}}, - } - err = canCreateCdcTask(ctx, ses, "Cluster", "", gen(pts)) - assert.Nil(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: sysAccountName, Database: moCatalog}}, - } - err = canCreateCdcTask(ctx, ses, "Cluster", "", gen(pts)) - assert.NotNil(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: sysAccountName}}, - } - err = canCreateCdcTask(ctx, ses, "Account", "acc1", gen(pts)) - assert.NotNil(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc2"}}, - } - err = canCreateCdcTask(ctx, ses, "Account", "acc1", gen(pts)) - assert.NotNil(t, err) - - pts = []*cdc2.PatternTuple{ - {}, - } - err = canCreateCdcTask(ctx, ses, "Account", "acc1", gen(pts)) - assert.Error(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc1"}}, - } - err = canCreateCdcTask(ctx, ses, "Account", "acc1", gen(pts)) - assert.Nil(t, err) - - tenantInfo = &TenantInfo{ - Tenant: "acc1", - DefaultRole: accountAdminRoleName, - } - ses.tenant = tenantInfo - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc1"}}, - {}, - } - err = canCreateCdcTask(ctx, ses, "Cluster", "", gen(pts)) - assert.NotNil(t, err) - - err = canCreateCdcTask(ctx, ses, "Account", "acc2", gen(pts)) - assert.NotNil(t, err) - - err = canCreateCdcTask(ctx, ses, "Account", "acc1", gen(pts)) - assert.Error(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc2"}}, - {Source: cdc2.PatternTable{Account: sysAccountName}}, - } - err = canCreateCdcTask(ctx, ses, "Account", "acc1", gen(pts)) - assert.NotNil(t, err) - -} - -func Test_attachAccoutForFilters(t *testing.T) { - var tenantInfo *TenantInfo - var err error - var pts []*cdc2.PatternTuple - ctx := context.Background() - ses := &Session{} - - gen := func(pts []*cdc2.PatternTuple) *cdc2.PatternTuples { - return &cdc2.PatternTuples{Pts: pts} - } - - tenantInfo = &TenantInfo{ - Tenant: sysAccountName, - DefaultRole: moAdminRoleName, - } - ses.tenant = tenantInfo - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc1"}}, - {Source: cdc2.PatternTable{Account: sysAccountName}}, - } - err = attachAccountToFilters(ctx, ses, "Cluster", "", gen(pts)) - assert.Nil(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: sysAccountName, Database: moCatalog}}, - } - err = attachAccountToFilters(ctx, ses, "Cluster", "", gen(pts)) - assert.Nil(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: sysAccountName}}, - } - err = attachAccountToFilters(ctx, ses, "Account", "acc1", gen(pts)) - assert.NotNil(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc2"}}, - } - err = attachAccountToFilters(ctx, ses, "Account", "acc1", gen(pts)) - assert.NotNil(t, err) - - pts = []*cdc2.PatternTuple{ - {}, - } - err = attachAccountToFilters(ctx, ses, "Account", "acc1", gen(pts)) - assert.Nil(t, err) - assert.Equalf(t, "acc1", pts[0].Source.Account, "different account") - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc1"}}, - } - err = attachAccountToFilters(ctx, ses, "Account", "acc1", gen(pts)) - assert.Nil(t, err) - - tenantInfo = &TenantInfo{ - Tenant: "acc1", - DefaultRole: accountAdminRoleName, - } - ses.tenant = tenantInfo - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc1"}}, - {}, - } - err = attachAccountToFilters(ctx, ses, "Cluster", "", gen(pts)) - assert.NotNil(t, err) - - err = attachAccountToFilters(ctx, ses, "Account", "acc2", gen(pts)) - assert.NotNil(t, err) - - err = attachAccountToFilters(ctx, ses, "Account", "acc1", gen(pts)) - assert.Nil(t, err) - - pts = []*cdc2.PatternTuple{ - {Source: cdc2.PatternTable{Account: "acc2"}}, - {Source: cdc2.PatternTable{Account: sysAccountName}}, - } - err = attachAccountToFilters(ctx, ses, "Account", "acc1", gen(pts)) - assert.NotNil(t, err) - -} - -func Test_extractTableInfo(t *testing.T) { - type args struct { - ctx context.Context - input string - mustBeConcreteTable bool - } - tests := []struct { - name string - args args - wantAccount string - wantDb string - wantTable string - wantErr assert.ErrorAssertionFunc - }{ - { - name: "t1-1", - args: args{ - input: "acc1.db.t1", - mustBeConcreteTable: true, - }, - wantAccount: "acc1", - wantDb: "db", - wantTable: "t1", - wantErr: nil, - }, - { - name: "t1-2", - args: args{ - input: "acc1.db.t*", - mustBeConcreteTable: true, - }, - wantAccount: "acc1", - wantDb: "db", - wantTable: "t*", - wantErr: nil, - }, - { - name: "t1-3-table pattern needs //", - args: args{ - input: "acc1.db.t*", - mustBeConcreteTable: false, - }, - wantAccount: "acc1", - wantDb: "db", - wantTable: "t*", - wantErr: nil, - }, - { - name: "t1-4", - args: args{ - input: "acc1.db./t*/", - mustBeConcreteTable: false, - }, - wantAccount: "acc1", - wantDb: "db", - wantTable: "/t*/", - wantErr: nil, - }, - { - name: "t2-1", - args: args{ - input: "db.t1", - mustBeConcreteTable: true, - }, - wantAccount: "", - wantDb: "db", - wantTable: "t1", - wantErr: nil, - }, - { - name: "t2-2-table pattern needs //", - args: args{ - input: "db.t*", - mustBeConcreteTable: true, - }, - wantAccount: "", - wantDb: "db", - wantTable: "t*", - wantErr: nil, - }, - { - name: "t2-3-table name can be 't*'", - args: args{ - input: "db.t*", - mustBeConcreteTable: false, - }, - wantAccount: "", - wantDb: "db", - wantTable: "t*", - wantErr: nil, - }, - { - name: "t2-4", - args: args{ - input: "db./t*/", - mustBeConcreteTable: false, - }, - wantAccount: "", - wantDb: "db", - wantTable: "/t*/", - wantErr: nil, - }, - { - name: "t2-5", - args: args{ - input: "db./t*/", - mustBeConcreteTable: true, - }, - wantAccount: "", - wantDb: "db", - wantTable: "/t*/", - wantErr: nil, - }, - { - name: "t3--invalid format", - args: args{ - input: "nodot", - mustBeConcreteTable: false, - }, - wantAccount: "", - wantDb: "", - wantTable: "", - wantErr: assert.Error, - }, - { - name: "t3--invalid account name", - args: args{ - input: "1234*90.db.t1", - mustBeConcreteTable: false, - }, - wantAccount: "1234*90", - wantDb: "db", - wantTable: "t1", - wantErr: nil, - }, - { - name: "t3--invalid database name", - args: args{ - input: "acc.12ddg.t1", - mustBeConcreteTable: false, - }, - wantAccount: "acc", - wantDb: "12ddg", - wantTable: "t1", - wantErr: nil, - }, - { - name: "t4--invalid database name", - args: args{ - input: "acc*./users|items/./t.*[12]/", - mustBeConcreteTable: false, - }, - wantAccount: "", - wantDb: "", - wantTable: "", - wantErr: assert.Error, - }, - { - name: "t4-- X ", - args: args{ - input: "/sys|acc.*/.*.t*", - mustBeConcreteTable: false, - }, - wantAccount: "", - wantDb: "", - wantTable: "", - wantErr: assert.Error, - }, - { - name: "t4-- XX", - args: args{ - input: "/sys|acc.*/.*./t.$/", - mustBeConcreteTable: false, - }, - wantAccount: "", - wantDb: "", - wantTable: "", - wantErr: assert.Error, - }, - { - name: "t4-- XXX", - args: args{ - input: "/sys|acc.*/.test*./t1{1,3}$/,/acc[23]/.items./.*/", - mustBeConcreteTable: false, - }, - wantAccount: "", - wantDb: "", - wantTable: "", - wantErr: assert.Error, - }, - { - name: "accountNameIsLegal", - args: args{ - input: "s:s.db.table", - mustBeConcreteTable: false, - }, - wantErr: assert.Error, - }, - { - name: "dbNameIsLegal", - args: args{ - input: "sys.d:b.table", - mustBeConcreteTable: false, - }, - wantErr: assert.Error, - }, - { - name: "tableNameIsLegal", - args: args{ - input: "sys.db.ta:le", - mustBeConcreteTable: false, - }, - wantErr: assert.Error, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotAccount, gotDb, gotTable, _, err := extractTableInfo(tt.args.ctx, tt.args.input, tt.args.mustBeConcreteTable) - if tt.wantErr != nil && tt.wantErr(t, err, fmt.Sprintf("extractTableInfo(%v, %v, %v)", tt.args.ctx, tt.args.input, tt.args.mustBeConcreteTable)) { - return - } else { - assert.Equalf(t, tt.wantAccount, gotAccount, "extractTableInfo(%v, %v, %v)", tt.args.ctx, tt.args.input, tt.args.mustBeConcreteTable) - assert.Equalf(t, tt.wantDb, gotDb, "extractTableInfo(%v, %v, %v)", tt.args.ctx, tt.args.input, tt.args.mustBeConcreteTable) - assert.Equalf(t, tt.wantTable, gotTable, "extractTableInfo(%v, %v, %v)", tt.args.ctx, tt.args.input, tt.args.mustBeConcreteTable) - } - }) - } -} - func Test_handleCreateCdc(t *testing.T) { type args struct { ses *Session @@ -647,25 +375,8 @@ func Test_handleCreateCdc(t *testing.T) { db, mock, err := sqlmock.New() assert.NoError(t, err) - ///////mock result - sql1 := "select account_id from `mo_catalog`.`mo_account` where account_name='sys'" - mock.ExpectQuery(sql1).WillReturnRows(sqlmock.NewRows([]string{"account_id"}).AddRow(uint64(sysAccountID))) - - sql2 := "select rel_id from `mo_catalog`.`mo_tables` where account_id = 0 and reldatabase ='db1' and relname = 't1'" - mock.ExpectQuery(sql2).WillReturnRows(sqlmock.NewRows([]string{"table_id"}).AddRow(uint64(11))) - - sql3 := "select count.*att_constraint_type.* from `mo_catalog`.`mo_columns` where account_id = 0 and att_database = 'db1' and att_relname = 't1' and att_constraint_type = 'p'" - mock.ExpectQuery(sql3).WillReturnRows(sqlmock.NewRows([]string{"count(att_constraint_type)"}).AddRow(uint64(1))) - - sql2_1 := "select rel_id from `mo_catalog`.`mo_tables` where account_id = 0 and reldatabase ='db1' and relname = 't2'" - mock.ExpectQuery(sql2_1).WillReturnRows(sqlmock.NewRows([]string{"table_id"}).AddRow(uint64(12))) - - sql3_1 := "select count.*att_constraint_type.* from `mo_catalog`.`mo_columns` where account_id = 0 and att_database = 'db1' and att_relname = 't2' and att_constraint_type = 'p'" - mock.ExpectQuery(sql3_1).WillReturnRows(sqlmock.NewRows([]string{"count(att_constraint_type)"}).AddRow(uint64(1))) - - sql4 := "insert into mo_catalog.mo_cdc_task values.*0,\".*\",\"task1\",\".*\",\"\",\".*\",\"mysql\",\".*\",\"\",\"\",\"\",\".*\",\".*\",\"\",\"common\",\"common\",\"\",\"\",\"\",\".*\",\"running\",0,\"0\",\"false\",\"\",'.*',\"\",\"\",\"\",\"\".*" + sql4 := "insert into mo_catalog.mo_cdc_task .*" mock.ExpectExec(sql4).WillReturnResult(sqlmock.NewResult(1, 1)) - //// pu := config.ParameterUnit{} pu.TaskService = &testTaskService{ @@ -682,10 +393,10 @@ func Test_handleCreateCdc(t *testing.T) { Tables: "db1.t1:db1.t1,db1.t2", Option: []string{ "Level", - cdc2.AccountLevel, + cdc2.TableLevel, "Account", sysAccountName, - "Rules", + "Exclude", "db2.t3,db2.t4", cdc2.InitSnapshotSplitTxn, "false", @@ -1081,9 +792,6 @@ const ( ) func TestRegisterCdcExecutor(t *testing.T) { - cdc2.AesKey = "test-aes-key-not-use-it-in-cloud" - defer func() { cdc2.AesKey = "" }() - type args struct { logger *zap.Logger ts *testTaskService @@ -1098,6 +806,9 @@ func TestRegisterCdcExecutor(t *testing.T) { curDTaskId int } + cdc2.AesKey = "test-aes-key-not-use-it-in-cloud" + defer func() { cdc2.AesKey = "" }() + cdc2.EnableConsoleSink = true defer func() { cdc2.EnableConsoleSink = false @@ -1120,15 +831,12 @@ func TestRegisterCdcExecutor(t *testing.T) { Pts: []*cdc2.PatternTuple{ { Source: cdc2.PatternTable{ - AccountId: uint64(sysAccountID), - Account: sysAccountName, - Database: "db1", - Table: "t1", + Database: "db1", + Table: "t1", }, }, }, - }, - ) + }) assert.NoError(t, err) filters, err := cdc2.JsonEncode(cdc2.PatternTuples{}) assert.NoError(t, err) @@ -1159,82 +867,21 @@ func TestRegisterCdcExecutor(t *testing.T) { cdc2.SendSqlTimeout, cdc2.DefaultSendSqlTimeout, cdc2.MaxSqlLength, cdc2.DefaultMaxSqlLength, ), - ), - ) - - sql2 := "select reldatabase_id,rel_id from mo_catalog.mo_tables where account_id = 0 and reldatabase = 'db1' and relname = 't1'" - mock.ExpectQuery(sql2).WillReturnRows(sqlmock.NewRows( - []string{ - "reldatabase_id", - "rel_id", - }).AddRow( - uint64(10), - uint64(1001), - ), - ) - - sql3 := "select table_id, watermark from mo_catalog.mo_cdc_watermark where account_id = 0 and task_id = '00000000-0000-0000-0000-000000000000'" - mock.ExpectQuery(sql3).WillReturnRows( - sqlmock.NewRows([]string{"table_id", "watermark"}). - AddRow("1001_1", "0-0")) - - sql4 := "insert into mo_catalog.mo_cdc_watermark values .*0, '00000000-0000-0000-0000-000000000000', '1001_0', 'db1', 't1', '0-0'.*" - mock.ExpectExec(sql4).WillReturnResult(sqlmock.NewResult(1, 1)) - - sql41 := "delete from mo_catalog.mo_cdc_watermark where account_id = 0 and task_id = '00000000-0000-0000-0000-000000000000' and table_id = '1001_1'" - mock.ExpectExec(sql41).WillReturnResult(sqlmock.NewResult(1, 1)) - - sql5 := "select watermark from mo_catalog.mo_cdc_watermark where account_id = 0 and task_id = '00000000-0000-0000-0000-000000000000' and table_id = '1001_0'" - mock.ExpectQuery(sql5).WillReturnRows(sqlmock.NewRows( - []string{ - "watermark", - }, - ).AddRow( - "0-0", )) sql7 := "update `mo_catalog`.`mo_cdc_task` set state = 'running', err_msg = '' where account_id = 0 and task_id = '00000000-0000-0000-0000-000000000000'" mock.ExpectExec(sql7).WillReturnResult(sqlmock.NewResult(1, 1)) - sql6 := "update mo_catalog.mo_cdc_watermark set watermark='0-0' where account_id = 0 and task_id = '00000000-0000-0000-0000-000000000000' and table_id = '1001_0'" - mock.ExpectExec(sql6).WillReturnResult(sqlmock.NewResult(1, 1)) - genSqlIdx := func(sql string) int { mSql1, err := regexp.MatchString(sql1, sql) assert.NoError(t, err) - mSql2, err := regexp.MatchString(sql2, sql) - assert.NoError(t, err) - mSql3, err := regexp.MatchString(sql3, sql) - assert.NoError(t, err) - mSql4, err := regexp.MatchString(sql4, sql) - assert.NoError(t, err) - mSql41, err := regexp.MatchString(sql41, sql) - assert.NoError(t, err) - mSql5, err := regexp.MatchString(sql5, sql) - assert.NoError(t, err) - mSql6, err := regexp.MatchString(sql6, sql) - assert.NoError(t, err) if mSql1 { return mSqlIdx1 - } else if mSql2 { - return mSqlIdx2 - } else if mSql3 { - return mSqlIdx3 - } else if mSql4 { - return mSqlIdx4 - } else if mSql41 { - return mSqlIdx41 - } else if mSql5 { - return mSqlIdx5 - } else if mSql6 { - return mSqlIdx6 } return -1 } - //////// - ///////////mock engine eng := mock_frontend.NewMockEngine(ctrl) eng.EXPECT().New(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() @@ -1243,21 +890,6 @@ func TestRegisterCdcExecutor(t *testing.T) { }).AnyTimes() eng.EXPECT().LatestLogtailAppliedTime().Return(timestamp.Timestamp{}).AnyTimes() - table := mock_frontend.NewMockRelation(ctrl) - - tableDef := &plan.TableDef{ - Pkey: &plan.PrimaryKeyDef{ - Names: []string{ - "a", - }, - }, - } - - table.EXPECT().CopyTableDef(gomock.Any()).Return(tableDef).AnyTimes() - table.EXPECT().CollectChanges(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, moerr.NewInternalErrorNoCtx("invalid handle")) - - eng.EXPECT().GetRelationById(gomock.Any(), gomock.Any(), gomock.Any()).Return("", "", table, nil).AnyTimes() - txnOperator := mock_frontend.NewMockTxnOperator(ctrl) txnOperator.EXPECT().Txn().Return(txn.TxnMeta{}).AnyTimes() eng.EXPECT().Database(ctx, gomock.Any(), txnOperator).Return(nil, nil).AnyTimes() @@ -1272,8 +904,6 @@ func TestRegisterCdcExecutor(t *testing.T) { txnClient := mock_frontend.NewMockTxnClient(ctrl) txnClient.EXPECT().New(gomock.Any(), gomock.Any(), gomock.Any()).Return(txnOperator, nil).AnyTimes() - ////////// - tIEFactory1 := func() ie.InternalExecutor { return &testIE{ db: db, @@ -1311,6 +941,14 @@ func TestRegisterCdcExecutor(t *testing.T) { assert.NoError(t, err) defer mpool.DeleteMPool(mp) + gostub.Stub(&cdc2.GetTableScanner, func(cnUUID string) *cdc2.TableScanner { + return &cdc2.TableScanner{ + Mutex: sync.Mutex{}, + Mp: make(map[uint32]cdc2.TblMap), + Callbacks: map[string]func(map[uint32]cdc2.TblMap){"id": func(mp map[uint32]cdc2.TblMap) {}}, + } + }) + tests := []struct { name string args args @@ -2497,85 +2135,21 @@ func Test_handleShowCdc(t *testing.T) { } func TestCdcTask_ResetWatermarkForTable(t *testing.T) { - type fields struct { - logger *zap.Logger - ie ie.InternalExecutor - cnUUID string - cnTxnClient client.TxnClient - cnEngine engine.Engine - fileService fileservice.FileService - cdcTask *task.CreateCdcDetails - mp *mpool.MPool - packerPool *fileservice.Pool[*types.Packer] - sinkUri cdc2.UriInfo - tables cdc2.PatternTuples - filters cdc2.PatternTuples - startTs types.TS - noFull string - activeRoutine *cdc2.ActiveRoutine - sunkWatermarkUpdater *cdc2.WatermarkUpdater - } - type args struct { - info *cdc2.DbTableInfo + cdc := &CdcTask{ + watermarkUpdater: &mockWatermarkUpdater{}, } - db, mock, err := sqlmock.New() - assert.NoError(t, err) - tie := &testIE{ - db: db, + info := &cdc2.DbTableInfo{ + SourceDbId: 0, + SourceDbName: "", + SourceTblId: 0, + SourceTblName: "", + SourceCreateSql: "", + SinkDbName: "", + SinkTblName: "", } - sqlx := "delete from mo_catalog.mo_cdc_watermark where account_id = .* and task_id = .* and table_id = .*" - mock.ExpectExec(sqlx).WillReturnResult(sqlmock.NewResult(1, 1)) - - sqlx1 := "insert into mo_catalog.mo_cdc_watermark values .*" - mock.ExpectExec(sqlx1).WillReturnResult(sqlmock.NewResult(1, 1)) - - tests := []struct { - name string - fields fields - args args - wantErr assert.ErrorAssertionFunc - }{ - { - name: "t1", - fields: fields{ - sunkWatermarkUpdater: cdc2.NewWatermarkUpdater( - sysAccountID, "taskID-1", tie, - ), - ie: tie, - }, - args: args{ - info: &cdc2.DbTableInfo{ - SourceTblId: 10, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cdc := &CdcTask{ - logger: tt.fields.logger, - ie: tt.fields.ie, - cnUUID: tt.fields.cnUUID, - cnTxnClient: tt.fields.cnTxnClient, - cnEngine: tt.fields.cnEngine, - fileService: tt.fields.fileService, - cdcTask: tt.fields.cdcTask, - mp: tt.fields.mp, - packerPool: tt.fields.packerPool, - sinkUri: tt.fields.sinkUri, - tables: tt.fields.tables, - filters: tt.fields.filters, - startTs: tt.fields.startTs, - noFull: tt.fields.noFull, - activeRoutine: tt.fields.activeRoutine, - sunkWatermarkUpdater: tt.fields.sunkWatermarkUpdater, - } - err := cdc.resetWatermarkForTable(tt.args.info) - assert.NoErrorf(t, err, fmt.Sprintf("resetWatermarkForTable(%v)", tt.args.info)) - }) - } + assert.NoError(t, cdc.resetWatermarkForTable(info)) } func TestCdcTask_Resume(t *testing.T) { @@ -2606,7 +2180,7 @@ func TestCdcTask_Restart(t *testing.T) { cdc := &CdcTask{ activeRoutine: cdc2.NewCdcActiveRoutine(), - sunkWatermarkUpdater: cdc2.NewWatermarkUpdater( + watermarkUpdater: cdc2.NewWatermarkUpdater( sysAccountID, "taskID-0", tie, @@ -2658,7 +2232,7 @@ func TestCdcTask_Cancel(t *testing.T) { } cdc := &CdcTask{ activeRoutine: cdc2.NewCdcActiveRoutine(), - sunkWatermarkUpdater: cdc2.NewWatermarkUpdater( + watermarkUpdater: cdc2.NewWatermarkUpdater( sysAccountID, "taskID-1", tie, @@ -2686,9 +2260,9 @@ func TestCdcTask_retrieveCdcTask(t *testing.T) { packerPool *fileservice.Pool[*types.Packer] sinkUri cdc2.UriInfo tables cdc2.PatternTuples - filters cdc2.PatternTuples + exclude *regexp.Regexp startTs types.TS - noFull string + noFull bool activeRoutine *cdc2.ActiveRoutine sunkWatermarkUpdater *cdc2.WatermarkUpdater } @@ -2715,10 +2289,8 @@ func TestCdcTask_retrieveCdcTask(t *testing.T) { Pts: []*cdc2.PatternTuple{ { Source: cdc2.PatternTable{ - AccountId: uint64(sysAccountID), - Account: sysAccountName, - Database: "db1", - Table: "t1", + Database: "db1", + Table: "t1", }, }, }, @@ -2792,22 +2364,22 @@ func TestCdcTask_retrieveCdcTask(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cdc := &CdcTask{ - logger: tt.fields.logger, - ie: tt.fields.ie, - cnUUID: tt.fields.cnUUID, - cnTxnClient: tt.fields.cnTxnClient, - cnEngine: tt.fields.cnEngine, - fileService: tt.fields.fileService, - cdcTask: tt.fields.cdcTask, - mp: tt.fields.mp, - packerPool: tt.fields.packerPool, - sinkUri: tt.fields.sinkUri, - tables: tt.fields.tables, - filters: tt.fields.filters, - startTs: tt.fields.startTs, - noFull: tt.fields.noFull, - activeRoutine: tt.fields.activeRoutine, - sunkWatermarkUpdater: tt.fields.sunkWatermarkUpdater, + logger: tt.fields.logger, + ie: tt.fields.ie, + cnUUID: tt.fields.cnUUID, + cnTxnClient: tt.fields.cnTxnClient, + cnEngine: tt.fields.cnEngine, + fileService: tt.fields.fileService, + cdcTask: tt.fields.cdcTask, + mp: tt.fields.mp, + packerPool: tt.fields.packerPool, + sinkUri: tt.fields.sinkUri, + tables: tt.fields.tables, + exclude: tt.fields.exclude, + startTs: tt.fields.startTs, + noFull: tt.fields.noFull, + activeRoutine: tt.fields.activeRoutine, + watermarkUpdater: tt.fields.sunkWatermarkUpdater, } err := cdc.retrieveCdcTask(tt.args.ctx) assert.NoError(t, err, fmt.Sprintf("retrieveCdcTask(%v)", tt.args.ctx)) @@ -2946,39 +2518,6 @@ func Test_initAesKey(t *testing.T) { } } -func Test_extractTablePair(t *testing.T) { - type args struct { - ctx context.Context - pattern string - defaultAcc string - } - tests := []struct { - name string - args args - want *cdc2.PatternTuple - wantErr assert.ErrorAssertionFunc - }{ - { - name: "t1", - args: args{ - ctx: context.Background(), - pattern: "source:sink:other", - defaultAcc: "sys", - }, - wantErr: assert.Error, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := extractTablePair(tt.args.ctx, tt.args.pattern, tt.args.defaultAcc) - if !tt.wantErr(t, err, fmt.Sprintf("extractTablePair(%v, %v, %v)", tt.args.ctx, tt.args.pattern, tt.args.defaultAcc)) { - return - } - assert.Equalf(t, tt.want, got, "extractTablePair(%v, %v, %v)", tt.args.ctx, tt.args.pattern, tt.args.defaultAcc) - }) - } -} - var _ ie.InternalExecutor = &mockIe{} type mockIe struct { @@ -3106,3 +2645,181 @@ func TestCdcTask_initAesKeyByInternalExecutor(t *testing.T) { err = initAesKeyByInternalExecutor(context.Background(), cdcTask, 0) assert.Error(t, err) } + +func TestCdcTask_handleNewTables(t *testing.T) { + stub1 := gostub.Stub(&cdc2.GetTxnOp, func(context.Context, engine.Engine, client.TxnClient, string) (client.TxnOperator, error) { + return nil, nil + }) + defer stub1.Reset() + + stub2 := gostub.Stub(&cdc2.FinishTxnOp, func(context.Context, error, client.TxnOperator, engine.Engine) {}) + defer stub2.Reset() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + eng := mock_frontend.NewMockEngine(ctrl) + eng.EXPECT().New(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + cdc := &CdcTask{ + cdcTask: &task.CreateCdcDetails{ + Accounts: []*task.Account{{Id: 0}}, + }, + tables: cdc2.PatternTuples{ + Pts: []*cdc2.PatternTuple{ + { + Source: cdc2.PatternTable{ + Database: "db1", + Table: cdc2.MatchAll, + }, + }, + }, + }, + exclude: regexp.MustCompile("db1.tb1"), + cnEngine: eng, + runningReaders: &sync.Map{}, + } + + mp := map[uint32]cdc2.TblMap{ + 0: { + "db1.tb1": &cdc2.DbTableInfo{}, + "db2.tb1": &cdc2.DbTableInfo{}, + }, + } + cdc.handleNewTables(mp) +} + +type mockWatermarkUpdater struct{} + +func (m mockWatermarkUpdater) Run(context.Context, *cdc2.ActiveRoutine) { + //TODO implement me + panic("implement me") +} + +func (m mockWatermarkUpdater) InsertIntoDb(*cdc2.DbTableInfo, types.TS) error { + return nil +} + +func (m mockWatermarkUpdater) GetFromMem(string, string) types.TS { + //TODO implement me + panic("implement me") +} + +func (m mockWatermarkUpdater) GetFromDb(dbName, tblName string) (watermark types.TS, err error) { + err = moerr.NewErrNoWatermarkFoundNoCtx(dbName, tblName) + return +} + +func (m mockWatermarkUpdater) UpdateMem(string, string, types.TS) {} + +func (m mockWatermarkUpdater) DeleteFromMem(string, string) {} + +func (m mockWatermarkUpdater) DeleteFromDb(string, string) error { + return nil +} + +func (m mockWatermarkUpdater) DeleteAllFromDb() error { + //TODO implement me + panic("implement me") +} + +func (m mockWatermarkUpdater) SaveErrMsg(string, string, string) error { + return nil +} + +type mockReader struct{} + +func (m mockReader) Run(ctx context.Context, ar *cdc2.ActiveRoutine) {} + +func (m mockReader) Close() {} + +type mockSinker struct{} + +func (m mockSinker) Run(ctx context.Context, ar *cdc2.ActiveRoutine) {} + +func (m mockSinker) Sink(ctx context.Context, data *cdc2.DecoderOutput) { + //TODO implement me + panic("implement me") +} + +func (m mockSinker) SendBegin() { + //TODO implement me + panic("implement me") +} + +func (m mockSinker) SendCommit() { + //TODO implement me + panic("implement me") +} + +func (m mockSinker) SendRollback() { + //TODO implement me + panic("implement me") +} + +func (m mockSinker) SendDummy() { + //TODO implement me + panic("implement me") +} + +func (m mockSinker) Error() error { + //TODO implement me + panic("implement me") +} + +func (m mockSinker) Reset() { + //TODO implement me + panic("implement me") +} + +func (m mockSinker) Close() { + //TODO implement me + panic("implement me") +} + +func TestCdcTask_addExecPipelineForTable(t *testing.T) { + cdc := &CdcTask{ + watermarkUpdater: &mockWatermarkUpdater{}, + runningReaders: &sync.Map{}, + noFull: true, + additionalConfig: map[string]interface{}{ + cdc2.MaxSqlLength: float64(cdc2.DefaultMaxSqlLength), + cdc2.SendSqlTimeout: cdc2.DefaultSendSqlTimeout, + cdc2.InitSnapshotSplitTxn: cdc2.DefaultInitSnapshotSplitTxn, + }, + } + + info := &cdc2.DbTableInfo{ + SourceDbId: 0, + SourceDbName: "", + SourceTblId: 0, + SourceTblName: "", + SourceCreateSql: "", + SinkDbName: "", + SinkTblName: "", + } + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + txnOperator := mock_frontend.NewMockTxnOperator(ctrl) + txnOperator.EXPECT().SnapshotTS().Return(timestamp.Timestamp{}).AnyTimes() + + stubGetTableDef := gostub.Stub(&cdc2.GetTableDef, func(context.Context, client.TxnOperator, engine.Engine, uint64) (*plan.TableDef, error) { + return nil, nil + }) + defer stubGetTableDef.Reset() + + stubSinker := gostub.Stub(&cdc2.NewSinker, func(cdc2.UriInfo, *cdc2.DbTableInfo, cdc2.IWatermarkUpdater, + *plan.TableDef, int, time.Duration, *cdc2.ActiveRoutine, uint64, string) (cdc2.Sinker, error) { + return &mockSinker{}, nil + }) + defer stubSinker.Reset() + + stubReader := gostub.Stub(&cdc2.NewTableReader, func(client.TxnClient, engine.Engine, *mpool.MPool, *fileservice.Pool[*types.Packer], + *cdc2.DbTableInfo, cdc2.Sinker, cdc2.IWatermarkUpdater, *plan.TableDef, func(*cdc2.DbTableInfo) error, bool, *sync.Map) cdc2.Reader { + return &mockReader{} + }) + defer stubReader.Reset() + + assert.NoError(t, cdc.addExecPipelineForTable(context.Background(), info, txnOperator)) +} diff --git a/pkg/frontend/compiler_context.go b/pkg/frontend/compiler_context.go index c0ef1a35beac7..622884fe4e365 100644 --- a/pkg/frontend/compiler_context.go +++ b/pkg/frontend/compiler_context.go @@ -509,6 +509,48 @@ func (tcc *TxnCompilerContext) Resolve(dbName string, tableName string, snapshot return obj, tableDef } +func (tcc *TxnCompilerContext) ResolveIndexTableByRef( + ref *plan.ObjectRef, + tblName string, + snapshot *plan2.Snapshot, +) (*plan2.ObjectRef, *plan2.TableDef) { + start := time.Now() + defer func() { + end := time.Since(start).Seconds() + v2.TxnStatementResolveDurationHistogram.Observe(end) + v2.TotalResolveDurationHistogram.Observe(end) + }() + + // no need to ensureDatabaseIsNotEmpty + + var subMeta *plan.SubscriptionMeta + if ref.PubInfo != nil { + subMeta = &plan.SubscriptionMeta{ + AccountId: ref.PubInfo.TenantId, + } + } + + ctx, table, err := tcc.getRelation(ref.SchemaName, tblName, subMeta, snapshot) + if err != nil { + return nil, nil + } + + obj := &plan2.ObjectRef{ + SchemaName: ref.SchemaName, + ObjName: tblName, + Obj: int64(table.GetTableID(ctx)), + SubscriptionName: ref.SubscriptionName, + PubInfo: ref.PubInfo, + } + + tableDef := table.CopyTableDef(ctx) + if tableDef.IsTemporary { + tableDef.Name = tblName + } + + return obj, tableDef +} + func (tcc *TxnCompilerContext) ResolveUdf(name string, args []*plan.Expr) (udf *function.Udf, err error) { var matchNum int var argstr string diff --git a/pkg/frontend/mysql_protocol.go b/pkg/frontend/mysql_protocol.go index a32c6730a84e0..4ec1d99dfe289 100644 --- a/pkg/frontend/mysql_protocol.go +++ b/pkg/frontend/mysql_protocol.go @@ -35,6 +35,7 @@ import ( "golang.org/x/exp/slices" "github.com/matrixorigin/matrixone/pkg/common/moerr" + util2 "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/config" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" @@ -1215,7 +1216,7 @@ func (mp *MysqlProtocolImpl) writeStringFix(data []byte, pos int, value string, // append a string with fixed length to the buffer // return the buffer func (mp *MysqlProtocolImpl) appendStringFix(value string, length int) error { - err := mp.append([]byte(value[:length])...) + err := mp.append(util2.UnsafeStringToBytes(value[:length])...) if err != nil { return err } diff --git a/pkg/frontend/predefined.go b/pkg/frontend/predefined.go index 7f9f538598db2..b82cc16f32dea 100644 --- a/pkg/frontend/predefined.go +++ b/pkg/frontend/predefined.go @@ -260,12 +260,11 @@ var ( MoCatalogMoCdcWatermarkDDL = `create table mo_catalog.mo_cdc_watermark ( account_id bigint unsigned, task_id uuid, - table_id varchar(64), db_name varchar(256), table_name varchar(256), watermark varchar(128), err_msg varchar(256), - primary key(account_id,task_id,table_id) + primary key(account_id,task_id,db_name,table_name) )` MoCatalogMoSessionsDDL = `CREATE VIEW mo_catalog.mo_sessions AS SELECT node_id, conn_id, session_id, account, user, host, db, session_start, command, info, txn_id, statement_id, statement_type, query_type, sql_source_type, query_start, client_host, role, proxy_host FROM mo_sessions() AS mo_sessions_tmp` diff --git a/pkg/frontend/session_test.go b/pkg/frontend/session_test.go index 60d24511628e0..60142350a19ba 100644 --- a/pkg/frontend/session_test.go +++ b/pkg/frontend/session_test.go @@ -384,6 +384,16 @@ func TestSession_TxnCompilerContext(t *testing.T) { convey.So(object, convey.ShouldNotBeNil) convey.So(tableRef, convey.ShouldNotBeNil) + ref := &plan.ObjectRef{ + SchemaName: "schema", + PubInfo: &plan.PubInfo{ + TenantId: 0, + }, + } + object, tableRef = tcc.ResolveIndexTableByRef(ref, "indexTable", &plan2.Snapshot{TS: ts}) + convey.So(object, convey.ShouldNotBeNil) + convey.So(tableRef, convey.ShouldNotBeNil) + pkd := tcc.GetPrimaryKeyDef("abc", "t1", &plan2.Snapshot{TS: ts}) convey.So(len(pkd), convey.ShouldBeZeroValue) diff --git a/pkg/frontend/util.go b/pkg/frontend/util.go index f0b5ab104570a..a2c2357528ed0 100644 --- a/pkg/frontend/util.go +++ b/pkg/frontend/util.go @@ -1560,10 +1560,14 @@ rule: it means all most all string can be legal. */ func dbNameIsLegal(name string) bool { + name = strings.TrimSpace(name) if hasSpecialChars(name) { return false } - name = strings.TrimSpace(name) + if name == cdc.MatchAll { + return true + } + createDBSqls := []string{ "create database " + name, "create database `" + name + "`", @@ -1581,10 +1585,14 @@ rule: it means all most all string can be legal. */ func tableNameIsLegal(name string) bool { + name = strings.TrimSpace(name) if hasSpecialChars(name) { return false } - name = strings.TrimSpace(name) + if name == cdc.MatchAll { + return true + } + createTableSqls := []string{ "create table " + name + "(a int)", "create table `" + name + "`(a int)", diff --git a/pkg/sql/colexec/multi_update/s3writer.go b/pkg/sql/colexec/multi_update/s3writer.go index 4242cc37647c5..8c3813aa6c789 100644 --- a/pkg/sql/colexec/multi_update/s3writer.go +++ b/pkg/sql/colexec/multi_update/s3writer.go @@ -188,9 +188,6 @@ func (writer *s3Writer) prepareDeleteBatchs( for _, bat := range src { rowIDVec := bat.GetVector(RowIDIdx) - if rowIDVec.IsConstNull() { - continue - } nulls := rowIDVec.GetNulls() if nulls.Count() == bat.RowCount() { continue @@ -272,7 +269,7 @@ func (writer *s3Writer) sortAndSync(proc *process.Process, analyzer process.Anal if len(updateCtx.DeleteCols) > 0 { var delBatchs []*batch.Batch if parititionCount == 0 { - bats, err = fetchSomeVecFromCompactBatchs(proc, writer.cacheBatchs, updateCtx.DeleteCols, DeleteBatchAttrs) + bats, err = fetchSomeVecFromCompactBatchs(writer.cacheBatchs, updateCtx.DeleteCols, DeleteBatchAttrs) if err != nil { return } @@ -341,7 +338,7 @@ func (writer *s3Writer) sortAndSync(proc *process.Process, analyzer process.Anal } } } - bats, err = fetchSomeVecFromCompactBatchs(proc, writer.cacheBatchs, updateCtx.InsertCols, insertAttrs) + bats, err = fetchSomeVecFromCompactBatchs(writer.cacheBatchs, updateCtx.InsertCols, insertAttrs) needSortBatch = false needCleanBatch = false } diff --git a/pkg/sql/colexec/multi_update/s3writer_util.go b/pkg/sql/colexec/multi_update/s3writer_util.go index d2d59205f157f..7a8bc85b9d1c5 100644 --- a/pkg/sql/colexec/multi_update/s3writer_util.go +++ b/pkg/sql/colexec/multi_update/s3writer_util.go @@ -245,11 +245,9 @@ func cloneSomeVecFromCompactBatchs( // fetchSomeVecFromCompactBatchs fetch some vectors from CompactBatchs // do not clean these batchs func fetchSomeVecFromCompactBatchs( - proc *process.Process, src *batch.CompactBatchs, cols []int, attrs []string) ([]*batch.Batch, error) { - mp := proc.GetMPool() var newBat *batch.Batch retBats := make([]*batch.Batch, src.Length()) for i := 0; i < src.Length(); i++ { @@ -258,18 +256,7 @@ func fetchSomeVecFromCompactBatchs( newBat.Attrs = attrs for j, idx := range cols { oldVec := oldBat.Vecs[idx] - //expand constant vector - if oldVec.IsConst() { - newVec := vector.NewVec(*oldVec.GetType()) - err := vector.GetUnionAllFunction(*oldVec.GetType(), mp)(newVec, oldVec) - if err != nil { - return nil, err - } - oldBat.ReplaceVector(oldVec, newVec, 0) - newBat.Vecs[j] = newVec - } else { - newBat.Vecs[j] = oldVec - } + newBat.Vecs[j] = oldVec } newBat.SetRowCount(newBat.Vecs[0].Length()) retBats[i] = newBat diff --git a/pkg/sql/compile/sql_executor_context.go b/pkg/sql/compile/sql_executor_context.go index 03cdb597ead88..bca4c3b1765d2 100644 --- a/pkg/sql/compile/sql_executor_context.go +++ b/pkg/sql/compile/sql_executor_context.go @@ -356,6 +356,10 @@ func (c *compilerContext) ResolveById(tableId uint64, snapshot *plan.Snapshot) ( return c.Resolve(dbName, tableName, snapshot) } +func (c *compilerContext) ResolveIndexTableByRef(ref *plan.ObjectRef, tblName string, snapshot *plan.Snapshot) (*plan.ObjectRef, *plan.TableDef) { + return c.Resolve(plan.DbNameOfObjRef(ref), tblName, snapshot) +} + func (c *compilerContext) Resolve(dbName string, tableName string, snapshot *plan.Snapshot) (*plan.ObjectRef, *plan.TableDef) { // In order to be compatible with various GUI clients and BI tools, lower case db and table name if it's a mysql system table if slices.Contains(mysql.CaseInsensitiveDbs, strings.ToLower(dbName)) { diff --git a/pkg/sql/plan/apply_indices.go b/pkg/sql/plan/apply_indices.go index e4f06a33b1931..ac3079dc7afe6 100644 --- a/pkg/sql/plan/apply_indices.go +++ b/pkg/sql/plan/apply_indices.go @@ -705,7 +705,7 @@ func (builder *QueryBuilder) tryIndexOnlyScan(idxDef *IndexDef, node *plan.Node, } idxTag := builder.genNewTag() - idxObjRef, idxTableDef := builder.compCtx.Resolve(node.ObjRef.SchemaName, idxDef.IndexTableName, scanSnapshot) + idxObjRef, idxTableDef := builder.compCtx.ResolveIndexTableByRef(node.ObjRef, idxDef.IndexTableName, scanSnapshot) builder.addNameByColRef(idxTag, idxTableDef) leadingColExpr := GetColExpr(idxTableDef.Cols[0].Typ, idxTag, 0) @@ -784,7 +784,7 @@ func (builder *QueryBuilder) getIndexForNonEquiCond(indexes []*IndexDef, node *p func (builder *QueryBuilder) applyIndexJoin(idxDef *IndexDef, node *plan.Node, filterType int, filterIdx []int32, scanSnapshot *Snapshot) (int32, int32) { idxTag := builder.genNewTag() - idxObjRef, idxTableDef := builder.compCtx.Resolve(node.ObjRef.SchemaName, idxDef.IndexTableName, scanSnapshot) + idxObjRef, idxTableDef := builder.compCtx.ResolveIndexTableByRef(node.ObjRef, idxDef.IndexTableName, scanSnapshot) builder.addNameByColRef(idxTag, idxTableDef) numParts := len(idxDef.Parts) @@ -1008,8 +1008,7 @@ func (builder *QueryBuilder) applyIndicesForJoins(nodeID int32, node *plan.Node, } idxTag := builder.genNewTag() - //idxObjRef, idxTableDef := builder.compCtx.Resolve(leftChild.ObjRef.SchemaName, idxDef.IndexTableName, *ts) - idxObjRef, idxTableDef := builder.compCtx.Resolve(leftChild.ObjRef.SchemaName, idxDef.IndexTableName, scanSnapshot) + idxObjRef, idxTableDef := builder.compCtx.ResolveIndexTableByRef(leftChild.ObjRef, idxDef.IndexTableName, scanSnapshot) builder.addNameByColRef(idxTag, idxTableDef) rfTag := builder.genNewMsgTag() diff --git a/pkg/sql/plan/apply_indices_master.go b/pkg/sql/plan/apply_indices_master.go index 910f14eb013c4..a0931eed0e729 100644 --- a/pkg/sql/plan/apply_indices_master.go +++ b/pkg/sql/plan/apply_indices_master.go @@ -39,8 +39,7 @@ func (builder *QueryBuilder) applyIndicesForFiltersUsingMasterIndex(nodeID int32 //ts1 := scanNode.ScanTS for i, filterExp := range scanNode.FilterList { // TODO: node should hold snapshot info and account info - //idxObjRef, idxTableDef := builder.compCtx.Resolve(scanNode.ObjRef.SchemaName, indexDef.IndexTableName, timestamp.Timestamp{}) - idxObjRef, idxTableDef := builder.compCtx.Resolve(scanNode.ObjRef.SchemaName, indexDef.IndexTableName, nil) + idxObjRef, idxTableDef := builder.compCtx.ResolveIndexTableByRef(scanNode.ObjRef, indexDef.IndexTableName, nil) // 1. SELECT pk from idx WHERE prefix_eq(`__mo_index_idx_col`,serial_full("0","value")) currIdxScanTag, currScanId := makeIndexTblScan(builder, builder.ctxByNode[nodeID], filterExp, idxTableDef, idxObjRef, scanNode.ScanSnapshot, colDefs) diff --git a/pkg/sql/plan/apply_indices_vector.go b/pkg/sql/plan/apply_indices_vector.go index 8c5b7f076e32e..0b50c18470481 100644 --- a/pkg/sql/plan/apply_indices_vector.go +++ b/pkg/sql/plan/apply_indices_vector.go @@ -87,9 +87,9 @@ func (builder *QueryBuilder) applyIndicesForSortUsingVectorIndex(nodeID int32, p scanSnapshot = &Snapshot{} } - idxObjRefs[0], idxTableDefs[0] = builder.compCtx.Resolve(scanNode.ObjRef.SchemaName, multiTableIndexWithSortDistFn.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Metadata].IndexTableName, scanSnapshot) - idxObjRefs[1], idxTableDefs[1] = builder.compCtx.Resolve(scanNode.ObjRef.SchemaName, multiTableIndexWithSortDistFn.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Centroids].IndexTableName, scanSnapshot) - idxObjRefs[2], idxTableDefs[2] = builder.compCtx.Resolve(scanNode.ObjRef.SchemaName, multiTableIndexWithSortDistFn.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Entries].IndexTableName, scanSnapshot) + idxObjRefs[0], idxTableDefs[0] = builder.compCtx.ResolveIndexTableByRef(scanNode.ObjRef, multiTableIndexWithSortDistFn.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Metadata].IndexTableName, scanSnapshot) + idxObjRefs[1], idxTableDefs[1] = builder.compCtx.ResolveIndexTableByRef(scanNode.ObjRef, multiTableIndexWithSortDistFn.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Centroids].IndexTableName, scanSnapshot) + idxObjRefs[2], idxTableDefs[2] = builder.compCtx.ResolveIndexTableByRef(scanNode.ObjRef, multiTableIndexWithSortDistFn.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Entries].IndexTableName, scanSnapshot) builder.addNameByColRef(idxTags["meta.scan"], idxTableDefs[0]) builder.addNameByColRef(idxTags["centroids.scan"], idxTableDefs[1]) diff --git a/pkg/sql/plan/bind_delete.go b/pkg/sql/plan/bind_delete.go index 3ffda150b4cf1..43d75e1560b66 100644 --- a/pkg/sql/plan/bind_delete.go +++ b/pkg/sql/plan/bind_delete.go @@ -149,7 +149,7 @@ func (builder *QueryBuilder) bindDelete(stmt *tree.Delete, bindCtx *BindContext) return 0, moerr.NewUnsupportedDML(builder.GetContext(), "have vector index table") } - idxObjRef, idxTableDef := builder.compCtx.Resolve(dmlCtx.objRefs[0].SchemaName, idxDef.IndexTableName, bindCtx.snapshot) + idxObjRef, idxTableDef := builder.compCtx.ResolveIndexTableByRef(dmlCtx.objRefs[0], idxDef.IndexTableName, bindCtx.snapshot) if len(idxTableDef.Name2ColIndex) == 0 { idxTableDef.Name2ColIndex = make(map[string]int32) for colIdx, col := range idxTableDef.Cols { diff --git a/pkg/sql/plan/bind_insert.go b/pkg/sql/plan/bind_insert.go index a0aa707b57583..34a9cb208aa02 100644 --- a/pkg/sql/plan/bind_insert.go +++ b/pkg/sql/plan/bind_insert.go @@ -208,7 +208,7 @@ func (builder *QueryBuilder) appendDedupAndMultiUpdateNodesForBindInsert( if !idxDef.TableExist || skipUniqueIdx[j] || !idxDef.Unique { continue } - _, idxTableDef := builder.compCtx.Resolve(dmlCtx.objRefs[0].SchemaName, idxDef.IndexTableName, bindCtx.snapshot) + _, idxTableDef := builder.compCtx.ResolveIndexTableByRef(dmlCtx.objRefs[0], idxDef.IndexTableName, bindCtx.snapshot) var pkIdxInBat int32 if len(idxDef.Parts) == 1 { @@ -243,7 +243,7 @@ func (builder *QueryBuilder) appendDedupAndMultiUpdateNodesForBindInsert( continue } - idxObjRefs[i], idxTableDefs[i] = builder.compCtx.Resolve(objRef.SchemaName, idxDef.IndexTableName, bindCtx.snapshot) + idxObjRefs[i], idxTableDefs[i] = builder.compCtx.ResolveIndexTableByRef(objRef, idxDef.IndexTableName, bindCtx.snapshot) } } else { if pkName != catalog.FakePrimaryKeyColName { @@ -336,7 +336,7 @@ func (builder *QueryBuilder) appendDedupAndMultiUpdateNodesForBindInsert( continue } - idxObjRefs[i], idxTableDefs[i] = builder.compCtx.Resolve(objRef.SchemaName, idxDef.IndexTableName, bindCtx.snapshot) + idxObjRefs[i], idxTableDefs[i] = builder.compCtx.ResolveIndexTableByRef(objRef, idxDef.IndexTableName, bindCtx.snapshot) if !idxDef.Unique { continue diff --git a/pkg/sql/plan/bind_update.go b/pkg/sql/plan/bind_update.go index 279f764830740..929090ffcd698 100644 --- a/pkg/sql/plan/bind_update.go +++ b/pkg/sql/plan/bind_update.go @@ -244,7 +244,7 @@ func (builder *QueryBuilder) bindUpdate(stmt *tree.Update, bindCtx *BindContext) continue } - idxObjRef, idxTableDef := builder.compCtx.Resolve(dmlCtx.objRefs[i].SchemaName, idxDef.IndexTableName, bindCtx.snapshot) + idxObjRef, idxTableDef := builder.compCtx.ResolveIndexTableByRef(dmlCtx.objRefs[i], idxDef.IndexTableName, bindCtx.snapshot) idxTag := builder.genNewTag() builder.addNameByColRef(idxTag, idxTableDef) diff --git a/pkg/sql/plan/build_delete.go b/pkg/sql/plan/build_delete.go index 7d7919d830ef0..adeb3286682ff 100644 --- a/pkg/sql/plan/build_delete.go +++ b/pkg/sql/plan/build_delete.go @@ -88,7 +88,7 @@ func buildDelete(stmt *tree.Delete, ctx CompilerContext, isPrepareStmt bool) (*P partTableIds := make([]uint64, tableDef.Partition.PartitionNum) partTableNames := make([]string, tableDef.Partition.PartitionNum) for j, partition := range tableDef.Partition.Partitions { - _, partTableDef := ctx.Resolve(tblInfo.objRef[i].SchemaName, partition.PartitionTableName, nil) + _, partTableDef := ctx.Resolve(DbNameOfObjRef(tblInfo.objRef[i]), partition.PartitionTableName, nil) partTableIds[j] = partTableDef.TblId partTableNames[j] = partition.PartitionTableName } diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index 77ffafc09c1f4..ce0379d9c0c97 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -1256,7 +1256,7 @@ func makeDeleteNodeInfo(ctx CompilerContext, objRef *ObjectRef, tableDef *TableD partTableIds := make([]uint64, tableDef.Partition.PartitionNum) partTableNames := make([]string, tableDef.Partition.PartitionNum) for i, partition := range tableDef.Partition.Partitions { - _, partTableDef := ctx.Resolve(objRef.SchemaName, partition.PartitionTableName, nil) + _, partTableDef := ctx.Resolve(DbNameOfObjRef(objRef), partition.PartitionTableName, nil) partTableIds[i] = partTableDef.TblId partTableNames[i] = partition.PartitionTableName } @@ -1303,7 +1303,7 @@ func getPartTableIdsAndNames(ctx CompilerContext, objRef *ObjectRef, tableDef *T partTableIds = make([]uint64, tableDef.Partition.PartitionNum) partTableNames = make([]string, tableDef.Partition.PartitionNum) for i, partition := range tableDef.Partition.Partitions { - _, partTableDef := ctx.Resolve(objRef.SchemaName, partition.PartitionTableName, nil) + _, partTableDef := ctx.Resolve(DbNameOfObjRef(objRef), partition.PartitionTableName, nil) partTableIds[i] = partTableDef.TblId partTableNames[i] = partition.PartitionTableName } @@ -3594,9 +3594,9 @@ func buildPreInsertMultiTableIndexes(ctx CompilerContext, builder *QueryBuilder, //idxRefs[1], idxTableDefs[1] = ctx.Resolve(objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Centroids].IndexTableName, timestamp.Timestamp{}) //idxRefs[2], idxTableDefs[2] = ctx.Resolve(objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Entries].IndexTableName, timestamp.Timestamp{}) - idxRefs[0], idxTableDefs[0] = ctx.Resolve(objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Metadata].IndexTableName, nil) - idxRefs[1], idxTableDefs[1] = ctx.Resolve(objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Centroids].IndexTableName, nil) - idxRefs[2], idxTableDefs[2] = ctx.Resolve(objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Entries].IndexTableName, nil) + idxRefs[0], idxTableDefs[0] = ctx.ResolveIndexTableByRef(objRef, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Metadata].IndexTableName, nil) + idxRefs[1], idxTableDefs[1] = ctx.ResolveIndexTableByRef(objRef, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Centroids].IndexTableName, nil) + idxRefs[2], idxTableDefs[2] = ctx.ResolveIndexTableByRef(objRef, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Entries].IndexTableName, nil) // remove row_id for i := range idxTableDefs { @@ -3658,9 +3658,9 @@ func buildDeleteMultiTableIndexes(ctx CompilerContext, builder *QueryBuilder, bi //idxRefs[1], idxTableDefs[1] = ctx.Resolve(delCtx.objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Centroids].IndexTableName, timestamp.Timestamp{}) //idxRefs[2], idxTableDefs[2] = ctx.Resolve(delCtx.objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Entries].IndexTableName, timestamp.Timestamp{}) - idxRefs[0], idxTableDefs[0] = ctx.Resolve(delCtx.objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Metadata].IndexTableName, nil) - idxRefs[1], idxTableDefs[1] = ctx.Resolve(delCtx.objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Centroids].IndexTableName, nil) - idxRefs[2], idxTableDefs[2] = ctx.Resolve(delCtx.objRef.SchemaName, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Entries].IndexTableName, nil) + idxRefs[0], idxTableDefs[0] = ctx.ResolveIndexTableByRef(delCtx.objRef, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Metadata].IndexTableName, nil) + idxRefs[1], idxTableDefs[1] = ctx.ResolveIndexTableByRef(delCtx.objRef, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Centroids].IndexTableName, nil) + idxRefs[2], idxTableDefs[2] = ctx.ResolveIndexTableByRef(delCtx.objRef, multiTableIndex.IndexDefs[catalog.SystemSI_IVFFLAT_TblType_Entries].IndexTableName, nil) entriesObjRef, entriesTableDef := idxRefs[2], idxTableDefs[2] if entriesTableDef == nil { @@ -3791,7 +3791,7 @@ func buildDeleteMultiTableIndexes(ctx CompilerContext, builder *QueryBuilder, bi func buildPreInsertRegularIndex(stmt *tree.Insert, ctx CompilerContext, builder *QueryBuilder, bindCtx *BindContext, objRef *ObjectRef, tableDef *TableDef, sourceStep int32, ifInsertFromUniqueColMap map[string]bool, indexdef *plan.IndexDef, idx int) error { - idxRef, idxTableDef := ctx.Resolve(objRef.SchemaName, indexdef.IndexTableName, nil) + idxRef, idxTableDef := ctx.ResolveIndexTableByRef(objRef, indexdef.IndexTableName, nil) // remove row_id idxTableDef.Cols = RemoveIf[*ColDef](idxTableDef.Cols, func(col *ColDef) bool { return col.Name == catalog.Row_ID @@ -3813,7 +3813,7 @@ func buildPreInsertRegularIndex(stmt *tree.Insert, ctx CompilerContext, builder // with the primary key of the hidden table as the unique key. // package contains some information needed by the fuzzy filter to run background SQL. if indexdef.GetUnique() { - _, idxTableDef := ctx.Resolve(objRef.SchemaName, indexdef.IndexTableName, nil) + _, idxTableDef := ctx.ResolveIndexTableByRef(objRef, indexdef.IndexTableName, nil) // remove row_id idxTableDef.Cols = RemoveIf[*ColDef](idxTableDef.Cols, func(colVal *ColDef) bool { return colVal.Name == catalog.Row_ID @@ -3897,7 +3897,7 @@ func buildPreInsertRegularIndex(stmt *tree.Insert, ctx CompilerContext, builder func buildPreInsertMasterIndex(stmt *tree.Insert, ctx CompilerContext, builder *QueryBuilder, bindCtx *BindContext, objRef *ObjectRef, tableDef *TableDef, sourceStep int32, ifInsertFromUniqueColMap map[string]bool, indexdef *plan.IndexDef, idx int) error { - idxRef, idxTableDef := ctx.Resolve(objRef.SchemaName, indexdef.IndexTableName, nil) + idxRef, idxTableDef := ctx.ResolveIndexTableByRef(objRef, indexdef.IndexTableName, nil) // remove row_id idxTableDef.Cols = RemoveIf[*ColDef](idxTableDef.Cols, func(colVal *ColDef) bool { return colVal.Name == catalog.Row_ID @@ -3947,7 +3947,7 @@ func buildDeleteRegularIndex(ctx CompilerContext, builder *QueryBuilder, bindCtx var isUk = indexdef.Unique var isSK = !isUk && catalog.IsRegularIndexAlgo(indexdef.IndexAlgo) - uniqueObjRef, uniqueTableDef := builder.compCtx.Resolve(delCtx.objRef.SchemaName, indexdef.IndexTableName, nil) + uniqueObjRef, uniqueTableDef := builder.compCtx.ResolveIndexTableByRef(delCtx.objRef, indexdef.IndexTableName, nil) if uniqueTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), delCtx.objRef.SchemaName, indexdef.IndexTableName) } @@ -4058,7 +4058,7 @@ func buildDeleteMasterIndex(ctx CompilerContext, builder *QueryBuilder, bindCtx indexdef *plan.IndexDef, idx int, typMap map[string]plan.Type, posMap map[string]int) error { isUpdate := delCtx.updateColLength > 0 // Used by pre-insert vector index. - masterObjRef, masterTableDef := ctx.Resolve(delCtx.objRef.SchemaName, indexdef.IndexTableName, nil) + masterObjRef, masterTableDef := ctx.ResolveIndexTableByRef(delCtx.objRef, indexdef.IndexTableName, nil) if masterTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), delCtx.objRef.SchemaName, indexdef.IndexName) } @@ -4471,7 +4471,7 @@ func buildPreInsertFullTextIndex(stmt *tree.Insert, ctx CompilerContext, builder lastNodeId = builder.appendNode(projectNode, bindCtx) - indexObjRef, indexTableDef := ctx.Resolve(objRef.SchemaName, indexdef.IndexTableName, nil) + indexObjRef, indexTableDef := ctx.ResolveIndexTableByRef(objRef, indexdef.IndexTableName, nil) if indexTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), objRef.SchemaName, indexdef.IndexName) } @@ -4717,7 +4717,7 @@ func buildPreDeleteFullTextIndex(ctx CompilerContext, builder *QueryBuilder, bin indexdef *plan.IndexDef, idx int, typMap map[string]plan.Type, posMap map[string]int) error { //isUpdate := delCtx.updateColLength > 0 - indexObjRef, indexTableDef := ctx.Resolve(delCtx.objRef.SchemaName, indexdef.IndexTableName, nil) + indexObjRef, indexTableDef := ctx.ResolveIndexTableByRef(delCtx.objRef, indexdef.IndexTableName, nil) if indexTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), delCtx.objRef.SchemaName, indexdef.IndexName) } @@ -4784,7 +4784,7 @@ func buildPostDeleteFullTextIndex(ctx CompilerContext, builder *QueryBuilder, bi isDelete := true isInsert := delCtx.updateColLength > 0 - indexObjRef, indexTableDef := ctx.Resolve(delCtx.objRef.SchemaName, indexdef.IndexTableName, nil) + indexObjRef, indexTableDef := ctx.ResolveIndexTableByRef(delCtx.objRef, indexdef.IndexTableName, nil) if indexTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), delCtx.objRef.SchemaName, indexdef.IndexName) } @@ -4802,7 +4802,7 @@ func buildPostInsertFullTextIndex(stmt *tree.Insert, ctx CompilerContext, builde isInsert := true isDeleteWithoutFilters := false - indexObjRef, indexTableDef := ctx.Resolve(objRef.SchemaName, indexdef.IndexTableName, nil) + indexObjRef, indexTableDef := ctx.ResolveIndexTableByRef(objRef, indexdef.IndexTableName, nil) if indexTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), objRef.SchemaName, indexdef.IndexName) } diff --git a/pkg/sql/plan/build_util.go b/pkg/sql/plan/build_util.go index 3075d5f70f4dd..e15a842a5e586 100644 --- a/pkg/sql/plan/build_util.go +++ b/pkg/sql/plan/build_util.go @@ -669,7 +669,7 @@ func getPartitionInfos(ctx CompilerContext, objRef *ObjectRef, tableDef *TableDe partTableIds := make([]uint64, tableDef.Partition.PartitionNum) partTableNames := make([]string, tableDef.Partition.PartitionNum) for i, partition := range tableDef.Partition.Partitions { - _, partTableDef := ctx.Resolve(objRef.SchemaName, partition.PartitionTableName, nil) + _, partTableDef := ctx.Resolve(DbNameOfObjRef(objRef), partition.PartitionTableName, nil) partTableIds[i] = partTableDef.TblId partTableNames[i] = partition.PartitionTableName } diff --git a/pkg/sql/plan/mock.go b/pkg/sql/plan/mock.go index 95a7c4ff1314c..55814a71cc3be 100644 --- a/pkg/sql/plan/mock.go +++ b/pkg/sql/plan/mock.go @@ -71,6 +71,10 @@ func (m *MockCompilerContext) CheckSubscriptionValid(subName, accName string, pu panic("implement me") } +func (m *MockCompilerContext) ResolveIndexTableByRef(ref *ObjectRef, tblName string, snapshot *Snapshot) (*ObjectRef, *TableDef) { + return m.Resolve(DbNameOfObjRef(ref), tblName, snapshot) +} + func (m *MockCompilerContext) ResolveSubscriptionTableById(tableId uint64, pubmeta *SubscriptionMeta) (*ObjectRef, *TableDef) { return nil, nil } diff --git a/pkg/sql/plan/opt_misc.go b/pkg/sql/plan/opt_misc.go index bba0aa2398f0c..0bf0d6fb19cef 100644 --- a/pkg/sql/plan/opt_misc.go +++ b/pkg/sql/plan/opt_misc.go @@ -1161,7 +1161,7 @@ func (builder *QueryBuilder) lockTableIfLockNoRowsAtTheEndForDelAndUpdate() (err tableIDs[tableDef.TblId] = true for _, idx := range tableDef.Indexes { if idx.TableExist { - _, idxTableDef := builder.compCtx.Resolve(objRef.SchemaName, idx.IndexTableName, nil) + _, idxTableDef := builder.compCtx.ResolveIndexTableByRef(objRef, idx.IndexTableName, nil) if idxTableDef == nil { return } diff --git a/pkg/sql/plan/types.go b/pkg/sql/plan/types.go index 98e607fc9ac68..6178768606da4 100644 --- a/pkg/sql/plan/types.go +++ b/pkg/sql/plan/types.go @@ -82,6 +82,8 @@ type CompilerContext interface { DatabaseExists(name string, snapshot *Snapshot) bool // get table definition by database/schema Resolve(schemaName string, tableName string, snapshot *Snapshot) (*ObjectRef, *TableDef) + // get index table definition by an ObjectRef, will skip unnecessary subscription check + ResolveIndexTableByRef(ref *ObjectRef, tblName string, snapshot *Snapshot) (*ObjectRef, *TableDef) // get table definition by table id ResolveById(tableId uint64, snapshot *Snapshot) (*ObjectRef, *TableDef) // get the value of variable diff --git a/pkg/sql/plan/types_mock.go b/pkg/sql/plan/types_mock.go index e062399894ca2..ed9accd307f2a 100644 --- a/pkg/sql/plan/types_mock.go +++ b/pkg/sql/plan/types_mock.go @@ -385,6 +385,14 @@ func (mr *MockCompilerContext2MockRecorder) ReplacePlan(execPlan interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReplacePlan", reflect.TypeOf((*MockCompilerContext2)(nil).InitExecuteStmtParam), execPlan) } +func (m *MockCompilerContext2) ResolveIndexTableByRef(ref *ObjectRef, tblName string, snapshot *Snapshot) (*ObjectRef, *TableDef) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveIndexTableByRef", ref, tblName, snapshot) + ret0, _ := ret[0].(*ObjectRef) + ret1, _ := ret[1].(*TableDef) + return ret0, ret1 +} + // Resolve mocks base method. func (m *MockCompilerContext2) Resolve(schemaName, tableName string, snapshot *Snapshot) (*ObjectRef, *TableDef) { m.ctrl.T.Helper() diff --git a/pkg/sql/plan/utils.go b/pkg/sql/plan/utils.go index 50eee5164e7a6..cdb10f7a14feb 100644 --- a/pkg/sql/plan/utils.go +++ b/pkg/sql/plan/utils.go @@ -2720,3 +2720,11 @@ func offsetToString(offset int) string { // } // return !strings.HasPrefix(tableDef.Name, catalog.IndexTableNamePrefix) // } + +// DbNameOfObjRef return subscription name of ObjectRef if exists, to avoid the mismatching of account id and db name +func DbNameOfObjRef(objRef *ObjectRef) string { + if objRef.SubscriptionName == "" { + return objRef.SchemaName + } + return objRef.SubscriptionName +} diff --git a/pkg/sql/plan/utils_test.go b/pkg/sql/plan/utils_test.go index 2879e4c17cc8a..45d532946236b 100644 --- a/pkg/sql/plan/utils_test.go +++ b/pkg/sql/plan/utils_test.go @@ -101,3 +101,39 @@ func TestHandleOptimizerHints(t *testing.T) { handleOptimizerHints("skipDedup=1", builder) require.Equal(t, 1, builder.optimizerHints.skipDedup) } + +func TestDbNameOfObjRef(t *testing.T) { + type args struct { + objRef *ObjectRef + } + tests := []struct { + name string + args args + want string + }{ + { + name: "case 1", + args: args{ + objRef: &ObjectRef{ + SchemaName: "db", + }, + }, + want: "db", + }, + { + name: "case 2", + args: args{ + objRef: &ObjectRef{ + SchemaName: "whatever", + SubscriptionName: "sub", + }, + }, + want: "sub", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, DbNameOfObjRef(tt.args.objRef), "DbNameOfObjRef(%v)", tt.args.objRef) + }) + } +} diff --git a/pkg/taskservice/mysql_task_storage.go b/pkg/taskservice/mysql_task_storage.go index 9dc580753c6f8..b52eb1b325283 100644 --- a/pkg/taskservice/mysql_task_storage.go +++ b/pkg/taskservice/mysql_task_storage.go @@ -270,7 +270,9 @@ func (m *mysqlTaskStorage) UpdateAsyncTask(ctx context.Context, tasks []task.Asy } if t.ExecuteResult != nil { execResult.Code = t.ExecuteResult.Code - execResult.Error = t.ExecuteResult.Error + if len(execResult.Error) > 1000 { + execResult.Error = execResult.Error[:1000] + } } j, err := json.Marshal(t.Metadata.Options) diff --git a/pkg/vm/engine/disttae/txn_table.go b/pkg/vm/engine/disttae/txn_table.go index 287f7e260a456..e40ba04fcfa61 100644 --- a/pkg/vm/engine/disttae/txn_table.go +++ b/pkg/vm/engine/disttae/txn_table.go @@ -2172,13 +2172,15 @@ func (tbl *txnTable) MergeObjects( sortKeyPos, sortKeyIsPK := tbl.getSortKeyPosAndSortKeyIsPK() - // check object visibility - for _, objstat := range objStats { + // check object visibility and set object stats. + for i, objstat := range objStats { info, exist := state.GetObject(*objstat.ObjectShortName()) if !exist || (!info.DeleteTime.IsEmpty() && info.DeleteTime.LE(&snapshot)) { logutil.Errorf("object not visible: %s", info.String()) return nil, moerr.NewInternalErrorNoCtxf("object %s not exist", objstat.ObjectName().String()) } + objectio.SetObjectStats(&objstat, &info.ObjectStats) + objStats[i] = objstat } tbl.ensureSeqnumsAndTypesExpectRowid() @@ -2192,7 +2194,7 @@ func (tbl *txnTable) MergeObjects( return nil, err } - err = mergesort.DoMergeAndWrite(ctx, tbl.getTxn().op.Txn().DebugString(), sortKeyPos, taskHost, false) + err = mergesort.DoMergeAndWrite(ctx, tbl.getTxn().op.Txn().DebugString(), sortKeyPos, taskHost) if err != nil { taskHost.commitEntry.Err = err.Error() return taskHost.commitEntry, err diff --git a/pkg/vm/engine/memoryengine/compiler_context.go b/pkg/vm/engine/memoryengine/compiler_context.go index c6195eabea2c1..473c1ee77a190 100644 --- a/pkg/vm/engine/memoryengine/compiler_context.go +++ b/pkg/vm/engine/memoryengine/compiler_context.go @@ -240,6 +240,10 @@ func (c *CompilerContext) ResolveById(tableId uint64, snapshot *plan.Snapshot) ( return c.Resolve(dbName, tableName, snapshot) } +func (c *CompilerContext) ResolveIndexTableByRef(ref *plan.ObjectRef, tblName string, snapshot *plan.Snapshot) (*plan.ObjectRef, *plan.TableDef) { + return c.Resolve(plan.DbNameOfObjRef(ref), tblName, snapshot) +} + func (c *CompilerContext) Resolve(schemaName string, tableName string, snapshot *plan.Snapshot) (objRef *plan.ObjectRef, tableDef *plan.TableDef) { if schemaName == "" { schemaName = c.defaultDB diff --git a/pkg/vm/engine/tae/catalog/table.go b/pkg/vm/engine/tae/catalog/table.go index 9d010e201d0a4..9d3b09aa6e481 100644 --- a/pkg/vm/engine/tae/catalog/table.go +++ b/pkg/vm/engine/tae/catalog/table.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "math" + "slices" "sync/atomic" pkgcatalog "github.com/matrixorigin/matrixone/pkg/catalog" @@ -379,6 +380,7 @@ func (entry *TableEntry) ObjectStats(level common.PPLevel, start, end int, isTom needCount = math.MaxInt } + objEntries := make([]*ObjectEntry, 0) for it.Next() { objectEntry := it.Item() if !objectEntry.IsActive() { @@ -392,6 +394,8 @@ func (entry *TableEntry) ObjectStats(level common.PPLevel, start, end int, isTom break } needCount-- + objEntries = append(objEntries, objectEntry) + stat.ObjectCnt += 1 if objectEntry.GetLoaded() { stat.Loaded += 1 @@ -399,21 +403,36 @@ func (entry *TableEntry) ObjectStats(level common.PPLevel, start, end int, isTom stat.OSize += int(objectEntry.OriginSize()) stat.Csize += int(objectEntry.Size()) } - if level > common.PPL0 { - _ = w.WriteByte('\n') - _, _ = w.WriteString(objectEntry.ID().String()) + } + + slices.SortFunc(objEntries, func(a, b *ObjectEntry) int { + zmA := a.SortKeyZoneMap() + zmB := b.SortKeyZoneMap() + + c := zmA.CompareMin(zmB) + if c != 0 { + return c + } + return zmA.CompareMax(zmB) + }) + + if level > common.PPL0 { + for _, objEntry := range objEntries { _ = w.WriteByte('\n') - _, _ = w.WriteString(" ") - _, _ = w.WriteString(objectEntry.StatsString(zonemapKind)) + _, _ = w.WriteString(objEntry.ID().String()) + _, _ = w.WriteString("\n ") + _, _ = w.WriteString(objEntry.StatsString(zonemapKind)) + + if w.Len() > 8*common.Const1MBytes { + w.WriteString("\n...(truncated for too long, more than 8 MB)") + break + } } - if w.Len() > 8*common.Const1MBytes { - w.WriteString("\n...(truncated for too long, more than 8 MB)") - break + if stat.ObjectCnt > 0 { + w.WriteByte('\n') } } - if level > common.PPL0 && stat.ObjectCnt > 0 { - w.WriteByte('\n') - } + return } @@ -428,8 +447,10 @@ func (entry *TableEntry) ObjectStatsString(level common.PPLevel, start, end int, } summary := fmt.Sprintf( - "summary: %d total, %d unknown, avgRow %d, avgOsize %s, avgCsize %v", - stat.ObjectCnt, stat.ObjectCnt-stat.Loaded, avgRow, common.HumanReadableBytes(avgOsize), common.HumanReadableBytes(avgCsize), + "summary: %d objs, %d unloaded, total orignal size:%s, average orignal size:%s, average rows:%d, average compressed size:%s", + stat.ObjectCnt, stat.ObjectCnt-stat.Loaded, + common.HumanReadableBytes(stat.OSize), common.HumanReadableBytes(avgOsize), + avgRow, common.HumanReadableBytes(avgCsize), ) detail.WriteString(summary) return detail.String() diff --git a/pkg/vm/engine/tae/common/stats.go b/pkg/vm/engine/tae/common/stats.go index 525d14aacad93..453ee917426a3 100644 --- a/pkg/vm/engine/tae/common/stats.go +++ b/pkg/vm/engine/tae/common/stats.go @@ -24,7 +24,7 @@ import ( ) const ( - DefaultMinOsizeQualifiedMB = 110 // MB + DefaultMinOsizeQualifiedMB = 90 // MB DefaultMaxOsizeObjMB = 128 // MB DefaultMinCNMergeSize = 80000 // MB DefaultCNMergeMemControlHint = 8192 // MB diff --git a/pkg/vm/engine/tae/db/db.go b/pkg/vm/engine/tae/db/db.go index 6f8d5032021ba..addbd017ac863 100644 --- a/pkg/vm/engine/tae/db/db.go +++ b/pkg/vm/engine/tae/db/db.go @@ -71,8 +71,6 @@ type DB struct { DBLocker io.Closer - CNMergeSched merge.CNMergeScheduler - Closed *atomic.Value } diff --git a/pkg/vm/engine/tae/db/merge/cnScheduler.go b/pkg/vm/engine/tae/db/merge/cnScheduler.go new file mode 100644 index 0000000000000..703ecf2fa6ad7 --- /dev/null +++ b/pkg/vm/engine/tae/db/merge/cnScheduler.go @@ -0,0 +1,154 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package merge + +import ( + "bytes" + "context" + "fmt" + "strconv" + "sync" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/pb/api" + taskpb "github.com/matrixorigin/matrixone/pkg/pb/task" + "github.com/matrixorigin/matrixone/pkg/taskservice" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" +) + +func NewTaskServiceGetter(getter taskservice.Getter) *CNMergeScheduler { + return &CNMergeScheduler{ + getter: getter, + activeObjects: struct { + sync.Mutex + o map[objectio.ObjectId]activeEntry + }{o: make(map[objectio.ObjectId]activeEntry)}, + } +} + +type CNMergeScheduler struct { + getter taskservice.Getter + + activeObjects struct { + sync.Mutex + o map[objectio.ObjectId]activeEntry + } +} + +func (s *CNMergeScheduler) sendMergeTask(ctx context.Context, task *api.MergeTaskEntry) error { + ts, ok := s.getter() + if !ok { + return taskservice.ErrNotReady + } + taskIDPrefix := "Merge:" + strconv.Itoa(int(task.TblId)) + asyncTask, err := ts.QueryAsyncTask(ctx, + taskservice.WithTaskMetadataId(taskservice.LIKE, taskIDPrefix+"%"), + taskservice.WithTaskStatusCond(taskpb.TaskStatus_Created, taskpb.TaskStatus_Running)) + if err != nil { + return err + } + if len(asyncTask) != 0 { + return moerr.NewInternalError(ctx, fmt.Sprintf("table %q is merging", task.TableName)) + } + b, err := task.Marshal() + if err != nil { + return err + } + return ts.CreateAsyncTask(ctx, + taskpb.TaskMetadata{ + ID: taskIDPrefix + ":" + strconv.FormatInt(time.Now().Unix(), 10), + Executor: taskpb.TaskCode_MergeObject, + Context: b, + Options: taskpb.TaskOptions{Resource: &taskpb.Resource{Memory: task.EstimatedMemUsage}}, + }) +} + +func (s *CNMergeScheduler) addActiveObjects(entries []*catalog.ObjectEntry) { + s.activeObjects.Lock() + for _, entry := range entries { + s.activeObjects.o[*entry.ID()] = activeEntry{ + entry.GetTable().ID, + time.Now(), + } + } + s.activeObjects.Unlock() +} + +func (s *CNMergeScheduler) checkOverlapOnCNActive(entries []*catalog.ObjectEntry) bool { + s.activeObjects.Lock() + defer s.activeObjects.Unlock() + for _, entry := range entries { + if _, ok := s.activeObjects.o[*entry.ID()]; ok { + return true + } + } + return false +} + +func (s *CNMergeScheduler) activeObjsString() string { + s.activeObjects.Lock() + defer s.activeObjects.Unlock() + + b := &bytes.Buffer{} + now := time.Now() + for k, v := range s.activeObjects.o { + b.WriteString(fmt.Sprintf(" id: %v, table: %v, insertAt: %s ago\n", + k.String(), v.tid, now.Sub(v.insertAt).String())) + } + return b.String() +} + +func (s *CNMergeScheduler) removeActiveObject(ids []objectio.ObjectId) { + s.activeObjects.Lock() + defer s.activeObjects.Unlock() + for _, id := range ids { + delete(s.activeObjects.o, id) + } +} + +func (s *CNMergeScheduler) prune(id uint64, ago time.Duration) { + s.activeObjects.Lock() + defer s.activeObjects.Unlock() + now := time.Now() + if ago == 0 { + for k, v := range s.activeObjects.o { + if v.tid == id { + delete(s.activeObjects.o, k) + } + } + return + } + + if id == 0 && ago > 1*time.Second { + for k, v := range s.activeObjects.o { + if now.Sub(v.insertAt) > ago { + delete(s.activeObjects.o, k) + } + } + return + } + for k, v := range s.activeObjects.o { + if v.tid == id && now.Sub(v.insertAt) > ago { + delete(s.activeObjects.o, k) + } + } +} + +type activeEntry struct { + tid uint64 + insertAt time.Time +} diff --git a/pkg/vm/engine/tae/db/merge/cnScheduler_test.go b/pkg/vm/engine/tae/db/merge/cnScheduler_test.go new file mode 100644 index 0000000000000..1a947dc4492ef --- /dev/null +++ b/pkg/vm/engine/tae/db/merge/cnScheduler_test.go @@ -0,0 +1,117 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package merge + +import ( + "context" + "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/pb/api" + "github.com/matrixorigin/matrixone/pkg/taskservice" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/dbutils" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/txn/txnbase" + "github.com/stretchr/testify/require" + "slices" + "testing" + "time" +) + +func TestScheduler_CNActiveObjectsString(t *testing.T) { + memStorage := taskservice.NewMemTaskStorage() + cnScheduler := NewTaskServiceGetter(func() (taskservice.TaskService, bool) { + return taskservice.NewTaskService(runtime.DefaultRuntime(), memStorage), true + }) + + cata := catalog.MockCatalog() + defer cata.Close() + txnMgr := txnbase.NewTxnManager(catalog.MockTxnStoreFactory(cata), catalog.MockTxnFactory(cata), types.NewMockHLCClock(1)) + txnMgr.Start(context.Background()) + defer txnMgr.Stop() + txn1, _ := txnMgr.StartTxn(nil) + db, err := cata.CreateDBEntry("db", "", "", txn1) + require.NoError(t, err) + catalog.MockSchema(1, 0) + tbl, err := db.CreateTableEntry(catalog.MockSchema(1, 0), txn1, nil) + require.NoError(t, err) + require.NoError(t, txn1.Commit(context.Background())) + schema := tbl.GetLastestSchema(false) + + txn2, _ := txnMgr.StartTxn(nil) + entry := newSortedDataEntryWithTableEntry(t, tbl, txn2, 0, 1, overlapSizeThreshold) + stat := *entry.GetObjectStats() + cnScheduler.addActiveObjects([]*catalog.ObjectEntry{entry}) + require.NotEmpty(t, cnScheduler.activeObjsString()) + + cnScheduler.removeActiveObject([]objectio.ObjectId{*entry.ID()}) + require.Empty(t, cnScheduler.activeObjsString()) + + taskEntry := &api.MergeTaskEntry{ + AccountId: schema.AcInfo.TenantID, + UserId: schema.AcInfo.UserID, + RoleId: schema.AcInfo.RoleID, + TblId: tbl.ID, + DbId: tbl.GetDB().GetID(), + TableName: tbl.GetLastestSchema(false).Name, + DbName: tbl.GetDB().GetName(), + ToMergeObjs: [][]byte{stat[:]}, + } + + require.NoError(t, cnScheduler.sendMergeTask(context.Background(), taskEntry)) + tasks, err := memStorage.QueryAsyncTask(context.Background()) + if err != nil { + return + } + require.Equal(t, 1, len(tasks)) + meta := new(api.MergeTaskEntry) + require.NoError(t, meta.Unmarshal(tasks[0].Metadata.Context)) + require.Equal(t, meta.DbName, tbl.GetDB().GetName()) + require.Error(t, cnScheduler.sendMergeTask(context.Background(), taskEntry)) +} + +func TestExecutorCNMerge(t *testing.T) { + + cata := catalog.MockCatalog() + defer cata.Close() + txnMgr := txnbase.NewTxnManager(catalog.MockTxnStoreFactory(cata), catalog.MockTxnFactory(cata), types.NewMockHLCClock(1)) + txnMgr.Start(context.Background()) + defer txnMgr.Stop() + txn1, _ := txnMgr.StartTxn(nil) + db, err := cata.CreateDBEntry("db", "", "", txn1) + require.NoError(t, err) + catalog.MockSchema(1, 0) + tbl, err := db.CreateTableEntry(catalog.MockSchema(1, 0), txn1, nil) + require.NoError(t, err) + require.NoError(t, txn1.Commit(context.Background())) + + txn2, _ := txnMgr.StartTxn(nil) + entry := newSortedDataEntryWithTableEntry(t, tbl, txn2, 0, 1, overlapSizeThreshold) + + memStorage := taskservice.NewMemTaskStorage() + cnScheduler := NewTaskServiceGetter(func() (taskservice.TaskService, bool) { + return taskservice.NewTaskService(runtime.DefaultRuntime(), memStorage), true + }) + executor := newMergeExecutor(&dbutils.Runtime{}, cnScheduler) + executor.executeFor(tbl, []*catalog.ObjectEntry{entry}, taskHostCN) + require.NotEmpty(t, cnScheduler.activeObjsString()) + + executor.executeFor(tbl, []*catalog.ObjectEntry{entry}, taskHostCN) + entry2 := newSortedDataEntryWithTableEntry(t, tbl, txn2, 0, 1, overlapSizeThreshold) + executor.executeFor(tbl, slices.Repeat([]*catalog.ObjectEntry{entry2}, 31), taskHostCN) + + executor.cnSched.prune(0, 0) + executor.cnSched.prune(0, time.Hour) +} diff --git a/pkg/vm/engine/tae/db/merge/config.go b/pkg/vm/engine/tae/db/merge/config.go index 75c0a96b80296..5b54dd19b38bb 100644 --- a/pkg/vm/engine/tae/db/merge/config.go +++ b/pkg/vm/engine/tae/db/merge/config.go @@ -28,8 +28,7 @@ import ( ) var ( - _ policy = (*basic)(nil) - defaultBasicConfig = &BasicPolicyConfig{ + defaultBasicConfig = &BasicPolicyConfig{ MergeMaxOneRun: common.DefaultMaxMergeObjN, MaxOsizeMergedObj: common.DefaultMaxOsizeObjMB * common.Const1MBytes, ObjectMinOsize: common.DefaultMinOsizeQualifiedMB * common.Const1MBytes, diff --git a/pkg/vm/engine/tae/db/merge/config_test.go b/pkg/vm/engine/tae/db/merge/config_test.go index eea9882592b6b..8b1cf5cd4f0fe 100644 --- a/pkg/vm/engine/tae/db/merge/config_test.go +++ b/pkg/vm/engine/tae/db/merge/config_test.go @@ -35,7 +35,7 @@ func newTestSchema(colCnt, pkIdx int, config *BasicPolicyConfig) *catalog.Schema } func TestString(t *testing.T) { - require.Equal(t, "minOsizeObj:110MiB, maxOneRun:16, maxOsizeMergedObj: 128MiB, offloadToCNSize:78.12GiB, hints: []", defaultBasicConfig.String()) + require.Equal(t, "minOsizeObj:90MiB, maxOneRun:16, maxOsizeMergedObj: 128MiB, offloadToCNSize:78.12GiB, hints: []", defaultBasicConfig.String()) } func TestConfigForTable(t *testing.T) { @@ -77,5 +77,5 @@ func TestConfigForTable(t *testing.T) { config = configProvider.getConfig(tbl3) require.Equal(t, defaultBasicConfig.MaxOsizeMergedObj, config.MaxOsizeMergedObj) - require.Equal(t, "customConfigProvider: 0-:115343360,2 | ", configProvider.String()) + require.Equal(t, "customConfigProvider: 0-:94371840,2 | ", configProvider.String()) } diff --git a/pkg/vm/engine/tae/db/merge/executor.go b/pkg/vm/engine/tae/db/merge/executor.go index ab1e44ba60587..34ac1fe1a70d2 100644 --- a/pkg/vm/engine/tae/db/merge/executor.go +++ b/pkg/vm/engine/tae/db/merge/executor.go @@ -15,237 +15,111 @@ package merge import ( - "bytes" "context" "errors" "fmt" - "math" - "os" - "sync" - "sync/atomic" + "slices" + "time" - "github.com/KimMachineGun/automemlimit/memlimit" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/logutil" - "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/api" - v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/dbutils" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/iface/txnif" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/tables/jobs" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/tasks" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/mem" - "github.com/shirou/gopsutil/v3/process" ) -type activeTaskStats map[uint64]struct { - blk int - estBytes int -} - // executor consider resources to decide to merge or not. type executor struct { - tableName string - rt *dbutils.Runtime - cnSched CNMergeScheduler - memLimit int - memUsing int - transPageLimit uint64 - cpuPercent float64 - activeMergeBlkCount atomic.Int32 - activeEstimateBytes atomic.Int64 - roundMergeRows uint64 - taskConsume struct { - sync.Mutex - o map[objectio.ObjectId]struct{} - m activeTaskStats - } + rt *dbutils.Runtime + cnSched *CNMergeScheduler } -func newMergeExecutor(rt *dbutils.Runtime, sched CNMergeScheduler) *executor { +func newMergeExecutor(rt *dbutils.Runtime, sched *CNMergeScheduler) *executor { return &executor{ rt: rt, cnSched: sched, } } -func (e *executor) setMemLimit(total uint64) { - containerMLimit, err := memlimit.FromCgroup() - if containerMLimit > 0 && containerMLimit < total { - e.memLimit = int(float64(containerMLimit) * 0.9) - } else { - e.memLimit = int(float64(total) * 0.9) - } - - if e.memLimit > 200*common.Const1GBytes { - e.transPageLimit = uint64(e.memLimit / 25 * 2) // 8% - } else if e.memLimit > 100*common.Const1GBytes { - e.transPageLimit = uint64(e.memLimit / 25 * 3) // 12% - } else if e.memLimit > 40*common.Const1GBytes { - e.transPageLimit = uint64(e.memLimit / 25 * 4) // 16% - } else { - e.transPageLimit = math.MaxUint64 // no limit +func (e *executor) executeFor(entry *catalog.TableEntry, objs []*catalog.ObjectEntry, kind taskHostKind) { + if len(objs) == 0 { + return } - - logutil.Info( - "MergeExecutorMemoryInfo", - common.AnyField("container-limit", common.HumanReadableBytes(int(containerMLimit))), - common.AnyField("host-memory", common.HumanReadableBytes(int(total))), - common.AnyField("process-limit", common.HumanReadableBytes(e.memLimit)), - common.AnyField("transfer-page-limit", common.HumanReadableBytes(int(e.transPageLimit))), - common.AnyField("error", err), - ) -} - -var proc *process.Process - -func (e *executor) refreshMemInfo() { - if proc == nil { - proc, _ = process.NewProcess(int32(os.Getpid())) - } else if mem, err := proc.MemoryInfo(); err == nil { - e.memUsing = int(mem.RSS) + // check objects are merging by CNs. + if e.cnSched.checkOverlapOnCNActive(objs) { + return } - if e.memLimit == 0 { - if stats, err := mem.VirtualMemory(); err == nil { - e.setMemLimit(stats.Total) + isTombstone := objs[0].IsTombstone + for _, o := range objs { + if o.IsTombstone != isTombstone { + panic("merging tombstone and data objects in one merge") } } - if percents, err := cpu.Percent(0, false); err == nil { - e.cpuPercent = percents[0] - } - e.roundMergeRows = 0 -} - -func (e *executor) printStats() { - cnt := e.activeMergeBlkCount.Load() - if cnt == 0 && e.memAvailBytes() > 512*common.Const1MBytes { + if kind == taskHostDN { + e.scheduleMergeObjects(slices.Clone(objs), entry, isTombstone) return } - logutil.Info( - "MergeExecutorMemoryStats", - common.AnyField("process-limit", common.HumanReadableBytes(e.memLimit)), - common.AnyField("process-mem", common.HumanReadableBytes(e.memUsing)), - common.AnyField("inuse-mem", common.HumanReadableBytes(int(e.activeEstimateBytes.Load()))), - common.AnyField("inuse-cnt", cnt), - ) -} - -func (e *executor) addActiveTask(taskId uint64, blkn, esize int) { - e.activeEstimateBytes.Add(int64(esize)) - e.activeMergeBlkCount.Add(int32(blkn)) - e.taskConsume.Lock() - if e.taskConsume.m == nil { - e.taskConsume.m = make(activeTaskStats) + // prevent CN OOM + if len(objs) > 30 { + objs = objs[:30] } - e.taskConsume.m[taskId] = struct { - blk int - estBytes int - }{blkn, esize} - e.taskConsume.Unlock() -} -func (e *executor) OnExecDone(v any) { - task := v.(tasks.MScopedTask) - - e.taskConsume.Lock() - stat := e.taskConsume.m[task.ID()] - delete(e.taskConsume.m, task.ID()) - e.taskConsume.Unlock() - - e.activeMergeBlkCount.Add(-int32(stat.blk)) - e.activeEstimateBytes.Add(-int64(stat.estBytes)) -} - -func (e *executor) executeFor(entry *catalog.TableEntry, mobjs []*catalog.ObjectEntry, kind TaskHostKind) { - if e.roundMergeRows*36 /*28 * 1.3 */ > e.transPageLimit/8 { + stats := make([][]byte, 0, len(objs)) + cids := make([]common.ID, 0, len(objs)) + for _, obj := range objs { + stat := *obj.GetObjectStats() + stats = append(stats, stat[:]) + cids = append(cids, *obj.AsCommonID()) + } + // check objects are merging by TN. + if e.rt.Scheduler != nil && e.rt.Scheduler.CheckAsyncScopes(cids) != nil { return } - e.tableName = fmt.Sprintf("%v-%v", entry.ID, entry.GetLastestSchema(false).Name) - - if ActiveCNObj.CheckOverlapOnCNActive(mobjs) { + schema := entry.GetLastestSchema(false) + cntask := &api.MergeTaskEntry{ + AccountId: schema.AcInfo.TenantID, + UserId: schema.AcInfo.UserID, + RoleId: schema.AcInfo.RoleID, + TblId: entry.ID, + DbId: entry.GetDB().GetID(), + TableName: entry.GetLastestSchema(isTombstone).Name, + DbName: entry.GetDB().GetName(), + ToMergeObjs: stats, + EstimatedMemUsage: uint64(estimateMergeSize(objs)), + } + ctx, cancel := context.WithTimeoutCause(context.Background(), 10*time.Second, moerr.CauseCreateCNMerge) + defer cancel() + err := e.cnSched.sendMergeTask(ctx, cntask) + if err != nil { + logutil.Info("MergeExecutorError", + common.OperationField("send-cn-task"), + common.AnyField("task", fmt.Sprintf("table-%d-%s", cntask.TblId, cntask.TableName)), + common.AnyField("error", err), + ) return } - if kind == TaskHostCN { - osize, esize := estimateMergeConsume(mobjs) - blkCnt := 0 - for _, obj := range mobjs { - blkCnt += obj.BlockCnt() - } - stats := make([][]byte, 0, len(mobjs)) - cids := make([]common.ID, 0, len(mobjs)) - for _, obj := range mobjs { - stat := *obj.GetObjectStats() - stats = append(stats, stat[:]) - cids = append(cids, *obj.AsCommonID()) - } - if e.rt.Scheduler.CheckAsyncScopes(cids) != nil { - return - } - schema := entry.GetLastestSchema(false) - cntask := &api.MergeTaskEntry{ - AccountId: schema.AcInfo.TenantID, - UserId: schema.AcInfo.UserID, - RoleId: schema.AcInfo.RoleID, - TblId: entry.ID, - DbId: entry.GetDB().GetID(), - TableName: entry.GetLastestSchema(false).Name, - DbName: entry.GetDB().GetName(), - ToMergeObjs: stats, - EstimatedMemUsage: uint64(esize), - } - if err := e.cnSched.SendMergeTask(context.TODO(), cntask); err == nil { - ActiveCNObj.AddActiveCNObj(mobjs) - logMergeTask(e.tableName, math.MaxUint64, mobjs, blkCnt, osize, esize) - } else { - logutil.Info( - "MergeExecutorError", - common.OperationField("send-cn-task"), - common.AnyField("task", fmt.Sprintf("table-%d-%s", cntask.TblId, cntask.TableName)), - common.AnyField("error", err), - ) - return - } - entry.Stats.SetLastMergeTime() - } else { - objScopes := make([]common.ID, 0) - tombstoneScopes := make([]common.ID, 0) - objs := make([]*catalog.ObjectEntry, 0) - tombstones := make([]*catalog.ObjectEntry, 0) - objectBlkCnt := 0 - tombstoneBlkCnt := 0 - for _, obj := range mobjs { - if obj.IsTombstone { - tombstoneBlkCnt += obj.BlockCnt() - tombstones = append(tombstones, obj) - tombstoneScopes = append(tombstoneScopes, *obj.AsCommonID()) - } else { - objectBlkCnt += obj.BlockCnt() - objs = append(objs, obj) - objScopes = append(objScopes, *obj.AsCommonID()) - } - } + e.cnSched.addActiveObjects(objs) + entry.Stats.SetLastMergeTime() +} - if len(objs) > 0 { - e.scheduleMergeObjects(objScopes, objs, objectBlkCnt, entry, false) - } - if len(tombstones) > 1 { - e.scheduleMergeObjects(tombstoneScopes, tombstones, tombstoneBlkCnt, entry, true) - } +func (e *executor) scheduleMergeObjects(mObjs []*catalog.ObjectEntry, entry *catalog.TableEntry, isTombstone bool) { + scopes := make([]common.ID, 0, len(mObjs)) + for _, obj := range mObjs { + scopes = append(scopes, *obj.AsCommonID()) } -} -func (e *executor) scheduleMergeObjects(scopes []common.ID, mobjs []*catalog.ObjectEntry, blkCnt int, entry *catalog.TableEntry, isTombstone bool) { - osize, esize := estimateMergeConsume(mobjs) factory := func(ctx *tasks.Context, txn txnif.AsyncTxn) (tasks.Task, error) { txn.GetMemo().IsFlushOrMerge = true - return jobs.NewMergeObjectsTask(ctx, txn, mobjs, e.rt, common.DefaultMaxOsizeObjMB*common.Const1MBytes, isTombstone) + return jobs.NewMergeObjectsTask(ctx, txn, mObjs, e.rt, common.DefaultMaxOsizeObjMB*common.Const1MBytes, isTombstone) } - task, err := e.rt.Scheduler.ScheduleMultiScopedTxnTaskWithObserver(nil, tasks.DataCompactionTask, scopes, factory, e) + task, err := e.rt.Scheduler.ScheduleMultiScopedTxnTask(nil, tasks.DataCompactionTask, scopes, factory) if err != nil { if !errors.Is(err, tasks.ErrScheduleScopeConflict) { logutil.Info( @@ -257,58 +131,5 @@ func (e *executor) scheduleMergeObjects(scopes []common.ID, mobjs []*catalog.Obj } return } - e.addActiveTask(task.ID(), blkCnt, esize) - for _, obj := range mobjs { - e.roundMergeRows += uint64(obj.Rows()) - } - logMergeTask(e.tableName, task.ID(), mobjs, blkCnt, osize, esize) entry.Stats.SetLastMergeTime() } - -func (e *executor) memAvailBytes() int { - merging := int(e.activeEstimateBytes.Load()) - avail := e.memLimit - e.memUsing - merging - if avail < 0 { - avail = 0 - } - return avail -} - -func (e *executor) transferPageSizeLimit() uint64 { - return e.transPageLimit -} - -func (e *executor) CPUPercent() int64 { - return int64(e.cpuPercent) -} - -func logMergeTask(name string, taskId uint64, merges []*catalog.ObjectEntry, blkn, osize, esize int) { - rows := 0 - infoBuf := &bytes.Buffer{} - for _, obj := range merges { - r := int(obj.Rows()) - rows += r - infoBuf.WriteString(fmt.Sprintf(" %d(%s)", r, obj.ID().ShortStringEx())) - } - platform := fmt.Sprintf("t%d", taskId) - if taskId == math.MaxUint64 { - platform = "CN" - v2.TaskCNMergeScheduledByCounter.Inc() - v2.TaskCNMergedSizeCounter.Add(float64(osize)) - } else { - v2.TaskDNMergeScheduledByCounter.Inc() - v2.TaskDNMergedSizeCounter.Add(float64(osize)) - } - logutil.Info( - "MergeExecutor", - common.OperationField("schedule-merge-task"), - common.AnyField("name", name), - common.AnyField("platform", platform), - common.AnyField("num-obj", len(merges)), - common.AnyField("num-blk", blkn), - common.AnyField("orig-size", common.HumanReadableBytes(osize)), - common.AnyField("est-size", common.HumanReadableBytes(esize)), - common.AnyField("rows", rows), - common.AnyField("info", infoBuf.String()), - ) -} diff --git a/pkg/vm/engine/tae/db/merge/meminfo_darwin.go b/pkg/vm/engine/tae/db/merge/meminfo_darwin.go new file mode 100644 index 0000000000000..6d7dee835ef34 --- /dev/null +++ b/pkg/vm/engine/tae/db/merge/meminfo_darwin.go @@ -0,0 +1,33 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package merge + +import ( + "syscall" + "unsafe" +) + +func totalMem() uint64 { + s, err := syscall.Sysctl("hw.memsize") + if err != nil { + return 0 + } + // hack because the string conversion above drops a \0 + b := []byte(s) + if len(b) < 8 { + b = append(b, 0) + } + return *(*uint64)(unsafe.Pointer(&b[0])) +} diff --git a/pkg/vm/engine/tae/db/merge/meminfo_linux.go b/pkg/vm/engine/tae/db/merge/meminfo_linux.go new file mode 100644 index 0000000000000..57a10daf214ac --- /dev/null +++ b/pkg/vm/engine/tae/db/merge/meminfo_linux.go @@ -0,0 +1,26 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package merge + +import "syscall" + +func totalMem() uint64 { + in := new(syscall.Sysinfo_t) + err := syscall.Sysinfo(in) + if err != nil { + return 0 + } + return in.Totalram * uint64(in.Unit) +} diff --git a/pkg/vm/engine/tae/db/merge/mod.go b/pkg/vm/engine/tae/db/merge/mod.go deleted file mode 100644 index 6c72813f1a251..0000000000000 --- a/pkg/vm/engine/tae/db/merge/mod.go +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2023 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package merge - -import ( - "bytes" - "context" - "fmt" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/matrixorigin/matrixone/pkg/common/moerr" - "github.com/matrixorigin/matrixone/pkg/fileservice" - "github.com/matrixorigin/matrixone/pkg/objectio" - "github.com/matrixorigin/matrixone/pkg/pb/api" - taskpb "github.com/matrixorigin/matrixone/pkg/pb/task" - "github.com/matrixorigin/matrixone/pkg/taskservice" - "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" - "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" -) - -var StopMerge atomic.Bool - -type CNMergeScheduler interface { - SendMergeTask(ctx context.Context, task *api.MergeTaskEntry) error -} - -func NewTaskServiceGetter(getter taskservice.Getter) CNMergeScheduler { - return &taskServiceGetter{ - Getter: getter, - } -} - -type taskServiceGetter struct { - taskservice.Getter -} - -func (tsg *taskServiceGetter) SendMergeTask(ctx context.Context, task *api.MergeTaskEntry) error { - ts, ok := tsg.Getter() - if !ok { - return taskservice.ErrNotReady - } - taskIDPrefix := "Merge:" + task.TableName - asyncTask, err := ts.QueryAsyncTask(ctx, - taskservice.WithTaskMetadataId(taskservice.LIKE, taskIDPrefix+"%"), - taskservice.WithTaskStatusCond(taskpb.TaskStatus_Created, taskpb.TaskStatus_Running)) - if err != nil { - return err - } - if len(asyncTask) != 0 { - return moerr.NewInternalError(ctx, fmt.Sprintf("table %q is merging", task.TableName)) - } - b, err := task.Marshal() - if err != nil { - return err - } - return ts.CreateAsyncTask(ctx, - taskpb.TaskMetadata{ - ID: taskIDPrefix + ":" + strconv.FormatInt(time.Now().Unix(), 10), - Executor: taskpb.TaskCode_MergeObject, - Context: b, - Options: taskpb.TaskOptions{Resource: &taskpb.Resource{Memory: task.EstimatedMemUsage}}, - }) -} - -type TaskHostKind int - -const ( - TaskHostCN TaskHostKind = iota - TaskHostDN -) - -type activeEntry struct { - tid uint64 - insertAt time.Time -} - -var ActiveCNObj = ActiveCNObjMap{ - o: make(map[objectio.ObjectId]activeEntry), -} - -type ActiveCNObjMap struct { - sync.Mutex - o map[objectio.ObjectId]activeEntry -} - -func (e *ActiveCNObjMap) Prune(id uint64, ago time.Duration) { - e.Lock() - defer e.Unlock() - now := time.Now() - if ago == 0 { - for k, v := range e.o { - if v.tid == id { - delete(e.o, k) - } - } - return - } - - if id == 0 && ago > 1*time.Second { - for k, v := range e.o { - if now.Sub(v.insertAt) > ago { - delete(e.o, k) - } - } - return - } - for k, v := range e.o { - if v.tid == id && now.Sub(v.insertAt) > ago { - delete(e.o, k) - } - } -} - -func (e *ActiveCNObjMap) String() string { - e.Lock() - defer e.Unlock() - - b := &bytes.Buffer{} - now := time.Now() - for k, v := range e.o { - b.WriteString(fmt.Sprintf(" id: %v, table: %v, insertAt: %s ago\n", - k.String(), v.tid, now.Sub(v.insertAt).String())) - } - return b.String() -} - -func (e *ActiveCNObjMap) AddActiveCNObj(entries []*catalog.ObjectEntry) { - e.Lock() - for _, entry := range entries { - e.o[*entry.ID()] = activeEntry{ - entry.GetTable().ID, - time.Now(), - } - } - e.Unlock() -} - -func (e *ActiveCNObjMap) RemoveActiveCNObj(ids []objectio.ObjectId) { - e.Lock() - defer e.Unlock() - for _, id := range ids { - delete(e.o, id) - } -} - -func (e *ActiveCNObjMap) CheckOverlapOnCNActive(entries []*catalog.ObjectEntry) bool { - e.Lock() - defer e.Unlock() - for _, entry := range entries { - if _, ok := e.o[*entry.ID()]; ok { - return true - } - } - return false -} - -func CleanUpUselessFiles(entry *api.MergeCommitEntry, fs fileservice.FileService) { - if entry == nil { - return - } - ctx, cancel := context.WithTimeoutCause(context.Background(), 2*time.Minute, moerr.CauseCleanUpUselessFiles) - defer cancel() - for _, filepath := range entry.BookingLoc { - _ = fs.Delete(ctx, filepath) - } - if len(entry.CreatedObjs) != 0 { - for _, obj := range entry.CreatedObjs { - if len(obj) == 0 { - continue - } - s := objectio.ObjectStats(obj) - _ = fs.Delete(ctx, s.ObjectName().String()) - } - } -} - -const ( - constMaxMemCap = 12 * common.Const1GBytes // max original memory for an object - constSmallMergeGap = 3 * time.Minute -) - -type policy interface { - onObject(*catalog.ObjectEntry, *BasicPolicyConfig) bool - revise(cpu, mem int64, config *BasicPolicyConfig) []reviseResult - resetForTable(*catalog.TableEntry) -} - -func NewUpdatePolicyReq(c *BasicPolicyConfig) *api.AlterTableReq { - return &api.AlterTableReq{ - Kind: api.AlterKind_UpdatePolicy, - Operation: &api.AlterTableReq_UpdatePolicy{ - UpdatePolicy: &api.AlterTablePolicy{ - MinOsizeQuailifed: c.ObjectMinOsize, - MaxObjOnerun: uint32(c.MergeMaxOneRun), - MaxOsizeMergedObj: c.MaxOsizeMergedObj, - MinCnMergeSize: c.MinCNMergeSize, - Hints: c.MergeHints, - }, - }, - } -} diff --git a/pkg/vm/engine/tae/db/merge/mod_test.go b/pkg/vm/engine/tae/db/merge/mod_test.go deleted file mode 100644 index db0d647588de5..0000000000000 --- a/pkg/vm/engine/tae/db/merge/mod_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package merge - -import ( - "context" - "os" - "path" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/matrixorigin/matrixone/pkg/defines" - "github.com/matrixorigin/matrixone/pkg/fileservice" - "github.com/matrixorigin/matrixone/pkg/pb/api" -) - -func Test_CleanUpUselessFiles(t *testing.T) { - tDir := os.TempDir() - dir := path.Join(tDir, "/local") - assert.NoError(t, os.RemoveAll(dir)) - defer func() { - _ = os.RemoveAll(dir) - }() - - c := fileservice.Config{ - Name: defines.ETLFileServiceName, - Backend: "DISK", - DataDir: dir, - Cache: fileservice.DisabledCacheConfig, - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - fs, err := fileservice.NewFileService(ctx, c, nil) - assert.Nil(t, err) - defer fs.Close(ctx) - - ent := &api.MergeCommitEntry{ - BookingLoc: []string{"abc"}, - } - - CleanUpUselessFiles(ent, fs) -} diff --git a/pkg/vm/engine/tae/db/merge/policyCompact.go b/pkg/vm/engine/tae/db/merge/policyCompact.go index f7719bf252d49..2e9db4e65cc6b 100644 --- a/pkg/vm/engine/tae/db/merge/policyCompact.go +++ b/pkg/vm/engine/tae/db/merge/policyCompact.go @@ -16,39 +16,28 @@ package merge import ( "context" + "sort" "time" - "go.uber.org/zap" - - "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/fileservice" - "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" - "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/blockio" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" ) +var _ policy = (*objCompactPolicy)(nil) + type objCompactPolicy struct { tblEntry *catalog.TableEntry - objects []*catalog.ObjectEntry fs fileservice.FileService - tombstones []*catalog.ObjectEntry - tombstoneStats []objectio.ObjectStats + objects []*catalog.ObjectEntry - validTombstones map[*catalog.ObjectEntry]struct{} + tombstoneMetas []objectio.ObjectDataMeta } func newObjCompactPolicy(fs fileservice.FileService) *objCompactPolicy { - return &objCompactPolicy{ - objects: make([]*catalog.ObjectEntry, 0), - fs: fs, - - tombstones: make([]*catalog.ObjectEntry, 0), - tombstoneStats: make([]objectio.ObjectStats, 0), - validTombstones: make(map[*catalog.ObjectEntry]struct{}), - } + return &objCompactPolicy{fs: fs} } func (o *objCompactPolicy) onObject(entry *catalog.ObjectEntry, config *BasicPolicyConfig) bool { @@ -61,60 +50,34 @@ func (o *objCompactPolicy) onObject(entry *catalog.ObjectEntry, config *BasicPol if entry.OriginSize() < config.ObjectMinOsize { return false } - if len(o.tombstoneStats) == 0 { - return false - } - ctx, cancel := context.WithTimeoutCause(context.Background(), time.Second*10, moerr.CauseOnObject) - sels, err := blockio.FindTombstonesOfObject(ctx, entry.ID(), o.tombstoneStats, o.fs) - cancel() - if err != nil { - return false - } - iter := sels.Iterator() - for iter.HasNext() { - i := iter.Next() - tombstone := o.tombstones[i] - - o.validTombstones[tombstone] = struct{}{} - - if (entryOutdated(tombstone, config.TombstoneLifetime) && tombstone.OriginSize() > 10*common.Const1MBytes) || - tombstone.OriginSize() > common.DefaultMinOsizeQualifiedMB*common.Const1MBytes { - logutil.Info("[MERGE-POLICY]", - zap.String("policy", "compact"), - zap.String("table", o.tblEntry.GetFullName()), - zap.String("data object", entry.ID().ShortStringEx()), - zap.Uint32("data rows", entry.Rows()), - zap.Uint32("tombstone size", tombstone.OriginSize()), - ) - o.objects = append(o.objects, entry) - return true + for _, meta := range o.tombstoneMetas { + if !checkTombstoneMeta(meta, entry.ID()) { + continue } + o.objects = append(o.objects, entry) } return false } -func (o *objCompactPolicy) revise(cpu, mem int64, config *BasicPolicyConfig) []reviseResult { +func (o *objCompactPolicy) revise(rc *resourceController) []reviseResult { if o.tblEntry == nil { return nil } - o.filterValidTombstones() - results := make([]reviseResult, 0, len(o.objects)+len(o.tombstones)) + results := make([]reviseResult, 0, len(o.objects)) for _, obj := range o.objects { - results = append(results, reviseResult{[]*catalog.ObjectEntry{obj}, TaskHostDN}) - } - if len(o.tombstoneStats) > 0 { - results = append(results, reviseResult{o.tombstones, TaskHostDN}) + if rc.resourceAvailable([]*catalog.ObjectEntry{obj}) { + rc.reserveResources([]*catalog.ObjectEntry{obj}) + results = append(results, reviseResult{[]*catalog.ObjectEntry{obj}, taskHostDN}) + } } return results } -func (o *objCompactPolicy) resetForTable(entry *catalog.TableEntry) { +func (o *objCompactPolicy) resetForTable(entry *catalog.TableEntry, config *BasicPolicyConfig) { o.tblEntry = entry + o.tombstoneMetas = o.tombstoneMetas[:0] o.objects = o.objects[:0] - o.tombstones = o.tombstones[:0] - o.tombstoneStats = o.tombstoneStats[:0] - clear(o.validTombstones) tIter := entry.MakeTombstoneObjectIt() for tIter.Next() { @@ -124,21 +87,50 @@ func (o *objCompactPolicy) resetForTable(entry *catalog.TableEntry) { continue } - o.tombstones = append(o.tombstones, tEntry) - o.tombstoneStats = append(o.tombstoneStats, *tEntry.GetObjectStats()) + if tEntry.OriginSize() > common.DefaultMaxOsizeObjMB*common.Const1MBytes { + meta, err := loadTombstoneMeta(tEntry.GetObjectStats(), o.fs) + if err != nil { + continue + } + o.tombstoneMetas = append(o.tombstoneMetas, meta) + } } } -func (o *objCompactPolicy) filterValidTombstones() { - i := 0 - for _, x := range o.tombstones { - if _, ok := o.validTombstones[x]; !ok { - o.tombstones[i] = x - i++ - } +func loadTombstoneMeta(tombstoneObject *objectio.ObjectStats, fs fileservice.FileService) (objectio.ObjectDataMeta, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + location := tombstoneObject.ObjectLocation() + objMeta, err := objectio.FastLoadObjectMeta( + ctx, &location, false, fs, + ) + if err != nil { + return nil, err } - for j := i; j < len(o.tombstones); j++ { - o.tombstones[j] = nil + return objMeta.MustDataMeta(), nil +} + +func checkTombstoneMeta(tombstoneMeta objectio.ObjectDataMeta, objectId *objectio.ObjectId) bool { + prefixPattern := objectId[:] + blkCnt := int(tombstoneMeta.BlockCount()) + + startIdx := sort.Search(blkCnt, func(i int) bool { + return tombstoneMeta.GetBlockMeta(uint32(i)).MustGetColumn(0).ZoneMap().AnyGEByValue(prefixPattern) + }) + + for pos := startIdx; pos < blkCnt; pos++ { + blkMeta := tombstoneMeta.GetBlockMeta(uint32(pos)) + columnZonemap := blkMeta.MustGetColumn(0).ZoneMap() + // block id is the prefixPattern of the rowid and zonemap is min-max of rowid + // !PrefixEq means there is no rowid of this block in this zonemap, so skip + if columnZonemap.RowidPrefixEq(prefixPattern) { + return true + } + if columnZonemap.RowidPrefixGT(prefixPattern) { + // all zone maps are sorted by the rowid + // if the block id is less than the prefixPattern of the min rowid, skip the rest blocks + break + } } - o.tombstones = o.tombstones[:i] + return false } diff --git a/pkg/vm/engine/tae/db/merge/policyBasic.go b/pkg/vm/engine/tae/db/merge/policyGroup.go similarity index 55% rename from pkg/vm/engine/tae/db/merge/policyBasic.go rename to pkg/vm/engine/tae/db/merge/policyGroup.go index c3ac00c397ae0..d90e7acc56004 100644 --- a/pkg/vm/engine/tae/db/merge/policyBasic.go +++ b/pkg/vm/engine/tae/db/merge/policyGroup.go @@ -15,10 +15,7 @@ package merge import ( - "cmp" "context" - "slices" - "time" pkgcatalog "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/container/types" @@ -32,7 +29,7 @@ import ( type reviseResult struct { objs []*catalog.ObjectEntry - kind TaskHostKind + kind taskHostKind } type policyGroup struct { @@ -57,10 +54,10 @@ func (g *policyGroup) onObject(obj *catalog.ObjectEntry) { } } -func (g *policyGroup) revise(cpu, mem int64) []reviseResult { +func (g *policyGroup) revise(rc *resourceController) []reviseResult { results := make([]reviseResult, 0, len(g.policies)) for _, p := range g.policies { - pResult := p.revise(cpu, mem, g.config) + pResult := p.revise(rc) for _, r := range pResult { if len(r.objs) > 0 { results = append(results, r) @@ -71,10 +68,10 @@ func (g *policyGroup) revise(cpu, mem int64) []reviseResult { } func (g *policyGroup) resetForTable(entry *catalog.TableEntry) { + g.config = g.configProvider.getConfig(entry) for _, p := range g.policies { - p.resetForTable(entry) + p.resetForTable(entry, g.config) } - g.config = g.configProvider.getConfig(entry) } func (g *policyGroup) setConfig(tbl *catalog.TableEntry, txn txnif.AsyncTxn, cfg *BasicPolicyConfig) (err error) { @@ -166,7 +163,7 @@ func (g *policyGroup) setConfig(tbl *catalog.TableEntry, txn txnif.AsyncTxn, cfg } return tblHandle.AlterTable( ctx, - NewUpdatePolicyReq(cfg), + newUpdatePolicyReq(cfg), ) } @@ -182,159 +179,3 @@ func (g *policyGroup) getConfig(tbl *catalog.TableEntry) *BasicPolicyConfig { } return r } - -type basic struct { - schema *catalog.Schema - objects []*catalog.ObjectEntry - - lastMergeTime time.Time - - objectsSize int - accBuf []int -} - -func newBasicPolicy() policy { - return &basic{ - objects: make([]*catalog.ObjectEntry, 0, 16), - accBuf: make([]int, 1, 32), - } -} - -// impl policy for Basic -func (o *basic) onObject(obj *catalog.ObjectEntry, config *BasicPolicyConfig) bool { - if obj.IsTombstone { - return false - } - - osize := int(obj.OriginSize()) - - isCandidate := func() bool { - if len(o.objects) >= config.MergeMaxOneRun { - return false - } - if osize < int(config.ObjectMinOsize) { - if o.objectsSize > 2*common.DefaultMaxOsizeObjMB*common.Const1MBytes { - return false - } - o.objectsSize += osize - return true - } - // skip big object as an insurance - if osize > 110*common.Const1MBytes { - return false - } - - return false - } - - if isCandidate() { - o.objects = append(o.objects, obj) - return true - } - return false -} - -func (o *basic) revise(cpu, mem int64, config *BasicPolicyConfig) []reviseResult { - slices.SortFunc(o.objects, func(a, b *catalog.ObjectEntry) int { - return cmp.Compare(a.Rows(), b.Rows()) - }) - objs := o.objects - - isStandalone := common.IsStandaloneBoost.Load() - mergeOnDNIfStandalone := !common.ShouldStandaloneCNTakeOver.Load() - - dnobjs := controlMem(objs, mem) - dnobjs = o.optimize(dnobjs, config) - - dnosize, _ := estimateMergeConsume(dnobjs) - - schedDN := func() []reviseResult { - if cpu > 85 { - if dnosize > 25*common.Const1MBytes { - logutil.Infof("mergeblocks skip big merge for high level cpu usage, %d", cpu) - return nil - } - } - if len(dnobjs) > 1 { - return []reviseResult{{dnobjs, TaskHostDN}} - } - return nil - } - - schedCN := func() []reviseResult { - cnobjs := controlMem(objs, int64(common.RuntimeCNMergeMemControl.Load())) - cnobjs = o.optimize(cnobjs, config) - return []reviseResult{{cnobjs, TaskHostCN}} - } - - if isStandalone && mergeOnDNIfStandalone { - return schedDN() - } - - // CNs come into the picture in two cases: - // 1.cluster deployed - // 2.standalone deployed but it's asked to merge on cn - if common.RuntimeCNTakeOverAll.Load() || dnosize > int(common.RuntimeMinCNMergeSize.Load()) { - return schedCN() - } - - // CNs don't take over the task, leave it on dn. - return schedDN() -} - -func (o *basic) optimize(objs []*catalog.ObjectEntry, config *BasicPolicyConfig) []*catalog.ObjectEntry { - // objs are sorted by remaining rows - o.accBuf = o.accBuf[:1] - for i, obj := range objs { - o.accBuf = append(o.accBuf, o.accBuf[i]+int(obj.Rows())) - } - acc := o.accBuf - - isBigGap := func(small, big int) bool { - if big < int(o.schema.Extra.BlockMaxRows) { - return false - } - return big-small > 3*small - } - - var i int - // skip merging objects with big row count gaps, 3x and more - for i = len(acc) - 1; i > 1 && isBigGap(acc[i-1], acc[i]); i-- { - } - - readyToMergeRows := acc[i] - - // avoid frequent small object merge - if readyToMergeRows < int(o.schema.Extra.BlockMaxRows) && - !o.lastMergeTime.Before(time.Now().Add(-constSmallMergeGap)) && - i < config.MergeMaxOneRun { - return nil - } - - objs = objs[:i] - - return objs -} - -func controlMem(objs []*catalog.ObjectEntry, mem int64) []*catalog.ObjectEntry { - if mem > constMaxMemCap { - mem = constMaxMemCap - } - - needPopout := func(ss []*catalog.ObjectEntry) bool { - _, esize := estimateMergeConsume(ss) - return esize > int(2*mem/3) - } - for needPopout(objs) { - objs = objs[:len(objs)-1] - } - - return objs -} - -func (o *basic) resetForTable(entry *catalog.TableEntry) { - o.schema = entry.GetLastestSchemaLocked(false) - o.lastMergeTime = entry.Stats.GetLastMergeTime() - o.objects = o.objects[:0] - o.objectsSize = 0 -} diff --git a/pkg/vm/engine/tae/db/merge/policyOverlap.go b/pkg/vm/engine/tae/db/merge/policyOverlap.go index a12c5a895ba90..585c5a3adefeb 100644 --- a/pkg/vm/engine/tae/db/merge/policyOverlap.go +++ b/pkg/vm/engine/tae/db/merge/policyOverlap.go @@ -15,28 +15,28 @@ package merge import ( - "cmp" "slices" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" - "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/compute" - "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" ) var _ policy = (*objOverlapPolicy)(nil) +var levels = [6]int{ + 1, 2, 4, 16, 64, 256, +} + type objOverlapPolicy struct { - objects []*catalog.ObjectEntry - objectsSize int + leveledObjects [len(levels)][]*catalog.ObjectEntry - overlappingObjsSet [][]*catalog.ObjectEntry + segments map[objectio.Segmentid]map[*catalog.ObjectEntry]struct{} } func newObjOverlapPolicy() *objOverlapPolicy { return &objOverlapPolicy{ - objects: make([]*catalog.ObjectEntry, 0), - overlappingObjsSet: make([][]*catalog.ObjectEntry, 0), + segments: make(map[objectio.Segmentid]map[*catalog.ObjectEntry]struct{}), } } @@ -44,117 +44,126 @@ func (m *objOverlapPolicy) onObject(obj *catalog.ObjectEntry, config *BasicPolic if obj.IsTombstone { return false } - if obj.OriginSize() < config.ObjectMinOsize { - return false - } if !obj.SortKeyZoneMap().IsInited() { return false } - if m.objectsSize > 10*common.DefaultMaxOsizeObjMB*common.Const1MBytes { - return false + if m.segments[obj.ObjectName().SegmentId()] == nil { + m.segments[obj.ObjectName().SegmentId()] = make(map[*catalog.ObjectEntry]struct{}) } - m.objects = append(m.objects, obj) - m.objectsSize += int(obj.OriginSize()) + m.segments[obj.ObjectName().SegmentId()][obj] = struct{}{} return true } -func (m *objOverlapPolicy) revise(cpu, mem int64, config *BasicPolicyConfig) []reviseResult { - if len(m.objects) < 2 { - return nil - } - if cpu > 80 { - return nil - } - objs, taskHostKind := m.reviseDataObjs(config) - objs = controlMem(objs, mem) - if len(objs) > 1 { - return []reviseResult{{objs, taskHostKind}} - } - return nil -} - -func (m *objOverlapPolicy) reviseDataObjs(config *BasicPolicyConfig) ([]*catalog.ObjectEntry, TaskHostKind) { - slices.SortFunc(m.objects, func(a, b *catalog.ObjectEntry) int { - zmA := a.SortKeyZoneMap() - zmB := b.SortKeyZoneMap() - if c := zmA.CompareMin(zmB); c != 0 { - return c - } - return zmA.CompareMax(zmB) - }) - set := entrySet{entries: make([]*catalog.ObjectEntry, 0), maxValue: []byte{}} - for _, obj := range m.objects { - if len(set.entries) == 0 { - set.add(obj) - continue +func (m *objOverlapPolicy) revise(rc *resourceController) []reviseResult { + for _, objects := range m.segments { + l := segLevel(len(objects)) + for obj := range objects { + m.leveledObjects[l] = append(m.leveledObjects[l], obj) } + } - if zm := obj.SortKeyZoneMap(); index.StrictlyCompareZmMaxAndMin(set.maxValue, zm.GetMinBuf(), zm.GetType(), zm.GetScale(), zm.GetScale()) > 0 { - // zm is overlapped - set.add(obj) + reviseResults := make([]reviseResult, 0, len(levels)) + for i := range 4 { + if len(m.leveledObjects[i]) < 2 { continue } - // obj is not added in the set. - // either dismiss the set or add the set in m.overlappingObjsSet - if len(set.entries) > 1 { - objs := make([]*catalog.ObjectEntry, len(set.entries)) - copy(objs, set.entries) - m.overlappingObjsSet = append(m.overlappingObjsSet, objs) + for _, objs := range objectsWithGivenOverlaps(m.leveledObjects[i], 5) { + objs = removeOversize(objs) + if len(objs) < 2 || score(objs) < 1.1 { + continue + } + result := reviseResult{objs: objs, kind: taskHostDN} + if result.kind == taskHostDN { + if rc.cpuPercent > 80 { + continue + } + + if rc.resourceAvailable(result.objs) { + rc.reserveResources(result.objs) + } else { + result.kind = taskHostCN + } + } + reviseResults = append(reviseResults, result) } - - set.reset() - set.add(obj) } - // there is still more than one entry in set. - if len(set.entries) > 1 { - objs := make([]*catalog.ObjectEntry, len(set.entries)) - copy(objs, set.entries) - m.overlappingObjsSet = append(m.overlappingObjsSet, objs) - set.reset() - } - if len(m.overlappingObjsSet) == 0 { - return nil, TaskHostDN - } - - slices.SortFunc(m.overlappingObjsSet, func(a, b []*catalog.ObjectEntry) int { - return cmp.Compare(len(a), len(b)) - }) + return reviseResults +} - // get the overlapping set with most objs. - objs := m.overlappingObjsSet[len(m.overlappingObjsSet)-1] - if len(objs) < 2 { - return nil, TaskHostDN +func (m *objOverlapPolicy) resetForTable(*catalog.TableEntry, *BasicPolicyConfig) { + for i := range m.leveledObjects { + m.leveledObjects[i] = m.leveledObjects[i][:0] } - if len(objs) > config.MergeMaxOneRun { - objs = objs[:config.MergeMaxOneRun] - } - return objs, TaskHostDN + clear(m.segments) } -func (m *objOverlapPolicy) resetForTable(*catalog.TableEntry) { - m.objects = m.objects[:0] - m.overlappingObjsSet = m.overlappingObjsSet[:0] - m.objectsSize = 0 +func segLevel(length int) int { + l := len(levels) - 1 + for i, level := range levels { + if length < level { + l = i - 1 + break + } + } + return l } -type entrySet struct { - entries []*catalog.ObjectEntry - maxValue []byte - size int -} +type endPoint struct { + val []byte + s int -func (s *entrySet) reset() { - s.entries = s.entries[:0] - s.maxValue = []byte{} - s.size = 0 + obj *catalog.ObjectEntry } -func (s *entrySet) add(obj *catalog.ObjectEntry) { - s.entries = append(s.entries, obj) - s.size += int(obj.OriginSize()) - if zm := obj.SortKeyZoneMap(); len(s.maxValue) == 0 || - compute.Compare(s.maxValue, zm.GetMaxBuf(), zm.GetType(), zm.GetScale(), zm.GetScale()) < 0 { - s.maxValue = zm.GetMaxBuf() +func objectsWithGivenOverlaps(objects []*catalog.ObjectEntry, overlaps int) [][]*catalog.ObjectEntry { + if len(objects) < 2 { + return nil + } + points := make([]endPoint, 0, 2*len(objects)) + for _, obj := range objects { + zm := obj.SortKeyZoneMap() + points = append(points, endPoint{val: zm.GetMinBuf(), s: 1, obj: obj}) + points = append(points, endPoint{val: zm.GetMaxBuf(), s: -1, obj: obj}) + } + slices.SortFunc(points, func(a, b endPoint) int { + c := compute.Compare(a.val, b.val, objects[0].SortKeyZoneMap().GetType(), + a.obj.SortKeyZoneMap().GetScale(), b.obj.SortKeyZoneMap().GetScale()) + if c != 0 { + return c + } + // left node is first + return -a.s + }) + + globalMax := 0 + + res := make([][]*catalog.ObjectEntry, 0) + tmp := make(map[*catalog.ObjectEntry]struct{}) + for { + objs := make([]*catalog.ObjectEntry, 0, len(points)/2) + clear(tmp) + for _, p := range points { + if p.s == 1 { + tmp[p.obj] = struct{}{} + } else { + delete(tmp, p.obj) + } + if len(tmp) > globalMax { + globalMax = len(tmp) + objs = objs[:0] + for obj := range tmp { + objs = append(objs, obj) + } + } + } + if len(objs) < overlaps { + return res + } + res = append(res, objs) + points = slices.DeleteFunc(points, func(point endPoint) bool { + return slices.Contains(objs, point.obj) + }) + globalMax = 0 } } diff --git a/pkg/vm/engine/tae/db/merge/policyTombstone.go b/pkg/vm/engine/tae/db/merge/policyTombstone.go index f36f7a04b28ad..f5e89d01e7edd 100644 --- a/pkg/vm/engine/tae/db/merge/policyTombstone.go +++ b/pkg/vm/engine/tae/db/merge/policyTombstone.go @@ -15,9 +15,13 @@ package merge import ( + "slices" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" ) +var _ policy = (*tombstonePolicy)(nil) + type tombstonePolicy struct { tombstones []*catalog.ObjectEntry } @@ -33,19 +37,19 @@ func (t *tombstonePolicy) onObject(entry *catalog.ObjectEntry, config *BasicPoli return true } -func (t *tombstonePolicy) revise(cpu, mem int64, config *BasicPolicyConfig) []reviseResult { +func (t *tombstonePolicy) revise(*resourceController) []reviseResult { if len(t.tombstones) < 2 { return nil } - return []reviseResult{{t.tombstones, TaskHostDN}} + return []reviseResult{{slices.Clone(t.tombstones), taskHostDN}} } -func (t *tombstonePolicy) resetForTable(*catalog.TableEntry) { +func (t *tombstonePolicy) resetForTable(*catalog.TableEntry, *BasicPolicyConfig) { t.tombstones = t.tombstones[:0] } func newTombstonePolicy() policy { return &tombstonePolicy{ - tombstones: make([]*catalog.ObjectEntry, 0), + tombstones: make([]*catalog.ObjectEntry, 0, 5), } } diff --git a/pkg/vm/engine/tae/db/merge/policy_test.go b/pkg/vm/engine/tae/db/merge/policy_test.go index 9759150a6c1bc..96a663e1f218f 100644 --- a/pkg/vm/engine/tae/db/merge/policy_test.go +++ b/pkg/vm/engine/tae/db/merge/policy_test.go @@ -17,20 +17,26 @@ package merge import ( "context" "math" + "math/rand/v2" "testing" - "github.com/stretchr/testify/require" - + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/api" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/blockio" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/iface/txnif" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/options" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/txn/txnbase" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func testConfig(objectMinOSize uint32, maxOneRun int) *BasicPolicyConfig { @@ -44,7 +50,7 @@ func newSortedDataEntryWithTableEntry(t *testing.T, tbl *catalog.TableEntry, txn zm := index.NewZM(types.T_int32, 0) index.UpdateZM(zm, types.EncodeInt32(&v1)) index.UpdateZM(zm, types.EncodeInt32(&v2)) - stats := objectio.NewObjectStats() + stats := objectio.NewObjectStatsWithObjectID(objectio.NewObjectid(), false, true, false) require.NoError(t, objectio.SetObjectStatsSortKeyZoneMap(stats, zm)) require.NoError(t, objectio.SetObjectStatsOriginSize(stats, size)) require.NoError(t, objectio.SetObjectStatsRowCnt(stats, 2)) @@ -69,11 +75,13 @@ func newSortedTombstoneEntryWithTableEntry(t *testing.T, tbl *catalog.TableEntry return entry } -func newSortedTestObjectEntry(t *testing.T, v1, v2 int32, size uint32) *catalog.ObjectEntry { +func newSortedTestObjectEntry(t testing.TB, v1, v2 int32, size uint32) *catalog.ObjectEntry { zm := index.NewZM(types.T_int32, 0) index.UpdateZM(zm, types.EncodeInt32(&v1)) index.UpdateZM(zm, types.EncodeInt32(&v2)) stats := objectio.NewObjectStats() + objName := objectio.BuildObjectNameWithObjectID(objectio.NewObjectid()) + require.NoError(t, objectio.SetObjectStatsObjectName(stats, objName)) require.NoError(t, objectio.SetObjectStatsSortKeyZoneMap(stats, zm)) require.NoError(t, objectio.SetObjectStatsOriginSize(stats, size)) require.NoError(t, objectio.SetObjectStatsRowCnt(stats, 2)) @@ -82,14 +90,18 @@ func newSortedTestObjectEntry(t *testing.T, v1, v2 int32, size uint32) *catalog. } } -func newTestObjectEntryWithRowCnt(t *testing.T, size, rowCnt uint32, isTombstone bool) *catalog.ObjectEntry { +func newTestVarcharObjectEntry(t testing.TB, v1, v2 string, size uint32) *catalog.ObjectEntry { + zm := index.NewZM(types.T_varchar, 0) + index.UpdateZM(zm, []byte(v1)) + index.UpdateZM(zm, []byte(v2)) stats := objectio.NewObjectStats() + objName := objectio.BuildObjectNameWithObjectID(objectio.NewObjectid()) + require.NoError(t, objectio.SetObjectStatsObjectName(stats, objName)) + require.NoError(t, objectio.SetObjectStatsSortKeyZoneMap(stats, zm)) require.NoError(t, objectio.SetObjectStatsOriginSize(stats, size)) - require.NoError(t, objectio.SetObjectStatsRowCnt(stats, rowCnt)) - + require.NoError(t, objectio.SetObjectStatsRowCnt(stats, 2)) return &catalog.ObjectEntry{ ObjectMVCCNode: catalog.ObjectMVCCNode{ObjectStats: *stats}, - ObjectNode: catalog.ObjectNode{IsTombstone: isTombstone}, } } @@ -103,101 +115,57 @@ func newTestObjectEntry(t *testing.T, size uint32, isTombstone bool) *catalog.Ob } } -func TestPolicyBasic(t *testing.T) { - common.IsStandaloneBoost.Store(true) - p := newBasicPolicy() - - // only schedule objects whose size < cfg.objectMinOSize - p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) - cfg := testConfig(100, 3) - require.True(t, p.onObject(newTestObjectEntry(t, 10, false), cfg)) - require.True(t, p.onObject(newTestObjectEntry(t, 20, false), cfg)) - require.False(t, p.onObject(newTestObjectEntry(t, 120, false), cfg)) - result := p.revise(0, math.MaxInt64, cfg) - require.Equal(t, 1, len(result)) - require.Equal(t, 2, len(result[0].objs)) - require.Equal(t, TaskHostDN, result[0].kind) - - // only schedule objects less than cfg.maxOneRun - p.resetForTable(catalog.MockStaloneTableEntry(1, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) - cfg = testConfig(100, 2) - require.True(t, p.onObject(newTestObjectEntry(t, 10, false), cfg)) - require.True(t, p.onObject(newTestObjectEntry(t, 20, false), cfg)) - require.False(t, p.onObject(newTestObjectEntry(t, 30, false), cfg)) - result = p.revise(0, math.MaxInt64, cfg) - require.Equal(t, 1, len(result)) - require.Equal(t, 2, len(result[0].objs)) - require.Equal(t, TaskHostDN, result[0].kind) - - // basic policy do not schedule tombstones - p.resetForTable(catalog.MockStaloneTableEntry(2, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) - cfg = testConfig(100, 2) - require.False(t, p.onObject(newTestObjectEntry(t, 10, true), cfg)) - require.False(t, p.onObject(newTestObjectEntry(t, 20, true), cfg)) - result = p.revise(0, math.MaxInt64, cfg) - require.Equal(t, 0, len(result)) - - // memory limit - p.resetForTable(catalog.MockStaloneTableEntry(2, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) - cfg = testConfig(100, 3) - require.True(t, p.onObject(newTestObjectEntryWithRowCnt(t, 10, 1, false), cfg)) - require.True(t, p.onObject(newTestObjectEntryWithRowCnt(t, 20, 1, false), cfg)) - require.True(t, p.onObject(newTestObjectEntryWithRowCnt(t, 20, 1, false), cfg)) - result = p.revise(0, 36, cfg) - require.Equal(t, 1, len(result)) - require.Equal(t, 2, len(result[0].objs)) - require.Equal(t, TaskHostDN, result[0].kind) -} - func TestPolicyTombstone(t *testing.T) { common.IsStandaloneBoost.Store(true) p := newTombstonePolicy() + rc := new(resourceController) // tombstone policy do not schedule data objects - p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) + p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}}), nil) cfg := testConfig(100, 2) require.False(t, p.onObject(newTestObjectEntry(t, 10, false), cfg)) require.False(t, p.onObject(newTestObjectEntry(t, 20, false), cfg)) - result := p.revise(0, math.MaxInt64, cfg) + result := p.revise(rc) require.Equal(t, 0, len(result)) - p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) + p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}}), nil) cfg = testConfig(100, 2) require.True(t, p.onObject(newTestObjectEntry(t, 10, true), cfg)) require.True(t, p.onObject(newTestObjectEntry(t, 20, true), cfg)) - result = p.revise(0, math.MaxInt64, cfg) + result = p.revise(rc) require.Equal(t, 1, len(result)) require.Equal(t, 2, len(result[0].objs)) - require.Equal(t, TaskHostDN, result[0].kind) + require.Equal(t, taskHostDN, result[0].kind) // only schedule objects less than cfg.maxOneRun - p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) + p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}}), nil) cfg = testConfig(100, 2) require.True(t, p.onObject(newTestObjectEntry(t, 10, true), cfg)) require.True(t, p.onObject(newTestObjectEntry(t, 20, true), cfg)) require.False(t, p.onObject(newTestObjectEntry(t, 30, true), cfg)) - result = p.revise(0, math.MaxInt64, cfg) + result = p.revise(rc) require.Equal(t, 1, len(result)) require.Equal(t, 2, len(result[0].objs)) - require.Equal(t, TaskHostDN, result[0].kind) + require.Equal(t, taskHostDN, result[0].kind) // tombstone do not consider size limit - p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) + p.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}}), nil) cfg = testConfig(100, 3) require.True(t, p.onObject(newTestObjectEntry(t, 10, true), cfg)) require.True(t, p.onObject(newTestObjectEntry(t, 20, true), cfg)) require.True(t, p.onObject(newTestObjectEntry(t, 120, true), cfg)) - result = p.revise(0, math.MaxInt64, cfg) + result = p.revise(rc) require.Equal(t, 1, len(result)) require.Equal(t, 3, len(result[0].objs)) - require.Equal(t, TaskHostDN, result[0].kind) + require.Equal(t, taskHostDN, result[0].kind) } func TestPolicyGroup(t *testing.T) { common.IsStandaloneBoost.Store(true) - g := newPolicyGroup(newBasicPolicy(), newTombstonePolicy()) + g := newPolicyGroup(newTombstonePolicy()) g.resetForTable(catalog.MockStaloneTableEntry(0, &catalog.Schema{Extra: &api.SchemaExtra{BlockMaxRows: options.DefaultBlockMaxRows}})) g.config = &BasicPolicyConfig{MergeMaxOneRun: 2, ObjectMinOsize: 100} + rc := new(resourceController) g.onObject(newTestObjectEntry(t, 10, false)) g.onObject(newTestObjectEntry(t, 20, false)) @@ -206,13 +174,11 @@ func TestPolicyGroup(t *testing.T) { g.onObject(newTestObjectEntry(t, 20, true)) g.onObject(newTestObjectEntry(t, 30, true)) - results := g.revise(0, math.MaxInt64) - require.Equal(t, 2, len(results)) - require.Equal(t, TaskHostDN, results[0].kind) - require.Equal(t, TaskHostDN, results[1].kind) + results := g.revise(rc) + require.Equal(t, 1, len(results)) + require.Equal(t, taskHostDN, results[0].kind) require.Equal(t, 2, len(results[0].objs)) - require.Equal(t, 2, len(results[1].objs)) } const overlapSizeThreshold = common.DefaultMinOsizeQualifiedMB * common.Const1MBytes @@ -221,43 +187,48 @@ func TestObjOverlap(t *testing.T) { // empty policy policy := newObjOverlapPolicy() - objs := policy.revise(0, math.MaxInt64, defaultBasicConfig) - require.Equal(t, 0, len(objs)) + rc := new(resourceController) + rc.setMemLimit(estimateMemUsagePerRow * 20) + objs := policy.revise(rc) + for _, obj := range objs { + require.Equal(t, 0, len(obj.objs)) + } - policy.resetForTable(nil) + policy.resetForTable(nil, nil) // no overlap entry1 := newSortedTestObjectEntry(t, 1, 2, overlapSizeThreshold) entry2 := newSortedTestObjectEntry(t, 3, 4, overlapSizeThreshold) require.True(t, policy.onObject(entry1, defaultBasicConfig)) require.True(t, policy.onObject(entry2, defaultBasicConfig)) - objs = policy.revise(0, math.MaxInt64, defaultBasicConfig) - require.Equal(t, 0, len(objs)) + objs = policy.revise(rc) + for _, obj := range objs { + require.Equal(t, 0, len(obj.objs)) + } - policy.resetForTable(nil) + policy.resetForTable(nil, nil) // overlap entry3 := newSortedTestObjectEntry(t, 1, 4, overlapSizeThreshold) entry4 := newSortedTestObjectEntry(t, 2, 3, overlapSizeThreshold) require.True(t, policy.onObject(entry3, defaultBasicConfig)) require.True(t, policy.onObject(entry4, defaultBasicConfig)) - objs = policy.revise(0, math.MaxInt64, defaultBasicConfig) - require.Equal(t, 1, len(objs)) - require.Equal(t, 2, len(objs[0].objs)) - require.Equal(t, TaskHostDN, objs[0].kind) - - policy.resetForTable(nil) + objs = policy.revise(rc) + require.Zero(t, len(objs)) + policy.resetForTable(nil, nil) // entry is not sorted entry5 := newTestObjectEntry(t, overlapSizeThreshold, false) entry6 := newTestObjectEntry(t, overlapSizeThreshold, false) require.False(t, policy.onObject(entry5, defaultBasicConfig)) require.False(t, policy.onObject(entry6, defaultBasicConfig)) - require.Equal(t, 0, len(policy.objects)) - objs = policy.revise(0, math.MaxInt64, defaultBasicConfig) - require.Equal(t, 0, len(objs)) + require.Equal(t, 6, len(policy.leveledObjects)) + objs = policy.revise(rc) + for _, obj := range objs { + require.Equal(t, 0, len(obj.objs)) + } - policy.resetForTable(nil) + policy.resetForTable(nil, nil) // two overlap set: // {entry7, entry8} @@ -275,12 +246,10 @@ func TestObjOverlap(t *testing.T) { require.True(t, policy.onObject(entry10, defaultBasicConfig)) require.True(t, policy.onObject(entry11, defaultBasicConfig)) - objs = policy.revise(0, math.MaxInt64, defaultBasicConfig) - require.Equal(t, 1, len(objs)) - require.Equal(t, 3, len(objs[0].objs)) - require.Equal(t, TaskHostDN, objs[0].kind) + objs = policy.revise(rc) + require.Zero(t, len(objs)) - policy.resetForTable(nil) + policy.resetForTable(nil, nil) // no enough memory entry12 := newSortedTestObjectEntry(t, 1, 4, overlapSizeThreshold) @@ -289,16 +258,19 @@ func TestObjOverlap(t *testing.T) { require.True(t, policy.onObject(entry12, defaultBasicConfig)) require.True(t, policy.onObject(entry13, defaultBasicConfig)) - objs = policy.revise(0, 0, defaultBasicConfig) - require.Equal(t, 0, len(objs)) + objs = policy.revise(rc) + for _, obj := range objs { + require.Equal(t, 0, len(obj.objs)) + } - policy.resetForTable(nil) + policy.resetForTable(nil, nil) } func TestPolicyCompact(t *testing.T) { fs, err := fileservice.NewMemoryFS("memory", fileservice.DisabledCacheConfig, nil) require.NoError(t, err) p := newObjCompactPolicy(fs) + rc := new(resourceController) cata := catalog.MockCatalog() defer cata.Close() @@ -313,9 +285,15 @@ func TestPolicyCompact(t *testing.T) { require.NoError(t, err) require.NoError(t, txn1.Commit(context.Background())) - p.resetForTable(tbl) + obj := catalog.MockObjEntryWithTbl(tbl, math.MaxUint32, false) + tombstone := catalog.MockObjEntryWithTbl(tbl, math.MaxUint32, true) + require.NoError(t, objectio.SetObjectStatsOriginSize(tombstone.GetObjectStats(), math.MaxUint32)) + tbl.AddEntryLocked(obj) + tbl.AddEntryLocked(tombstone) + p.resetForTable(tbl, &BasicPolicyConfig{}) + p.onObject(obj, &BasicPolicyConfig{}) - objs := p.revise(0, math.MaxInt64, defaultBasicConfig) + objs := p.revise(rc) require.Equal(t, 0, len(objs)) txn2, _ := txnMgr.StartTxn(nil) @@ -347,7 +325,7 @@ func Test_timeout(t *testing.T) { require.NoError(t, err) require.NoError(t, txn1.Commit(context.Background())) - p.resetForTable(tbl) + p.resetForTable(tbl, defaultBasicConfig) txn3, _ := txnMgr.StartTxn(nil) ent3 := newSortedTombstoneEntryWithTableEntry(t, tbl, txn3, types.Rowid{0}, types.Rowid{1}) @@ -357,8 +335,120 @@ func Test_timeout(t *testing.T) { copy(originSizes, minSizeBytes) require.NoError(t, txn3.Commit(context.Background())) - p.tombstoneStats = []objectio.ObjectStats{ - ent3.ObjectStats, - } require.False(t, p.onObject(ent3, defaultBasicConfig)) } + +func TestSegLevel(t *testing.T) { + require.Equal(t, 0, segLevel(1)) + require.Equal(t, 1, segLevel(2)) + require.Equal(t, 1, segLevel(3)) + require.Equal(t, 2, segLevel(4)) + require.Equal(t, 2, segLevel(5)) + require.Equal(t, 2, segLevel(6)) + require.Equal(t, 2, segLevel(14)) + require.Equal(t, 2, segLevel(15)) + require.Equal(t, 3, segLevel(16)) + require.Equal(t, 3, segLevel(17)) + require.Equal(t, 3, segLevel(63)) + require.Equal(t, 4, segLevel(64)) + require.Equal(t, 4, segLevel(65)) + require.Equal(t, 4, segLevel(255)) + require.Equal(t, 5, segLevel(256)) + require.Equal(t, 5, segLevel(257)) +} + +func TestCheckTombstone(t *testing.T) { + mp := mpool.MustNewZero() + + fs := testutil.NewSharedFS() + + rowCnt := 100 + ssCnt := 2 + + rowids := make([]types.Rowid, rowCnt) + metas := make([]objectio.ObjectDataMeta, ssCnt) + for i := 0; i < ssCnt; i++ { + writer := blockio.ConstructTombstoneWriter(objectio.HiddenColumnSelection_None, fs) + assert.NotNil(t, writer) + + bat := batch.NewWithSize(2) + bat.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + bat.Vecs[1] = vector.NewVec(types.T_int32.ToType()) + + for j := 0; j < rowCnt; j++ { + row := types.RandomRowid() + rowids[j] = row + pk := rand.Int() + + err := vector.AppendFixed[types.Rowid](bat.Vecs[0], row, false, mp) + require.NoError(t, err) + + err = vector.AppendFixed[int32](bat.Vecs[1], int32(pk), false, mp) + require.NoError(t, err) + } + + _, err := writer.WriteBatch(bat) + require.NoError(t, err) + + _, _, err = writer.Sync(context.Background()) + require.NoError(t, err) + + ss := writer.GetObjectStats() + require.Equal(t, rowCnt, int(ss.Rows())) + meta, err := loadTombstoneMeta(&ss, fs) + require.NoError(t, err) + metas[i] = meta + } + for _, rowID := range rowids { + id := rowID.BorrowObjectID() + for i := range metas { + ok := checkTombstoneMeta(metas[i], id) + if i == 0 { + require.False(t, ok) + } else { + require.True(t, ok) + } + } + } +} + +func TestObjectsWithMaximumOverlaps(t *testing.T) { + o1 := newSortedTestObjectEntry(t, 0, 50, math.MaxInt32) + o2 := newSortedTestObjectEntry(t, 51, 100, math.MaxInt32) + o3 := newSortedTestObjectEntry(t, 49, 52, math.MaxInt32) + o4 := newSortedTestObjectEntry(t, 1, 52, math.MaxInt32) + o5 := newSortedTestObjectEntry(t, 50, 51, math.MaxInt32) + o6 := newSortedTestObjectEntry(t, 55, 60, math.MaxInt32) + + res1 := objectsWithGivenOverlaps([]*catalog.ObjectEntry{o1, o2}, 2) + require.Equal(t, 0, len(res1)) + + res2 := objectsWithGivenOverlaps([]*catalog.ObjectEntry{o1, o3}, 2) + require.Equal(t, 1, len(res2)) + require.ElementsMatch(t, []*catalog.ObjectEntry{o1, o3}, res2[0]) + + res3 := objectsWithGivenOverlaps([]*catalog.ObjectEntry{o2, o3}, 2) + require.Equal(t, 1, len(res3)) + require.ElementsMatch(t, []*catalog.ObjectEntry{o2, o3}, res3[0]) + + res4 := objectsWithGivenOverlaps([]*catalog.ObjectEntry{o1, o2, o3}, 2) + require.Equal(t, 1, len(res4)) + require.ElementsMatch(t, []*catalog.ObjectEntry{o1, o3}, res4[0]) + + res5 := objectsWithGivenOverlaps([]*catalog.ObjectEntry{o1, o2, o3}, 2) + require.Equal(t, 1, len(res5)) + require.ElementsMatch(t, []*catalog.ObjectEntry{o1, o3}, res5[0]) + + res6 := objectsWithGivenOverlaps([]*catalog.ObjectEntry{o1, o2, o3, o4}, 2) + require.Equal(t, 1, len(res6)) + require.ElementsMatch(t, []*catalog.ObjectEntry{o1, o3, o4}, res6[0]) + + res7 := objectsWithGivenOverlaps([]*catalog.ObjectEntry{o1, o5}, 2) + require.Equal(t, 1, len(res7)) + require.ElementsMatch(t, []*catalog.ObjectEntry{o1, o5}, res7[0]) + + res8 := objectsWithGivenOverlaps([]*catalog.ObjectEntry{o1, o2, o3, o4, o5, o6}, 2) + require.Equal(t, 2, len(res8)) + require.ElementsMatch(t, []*catalog.ObjectEntry{o1, o3, o4, o5}, res8[0]) + require.ElementsMatch(t, []*catalog.ObjectEntry{o2, o6}, res8[1]) +} diff --git a/pkg/vm/engine/tae/db/merge/scheduler.go b/pkg/vm/engine/tae/db/merge/scheduler.go index 6deccb16d0cce..1f7c685618251 100644 --- a/pkg/vm/engine/tae/db/merge/scheduler.go +++ b/pkg/vm/engine/tae/db/merge/scheduler.go @@ -17,6 +17,7 @@ package merge import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/objectio" v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" @@ -24,6 +25,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/dbutils" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/iface/txnif" dto "github.com/prometheus/client_model/go" + "time" ) type Scheduler struct { @@ -34,20 +36,21 @@ type Scheduler struct { executor *executor skipForTransPageLimit bool + + rc *resourceController } -func NewScheduler(rt *dbutils.Runtime, sched CNMergeScheduler) *Scheduler { - policySlice := make([]policy, 0, 4) - policySlice = append(policySlice, newBasicPolicy(), newObjCompactPolicy(rt.Fs.Service)) - if !common.RuntimeDisableZMBasedMerge.Load() { - policySlice = append(policySlice, newObjOverlapPolicy()) +func NewScheduler(rt *dbutils.Runtime, sched *CNMergeScheduler) *Scheduler { + policySlice := []policy{ + newObjOverlapPolicy(), + newObjCompactPolicy(rt.Fs.Service), + newTombstonePolicy(), } - policySlice = append(policySlice, newTombstonePolicy()) - op := &Scheduler{ LoopProcessor: new(catalog.LoopProcessor), policies: newPolicyGroup(policySlice...), executor: newMergeExecutor(rt, sched), + rc: new(resourceController), } op.DatabaseFn = op.onDataBase @@ -76,22 +79,24 @@ func (s *Scheduler) resetForTable(entry *catalog.TableEntry) { } func (s *Scheduler) PreExecute() error { - s.executor.refreshMemInfo() + s.rc.refresh() s.skipForTransPageLimit = false m := &dto.Metric{} - v2.TaskMergeTransferPageLengthGauge.Write(m) + if err := v2.TaskMergeTransferPageLengthGauge.Write(m); err != nil { + return err + } pagesize := m.GetGauge().GetValue() * 28 /*int32 + rowid(24b)*/ - if pagesize > float64(s.executor.transferPageSizeLimit()) { + if pagesize > float64(s.rc.transferPageLimit) { logutil.Infof("[mergeblocks] skip merge scanning due to transfer page %s, limit %s", common.HumanReadableBytes(int(pagesize)), - common.HumanReadableBytes(int(s.executor.transferPageSizeLimit()))) + common.HumanReadableBytes(int(s.rc.transferPageLimit))) s.skipForTransPageLimit = true } return nil } func (s *Scheduler) PostExecute() error { - s.executor.printStats() + s.rc.printStats() return nil } @@ -99,7 +104,7 @@ func (s *Scheduler) onDataBase(*catalog.DBEntry) (err error) { if StopMerge.Load() { return moerr.GetOkStopCurrRecur() } - if s.executor.memAvailBytes() < 100*common.Const1MBytes { + if s.rc.availableMem() < 100*common.Const1MBytes { return moerr.GetOkStopCurrRecur() } @@ -143,7 +148,7 @@ func (s *Scheduler) onPostTable(tableEntry *catalog.TableEntry) (err error) { } // delObjs := s.ObjectHelper.finish() - results := s.policies.revise(s.executor.CPUPercent(), int64(s.executor.memAvailBytes())) + results := s.policies.revise(s.rc) for _, r := range results { if len(r.objs) > 0 { s.executor.executeFor(tableEntry, r.objs, r.kind) @@ -191,18 +196,14 @@ func (s *Scheduler) StartMerge(tblID uint64, reentrant bool) error { return s.executor.rt.LockMergeService.UnlockFromUser(tblID, reentrant) } -func objectValid(objectEntry *catalog.ObjectEntry) bool { - if objectEntry.IsAppendable() { - return false - } - if !objectEntry.IsActive() { - return false - } - if !objectEntry.IsCommitted() { - return false - } - if objectEntry.IsCreatingOrAborted() { - return false - } - return true +func (s *Scheduler) CNActiveObjectsString() string { + return s.executor.cnSched.activeObjsString() +} + +func (s *Scheduler) RemoveCNActiveObjects(ids []objectio.ObjectId) { + s.executor.cnSched.removeActiveObject(ids) +} + +func (s *Scheduler) PruneCNActiveObjects(id uint64, ago time.Duration) { + s.executor.cnSched.prune(id, ago) } diff --git a/pkg/vm/engine/tae/db/merge/scheduler_test.go b/pkg/vm/engine/tae/db/merge/scheduler_test.go index 1d50194496c7b..a360b8d93af70 100644 --- a/pkg/vm/engine/tae/db/merge/scheduler_test.go +++ b/pkg/vm/engine/tae/db/merge/scheduler_test.go @@ -16,9 +16,11 @@ package merge import ( "context" + "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/taskservice" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/dbutils" @@ -28,10 +30,15 @@ import ( ) func TestStopStartMerge(t *testing.T) { + memStorage := taskservice.NewMemTaskStorage() + cnScheduler := NewTaskServiceGetter(func() (taskservice.TaskService, bool) { + return taskservice.NewTaskService(runtime.DefaultRuntime(), memStorage), true + }) + scheduler := Scheduler{ executor: newMergeExecutor(&dbutils.Runtime{ LockMergeService: dbutils.NewLockMergeService(), - }, nil), + }, cnScheduler), } lockService := scheduler.executor.rt.LockMergeService @@ -76,6 +83,8 @@ func TestStopStartMerge(t *testing.T) { require.Error(t, scheduler.onTable(tblEntry1)) require.Error(t, scheduler.onTable(tblEntry2)) + require.Empty(t, scheduler.CNActiveObjectsString()) + scheduler.StartMerge(tblEntry1.GetID(), false) require.Equal(t, 1, len(lockService.LockedInfos())) scheduler.StartMerge(tblEntry2.GetID(), false) diff --git a/pkg/vm/engine/tae/db/merge/utils.go b/pkg/vm/engine/tae/db/merge/utils.go index a88a383a556d4..8327646f93978 100644 --- a/pkg/vm/engine/tae/db/merge/utils.go +++ b/pkg/vm/engine/tae/db/merge/utils.go @@ -15,27 +15,329 @@ package merge import ( + "cmp" + "context" + "math" + "os" + "slices" + "sync/atomic" "time" + "github.com/KimMachineGun/automemlimit/memlimit" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/process" ) -func estimateMergeConsume(mobjs []*catalog.ObjectEntry) (origSize, estSize int) { - if len(mobjs) == 0 { +var StopMerge atomic.Bool + +type taskHostKind int + +const ( + taskHostCN taskHostKind = iota + taskHostDN + + constMaxMemCap = 12 * common.Const1GBytes // max original memory for an object + estimateMemUsagePerRow = 30 +) + +func score(objs []*catalog.ObjectEntry) float64 { + if len(objs) < 2 { + return 0 + } + totalDiff := float64(0) + minMaxZM := objs[0].SortKeyZoneMap().Clone() + if !minMaxZM.GetType().IsFixedLen() { + return math.MaxFloat64 + } + for _, obj := range objs { + zm := obj.SortKeyZoneMap() + index.UpdateZM(minMaxZM, zm.GetMinBuf()) + index.UpdateZM(minMaxZM, zm.GetMaxBuf()) + w := diff(zm.GetMax(), zm.GetMin(), zm.GetType()) + if w == math.MaxUint64 { + return math.MaxFloat64 + } + totalDiff += float64(w) + } + maxDiff := diff(minMaxZM.GetMax(), minMaxZM.GetMin(), minMaxZM.GetType()) + if maxDiff == math.MaxUint64 { + return math.MaxFloat64 + } + return totalDiff / float64(maxDiff) +} + +func diff(a, b any, t types.T) uint64 { + switch t { + case types.T_bool: + if a == b { + return 0 + } + return 1 + case types.T_bit: + x, y := a.(uint64), b.(uint64) + return max(x, y) - min(x, y) + case types.T_int8: + x, y := a.(int8), b.(int8) + return uint64(max(x, y) - min(x, y)) + case types.T_int16: + x, y := a.(int16), b.(int16) + return uint64(max(x, y) - min(x, y)) + case types.T_int32: + x, y := a.(int32), b.(int32) + return uint64(max(x, y) - min(x, y)) + case types.T_int64: + x, y := a.(int64), b.(int64) + return uint64(max(x, y) - min(x, y)) + case types.T_uint8: + x, y := a.(uint8), b.(uint8) + return uint64(max(x, y) - min(x, y)) + case types.T_uint16: + x, y := a.(uint16), b.(uint16) + return uint64(max(x, y) - min(x, y)) + case types.T_uint32: + x, y := a.(uint32), b.(uint32) + return uint64(max(x, y) - min(x, y)) + case types.T_uint64: + x, y := a.(uint64), b.(uint64) + return max(x, y) - min(x, y) + case types.T_float32: + x, y := a.(float32), b.(float32) + return uint64(max(x, y) - min(x, y)) + case types.T_float64: + x, y := a.(float64), b.(float64) + return uint64(max(x, y) - min(x, y)) + case types.T_date: + x, y := a.(types.Date), b.(types.Date) + return uint64(max(x, y) - min(x, y)) + case types.T_time: + x, y := a.(types.Time), b.(types.Time) + return uint64(max(x, y) - min(x, y)) + case types.T_datetime: + x, y := a.(types.Datetime), b.(types.Datetime) + return uint64(max(x, y) - min(x, y)) + case types.T_timestamp: + x, y := a.(types.Timestamp), b.(types.Timestamp) + return uint64(max(x, y) - min(x, y)) + case types.T_enum: + x, y := a.(types.Enum), b.(types.Enum) + return uint64(max(x, y) - min(x, y)) + case types.T_decimal64: + x, y := a.(types.Decimal64), b.(types.Decimal64) + return uint64(max(x, y) - min(x, y)) + default: + } + return math.MaxUint64 +} + +func removeOversize(objs []*catalog.ObjectEntry) []*catalog.ObjectEntry { + if len(objs) < 2 { + return objs + } + slices.SortFunc(objs, func(a, b *catalog.ObjectEntry) int { + return cmp.Compare(a.OriginSize(), b.OriginSize()) + }) + + if len(objs) == 2 { + if uint(objs[1].OriginSize()) < 3*uint(objs[0].OriginSize()) { + return objs[:2] + } + return nil + } + + accSize := int(objs[0].OriginSize()) + int(objs[1].OriginSize()) + i := 2 + for i < len(objs) { + size := int(objs[i].OriginSize()) + if size > accSize { + break + } + accSize += size + i++ + } + for j := i; j < len(objs); j++ { + objs[j] = nil + } + if i == 2 { + if uint(objs[1].OriginSize()) < 3*uint(objs[0].OriginSize()) { + return objs[:2] + } + return nil + } + return objs[:i] +} + +func estimateMergeSize(objs []*catalog.ObjectEntry) int { + size := 0 + for _, o := range objs { + size += int(o.Rows()) * estimateMemUsagePerRow + } + return size +} + +type resourceController struct { + proc *process.Process + + limit int64 + using int64 + reserved int64 + + reservedMergeRows int64 + transferPageLimit int64 + + cpuPercent float64 +} + +func (c *resourceController) setMemLimit(total uint64) { + cgroup, err := memlimit.FromCgroup() + if cgroup != 0 && cgroup < total { + c.limit = int64(cgroup / 4 * 3) + } else if total != 0 { + c.limit = int64(total / 4 * 3) + } else { + panic("failed to get system total memory") + } + + if c.limit > 200*common.Const1GBytes { + c.transferPageLimit = c.limit / 25 * 2 // 8% + } else if c.limit > 100*common.Const1GBytes { + c.transferPageLimit = c.limit / 25 * 3 // 12% + } else if c.limit > 40*common.Const1GBytes { + c.transferPageLimit = c.limit / 25 * 4 // 16% + } else { + c.transferPageLimit = math.MaxInt64 // no limit + } + + logutil.Info( + "MergeExecutorMemoryInfo", + common.AnyField("container-limit", common.HumanReadableBytes(int(cgroup))), + common.AnyField("host-memory", common.HumanReadableBytes(int(total))), + common.AnyField("merge-limit", common.HumanReadableBytes(int(c.limit))), + common.AnyField("transfer-page-limit", common.HumanReadableBytes(int(c.transferPageLimit))), + common.AnyField("error", err), + ) +} + +func (c *resourceController) refresh() { + if c.limit == 0 { + c.setMemLimit(totalMem()) + } + + if c.proc == nil { + c.proc, _ = process.NewProcess(int32(os.Getpid())) + } + if m, err := c.proc.MemoryInfo(); err == nil { + c.using = int64(m.RSS) + } + + if percents, err := cpu.Percent(0, false); err == nil { + c.cpuPercent = percents[0] + } + c.reservedMergeRows = 0 + c.reserved = 0 +} + +func (c *resourceController) availableMem() int64 { + avail := c.limit - c.using - c.reserved + if avail < 0 { + avail = 0 + } + return avail +} + +func (c *resourceController) printStats() { + if c.reservedMergeRows == 0 && c.availableMem() > 512*common.Const1MBytes { return } - rows := 0 - for _, m := range mobjs { - rows += int(m.Rows()) - origSize += int(m.OriginSize()) + + logutil.Info("MergeExecutorMemoryStats", + common.AnyField("merge-limit", common.HumanReadableBytes(int(c.limit))), + common.AnyField("process-mem", common.HumanReadableBytes(int(c.using))), + common.AnyField("reserving-rows", common.HumanReadableBytes(int(c.reservedMergeRows))), + common.AnyField("reserving-mem", common.HumanReadableBytes(int(c.reserved))), + ) +} + +func (c *resourceController) reserveResources(objs []*catalog.ObjectEntry) { + for _, obj := range objs { + c.reservedMergeRows += int64(obj.Rows()) + c.reserved += estimateMemUsagePerRow * int64(obj.Rows()) } - // the main memory consumption is transfer table. - // each row uses 12B, so estimate size is 12 * rows. - estSize = rows * 12 - return } -func entryOutdated(entry *catalog.ObjectEntry, lifetime time.Duration) bool { - createdAt := entry.CreatedAt.Physical() - return time.Unix(0, createdAt).Add(lifetime).Before(time.Now()) +func (c *resourceController) resourceAvailable(objs []*catalog.ObjectEntry) bool { + if c.reservedMergeRows*36 /*28 * 1.3 */ > c.transferPageLimit/8 { + return false + } + + mem := c.availableMem() + if mem > constMaxMemCap { + mem = constMaxMemCap + } + return estimateMergeSize(objs) <= int(2*mem/3) +} + +func objectValid(objectEntry *catalog.ObjectEntry) bool { + if objectEntry.IsAppendable() { + return false + } + if !objectEntry.IsActive() { + return false + } + if !objectEntry.IsCommitted() { + return false + } + if objectEntry.IsCreatingOrAborted() { + return false + } + return true +} + +func CleanUpUselessFiles(entry *api.MergeCommitEntry, fs fileservice.FileService) { + if entry == nil { + return + } + ctx, cancel := context.WithTimeoutCause(context.Background(), 2*time.Minute, moerr.CauseCleanUpUselessFiles) + defer cancel() + for _, filepath := range entry.BookingLoc { + _ = fs.Delete(ctx, filepath) + } + if len(entry.CreatedObjs) != 0 { + for _, obj := range entry.CreatedObjs { + if len(obj) == 0 { + continue + } + s := objectio.ObjectStats(obj) + _ = fs.Delete(ctx, s.ObjectName().String()) + } + } +} + +type policy interface { + onObject(*catalog.ObjectEntry, *BasicPolicyConfig) bool + revise(*resourceController) []reviseResult + resetForTable(*catalog.TableEntry, *BasicPolicyConfig) +} + +func newUpdatePolicyReq(c *BasicPolicyConfig) *api.AlterTableReq { + return &api.AlterTableReq{ + Kind: api.AlterKind_UpdatePolicy, + Operation: &api.AlterTableReq_UpdatePolicy{ + UpdatePolicy: &api.AlterTablePolicy{ + MinOsizeQuailifed: c.ObjectMinOsize, + MaxObjOnerun: uint32(c.MergeMaxOneRun), + MaxOsizeMergedObj: c.MaxOsizeMergedObj, + MinCnMergeSize: c.MinCNMergeSize, + Hints: c.MergeHints, + }, + }, + } } diff --git a/pkg/vm/engine/tae/db/merge/utils_test.go b/pkg/vm/engine/tae/db/merge/utils_test.go new file mode 100644 index 0000000000000..1745a76f7dc88 --- /dev/null +++ b/pkg/vm/engine/tae/db/merge/utils_test.go @@ -0,0 +1,141 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package merge + +import ( + "context" + "math" + "os" + "path" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/pb/api" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScore(t *testing.T) { + o1 := newSortedTestObjectEntry(t, 0, 3, 1) + o2 := newSortedTestObjectEntry(t, 1, 4, 2) + o3 := newSortedTestObjectEntry(t, 1, 5, 4) + o4 := newSortedTestObjectEntry(t, 1, 100, 4) + o5 := newSortedTestObjectEntry(t, 5, 10, math.MaxInt32) + + // should merge + require.Less(t, 1.1, score([]*catalog.ObjectEntry{o1, o1})) + require.Less(t, 1.1, score([]*catalog.ObjectEntry{o1, o2})) + require.Less(t, 1.1, score([]*catalog.ObjectEntry{o1, o3})) + // should not merge + require.Greater(t, 1.1, score([]*catalog.ObjectEntry{o1, o4})) + require.Greater(t, 1.1, score([]*catalog.ObjectEntry{o1, o2, o4})) + require.Greater(t, 1.1, score([]*catalog.ObjectEntry{o1, o5})) + + o6 := newTestVarcharObjectEntry(t, "a", "z", 1) + o7 := newTestVarcharObjectEntry(t, "b", "y", 1) + + require.Less(t, 1.1, score([]*catalog.ObjectEntry{o6, o7})) +} + +func TestRemoveOversize(t *testing.T) { + o1 := newSortedTestObjectEntry(t, 0, 0, 1) + o2 := newSortedTestObjectEntry(t, 0, 0, 2) + o3 := newSortedTestObjectEntry(t, 0, 0, 4) + o5 := newSortedTestObjectEntry(t, 0, 0, math.MaxInt32) + + require.ElementsMatch(t, []*catalog.ObjectEntry{o1, o2}, removeOversize([]*catalog.ObjectEntry{o1, o2})) + require.ElementsMatch(t, []*catalog.ObjectEntry{o1, o2}, removeOversize([]*catalog.ObjectEntry{o5, o1, o2})) + require.ElementsMatch(t, nil, removeOversize([]*catalog.ObjectEntry{o1, o3})) +} + +func TestDiff(t *testing.T) { + require.Equal(t, uint64(0), diff(false, false, types.T_bool)) + require.Equal(t, uint64(1), diff(false, true, types.T_bool)) + require.Equal(t, uint64(10), diff(int8(1), int8(11), types.T_int8)) + require.Equal(t, uint64(10), diff(uint8(1), uint8(11), types.T_uint8)) + require.Equal(t, uint64(10), diff(int16(1), int16(11), types.T_int16)) + require.Equal(t, uint64(10), diff(uint16(1), uint16(11), types.T_uint16)) + require.Equal(t, uint64(10), diff(int32(1), int32(11), types.T_int32)) + require.Equal(t, uint64(10), diff(uint32(1), uint32(11), types.T_uint32)) + require.Equal(t, uint64(10), diff(int64(1), int64(11), types.T_int64)) + require.Equal(t, uint64(10), diff(uint64(1), uint64(11), types.T_uint64)) + require.Equal(t, uint64(10), diff(int8(1), int8(11), types.T_int8)) + require.Equal(t, uint64(10), diff(int8(1), int8(11), types.T_int8)) + require.Equal(t, uint64(10), diff(float32(1), float32(11), types.T_float32)) + require.Equal(t, uint64(10), diff(float64(1), float64(11), types.T_float64)) + require.Equal(t, uint64(10), diff(types.Date(1), types.Date(11), types.T_date)) + require.Equal(t, uint64(10), diff(types.Datetime(1), types.Datetime(11), types.T_datetime)) + require.Equal(t, uint64(10), diff(types.Time(1), types.Time(11), types.T_time)) + require.Equal(t, uint64(10), diff(types.Timestamp(1), types.Timestamp(11), types.T_timestamp)) + require.Equal(t, uint64(10), diff(types.Enum(1), types.Enum(11), types.T_enum)) + require.Equal(t, uint64(10), diff(types.Decimal64(1), types.Decimal64(11), types.T_decimal64)) + require.Equal(t, uint64(math.MaxUint64), diff(types.Decimal128{}, types.Decimal128{}, types.T_decimal128)) +} + +func BenchmarkRemoveOversize(b *testing.B) { + o1 := newSortedTestObjectEntry(b, 0, 50, math.MaxInt32) + o2 := newSortedTestObjectEntry(b, 51, 100, 1) + o3 := newSortedTestObjectEntry(b, 49, 52, 2) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + removeOversize([]*catalog.ObjectEntry{o1, o2, o3}) + } +} + +func TestResourceController(t *testing.T) { + rc := new(resourceController) + rc.setMemLimit(10000) + require.Equal(t, int64(7500), rc.limit) + require.Equal(t, int64(7500), rc.availableMem()) + + rc.refresh() + rc.limit = rc.using + 1 + require.Equal(t, int64(1), rc.availableMem()) + + require.Panics(t, func() { rc.setMemLimit(0) }) +} + +func Test_CleanUpUselessFiles(t *testing.T) { + tDir := os.TempDir() + dir := path.Join(tDir, "/local") + assert.NoError(t, os.RemoveAll(dir)) + defer func() { + _ = os.RemoveAll(dir) + }() + + c := fileservice.Config{ + Name: defines.ETLFileServiceName, + Backend: "DISK", + DataDir: dir, + Cache: fileservice.DisabledCacheConfig, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fs, err := fileservice.NewFileService(ctx, c, nil) + assert.Nil(t, err) + defer fs.Close(ctx) + + ent := &api.MergeCommitEntry{ + BookingLoc: []string{"abc"}, + } + + CleanUpUselessFiles(ent, fs) +} diff --git a/pkg/vm/engine/tae/db/open.go b/pkg/vm/engine/tae/db/open.go index 6d4ca5d11f873..3ad4eb740f7bb 100644 --- a/pkg/vm/engine/tae/db/open.go +++ b/pkg/vm/engine/tae/db/open.go @@ -114,11 +114,10 @@ func Open(ctx context.Context, dirname string, opts *options.Options) (db *DB, e } db = &DB{ - Dir: dirname, - Opts: opts, - Closed: new(atomic.Value), - usageMemo: logtail.NewTNUsageMemo(nil), - CNMergeSched: merge.NewTaskServiceGetter(opts.TaskServiceGetter), + Dir: dirname, + Opts: opts, + Closed: new(atomic.Value), + usageMemo: logtail.NewTNUsageMemo(nil), } fs := objectio.NewObjectFS(opts.Fs, serviceDir) localFs := objectio.NewObjectFS(opts.LocalFs, serviceDir) @@ -242,7 +241,7 @@ func Open(ctx context.Context, dirname string, opts *options.Options) (db *DB, e // Init timed scanner scanner := NewDBScanner(db, nil) - db.MergeScheduler = merge.NewScheduler(db.Runtime, db.CNMergeSched) + db.MergeScheduler = merge.NewScheduler(db.Runtime, merge.NewTaskServiceGetter(opts.TaskServiceGetter)) scanner.RegisterOp(db.MergeScheduler) db.Wal.Start() db.BGCheckpointRunner.Start() diff --git a/pkg/vm/engine/tae/db/test/db_test.go b/pkg/vm/engine/tae/db/test/db_test.go index f3fd527f29d42..e688f8aca3619 100644 --- a/pkg/vm/engine/tae/db/test/db_test.go +++ b/pkg/vm/engine/tae/db/test/db_test.go @@ -10025,7 +10025,7 @@ func TestStartStopTableMerge(t *testing.T) { db := testutil.InitTestDB(context.Background(), "MergeTest", t, nil) defer db.Close() - scheduler := merge.NewScheduler(db.Runtime, db.CNMergeSched) + scheduler := merge.NewScheduler(db.Runtime, nil) schema := catalog.MockSchema(2, 0) schema.Extra.BlockMaxRows = 1000 diff --git a/pkg/vm/engine/tae/index/zm.go b/pkg/vm/engine/tae/index/zm.go index 12f89cb61384b..fd56a8bac0645 100644 --- a/pkg/vm/engine/tae/index/zm.go +++ b/pkg/vm/engine/tae/index/zm.go @@ -1243,7 +1243,7 @@ func ZMPlus(v1, v2, res ZM) ZM { } // max = v1.max-v2.min -// min = v1.max-v2.min +// min = v1.min-v2.max func ZMMinus(v1, v2, res ZM) ZM { res.Reset() if !v1.compareCheck(v2) { diff --git a/pkg/vm/engine/tae/mergesort/merger.go b/pkg/vm/engine/tae/mergesort/merger.go index e0a2a07153f21..e2825b77ab126 100644 --- a/pkg/vm/engine/tae/mergesort/merger.go +++ b/pkg/vm/engine/tae/mergesort/merger.go @@ -100,12 +100,11 @@ type merger[T comparable] struct { sortKeyIdx int - isTombstone bool - rowPerBlk uint32 - stats mergeStats + rowPerBlk uint32 + stats mergeStats } -func newMerger[T comparable](host MergeTaskHost, lessFunc sort.LessFunc[T], sortKeyPos int, isTombstone bool, df dataFetcher[T]) Merger { +func newMerger[T comparable](host MergeTaskHost, lessFunc sort.LessFunc[T], sortKeyPos int, df dataFetcher[T]) Merger { size := host.GetObjectCnt() rowSizeU64 := host.GetTotalSize() / uint64(host.GetTotalRowCnt()) m := &merger[T]{ @@ -130,7 +129,6 @@ func newMerger[T comparable](host MergeTaskHost, lessFunc sort.LessFunc[T], sort blkPerObj: host.GetObjectMaxBlocks(), }, loadedObjBlkCnts: make([]int, size), - isTombstone: isTombstone, } totalBlkCnt := 0 for _, cnt := range m.objBlkCnts { @@ -215,14 +213,8 @@ func (m *merger[T]) merge(ctx context.Context) error { if m.writer == nil { m.writer = m.host.PrepareNewWriter() } - if m.isTombstone { - if _, err := m.writer.WriteBatch(m.buffer); err != nil { - return err - } - } else { - if _, err := m.writer.WriteBatch(m.buffer); err != nil { - return err - } + if _, err := m.writer.WriteBatch(m.buffer); err != nil { + return err } // force clean m.buffer.CleanOnlyData() @@ -252,15 +244,8 @@ func (m *merger[T]) merge(ctx context.Context) error { if m.writer == nil { m.writer = m.host.PrepareNewWriter() } - if m.isTombstone { - if _, err := m.writer.WriteBatch(m.buffer); err != nil { - return err - } - } else { - - if _, err := m.writer.WriteBatch(m.buffer); err != nil { - return err - } + if _, err := m.writer.WriteBatch(m.buffer); err != nil { + return err } m.buffer.CleanOnlyData() } @@ -345,7 +330,7 @@ func (m *merger[T]) release() { } } -func mergeObjs(ctx context.Context, mergeHost MergeTaskHost, sortKeyPos int, isTombstone bool) error { +func mergeObjs(ctx context.Context, mergeHost MergeTaskHost, sortKeyPos int) error { var merger Merger typ := mergeHost.GetSortKeyType() size := mergeHost.GetObjectCnt() @@ -356,7 +341,7 @@ func mergeObjs(ctx context.Context, mergeHost MergeTaskHost, sortKeyPos int, isT area []byte }, size), } - merger = newMerger(mergeHost, sort.GenericLess[string], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[string], sortKeyPos, df) } else { switch typ.Oid { case types.T_bool: @@ -364,139 +349,139 @@ func mergeObjs(ctx context.Context, mergeHost MergeTaskHost, sortKeyPos int, isT mustColFunc: vector.MustFixedColNoTypeCheck[bool], cols: make([][]bool, size), } - merger = newMerger(mergeHost, sort.BoolLess, sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.BoolLess, sortKeyPos, df) case types.T_bit: df := &fixedDataFetcher[uint64]{ mustColFunc: vector.MustFixedColNoTypeCheck[uint64], cols: make([][]uint64, size), } - merger = newMerger(mergeHost, sort.GenericLess[uint64], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[uint64], sortKeyPos, df) case types.T_int8: df := &fixedDataFetcher[int8]{ mustColFunc: vector.MustFixedColNoTypeCheck[int8], cols: make([][]int8, size), } - merger = newMerger(mergeHost, sort.GenericLess[int8], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[int8], sortKeyPos, df) case types.T_int16: df := &fixedDataFetcher[int16]{ mustColFunc: vector.MustFixedColNoTypeCheck[int16], cols: make([][]int16, size), } - merger = newMerger(mergeHost, sort.GenericLess[int16], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[int16], sortKeyPos, df) case types.T_int32: df := &fixedDataFetcher[int32]{ mustColFunc: vector.MustFixedColNoTypeCheck[int32], cols: make([][]int32, size), } - merger = newMerger(mergeHost, sort.GenericLess[int32], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[int32], sortKeyPos, df) case types.T_int64: df := &fixedDataFetcher[int64]{ mustColFunc: vector.MustFixedColNoTypeCheck[int64], cols: make([][]int64, size), } - merger = newMerger(mergeHost, sort.GenericLess[int64], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[int64], sortKeyPos, df) case types.T_float32: df := &fixedDataFetcher[float32]{ mustColFunc: vector.MustFixedColNoTypeCheck[float32], cols: make([][]float32, size), } - merger = newMerger(mergeHost, sort.GenericLess[float32], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[float32], sortKeyPos, df) case types.T_float64: df := &fixedDataFetcher[float64]{ mustColFunc: vector.MustFixedColNoTypeCheck[float64], cols: make([][]float64, size), } - merger = newMerger(mergeHost, sort.GenericLess[float64], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[float64], sortKeyPos, df) case types.T_uint8: df := &fixedDataFetcher[uint8]{ mustColFunc: vector.MustFixedColNoTypeCheck[uint8], cols: make([][]uint8, size), } - merger = newMerger(mergeHost, sort.GenericLess[uint8], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[uint8], sortKeyPos, df) case types.T_uint16: df := &fixedDataFetcher[uint16]{ mustColFunc: vector.MustFixedColNoTypeCheck[uint16], cols: make([][]uint16, size), } - merger = newMerger(mergeHost, sort.GenericLess[uint16], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[uint16], sortKeyPos, df) case types.T_uint32: df := &fixedDataFetcher[uint32]{ mustColFunc: vector.MustFixedColNoTypeCheck[uint32], cols: make([][]uint32, size), } - merger = newMerger(mergeHost, sort.GenericLess[uint32], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[uint32], sortKeyPos, df) case types.T_uint64: df := &fixedDataFetcher[uint64]{ mustColFunc: vector.MustFixedColNoTypeCheck[uint64], cols: make([][]uint64, size), } - merger = newMerger(mergeHost, sort.GenericLess[uint64], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[uint64], sortKeyPos, df) case types.T_date: df := &fixedDataFetcher[types.Date]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Date], cols: make([][]types.Date, size), } - merger = newMerger(mergeHost, sort.GenericLess[types.Date], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[types.Date], sortKeyPos, df) case types.T_timestamp: df := &fixedDataFetcher[types.Timestamp]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Timestamp], cols: make([][]types.Timestamp, size), } - merger = newMerger(mergeHost, sort.GenericLess[types.Timestamp], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[types.Timestamp], sortKeyPos, df) case types.T_datetime: df := &fixedDataFetcher[types.Datetime]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Datetime], cols: make([][]types.Datetime, size), } - merger = newMerger(mergeHost, sort.GenericLess[types.Datetime], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[types.Datetime], sortKeyPos, df) case types.T_time: df := &fixedDataFetcher[types.Time]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Time], cols: make([][]types.Time, size), } - merger = newMerger(mergeHost, sort.GenericLess[types.Time], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[types.Time], sortKeyPos, df) case types.T_enum: df := &fixedDataFetcher[types.Enum]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Enum], cols: make([][]types.Enum, size), } - merger = newMerger(mergeHost, sort.GenericLess[types.Enum], sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.GenericLess[types.Enum], sortKeyPos, df) case types.T_decimal64: df := &fixedDataFetcher[types.Decimal64]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Decimal64], cols: make([][]types.Decimal64, size), } - merger = newMerger(mergeHost, sort.Decimal64Less, sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.Decimal64Less, sortKeyPos, df) case types.T_decimal128: df := &fixedDataFetcher[types.Decimal128]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Decimal128], cols: make([][]types.Decimal128, size), } - merger = newMerger(mergeHost, sort.Decimal128Less, sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.Decimal128Less, sortKeyPos, df) case types.T_uuid: df := &fixedDataFetcher[types.Uuid]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Uuid], cols: make([][]types.Uuid, size), } - merger = newMerger(mergeHost, sort.UuidLess, sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.UuidLess, sortKeyPos, df) case types.T_TS: df := &fixedDataFetcher[types.TS]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.TS], cols: make([][]types.TS, size), } - merger = newMerger(mergeHost, sort.TsLess, sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.TsLess, sortKeyPos, df) case types.T_Rowid: df := &fixedDataFetcher[types.Rowid]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Rowid], cols: make([][]types.Rowid, size), } - merger = newMerger(mergeHost, sort.RowidLess, sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.RowidLess, sortKeyPos, df) case types.T_Blockid: df := &fixedDataFetcher[types.Blockid]{ mustColFunc: vector.MustFixedColNoTypeCheck[types.Blockid], cols: make([][]types.Blockid, size), } - merger = newMerger(mergeHost, sort.BlockidLess, sortKeyPos, isTombstone, df) + merger = newMerger(mergeHost, sort.BlockidLess, sortKeyPos, df) default: return moerr.NewErrUnsupportedDataType(ctx, typ) } diff --git a/pkg/vm/engine/tae/mergesort/task.go b/pkg/vm/engine/tae/mergesort/task.go index 1c3305f83352d..3b0d0c44bc453 100644 --- a/pkg/vm/engine/tae/mergesort/task.go +++ b/pkg/vm/engine/tae/mergesort/task.go @@ -17,6 +17,7 @@ package mergesort import ( "context" "fmt" + "strings" "time" "github.com/docker/go-units" @@ -29,6 +30,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/api" + v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/blockio" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" "go.uber.org/zap" @@ -88,30 +90,16 @@ func DoMergeAndWrite( txnInfo string, sortkeyPos int, mergehost MergeTaskHost, - isTombstone bool, ) (err error) { - now := time.Now() + start := time.Now() /*out args, keep the transfer information*/ commitEntry := mergehost.GetCommitEntry() - fromObjsDesc := "" - fromSize := uint32(0) - for _, o := range commitEntry.MergedObjs { - obj := objectio.ObjectStats(o) - fromObjsDesc += fmt.Sprintf("%s(%v, %s)Rows(%v),", - obj.ObjectName().ObjectId().ShortStringEx(), - obj.BlkCnt(), - units.BytesSize(float64(obj.OriginSize())), - obj.Rows()) - fromSize += obj.OriginSize() - } - logutil.Info( - "[MERGE-START]", - zap.String("task", mergehost.Name()), - common.AnyField("txn-info", txnInfo), - common.AnyField("host", mergehost.HostHintName()), - common.AnyField("timestamp", commitEntry.StartTs.DebugString()), - zap.String("from-objs", fromObjsDesc), - zap.String("from-size", units.BytesSize(float64(fromSize))), + logMergeStart( + mergehost.Name(), + txnInfo, + mergehost.HostHintName(), + commitEntry.StartTs.DebugString(), + commitEntry.MergedObjs, ) defer func() { if err != nil { @@ -123,13 +111,8 @@ func DoMergeAndWrite( } }() - hasSortKey := sortkeyPos >= 0 - if !hasSortKey { - sortkeyPos = 0 // no sort key, use the first column to do reshape - } - - if hasSortKey { - if err = mergeObjs(ctx, mergehost, sortkeyPos, isTombstone); err != nil { + if sortkeyPos >= 0 { + if err = mergeObjs(ctx, mergehost, sortkeyPos); err != nil { return err } } else { @@ -138,25 +121,7 @@ func DoMergeAndWrite( } } - toObjsDesc := "" - toSize := uint32(0) - for _, o := range commitEntry.CreatedObjs { - obj := objectio.ObjectStats(o) - toObjsDesc += fmt.Sprintf("%s(%v, %s)Rows(%v),", - obj.ObjectName().ObjectId().ShortStringEx(), - obj.BlkCnt(), - units.BytesSize(float64(obj.OriginSize())), - obj.Rows()) - toSize += obj.OriginSize() - } - - logutil.Info( - "[MERGE-END]", - zap.String("task", mergehost.Name()), - common.AnyField("to-objs", toObjsDesc), - common.AnyField("to-size", units.BytesSize(float64(toSize))), - common.DurationField(time.Since(now)), - ) + logMergeEnd(mergehost.Name(), start, commitEntry.CreatedObjs) return nil } @@ -238,3 +203,67 @@ func UpdateMappingAfterMerge(b api.TransferMaps, mapping []int, toLayout []uint3 totalHandledRows += uint32(size) } } + +func logMergeStart(name, txnInfo, host, startTS string, mergedObjs [][]byte) { + var fromObjsDescBuilder strings.Builder + fromSize, estSize := float64(0), float64(0) + rows, blkn := 0, 0 + for _, o := range mergedObjs { + obj := objectio.ObjectStats(o) + fromObjsDescBuilder.WriteString(fmt.Sprintf("%s(%v, %s)Rows(%v)[%v, %v],", + obj.ObjectName().ObjectId().ShortStringEx(), + obj.BlkCnt(), + units.BytesSize(float64(obj.OriginSize())), + obj.Rows(), + obj.SortKeyZoneMap().GetMin(), + obj.SortKeyZoneMap().GetMax())) + fromSize += float64(obj.OriginSize()) + estSize += float64(obj.Rows() * 20) + rows += int(obj.Rows()) + blkn += int(obj.BlkCnt()) + } + + logutil.Info( + "[MERGE-START]", + zap.String("task", name), + common.AnyField("txn-info", txnInfo), + common.AnyField("host", host), + common.AnyField("timestamp", startTS), + zap.String("from-objs", fromObjsDescBuilder.String()), + zap.String("from-size", units.BytesSize(fromSize)), + zap.String("est-size", units.BytesSize(estSize)), + zap.Int("num-obj", len(mergedObjs)), + common.AnyField("num-blk", blkn), + common.AnyField("rows", rows), + ) + + if host == "TN" { + v2.TaskDNMergeScheduledByCounter.Inc() + v2.TaskDNMergedSizeCounter.Add(fromSize) + } else if host == "CN" { + v2.TaskCNMergeScheduledByCounter.Inc() + v2.TaskCNMergedSizeCounter.Add(fromSize) + } +} + +func logMergeEnd(name string, start time.Time, objs [][]byte) { + toObjsDesc := "" + toSize := float64(0) + for _, o := range objs { + obj := objectio.ObjectStats(o) + toObjsDesc += fmt.Sprintf("%s(%v, %s)Rows(%v),", + obj.ObjectName().ObjectId().ShortStringEx(), + obj.BlkCnt(), + units.BytesSize(float64(obj.OriginSize())), + obj.Rows()) + toSize += float64(obj.OriginSize()) + } + + logutil.Info( + "[MERGE-END]", + zap.String("task", name), + common.AnyField("to-objs", toObjsDesc), + common.AnyField("to-size", units.BytesSize(toSize)), + common.DurationField(time.Since(start)), + ) +} diff --git a/pkg/vm/engine/tae/rpc/handle_debug.go b/pkg/vm/engine/tae/rpc/handle_debug.go index b2fc458423a9b..ad3c33dc743af 100644 --- a/pkg/vm/engine/tae/rpc/handle_debug.go +++ b/pkg/vm/engine/tae/rpc/handle_debug.go @@ -569,7 +569,7 @@ func (h *Handle) HandleCommitMerge( stat := objectio.ObjectStats(o) ids = append(ids, *stat.ObjectName().ObjectId()) } - merge.ActiveCNObj.RemoveActiveCNObj(ids) + h.GetDB().MergeScheduler.RemoveCNActiveObjects(ids) if req.Err != "" { resp.ReturnStr = req.Err err = moerr.NewInternalErrorf(ctx, "merge err in cn: %s", req.Err) diff --git a/pkg/vm/engine/tae/rpc/handle_test.go b/pkg/vm/engine/tae/rpc/handle_test.go index c065f30435c32..de28d4a318f94 100644 --- a/pkg/vm/engine/tae/rpc/handle_test.go +++ b/pkg/vm/engine/tae/rpc/handle_test.go @@ -78,7 +78,7 @@ func TestHandleInspectPolicy(t *testing.T) { Operation: "policy", }, resp) require.NoError(t, err) - require.Equal(t, "(*) maxMergeObjN: 16, maxOsizeObj: 128MB, minOsizeQualified: 110MB, offloadToCnSize: 80000MB, hints: [Auto]", resp.Message) + require.Equal(t, "(*) maxMergeObjN: 16, maxOsizeObj: 128MB, minOsizeQualified: 90MB, offloadToCnSize: 80000MB, hints: [Auto]", resp.Message) _, err = handle.HandleInspectTN(context.Background(), txn.TxnMeta{}, &cmd_util.InspectTN{ AccessInfo: cmd_util.AccessInfo{}, @@ -92,7 +92,7 @@ func TestHandleInspectPolicy(t *testing.T) { Operation: "policy -t db1.test1 -s true", }, resp) require.NoError(t, err) - require.Equal(t, "(1000-test1) maxMergeObjN: 16, maxOsizeObj: 128MB, minOsizeQualified: 110MB, offloadToCnSize: 80000MB, hints: [Auto]", resp.Message) + require.Equal(t, "(1000-test1) maxMergeObjN: 16, maxOsizeObj: 128MB, minOsizeQualified: 90MB, offloadToCnSize: 80000MB, hints: [Auto]", resp.Message) _, err = handle.HandleInspectTN(context.Background(), txn.TxnMeta{}, &cmd_util.InspectTN{ AccessInfo: cmd_util.AccessInfo{}, @@ -106,7 +106,7 @@ func TestHandleInspectPolicy(t *testing.T) { Operation: "policy -t db1.test1", }, resp) require.NoError(t, err) - require.Equal(t, "(1000-test1) maxMergeObjN: 16, maxOsizeObj: 128MB, minOsizeQualified: 110MB, offloadToCnSize: 80000MB, hints: [Auto]", resp.Message) + require.Equal(t, "(1000-test1) maxMergeObjN: 16, maxOsizeObj: 128MB, minOsizeQualified: 90MB, offloadToCnSize: 80000MB, hints: [Auto]", resp.Message) } func TestHandlePrecommitWriteError(t *testing.T) { diff --git a/pkg/vm/engine/tae/rpc/inspect.go b/pkg/vm/engine/tae/rpc/inspect.go index bf71ea35256b3..bb612c58a6c15 100644 --- a/pkg/vm/engine/tae/rpc/inspect.go +++ b/pkg/vm/engine/tae/rpc/inspect.go @@ -854,25 +854,22 @@ func (c *mergePolicyArg) Run() error { if err != nil { return err } - err = c.ctx.db.MergeScheduler.ConfigPolicy(c.tbl, txn, &merge.BasicPolicyConfig{ + if err = c.ctx.db.MergeScheduler.ConfigPolicy(c.tbl, txn, &merge.BasicPolicyConfig{ MergeMaxOneRun: int(c.maxMergeObjN), ObjectMinOsize: minosize, MaxOsizeMergedObj: maxosize, MinCNMergeSize: cnsize, MergeHints: c.hints, - }) - if err != nil { + }); err != nil { return err } if c.stopMerge { - err = c.ctx.db.MergeScheduler.StopMerge(c.tbl, false) - if err != nil { + if err = c.ctx.db.MergeScheduler.StopMerge(c.tbl, false); err != nil { return err } } else { if c.ctx.db.Runtime.LockMergeService.IsLockedByUser(c.tbl.GetID(), c.tbl.GetLastestSchema(false).Name) { - err = c.ctx.db.MergeScheduler.StartMerge(c.tbl.GetID(), false) - if err != nil { + if err = c.ctx.db.MergeScheduler.StartMerge(c.tbl.GetID(), false); err != nil { return err } } @@ -967,10 +964,10 @@ func (c *PolicyStatus) String() string { func (c *PolicyStatus) Run() (err error) { if c.pruneAgo == 0 && c.pruneId == 0 { - c.ctx.resp.Payload = []byte(merge.ActiveCNObj.String()) + c.ctx.resp.Payload = []byte(c.ctx.db.MergeScheduler.CNActiveObjectsString()) return nil } else { - merge.ActiveCNObj.Prune(c.pruneId, c.pruneAgo) + c.ctx.db.MergeScheduler.PruneCNActiveObjects(c.pruneId, c.pruneAgo) return nil } } diff --git a/pkg/vm/engine/tae/tables/jobs/mergeobjects.go b/pkg/vm/engine/tae/tables/jobs/mergeobjects.go index 84176a8c0e810..5636a586ad794 100644 --- a/pkg/vm/engine/tae/tables/jobs/mergeobjects.go +++ b/pkg/vm/engine/tae/tables/jobs/mergeobjects.go @@ -188,7 +188,7 @@ func (task *mergeObjectsTask) GetMPool() *mpool.MPool { return task.rt.VectorPool.Transient.GetMPool() } -func (task *mergeObjectsTask) HostHintName() string { return "DN" } +func (task *mergeObjectsTask) HostHintName() string { return "TN" } func (task *mergeObjectsTask) LoadNextBatch( ctx context.Context, objIdx uint32, @@ -369,7 +369,7 @@ func (task *mergeObjectsTask) Execute(ctx context.Context) (err error) { return moerr.NewInternalErrorNoCtxf("LockMerge give up in exec %v", task.Name()) } phaseDesc = "1-DoMergeAndWrite" - if err = mergesort.DoMergeAndWrite(ctx, task.txn.String(), sortkeyPos, task, task.isTombstone); err != nil { + if err = mergesort.DoMergeAndWrite(ctx, task.txn.String(), sortkeyPos, task); err != nil { return err } diff --git a/pkg/vm/engine/tae/tables/txnentries/mergeobjects.go b/pkg/vm/engine/tae/tables/txnentries/mergeobjects.go index c61335d769019..e62627bc06eaa 100644 --- a/pkg/vm/engine/tae/tables/txnentries/mergeobjects.go +++ b/pkg/vm/engine/tae/tables/txnentries/mergeobjects.go @@ -289,9 +289,9 @@ func (entry *mergeObjectsEntry) transferObjectDeletes( _max = k } } - panic(fmt.Sprintf( - "%s-%d find no transfer mapping for row %d, mapping range (%d, %d)", - dropped.ID().String(), blkOffsetInObj, row, _min, _max)) + err = moerr.NewInternalErrorNoCtxf("%s-%d find no transfer mapping for row %d, mapping range (%d, %d)", + dropped.ID().String(), blkOffsetInObj, row, _min, _max) + return } if entry.delTbls[*entry.createdObjs[destpos.ObjIdx].ID()] == nil { entry.delTbls[*entry.createdObjs[destpos.ObjIdx].ID()] = make(map[uint16]struct{}) diff --git a/pkg/vm/engine/tae/tasks/worker/worker.go b/pkg/vm/engine/tae/tasks/worker/worker.go index c8a33b849151d..60b420d96e485 100644 --- a/pkg/vm/engine/tae/tasks/worker/worker.go +++ b/pkg/vm/engine/tae/tasks/worker/worker.go @@ -29,7 +29,8 @@ import ( type Cmd = uint8 const ( - QUIT Cmd = iota + QUIT Cmd = iota + stuck // For tests only. Stop executing Ops. ) const ( @@ -198,8 +199,13 @@ func (w *OpWorker) SendOp(op iops.IOp) bool { w.Pending.Add(-1) return false } - w.OpC <- op - return true + select { + case w.OpC <- op: + return true + default: + } + w.Pending.Add(-1) + return false } func (w *OpWorker) opCancelOp(op iops.IOp) { @@ -228,6 +234,9 @@ func (w *OpWorker) onCmd(cmd Cmd) { panic("logic error") } w.ClosedCh <- struct{}{} + case stuck: // For test only + <-w.Ctx.Done() + return default: panic(fmt.Sprintf("Unsupported cmd %d", cmd)) } diff --git a/pkg/vm/engine/tae/tasks/worker/worker_test.go b/pkg/vm/engine/tae/tasks/worker/worker_test.go new file mode 100644 index 0000000000000..dd302273c7bd4 --- /dev/null +++ b/pkg/vm/engine/tae/tasks/worker/worker_test.go @@ -0,0 +1,75 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ops + +import ( + "context" + iops "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/tasks/ops/base" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +type testOp struct{} + +func (t testOp) OnExec(context.Context) error { + return nil +} + +func (t testOp) SetError(error) {} + +func (t testOp) GetError() error { + return nil +} + +func (t testOp) WaitDone(context.Context) error { + return nil +} + +func (t testOp) Waitable() bool { + return true +} + +func (t testOp) GetCreateTime() time.Time { + return time.Time{} +} + +func (t testOp) GetStartTime() time.Time { + return time.Time{} +} + +func (t testOp) GetEndTime() time.Time { + return time.Time{} +} + +func (t testOp) GetExecutTime() int64 { + return 0 +} + +func (t testOp) AddObserver(iops.Observer) {} + +func TestOpWorker(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + worker := NewOpWorker(ctx, "test", 100) + worker.Start() + worker.CmdC <- stuck + for len(worker.CmdC) > 0 { + } + for i := 0; i < 100; i++ { + require.True(t, worker.SendOp(testOp{})) + } + require.Falsef(t, worker.SendOp(testOp{}), "op channel should be full") +} diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_merge.result b/test/distributed/cases/function/mo_ctl/mo_ctl_merge.result index 0fb7b1408e35c..c0f19dbdd2a88 100644 --- a/test/distributed/cases/function/mo_ctl/mo_ctl_merge.result +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_merge.result @@ -242,4 +242,4 @@ rows_cnt 2 select mo_ctl('dn', 'inspect', 'policy'); mo_ctl(dn, inspect, policy) -\nmsg: (*) maxMergeObjN: 16, maxOsizeObj: 128MB, minOsizeQualified: 110MB, offloadToCnSize: 80000MB, hints: [Auto]\n\ngeneral setting has been refreshed +\nmsg: (*) maxMergeObjN: 16, maxOsizeObj: 128MB, minOsizeQualified: 90MB, offloadToCnSize: 80000MB, hints: [Auto]\n\ngeneral setting has been refreshed