diff --git a/pkg/frontend/pitr.go b/pkg/frontend/pitr.go index ff558e8283124..2219fb8c6a40b 100644 --- a/pkg/frontend/pitr.go +++ b/pkg/frontend/pitr.go @@ -891,11 +891,12 @@ func doRestorePitr(ctx context.Context, ses *Session, stmt *tree.RestorePitr) (s }() var ( - restoreLevel tree.RestoreLevel - ts int64 - pitrExist bool - sortedFkTbls []string - fkTableMap map[string]*tableInfo + restoreLevel tree.RestoreLevel + ts int64 + pitrExist bool + sortedFkTbls []string + fkTableMap map[string]*tableInfo + accountRecord *accountRecord ) // resolve timestamp ts, err = doResolveTimeStamp(stmt.TimeStamp) @@ -909,6 +910,9 @@ func doRestorePitr(ctx context.Context, ses *Session, stmt *tree.RestorePitr) (s dbName := string(stmt.DatabaseName) tblName := string(stmt.TableName) + isClusterRestore := false + isNeedToCleanToDatabase := true + // restore as a txn if err = bh.Exec(ctx, "begin;"); err != nil { return stats, err @@ -953,37 +957,61 @@ func doRestorePitr(ctx context.Context, ses *Session, stmt *tree.RestorePitr) (s restoreOtherAccount := func() (rtnErr error) { fromAccount := string(stmt.SrcAccountName) var ( - fromAccountId uint32 - toAccountId uint32 + toAccountId uint32 ) if len(fromAccount) == 0 { + // using account level pitr fromAccount = pitr.accountName - fromAccountId = uint32(pitr.accountId) + accountRecord, rtnErr = getAccountRecordByTs(ctx, ses, bh, pitrName, ts, fromAccount) + if rtnErr != nil { + return + } if fromAccount == accountName { // restore to the same account - toAccountId = fromAccountId + getLogger(ses.GetService()).Info("restore to the same account", zap.String("fromAccount", accountName), zap.String("toAccount", accountName)) + toAccountId, rtnErr = getAccountId(ctx, bh, accountName) + if rtnErr != nil { + // need create a new account + if rtnErr = createDroppedAccount(ctx, ses, bh, pitrName, *accountRecord); rtnErr != nil { + return + } + + if toAccountId, rtnErr = getAccountId(ctx, bh, accountRecord.accountName); rtnErr != nil { + return + } + } } else { // restore to new account + getLogger(ses.GetService()).Info("restore to the same account", zap.String("fromAccount", fromAccount), zap.String("toAccount", accountName)) toAccountId, rtnErr = getAccountId(ctx, bh, accountName) if rtnErr != nil { - return rtnErr + return } } } else { + // using cluster level pitr + accountRecord, rtnErr = getAccountRecordByTs(ctx, ses, bh, pitrName, ts, fromAccount) + if rtnErr != nil { + return + } if fromAccount == accountName { // restore to the same account - fromAccountId, rtnErr = getAccountId(ctx, bh, fromAccount) + getLogger(ses.GetService()).Info("restore to the same account", zap.String("fromAccount", accountName), zap.String("toAccount", accountName)) + toAccountId, rtnErr = getAccountId(ctx, bh, fromAccount) if rtnErr != nil { - return rtnErr + // need create a new account + if rtnErr = createDroppedAccount(ctx, ses, bh, pitrName, *accountRecord); rtnErr != nil { + return + } + + if toAccountId, rtnErr = getAccountId(ctx, bh, accountRecord.accountName); rtnErr != nil { + return + } } - toAccountId = fromAccountId } else { // restore to new account - fromAccountId, rtnErr = getAccountId(ctx, bh, fromAccount) - if rtnErr != nil { - return rtnErr - } + getLogger(ses.GetService()).Info("restore to the same account", zap.String("fromAccount", fromAccount), zap.String("toAccount", accountName)) toAccountId, rtnErr = getAccountId(ctx, bh, accountName) if rtnErr != nil { return rtnErr @@ -992,62 +1020,22 @@ func doRestorePitr(ctx context.Context, ses *Session, stmt *tree.RestorePitr) (s } // check account exists or not - var accountExist bool - if accountExist, rtnErr = doCheckAccountExistsInPitrRestore(ctx, ses.GetService(), bh, pitrName, ts, fromAccount, ses.GetAccountId()); rtnErr != nil { - return rtnErr - } - if !accountExist { - return moerr.NewInternalErrorf(ctx, "account `%s` does not exists at timestamp: %v", fromAccount, nanoTimeFormat(ts)) - } - // mock snapshot - var snapshotName string - snapshotName, rtnErr = insertSnapshotRecord(ctx, ses.GetService(), bh, pitrName, ts, uint64(fromAccountId), fromAccount) - defer func() { - deleteSnapshotRecord(ctx, ses.GetService(), bh, pitrName, snapshotName) - }() - if rtnErr != nil { - return rtnErr - } - - restoreAccount := fromAccountId - // drop foreign key related tables first - rtnErr = deleteCurFkTables(ctx, ses.GetService(), bh, dbName, tblName, toAccountId) - if err != nil { - return - } - - // get topo sorted tables with foreign key - sortedFkTbls, rtnErr = fkTablesTopoSort(ctx, bh, snapshotName, dbName, tblName) - if rtnErr != nil { - return - } - - // get foreign key table infos - fkTableMap, rtnErr = getTableInfoMap(ctx, ses.GetService(), bh, snapshotName, dbName, tblName, sortedFkTbls) - if rtnErr != nil { - return - } - - // collect views and tables during table restoration - viewMap := make(map[string]*tableInfo) - - rtnErr = restoreToAccount(ctx, ses.GetService(), bh, snapshotName, toAccountId, fkTableMap, viewMap, ts, restoreAccount, false, nil) + rtnErr = restoreAccountUsingClusterSnapshotToNew( + ctx, + ses, + bh, + pitrName, + ts, + *accountRecord, + uint64(toAccountId), + nil, + isClusterRestore, + isNeedToCleanToDatabase, + ) if rtnErr != nil { return rtnErr } - if len(fkTableMap) > 0 { - if rtnErr = restoreTablesWithFk(ctx, ses.GetService(), bh, snapshotName, sortedFkTbls, fkTableMap, toAccountId, ts); rtnErr != nil { - return - } - } - - if len(viewMap) > 0 { - if rtnErr = restoreViews(ctx, ses, bh, snapshotName, viewMap, toAccountId); rtnErr != nil { - return - } - } - // checks if the given context has been canceled. if rtnErr = CancelCheck(ctx); rtnErr != nil { return } diff --git a/pkg/frontend/pitr_test.go b/pkg/frontend/pitr_test.go index ff27c5eec708f..d7bf4ba10ef38 100644 --- a/pkg/frontend/pitr_test.go +++ b/pkg/frontend/pitr_test.go @@ -3037,3 +3037,363 @@ func Test_restoreViewsWithPitr(t *testing.T) { assert.NoError(t, err) }) } + +func Test_RestoreOtherAccount(t *testing.T) { + convey.Convey("doRestorePitr fail", t, func() { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ses := newTestSession(t, ctrl) + defer ses.Close() + + bh := &backgroundExecTest{} + bh.init() + + bhStub := gostub.StubFunc(&NewBackgroundExec, bh) + defer bhStub.Reset() + + pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) + pu.SV.SetDefaultValues() + pu.SV.KillRountinesInterval = 0 + setPu("", pu) + ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) + rm, _ := NewRoutineManager(ctx, "") + ses.rm = rm + + tenant := &TenantInfo{ + Tenant: sysAccountName, + User: rootName, + DefaultRole: moAdminRoleName, + TenantID: sysAccountID, + UserID: rootID, + DefaultRoleID: moAdminRoleID, + } + ses.SetTenantInfo(tenant) + + ts := time.Now().Add(time.Duration(-2) * time.Hour).UnixNano() + stmt := &tree.RestorePitr{ + Level: tree.RESTORELEVELACCOUNT, + Name: "pitr01", + + AccountName: "acc01", + TimeStamp: nanoTimeFormat(ts), + } + + ses.SetTenantInfo(tenant) + ctx = context.WithValue(ctx, defines.TenantIDKey{}, uint32(sysAccountID)) + + //no result set + bh.sql2result["begin;"] = nil + bh.sql2result["commit;"] = nil + bh.sql2result["rollback;"] = nil + + sql, err := getSqlForCheckPitr(ctx, "pitr01", sysAccountID) + assert.NoError(t, err) + mrs := newMrsForPitrRecord([][]interface{}{{"018ee4cd-5991-7caa-b75d-f9290144bd9f"}}) + bh.sql2result[sql] = mrs + + sql = "select * from mo_catalog.mo_pitr where pitr_name = 'pitr01' and create_account = 0" + mrs = newMrsForPitrRecord([][]interface{}{{ + "018ee4cd-5991-7caa-b75d-f9290144bd9f", + "pitr01", + uint64(0), + "2024-05-01 00:00:00", + "2024-05-01 00:00:00", + "ACCOUNT", + uint64(1), + "acc01", + "", + "", + uint64(1), + uint8(1), + "d", + }}) + bh.sql2result[sql] = mrs + + resovleTs, err := doResolveTimeStamp(stmt.TimeStamp) + assert.NoError(t, err) + + _, err = doRestorePitr(ctx, ses, stmt) + assert.Error(t, err) + + sql = fmt.Sprintf("select account_id, account_name, admin_name, comments from mo_catalog.mo_account {MO_TS = %d } where account_name = '%s';", resovleTs, "acc01") + mrs = newMrsForPitrRecord([][]interface{}{{uint64(1), "acc01", "root", ""}}) + bh.sql2result[sql] = mrs + + sql = "select account_id, account_name, status, version, suspended_time from mo_catalog.mo_account where 1=1 and account_name = 'acc01'" + mrs = newMrsForPitrRecord([][]interface{}{{uint64(1), "acc01", "open", uint64(1), nil}}) + bh.sql2result[sql] = mrs + + _, err = doRestorePitr(ctx, ses, stmt) + assert.Error(t, err) + }) + + convey.Convey("doRestorePitr fail", t, func() { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ses := newTestSession(t, ctrl) + defer ses.Close() + + bh := &backgroundExecTest{} + bh.init() + + bhStub := gostub.StubFunc(&NewBackgroundExec, bh) + defer bhStub.Reset() + + pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) + pu.SV.SetDefaultValues() + pu.SV.KillRountinesInterval = 0 + setPu("", pu) + ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) + rm, _ := NewRoutineManager(ctx, "") + ses.rm = rm + + tenant := &TenantInfo{ + Tenant: sysAccountName, + User: rootName, + DefaultRole: moAdminRoleName, + TenantID: sysAccountID, + UserID: rootID, + DefaultRoleID: moAdminRoleID, + } + ses.SetTenantInfo(tenant) + + ts := time.Now().Add(time.Duration(-2) * time.Hour).UnixNano() + stmt := &tree.RestorePitr{ + Level: tree.RESTORELEVELACCOUNT, + Name: "pitr01", + + AccountName: "acc02", + TimeStamp: nanoTimeFormat(ts), + } + + ses.SetTenantInfo(tenant) + ctx = context.WithValue(ctx, defines.TenantIDKey{}, uint32(sysAccountID)) + + //no result set + bh.sql2result["begin;"] = nil + bh.sql2result["commit;"] = nil + bh.sql2result["rollback;"] = nil + + sql, err := getSqlForCheckPitr(ctx, "pitr01", sysAccountID) + assert.NoError(t, err) + mrs := newMrsForPitrRecord([][]interface{}{{"018ee4cd-5991-7caa-b75d-f9290144bd9f"}}) + bh.sql2result[sql] = mrs + + sql = "select * from mo_catalog.mo_pitr where pitr_name = 'pitr01' and create_account = 0" + mrs = newMrsForPitrRecord([][]interface{}{{ + "018ee4cd-5991-7caa-b75d-f9290144bd9f", + "pitr01", + uint64(0), + "2024-05-01 00:00:00", + "2024-05-01 00:00:00", + "ACCOUNT", + uint64(1), + "acc01", + "", + "", + uint64(1), + uint8(1), + "d", + }}) + bh.sql2result[sql] = mrs + + resovleTs, err := doResolveTimeStamp(stmt.TimeStamp) + assert.NoError(t, err) + + _, err = doRestorePitr(ctx, ses, stmt) + assert.Error(t, err) + + sql = fmt.Sprintf("select account_id, account_name, admin_name, comments from mo_catalog.mo_account {MO_TS = %d } where account_name = '%s';", resovleTs, "acc01") + mrs = newMrsForPitrRecord([][]interface{}{{uint64(1), "acc01", "root", ""}}) + bh.sql2result[sql] = mrs + + sql = "select account_id, account_name, status, version, suspended_time from mo_catalog.mo_account where 1=1 and account_name = 'acc01'" + mrs = newMrsForPitrRecord([][]interface{}{{uint64(1), "acc01", "open", uint64(1), nil}}) + bh.sql2result[sql] = mrs + + _, err = doRestorePitr(ctx, ses, stmt) + assert.Error(t, err) + }) + + convey.Convey("doRestorePitr fail", t, func() { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ses := newTestSession(t, ctrl) + defer ses.Close() + + bh := &backgroundExecTest{} + bh.init() + + bhStub := gostub.StubFunc(&NewBackgroundExec, bh) + defer bhStub.Reset() + + pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) + pu.SV.SetDefaultValues() + pu.SV.KillRountinesInterval = 0 + setPu("", pu) + ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) + rm, _ := NewRoutineManager(ctx, "") + ses.rm = rm + + tenant := &TenantInfo{ + Tenant: sysAccountName, + User: rootName, + DefaultRole: moAdminRoleName, + TenantID: sysAccountID, + UserID: rootID, + DefaultRoleID: moAdminRoleID, + } + ses.SetTenantInfo(tenant) + + ts := time.Now().Add(time.Duration(-2) * time.Hour).UnixNano() + stmt := &tree.RestorePitr{ + Level: tree.RESTORELEVELACCOUNT, + Name: "pitr01", + + AccountName: "acc01", + SrcAccountName: "acc01", + TimeStamp: nanoTimeFormat(ts), + } + + ses.SetTenantInfo(tenant) + ctx = context.WithValue(ctx, defines.TenantIDKey{}, uint32(sysAccountID)) + + //no result set + bh.sql2result["begin;"] = nil + bh.sql2result["commit;"] = nil + bh.sql2result["rollback;"] = nil + + sql, err := getSqlForCheckPitr(ctx, "pitr01", sysAccountID) + assert.NoError(t, err) + mrs := newMrsForPitrRecord([][]interface{}{{"018ee4cd-5991-7caa-b75d-f9290144bd9f"}}) + bh.sql2result[sql] = mrs + + sql = "select * from mo_catalog.mo_pitr where pitr_name = 'pitr01' and create_account = 0" + mrs = newMrsForPitrRecord([][]interface{}{{ + "018ee4cd-5991-7caa-b75d-f9290144bd9f", + "pitr01", + uint64(0), + "2024-05-01 00:00:00", + "2024-05-01 00:00:00", + "CLUSTER", + uint64(1), + "acc01", + "", + "", + uint64(1), + uint8(1), + "d", + }}) + bh.sql2result[sql] = mrs + + resovleTs, err := doResolveTimeStamp(stmt.TimeStamp) + assert.NoError(t, err) + + _, err = doRestorePitr(ctx, ses, stmt) + assert.Error(t, err) + + sql = fmt.Sprintf("select account_id, account_name, admin_name, comments from mo_catalog.mo_account {MO_TS = %d } where account_name = '%s';", resovleTs, "acc01") + mrs = newMrsForPitrRecord([][]interface{}{{uint64(1), "acc01", "root", ""}}) + bh.sql2result[sql] = mrs + + sql = "select account_id, account_name, status, version, suspended_time from mo_catalog.mo_account where 1=1 and account_name = 'acc01'" + mrs = newMrsForPitrRecord([][]interface{}{{uint64(1), "acc01", "open", uint64(1), nil}}) + bh.sql2result[sql] = mrs + + _, err = doRestorePitr(ctx, ses, stmt) + assert.Error(t, err) + }) + + convey.Convey("doRestorePitr fail", t, func() { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ses := newTestSession(t, ctrl) + defer ses.Close() + + bh := &backgroundExecTest{} + bh.init() + + bhStub := gostub.StubFunc(&NewBackgroundExec, bh) + defer bhStub.Reset() + + pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) + pu.SV.SetDefaultValues() + pu.SV.KillRountinesInterval = 0 + setPu("", pu) + ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) + rm, _ := NewRoutineManager(ctx, "") + ses.rm = rm + + tenant := &TenantInfo{ + Tenant: sysAccountName, + User: rootName, + DefaultRole: moAdminRoleName, + TenantID: sysAccountID, + UserID: rootID, + DefaultRoleID: moAdminRoleID, + } + ses.SetTenantInfo(tenant) + + ts := time.Now().Add(time.Duration(-2) * time.Hour).UnixNano() + stmt := &tree.RestorePitr{ + Level: tree.RESTORELEVELACCOUNT, + Name: "pitr01", + + AccountName: "acc02", + SrcAccountName: "acc01", + TimeStamp: nanoTimeFormat(ts), + } + + ses.SetTenantInfo(tenant) + ctx = context.WithValue(ctx, defines.TenantIDKey{}, uint32(sysAccountID)) + + //no result set + bh.sql2result["begin;"] = nil + bh.sql2result["commit;"] = nil + bh.sql2result["rollback;"] = nil + + sql, err := getSqlForCheckPitr(ctx, "pitr01", sysAccountID) + assert.NoError(t, err) + mrs := newMrsForPitrRecord([][]interface{}{{"018ee4cd-5991-7caa-b75d-f9290144bd9f"}}) + bh.sql2result[sql] = mrs + + sql = "select * from mo_catalog.mo_pitr where pitr_name = 'pitr01' and create_account = 0" + mrs = newMrsForPitrRecord([][]interface{}{{ + "018ee4cd-5991-7caa-b75d-f9290144bd9f", + "pitr01", + uint64(0), + "2024-05-01 00:00:00", + "2024-05-01 00:00:00", + "CLUSTER", + uint64(1), + "acc01", + "", + "", + uint64(1), + uint8(1), + "d", + }}) + bh.sql2result[sql] = mrs + + resovleTs, err := doResolveTimeStamp(stmt.TimeStamp) + assert.NoError(t, err) + + _, err = doRestorePitr(ctx, ses, stmt) + assert.Error(t, err) + + sql = fmt.Sprintf("select account_id, account_name, admin_name, comments from mo_catalog.mo_account {MO_TS = %d } where account_name = '%s';", resovleTs, "acc01") + mrs = newMrsForPitrRecord([][]interface{}{{uint64(1), "acc01", "root", ""}}) + bh.sql2result[sql] = mrs + + sql = "select account_id, account_name, status, version, suspended_time from mo_catalog.mo_account where 1=1 and account_name = 'acc01'" + mrs = newMrsForPitrRecord([][]interface{}{{uint64(1), "acc01", "open", uint64(1), nil}}) + bh.sql2result[sql] = mrs + + _, err = doRestorePitr(ctx, ses, stmt) + assert.Error(t, err) + }) +} diff --git a/pkg/frontend/snapshot.go b/pkg/frontend/snapshot.go index 426b6c41ce731..d231cec7deba9 100644 --- a/pkg/frontend/snapshot.go +++ b/pkg/frontend/snapshot.go @@ -1706,7 +1706,7 @@ func restoreToCluster(ctx context.Context, return err } if newAccountId != uint32(account.accountId) { - if err = restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, snapshotName, snapshotTs, account, subDbToRestore, uint64(newAccountId), true, true); err != nil { + if err = restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, snapshotName, snapshotTs, account, uint64(newAccountId), subDbToRestore, true, true); err != nil { return err } } else { @@ -1739,7 +1739,7 @@ func restoreToCluster(ctx context.Context, } // 2.0 restore droped account to new account - err = restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, snapshotName, snapshotTs, account, subDbToRestore, uint64(newAccountId), true, false) + err = restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, snapshotName, snapshotTs, account, uint64(newAccountId), subDbToRestore, true, false) if err != nil { return err } @@ -1794,7 +1794,7 @@ func restoreToAccountUsingCluster( } } - err = restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, snapshotName, snapshotTs, *ar, nil, uint64(toAccountId), false, isNeedToCleanToDatabase) + err = restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, snapshotName, snapshotTs, *ar, uint64(toAccountId), nil, false, isNeedToCleanToDatabase) if err != nil { return err } @@ -1966,8 +1966,8 @@ func restoreAccountUsingClusterSnapshotToNew(ctx context.Context, snapshotName string, snapshotTs int64, account accountRecord, - subDbToRestore map[string]*subDbRestoreRecord, toAccountId uint64, + subDbToRestore map[string]*subDbRestoreRecord, isRestoreCluster bool, isNeedToCleanToDatabase bool, ) (err error) { @@ -1985,12 +1985,12 @@ func restoreAccountUsingClusterSnapshotToNew(ctx context.Context, // get topo sorted tables with foreign key var sortedFkTbls []string var fkTableMap map[string]*tableInfo - sortedFkTbls, err = fkTablesTopoSortWithDropped(ctx, bh, "", "", snapshotTs, uint32(fromAccount), uint32(toAccountId)) + sortedFkTbls, err = fkTablesTopoSortWithTS(ctx, bh, "", "", snapshotTs, uint32(fromAccount), uint32(toAccountId)) if err != nil { return err } // get foreign key table infos - fkTableMap, err = getTableInfoMapFromDropped(ctx, ses.GetService(), bh, "", "", sortedFkTbls, snapshotTs, uint32(fromAccount), uint32(toAccountId)) + fkTableMap, err = getTableInfoMapFromTS(ctx, ses.GetService(), bh, "", "", sortedFkTbls, snapshotTs, uint32(fromAccount), uint32(toAccountId)) if err != nil { return err } @@ -1999,7 +1999,7 @@ func restoreAccountUsingClusterSnapshotToNew(ctx context.Context, viewMap := make(map[string]*tableInfo) // restore to account - if err = restoreToAccountFromDropped( + if err = restoreToAccountFromTS( ctx, ses.GetService(), bh, @@ -2014,7 +2014,7 @@ func restoreAccountUsingClusterSnapshotToNew(ctx context.Context, } if len(fkTableMap) > 0 { - if err = restoreTablesWithFkFromDropped(ctx, + if err = restoreTablesWithFkFromTS(ctx, ses.GetService(), bh, snapshotTs, @@ -2027,7 +2027,7 @@ func restoreAccountUsingClusterSnapshotToNew(ctx context.Context, } if len(viewMap) > 0 { - if err = restoreViewsFromDropped( + if err = restoreViewsFromTS( ctx, ses, bh, diff --git a/pkg/frontend/snapshot_restore_with_ts.go b/pkg/frontend/snapshot_restore_with_ts.go index e8e6deafcfda6..f30aff087a415 100644 --- a/pkg/frontend/snapshot_restore_with_ts.go +++ b/pkg/frontend/snapshot_restore_with_ts.go @@ -29,11 +29,11 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/plan" ) -func fkTablesTopoSortWithDropped(ctx context.Context, bh BackgroundExec, dbName string, tblName string, ts int64, from, to uint32) (sortedTbls []string, err error) { +func fkTablesTopoSortWithTS(ctx context.Context, bh BackgroundExec, dbName string, tblName string, ts int64, from, to uint32) (sortedTbls []string, err error) { newCtx := defines.AttachAccountId(ctx, from) getLogger("").Info(fmt.Sprintf("[%d:%d] start to get fk tables topo sort from account %d", from, ts, from)) // get foreign key deps from mo_catalog.mo_foreign_keys - fkDeps, err := getFkDepsWithDropped(newCtx, bh, dbName, tblName, ts, from, to) + fkDeps, err := getFkDepsWithTS(newCtx, bh, dbName, tblName, ts, from, to) if err != nil { return } @@ -52,7 +52,7 @@ func fkTablesTopoSortWithDropped(ctx context.Context, bh BackgroundExec, dbName return } -func getFkDepsWithDropped(ctx context.Context, bh BackgroundExec, db string, tbl string, ts int64, from, to uint32) (ans map[string][]string, err error) { +func getFkDepsWithTS(ctx context.Context, bh BackgroundExec, db string, tbl string, ts int64, from, to uint32) (ans map[string][]string, err error) { sql := "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" if ts > 0 { sql += fmt.Sprintf(" {MO_TS = %d}", ts) @@ -103,7 +103,7 @@ func getFkDepsWithDropped(ctx context.Context, bh BackgroundExec, db string, tbl return } -func getTableInfoMapFromDropped( +func getTableInfoMapFromTS( ctx context.Context, sid string, bh BackgroundExec, @@ -135,14 +135,14 @@ func getTableInfoMapFromDropped( continue } - if tblInfoMap[key], err = getTableInfoFromDropped(newCtx, sid, bh, d, t, ts, from, to); err != nil { + if tblInfoMap[key], err = getTableInfoFromTS(newCtx, sid, bh, d, t, ts, from, to); err != nil { return } } return } -func getTableInfoFromDropped(ctx context.Context, +func getTableInfoFromTS(ctx context.Context, sid string, bh BackgroundExec, dbName, @@ -151,7 +151,7 @@ func getTableInfoFromDropped(ctx context.Context, from, to uint32) (*tableInfo, error) { getLogger(sid).Info(fmt.Sprintf("[%d:%d]start to get table info: datatabse `%s`, table `%s`", from, ts, dbName, tblName)) - tableInfos, err := getTableInfosFromDropped(ctx, sid, bh, dbName, tblName, ts, from, to) + tableInfos, err := getTableInfosFromTS(ctx, sid, bh, dbName, tblName, ts, from, to) if err != nil { return nil, err } @@ -163,7 +163,7 @@ func getTableInfoFromDropped(ctx context.Context, return tableInfos[0], nil } -func getTableInfosFromDropped(ctx context.Context, +func getTableInfosFromTS(ctx context.Context, sid string, bh BackgroundExec, dbName string, @@ -173,7 +173,7 @@ func getTableInfosFromDropped(ctx context.Context, to uint32) ([]*tableInfo, error) { newCtx := defines.AttachAccountId(ctx, from) getLogger(sid).Info(fmt.Sprintf("[%d:%d] start to get table info: datatabse `%s`, table `%s`", from, ts, dbName, tblName)) - tableInfos, err := showFullTablesFromDropped(newCtx, sid, bh, dbName, tblName, ts, from, to) + tableInfos, err := showFullTablesFromTS(newCtx, sid, bh, dbName, tblName, ts, from, to) if err != nil { return nil, err } @@ -181,7 +181,7 @@ func getTableInfosFromDropped(ctx context.Context, // only recreate snapshoted table need create sql if ts > 0 { for _, tblInfo := range tableInfos { - if tblInfo.createSql, err = getCreateTableSqlFromDropped(newCtx, bh, dbName, tblInfo.tblName, ts, from, to); err != nil { + if tblInfo.createSql, err = getCreateTableSqlFromTS(newCtx, bh, dbName, tblInfo.tblName, ts, from, to); err != nil { return nil, err } } @@ -189,7 +189,7 @@ func getTableInfosFromDropped(ctx context.Context, return tableInfos, nil } -func showFullTablesFromDropped(ctx context.Context, +func showFullTablesFromTS(ctx context.Context, sid string, bh BackgroundExec, dbName string, @@ -211,7 +211,7 @@ func showFullTablesFromDropped(ctx context.Context, getLogger(sid).Info(fmt.Sprintf("[%d:%d] show full table `%s.%s` sql: %s", from, ts, dbName, tblName, sql)) // cols: table name, table type - colsList, err := getStringColsListFromDropped(newCtx, bh, sql, from, to, 0, 1) + colsList, err := getStringColsListFromTS(newCtx, bh, sql, from, to, 0, 1) if err != nil { return nil, err } @@ -228,7 +228,7 @@ func showFullTablesFromDropped(ctx context.Context, return ans, nil } -func getStringColsListFromDropped(ctx context.Context, bh BackgroundExec, sql string, from, to uint32, colIndices ...uint64) (ans [][]string, err error) { +func getStringColsListFromTS(ctx context.Context, bh BackgroundExec, sql string, from, to uint32, colIndices ...uint64) (ans [][]string, err error) { bh.ClearExecResultSet() if err = bh.ExecRestore(ctx, sql, from, to); err != nil { return @@ -253,7 +253,7 @@ func getStringColsListFromDropped(ctx context.Context, bh BackgroundExec, sql st return } -func getCreateTableSqlFromDropped(ctx context.Context, bh BackgroundExec, dbName string, tblName string, ts int64, from, to uint32) (string, error) { +func getCreateTableSqlFromTS(ctx context.Context, bh BackgroundExec, dbName string, tblName string, ts int64, from, to uint32) (string, error) { getLogger("").Info(fmt.Sprintf("[%d:%d] start to get create table sql: datatabse `%s`, table `%s`", from, ts, dbName, tblName)) newCtx := defines.AttachAccountId(ctx, from) sql := fmt.Sprintf("show create table `%s`.`%s`", dbName, tblName) @@ -262,7 +262,7 @@ func getCreateTableSqlFromDropped(ctx context.Context, bh BackgroundExec, dbName } // cols: table_name, create_sql - colsList, err := getStringColsListFromDropped(newCtx, bh, sql, from, to, 1) + colsList, err := getStringColsListFromTS(newCtx, bh, sql, from, to, 1) if err != nil { return "", err } @@ -272,7 +272,7 @@ func getCreateTableSqlFromDropped(ctx context.Context, bh BackgroundExec, dbName return colsList[0][0], nil } -func restoreToAccountFromDropped( +func restoreToAccountFromTS( ctx context.Context, sid string, bh BackgroundExec, @@ -312,7 +312,7 @@ func restoreToAccountFromDropped( } // restore dbs - if dbNames, err = showDatabasesFromDropped(ctx, sid, bh, snapshotTs, restoreAccount, toAccountId); err != nil { + if dbNames, err = showDatabasesFromTS(ctx, sid, bh, snapshotTs, restoreAccount, toAccountId); err != nil { return } @@ -321,7 +321,7 @@ func restoreToAccountFromDropped( getLogger(sid).Info(fmt.Sprintf("[%d:%d] skip restore db: %v", restoreAccount, snapshotTs, dbName)) continue } - if err = restoreDatabaseFromDropped(ctx, + if err = restoreDatabaseFromTS(ctx, sid, bh, dbName, @@ -337,7 +337,7 @@ func restoreToAccountFromDropped( } // restore system db - if err = restoreSystemDatabaseFromDropped(ctx, + if err = restoreSystemDatabaseFromTS(ctx, sid, bh, snapshotTs, @@ -349,7 +349,7 @@ func restoreToAccountFromDropped( return } -func showDatabasesFromDropped(ctx context.Context, sid string, bh BackgroundExec, ts int64, from, to uint32) ([]string, error) { +func showDatabasesFromTS(ctx context.Context, sid string, bh BackgroundExec, ts int64, from, to uint32) ([]string, error) { getLogger(sid).Info(fmt.Sprintf("[%d:%d] start to get all database ", from, ts)) newCtx := defines.AttachAccountId(ctx, from) sql := "show databases" @@ -358,7 +358,7 @@ func showDatabasesFromDropped(ctx context.Context, sid string, bh BackgroundExec } // cols: dbname - colsList, err := getStringColsListFromDropped(newCtx, bh, sql, from, to, 0) + colsList, err := getStringColsListFromTS(newCtx, bh, sql, from, to, 0) if err != nil { return nil, err } @@ -370,7 +370,7 @@ func showDatabasesFromDropped(ctx context.Context, sid string, bh BackgroundExec return dbNames, nil } -func restoreDatabaseFromDropped( +func restoreDatabaseFromTS( ctx context.Context, sid string, bh BackgroundExec, @@ -386,7 +386,7 @@ func restoreDatabaseFromDropped( var createDbSql string var isSubDb bool - createDbSql, err = getCreateDatabaseSqlFromDropped(ctx, sid, bh, dbName, snapshotTs, restoreAccount, toAccountId) + createDbSql, err = getCreateDatabaseSqlFromTS(ctx, sid, bh, dbName, snapshotTs, restoreAccount, toAccountId) if err != nil { return } @@ -452,7 +452,7 @@ func restoreDatabaseFromDropped( } } - tableInfos, err := getTableInfosFromDropped(ctx, sid, bh, dbName, "", snapshotTs, restoreAccount, toAccountId) + tableInfos, err := getTableInfosFromTS(ctx, sid, bh, dbName, "", snapshotTs, restoreAccount, toAccountId) if err != nil { return } @@ -476,7 +476,7 @@ func restoreDatabaseFromDropped( return } - if err = recreateTableFromDropped(ctx, + if err = recreateTableFromTS(ctx, sid, bh, tblInfo, @@ -489,7 +489,7 @@ func restoreDatabaseFromDropped( return } -func getCreateDatabaseSqlFromDropped(ctx context.Context, +func getCreateDatabaseSqlFromTS(ctx context.Context, sid string, bh BackgroundExec, dbName string, @@ -506,7 +506,7 @@ func getCreateDatabaseSqlFromDropped(ctx context.Context, getLogger(sid).Info(fmt.Sprintf("[%d:%d] get create database `%s` sql: %s", from, ts, dbName, sql)) // cols: database_name, create_sql - colsList, err := getStringColsListFromDropped(newCtx, bh, sql, from, to, 0, 1) + colsList, err := getStringColsListFromTS(newCtx, bh, sql, from, to, 0, 1) if err != nil { return "", err } @@ -516,7 +516,7 @@ func getCreateDatabaseSqlFromDropped(ctx context.Context, return colsList[0][1], nil } -func recreateTableFromDropped( +func recreateTableFromTS( ctx context.Context, sid string, bh BackgroundExec, @@ -553,7 +553,7 @@ func recreateTableFromDropped( return } -func restoreSystemDatabaseFromDropped( +func restoreSystemDatabaseFromTS( ctx context.Context, sid string, bh BackgroundExec, @@ -568,7 +568,7 @@ func restoreSystemDatabaseFromDropped( tableInfos []*tableInfo ) - tableInfos, err = showFullTablesFromDropped(ctx, sid, bh, dbName, "", snapshotTs, restoreAccount, toAccountId) + tableInfos, err = showFullTablesFromTS(ctx, sid, bh, dbName, "", snapshotTs, restoreAccount, toAccountId) if err != nil { return err } @@ -581,7 +581,7 @@ func restoreSystemDatabaseFromDropped( } getLogger(sid).Info(fmt.Sprintf("[%d:%d] start to restore system table: %v.%v", restoreAccount, snapshotTs, moCatalog, tblInfo.tblName)) - tblInfo.createSql, err = getCreateTableSqlFromDropped(ctx, bh, dbName, tblInfo.tblName, snapshotTs, restoreAccount, toAccountId) + tblInfo.createSql, err = getCreateTableSqlFromTS(ctx, bh, dbName, tblInfo.tblName, snapshotTs, restoreAccount, toAccountId) if err != nil { return err } @@ -591,14 +591,14 @@ func restoreSystemDatabaseFromDropped( return } - if err = recreateTableFromDropped(ctx, sid, bh, tblInfo, snapshotTs, restoreAccount, toAccountId); err != nil { + if err = recreateTableFromTS(ctx, sid, bh, tblInfo, snapshotTs, restoreAccount, toAccountId); err != nil { return } } return } -func restoreTablesWithFkFromDropped( +func restoreTablesWithFkFromTS( ctx context.Context, sid string, bh BackgroundExec, @@ -616,7 +616,7 @@ func restoreTablesWithFkFromDropped( // e.g. t1.pk <- t2.fk, we only want to restore t2, fkTableMap[t1.key] is nil, ignore t1 if tblInfo := fkTableMap[key]; tblInfo != nil { getLogger(sid).Info(fmt.Sprintf("[%d:%d] start to restore table with fk: %v, restore timestamp: %d", restoreAccount, snapshotTs, tblInfo.tblName, snapshotTs)) - err = recreateTableFromDropped(ctx, sid, bh, tblInfo, snapshotTs, restoreAccount, toAccountId) + err = recreateTableFromTS(ctx, sid, bh, tblInfo, snapshotTs, restoreAccount, toAccountId) if err != nil { return } @@ -625,7 +625,7 @@ func restoreTablesWithFkFromDropped( return } -func restoreViewsFromDropped( +func restoreViewsFromTS( ctx context.Context, ses *Session, bh BackgroundExec, diff --git a/pkg/frontend/snapshot_test.go b/pkg/frontend/snapshot_test.go index fa31572b0a5db..30f52bd31ae25 100644 --- a/pkg/frontend/snapshot_test.go +++ b/pkg/frontend/snapshot_test.go @@ -26,8 +26,8 @@ import ( "github.com/smartystreets/goconvey/convey" ) -func Test_fkTablesTopoSortWithDropped(t *testing.T) { - convey.Convey("fkTablesTopoSortWithDropped ", t, func() { +func Test_fkTablesTopoSortWithTS(t *testing.T) { + convey.Convey("fkTablesTopoSortWithTS ", t, func() { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -59,26 +59,26 @@ func Test_fkTablesTopoSortWithDropped(t *testing.T) { ctx = context.WithValue(ctx, defines.TenantIDKey{}, uint32(sysAccountID)) - _, err := fkTablesTopoSortWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err := fkTablesTopoSortWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldNotBeNil) sql := "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs := newMrsForPitrRecord([][]interface{}{}) bh.sql2result[sql] = mrs - _, err = fkTablesTopoSortWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err = fkTablesTopoSortWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldBeNil) sql = "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs = newMrsForPitrRecord([][]interface{}{{"db1", "table1", "db2", "table2"}}) bh.sql2result[sql] = mrs - _, err = fkTablesTopoSortWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err = fkTablesTopoSortWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldBeNil) }) } -func Test_getFkDepsWithDropped(t *testing.T) { - convey.Convey("getFkDepsWithDropped ", t, func() { +func Test_getFkDepsWithTS(t *testing.T) { + convey.Convey("getFkDepsWithTS ", t, func() { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -110,49 +110,49 @@ func Test_getFkDepsWithDropped(t *testing.T) { ctx = context.WithValue(ctx, defines.TenantIDKey{}, uint32(sysAccountID)) - _, err := getFkDepsWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err := getFkDepsWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldNotBeNil) sql := "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs := newMrsForPitrRecord([][]interface{}{}) bh.sql2result[sql] = mrs - _, err = getFkDepsWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err = getFkDepsWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldBeNil) sql = "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs = newMrsForPitrRecord([][]interface{}{{"db1", "table1", "db2", "table2"}}) bh.sql2result[sql] = mrs - _, err = getFkDepsWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err = getFkDepsWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldBeNil) sql = "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs = newMrsForPitrRecord([][]interface{}{{types.Day_Hour, "table1", "db2", "table2"}}) bh.sql2result[sql] = mrs - _, err = getFkDepsWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err = getFkDepsWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldNotBeNil) sql = "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs = newMrsForPitrRecord([][]interface{}{{"db1", types.Day_Hour, "db2", "table2"}}) bh.sql2result[sql] = mrs - _, err = getFkDepsWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err = getFkDepsWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldNotBeNil) sql = "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs = newMrsForPitrRecord([][]interface{}{{"db1", "table1", types.Day_Hour, "table2"}}) bh.sql2result[sql] = mrs - _, err = getFkDepsWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err = getFkDepsWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldNotBeNil) sql = "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs = newMrsForPitrRecord([][]interface{}{{"db1", "table1", "db2", types.Day_Hour}}) bh.sql2result[sql] = mrs - _, err = getFkDepsWithDropped(ctx, bh, "", "", 0, 0, 0) + _, err = getFkDepsWithTS(ctx, bh, "", "", 0, 0, 0) convey.So(err, convey.ShouldNotBeNil) }) } @@ -190,14 +190,14 @@ func Test_restoreAccountUsingClusterSnapshotToNew(t *testing.T) { ctx = context.WithValue(ctx, defines.TenantIDKey{}, uint32(sysAccountID)) - err := restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, "sp01", 0, accountRecord{accountName: "sys", accountId: 0}, nil, 0, false, false) + err := restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, "sp01", 0, accountRecord{accountName: "sys", accountId: 0}, 0, nil, false, false) convey.So(err, convey.ShouldNotBeNil) sql := "select db_name, table_name, refer_db_name, refer_table_name from mo_catalog.mo_foreign_keys" mrs := newMrsForPitrRecord([][]interface{}{{"db1", "table1", "db2", "table2"}}) bh.sql2result[sql] = mrs - err = restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, "sp01", 0, accountRecord{accountName: "sys", accountId: 0}, nil, 0, false, false) + err = restoreAccountUsingClusterSnapshotToNew(ctx, ses, bh, "sp01", 0, accountRecord{accountName: "sys", accountId: 0}, 0, nil, false, false) convey.So(err, convey.ShouldNotBeNil) }) }