Skip to content

Commit

Permalink
backupccl: datadrivenify the restore OnFailOrCancel tests
Browse files Browse the repository at this point in the history 
This change ports all the "cleanup on failed or canceled restore"
tests to the datadriven framework. To make this possible new instructions
have been added to the datadriven framework, please refer to the header
comment for details about each one.

Informs: #77129

Release note: None
  • Loading branch information
@adityamaru
adityamaru committed Apr 1, 2022
1 parent fa02911 commit 1ba873a
Show file tree
Hide file tree
Showing 8 changed files with 631 additions and 325 deletions.
279 changes: 0 additions & 279 deletions pkg/ccl/backupccl/backup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1870,285 +1870,6 @@ func TestBackupRestoreControlJob(t *testing.T) {
})
}

func TestRestoreFailCleansUpTTLSchedules(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

injectedErr := errors.New("injected error")
testCases := []struct {
desc string
beforePublishingDescriptors func() error
afterPublishingDescriptors func() error
}{
{
desc: "error before publishing descriptors",
beforePublishingDescriptors: func() error {
return injectedErr
},
},
{
desc: "error after publishing descriptors",
afterPublishingDescriptors: func() error {
return injectedErr
},
},
}

ctx := context.Background()

for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
c, sqlDB, _, cleanup := backupRestoreTestSetup(t, singleNode, 0, InitManualReplication)
defer cleanup()
for _, server := range c.Servers {
registry := server.JobRegistry().(*jobs.Registry)
registry.TestingResumerCreationKnobs = map[jobspb.Type]func(raw jobs.Resumer) jobs.Resumer{
jobspb.TypeRestore: func(raw jobs.Resumer) jobs.Resumer {
r := raw.(*restoreResumer)
r.testingKnobs.beforePublishingDescriptors = tc.beforePublishingDescriptors
r.testingKnobs.afterPublishingDescriptors = tc.afterPublishingDescriptors
return r
},
}
}

// Create a database with a TTL table.
sqlDB.Exec(t, `
CREATE DATABASE d;
CREATE TABLE d.tb (id INT PRIMARY KEY) WITH (ttl_expire_after = '10 minutes')
`)

// Backup d.tb.
sqlDB.Exec(t, `BACKUP DATABASE d TO $1`, localFoo)

// Drop d so that it can be restored.
sqlDB.Exec(t, `DROP DATABASE d`)

// Attempt the restore and check it fails.
_, err := sqlDB.DB.ExecContext(ctx, `RESTORE DATABASE d FROM $1`, localFoo)
require.Error(t, err)
require.Regexp(t, injectedErr.Error(), err.Error())

var count int
sqlDB.QueryRow(t, `SELECT count(1) FROM [SHOW SCHEDULES] WHERE label LIKE 'row-level-ttl-%'`).Scan(&count)
require.Equal(t, 0, count)
})
}
}

func TestRestoreFailCleansUpTypeBackReferences(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

_, sqlDB, dir, cleanup := backupRestoreTestSetup(t, singleNode, 0, InitManualReplication)
defer cleanup()

dir = dir + "/foo"

// Create a database with a type and table.
sqlDB.Exec(t, `
CREATE DATABASE d;
CREATE TYPE d.ty AS ENUM ('hello');
CREATE TABLE d.tb (x d.ty);
INSERT INTO d.tb VALUES ('hello'), ('hello');
`)

// Backup d.tb.
sqlDB.Exec(t, `BACKUP TABLE d.tb TO $1`, localFoo)

// Drop d.tb so that it can be restored.
sqlDB.Exec(t, `DROP TABLE d.tb`)

// Bugger the backup by removing the SST files.
if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Fatal(err)
}
if info.Name() == backupManifestName || !strings.HasSuffix(path, ".sst") {
return nil
}
return os.Remove(path)
}); err != nil {
t.Fatal(err)
}

// We should get an error when restoring the table.
sqlDB.ExpectErr(t, "sst: no such file", `RESTORE d.tb FROM $1`, localFoo)

// The failed restore should clean up type back references so that we are able
// to drop d.ty.
sqlDB.Exec(t, `DROP TYPE d.ty`)
}

// TestRestoreFailCleanup tests that a failed RESTORE is cleaned up.
func TestRestoreFailCleanup(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

params := base.TestServerArgs{}
// Disable GC job so that the final check of crdb_internal.tables is
// guaranteed to not be cleaned up. Although this was never observed by a
// stress test, it is here for safety.
blockGC := make(chan struct{})
params.Knobs.GCJob = &sql.GCJobTestingKnobs{
RunBeforeResume: func(_ jobspb.JobID) error {
<-blockGC
return nil
},
}

const numAccounts = 1000
tc, sqlDB, dir, cleanup := backupRestoreTestSetupWithParams(t, singleNode, numAccounts,
InitManualReplication, base.TestClusterArgs{ServerArgs: params})
defer cleanup()
kvDB := tc.Server(0).DB()

dir = dir + "/foo"

sqlDB.Exec(t, `CREATE DATABASE restore`)

// Create a user defined type and check that it is cleaned up after the
// failed restore.
sqlDB.Exec(t, `CREATE TYPE data.myenum AS ENUM ('hello')`)
// Do the same with a user defined schema.
sqlDB.Exec(t, `USE data; CREATE SCHEMA myschema`)

sqlDB.Exec(t, `BACKUP DATABASE data TO $1`, localFoo)
// Bugger the backup by removing the SST files.
if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Fatal(err)
}
if info.Name() == backupManifestName || !strings.HasSuffix(path, ".sst") {
return nil
}
return os.Remove(path)
}); err != nil {
t.Fatal(err)
}
sqlDB.ExpectErr(
t, "sst: no such file",
`RESTORE data.* FROM $1 WITH OPTIONS (into_db='restore')`, localFoo,
)
// Verify the failed RESTORE added some DROP tables.
sqlDB.CheckQueryResults(t,
`SELECT name FROM crdb_internal.tables WHERE database_name = 'restore' AND state = 'DROP'`,
[][]string{{"bank"}},
)

// Verify that `myenum` was cleaned out from the failed restore. There should
// only be one namespace entry (data.myenum).
sqlDB.CheckQueryResults(t, `SELECT count(*) FROM system.namespace WHERE name = 'myenum'`, [][]string{{"1"}})
// Check the same for data.myschema.
sqlDB.CheckQueryResults(t, `SELECT count(*) FROM system.namespace WHERE name = 'myschema'`, [][]string{{"1"}})

// Verify that the only schema that appears is the public schema
dbDesc := desctestutils.TestingGetDatabaseDescriptor(kvDB, keys.SystemSQLCodec, "restore")
require.Equal(t, len(dbDesc.DatabaseDesc().Schemas), 1)
if _, found := dbDesc.DatabaseDesc().Schemas[tree.PublicSchema]; !found {
t.Error("public schema not found")
}
}

// TestRestoreFailDatabaseCleanup tests that a failed RESTORE is cleaned up
// when restoring an entire database.
func TestRestoreFailDatabaseCleanup(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

params := base.TestServerArgs{}
const numAccounts = 1000
_, sqlDB, dir, cleanup := backupRestoreTestSetupWithParams(t, singleNode, numAccounts,
InitManualReplication, base.TestClusterArgs{ServerArgs: params})
defer cleanup()

dir = dir + "/foo"

// Create a user defined type and check that it is cleaned up after the
// failed restore.
sqlDB.Exec(t, `CREATE TYPE data.myenum AS ENUM ('hello')`)
// Do the same with a user defined schema.
sqlDB.Exec(t, `USE data; CREATE SCHEMA myschema`)

sqlDB.Exec(t, `BACKUP DATABASE data TO $1`, localFoo)
// Bugger the backup by removing the SST files.
if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Fatal(err)
}
if info.Name() == backupManifestName || !strings.HasSuffix(path, ".sst") {
return nil
}
return os.Remove(path)
}); err != nil {
t.Fatal(err)
}
sqlDB.Exec(t, `DROP DATABASE data`)
sqlDB.ExpectErr(
t, "sst: no such file",
`RESTORE DATABASE data FROM $1`, localFoo,
)
sqlDB.ExpectErr(
t, `database "data" does not exist`,
`DROP DATABASE data`,
)
}

func TestRestoreFailCleansUpTempSystemDatabase(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

_, sqlDB, dir, cleanup := backupRestoreTestSetup(t, singleNode, 0, InitManualReplication)
defer cleanup()

// Create a database with a type and table.
sqlDB.Exec(t, `
CREATE DATABASE d;
CREATE TYPE d.ty AS ENUM ('hello');
CREATE TABLE d.tb (x d.ty);
INSERT INTO d.tb VALUES ('hello'), ('hello');
`)

// Cluster BACKUP.
sqlDB.Exec(t, `BACKUP TO $1`, localFoo)

// Bugger the backup by removing the SST files.
if err := filepath.Walk(dir+"/foo", func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Fatal(err)
}
if info.Name() == backupManifestName || !strings.HasSuffix(path, ".sst") {
return nil
}
return os.Remove(path)
}); err != nil {
t.Fatal(err)
}

_, sqlDBRestore, cleanupRestore := backupRestoreTestSetupEmpty(t, singleNode, dir, InitManualReplication,
base.TestClusterArgs{})
defer cleanupRestore()
var defaultDBID int
sqlDBRestore.QueryRow(
t, "SELECT id FROM system.namespace WHERE name = 'defaultdb'",
).Scan(&defaultDBID)

// We should get an error when restoring the table.
sqlDBRestore.ExpectErr(t, "sst: no such file", `RESTORE FROM $1`, localFoo)

// Make sure the temp system db is not present.
row := sqlDBRestore.QueryStr(t, fmt.Sprintf(`SELECT * FROM [SHOW DATABASES] WHERE database_name = '%s'`, restoreTempSystemDB))
require.Equal(t, 0, len(row))

// Make sure defaultdb and postgres are recreated with new IDs.
sqlDBRestore.CheckQueryResults(t,
fmt.Sprintf(`
SELECT name
FROM system.namespace
WHERE "parentID" = 0 AND id > %d`, defaultDBID,
), [][]string{{"defaultdb"}, {"postgres"}})
}

func TestBackupRestoreUserDefinedSchemas(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)
Expand Down
Loading

0 comments on commit 1ba873a

Please sign in to comment.