Backup transform tests migrated to go

go/test/endtoend/backup/backup_test.go

@@ -67,14 +67,12 @@ func TestMasterBackup(t *testing.T) {
 	assert.NotNil(t, err)
 	assert.Contains(t, output, "type MASTER cannot take backup. if you really need to do this, rerun the backup command with -allow_master")
 
-	backups := listBackups(t)
-	assert.Equal(t, len(backups), 0)
+	localCluster.VerifyBackupCount(t, shardKsName, 0)
 
 	err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", "-allow_master=true", master.Alias)
 	assert.Nil(t, err)
 
-	backups = listBackups(t)
-	assert.Equal(t, len(backups), 1)
+	backups := localCluster.VerifyBackupCount(t, shardKsName, 1)
 	assert.Contains(t, backups[0], master.Alias)
 
 	_, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true)
@@ -217,10 +215,8 @@ func restartMasterReplica(t *testing.T) {
 	stopAllTablets()
 
 	// remove all backups
-	backups := listBackups(t)
-	for _, backup := range backups {
-		localCluster.VtctlclientProcess.ExecuteCommand("RemoveBackup", shardKsName, backup)
-	}
+	localCluster.RemoveAllBackups(t, shardKsName)
+
 	// start all tablet and mysql instances
 	var mysqlProcs []*exec.Cmd
 	for _, tablet := range []*cluster.Vttablet{master, replica1} {
@@ -319,8 +315,7 @@ func testBackup(t *testing.T, tabletType string) {
 	err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias)
 	assert.Nil(t, err)
 
-	backups := listBackups(t)
-	assert.Equal(t, len(backups), 1)
+	backups := localCluster.VerifyBackupCount(t, shardKsName, 1)
 
 	_, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true)
 	assert.Nil(t, err)
@@ -371,12 +366,6 @@ func resetTabletDir(t *testing.T, tablet *cluster.Vttablet) {
 	assert.Nil(t, err)
 }
 
-func listBackups(t *testing.T) []string {
-	output, err := localCluster.ListBackups(shardKsName)
-	assert.Nil(t, err)
-	return output
-}
-
 func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []string) {
 	// Remove the backup
 	for _, backup := range backups {
@@ -385,8 +374,7 @@ func verifyAfterRemovingBackupNoBackupShouldBePresent(t *testing.T, backups []st
 	}
 
 	// Now, there should not be no backup
-	backups = listBackups(t)
-	assert.Equal(t, len(backups), 0)
+	localCluster.VerifyBackupCount(t, shardKsName, 0)
 }
 
 func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) {

go/test/endtoend/backup/main_test.go

@@ -73,6 +73,7 @@ func TestMain(m *testing.M) {
 		}
 		localCluster.Keyspaces = append(localCluster.Keyspaces, *keyspace)
 
+		// update password of mysql users
 		dbCredentialFile = initialsharding.WriteDbCredentialToTmp(localCluster.TmpDirectory)
 		initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql"))
 		sql := string(initDb)
@@ -87,6 +88,7 @@ func TestMain(m *testing.M) {
 			Name: shardName,
 		}
 
+		// start mysql process for all replicas and master
 		var mysqlProcs []*exec.Cmd
 		for i := 0; i < 3; i++ {
 			tabletType := "replica"
@@ -103,18 +105,21 @@ func TestMain(m *testing.M) {
 			tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory)
 			tablet.MysqlctlProcess.InitDBFile = newInitDBFile
 			tablet.MysqlctlProcess.ExtraArgs = extraArgs
-			if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil {
+			proc, err := tablet.MysqlctlProcess.StartProcess()
+			if err != nil {
 				return 1, err
-			} else {
-				mysqlProcs = append(mysqlProcs, proc)
 			}
+			mysqlProcs = append(mysqlProcs, proc)
+
 			shard.Vttablets = append(shard.Vttablets, tablet)
 		}
 		for _, proc := range mysqlProcs {
 			if err := proc.Wait(); err != nil {
 				return 1, err
 			}
 		}
+
+		// initialize tablets
 		master = shard.Vttablets[0]
 		replica1 = shard.Vttablets[1]
 		replica2 = shard.Vttablets[2]
@@ -126,6 +131,7 @@ func TestMain(m *testing.M) {
 			return 1, err
 		}
 
+		// create database direct in vtTablet
 		for _, tablet := range []cluster.Vttablet{*master, *replica1} {
 			if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil {
 				return 1, err
@@ -135,6 +141,7 @@ func TestMain(m *testing.M) {
 			}
 		}
 
+		// initialize master and start replication
 		if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard.Name, cell, master.TabletUID); err != nil {
 			return 1, err
 		}

go/test/endtoend/backuptransform/backup_transform_test.go

@@ -0,0 +1,193 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package backuptransform
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"vitess.io/vitess/go/test/endtoend/cluster"
+)
+
+// create query for test table creation
+var vtInsertTest = `create table vt_insert_test (
+		id bigint auto_increment,
+		msg varchar(64),
+		primary key (id)
+		) Engine=InnoDB`
+
+func TestBackupTransform(t *testing.T) {
+	// insert data in master, validate same in slave
+	verifyInitialReplication(t)
+
+	// restart the replica with transform hook parameter
+	replica1.VttabletProcess.TearDown()
+	replica1.VttabletProcess.ExtraArgs = []string{
+		"-db-credentials-file", dbCredentialFile,
+		"-backup_storage_hook", "test_backup_transform",
+		"-backup_storage_compress=false",
+		"-restore_from_backup",
+		"-backup_storage_implementation", "file",
+		"-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot}
+	replica1.VttabletProcess.ServingStatus = "SERVING"
+	err := replica1.VttabletProcess.Setup()
+	assert.Nil(t, err)
+
+	// take backup, it should not give any error
+	err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias)
+	assert.Nil(t, err)
+
+	// insert data in master
+	_, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true)
+	assert.Nil(t, err)
+
+	// validate backup_list, expecting 1 backup available
+	backups := localCluster.VerifyBackupCount(t, shardKsName, 1)
+
+	backupLocation := localCluster.CurrentVTDATAROOT + "/backups/" + shardKsName + "/" + backups[0]
+
+	// validate that MANIFEST has TransformHook
+	// every file should start with 'header'
+	validateManifestFile(t, backupLocation)
+
+	// restore replica2 from backup, should not give any error
+	// Note: we don't need to pass in the backup_storage_transform parameter,
+	// as it is read from the MANIFEST.
+	replica2.MysqlctlProcess.ExtraArgs = []string{
+		"-db-credentials-file", dbCredentialFile}
+	// clear replica2
+	replica2.MysqlctlProcess.Stop()
+	os.RemoveAll(replica2.VttabletProcess.Directory)
+
+	// start replica2 from backup
+	err = replica2.MysqlctlProcess.Start()
+	require.Nil(t, err)
+	err = localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shardName)
+	assert.Nil(t, err)
+	replica2.VttabletProcess.CreateDB(keyspaceName)
+	replica2.VttabletProcess.ExtraArgs = []string{
+		"-db-credentials-file", dbCredentialFile,
+		"-restore_from_backup",
+		"-backup_storage_implementation", "file",
+		"-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot}
+	replica2.VttabletProcess.ServingStatus = ""
+	err = replica2.VttabletProcess.Setup()
+	require.Nil(t, err)
+	err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second)
+	require.Nil(t, err)
+	defer replica2.VttabletProcess.TearDown()
+
+	// validate that semi-sync is enabled for replica, disable for rdOnly
+	if replica2.Type == "replica" {
+		verifyReplicationStatus(t, replica2, "ON")
+	} else if replica2.Type == "rdonly" {
+		verifyReplicationStatus(t, replica2, "OFF")
+	}
+
+	// validate that new slave has all the data
+	cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2)
+
+	// Remove all backups
+	localCluster.RemoveAllBackups(t, shardKsName)
+
+}
+
+// TestBackupTransformError validate backup with test_backup_error
+// backup_storage_hook, which should fail.
+func TestBackupTransformError(t *testing.T) {
+	// restart the replica with transform hook parameter
+	err := replica1.VttabletProcess.TearDown()
+	require.Nil(t, err)
+
+	replica1.VttabletProcess.ExtraArgs = []string{
+		"-db-credentials-file", dbCredentialFile,
+		"-backup_storage_hook", "test_backup_error",
+		"-restore_from_backup",
+		"-backup_storage_implementation", "file",
+		"-file_backup_storage_root", localCluster.VtctldProcess.FileBackupStorageRoot}
+	replica1.VttabletProcess.ServingStatus = "SERVING"
+	err = replica1.VttabletProcess.Setup()
+	assert.Nil(t, err)
+
+	// create backup, it should fail
+	out, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("Backup", replica1.Alias)
+	require.NotNil(t, err)
+	assert.Containsf(t, out, "backup is not usable, aborting it", "unexpected error received %v", err)
+
+	// validate there is no backup left
+	localCluster.VerifyBackupCount(t, shardKsName, 0)
+}
+
+// validateManifestFile reads manifest and validates that it
+// has a TransformHook, SkipCompress and FileEntries. It also
+// validates that backup_files available in FileEntries have
+// 'header' as their first line.
+func validateManifestFile(t *testing.T, backupLocation string) {
+
+	// reading manifest
+	data, err := ioutil.ReadFile(backupLocation + "/MANIFEST")
+	require.Nilf(t, err, "error while reading MANIFEST %v", err)
+	manifest := make(map[string]interface{})
+
+	// parsing manifest
+	err = json.Unmarshal(data, &manifest)
+	require.Nilf(t, err, "error while parsing MANIFEST %v", err)
+
+	// validate manifest
+	transformHook, _ := manifest["TransformHook"]
+	require.Equalf(t, "test_backup_transform", transformHook, "invalid transformHook in MANIFEST")
+	skipCompress, _ := manifest["SkipCompress"]
+	assert.Equalf(t, skipCompress, true, "invalid value of skipCompress")
+
+	// validate backup files
+	for i := range manifest["FileEntries"].([]interface{}) {
+		f, err := os.Open(fmt.Sprintf("%s/%d", backupLocation, i))
+		require.Nilf(t, err, "error while opening backup_file %d: %v", i, err)
+		var fileHeader string
+		_, err = fmt.Fscanln(f, &fileHeader)
+		f.Close()
+
+		require.Nilf(t, err, "error while reading backup_file %d: %v", i, err)
+		require.Equalf(t, "header", fileHeader, "wrong file contents for %d", i)
+	}
+
+}
+
+// verifyReplicationStatus validates the replication status in tablet.
+func verifyReplicationStatus(t *testing.T, vttablet *cluster.Vttablet, expectedStatus string) {
+	status, err := vttablet.VttabletProcess.GetDBVar("rpl_semi_sync_slave_enabled", keyspaceName)
+	assert.Nil(t, err)
+	assert.Equal(t, expectedStatus, status)
+	status, err = vttablet.VttabletProcess.GetDBStatus("rpl_semi_sync_slave_status", keyspaceName)
+	assert.Nil(t, err)
+	assert.Equal(t, expectedStatus, status)
+}
+
+// verifyInitialReplication creates schema in master, insert some data to master and verify the same data in replica
+func verifyInitialReplication(t *testing.T) {
+	_, err := master.VttabletProcess.QueryTablet(vtInsertTest, keyspaceName, true)
+	assert.Nil(t, err)
+	_, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test1')", keyspaceName, true)
+	assert.Nil(t, err)
+	cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 1)
+}