Merge pull request vitessio#10847 from planetscale/8.0.30_builtin_bac…

…kups

... removing the temporary workflow changes on main

Backups: Support InnoDB Redo Log Location With 8.0.30+
Signed-off-by: Matt Lord <[email protected]>

go/mysql/flavor.go

@@ -51,6 +51,7 @@ const (
 	InstantChangeColumnDefaultFlavorCapability
 	MySQLJSONFlavorCapability
 	MySQLUpgradeInServerFlavorCapability
+	DynamicRedoLogCapacityFlavorCapability // supported in MySQL 8.0.30 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-30.html
 )
 
 const (

go/mysql/flavor_mysql.go

@@ -395,6 +395,8 @@ func (mysqlFlavor80) supportsCapability(serverVersion string, capability FlavorC
 		return true, nil
 	case MySQLUpgradeInServerFlavorCapability:
 		return ServerVersionAtLeast(serverVersion, 8, 0, 16)
+	case DynamicRedoLogCapacityFlavorCapability:
+		return ServerVersionAtLeast(serverVersion, 8, 0, 30)
 	default:
 		return false, nil
 	}

go/mysql/flavor_mysqlgr.go

@@ -261,6 +261,8 @@ func (mysqlGRFlavor) supportsCapability(serverVersion string, capability FlavorC
 		return ServerVersionAtLeast(serverVersion, 5, 7, 0)
 	case MySQLUpgradeInServerFlavorCapability:
 		return ServerVersionAtLeast(serverVersion, 8, 0, 16)
+	case DynamicRedoLogCapacityFlavorCapability:
+		return ServerVersionAtLeast(serverVersion, 8, 0, 30)
 	default:
 		return false, nil
 	}

go/mysql/flavor_test.go

@@ -135,6 +135,21 @@ func TestGetFlavor(t *testing.T) {
 			capability: MySQLJSONFlavorCapability,
 			isCapable:  true,
 		},
+		{
+			version:    "8.0.30",
+			capability: DynamicRedoLogCapacityFlavorCapability,
+			isCapable:  true,
+		},
+		{
+			version:    "8.0.29",
+			capability: DynamicRedoLogCapacityFlavorCapability,
+			isCapable:  false,
+		},
+		{
+			version:    "5.7.38",
+			capability: DynamicRedoLogCapacityFlavorCapability,
+			isCapable:  false,
+		},
 	}
 	for _, tc := range testcases {
 		name := fmt.Sprintf("%s %v", tc.version, tc.capability)

go/mysql/innodb_constants.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2022 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+// This file contains the constant definitions for this package.
+
+const (
+	// The directory used for redo logs within innodb_log_group_home_dir
+	// in MySQL 8.0.30 and later.
+	// You would check to see if this is relevant using the Flavor's
+	// capability interface to check for DynamicRedoLogCapacityFlavorCapability.
+	DynamicRedoLogSubdir = "#innodb_redo"
+)

go/vt/mysqlctl/backup_test.go

@@ -22,11 +22,22 @@ import (
 	"reflect"
 	"sort"
 	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/mysql"
 )
 
 func TestFindFilesToBackup(t *testing.T) {
 	root := t.TempDir()
 
+	// get the flavor and version to deal with any behavioral differences
+	versionStr, err := GetVersionString()
+	require.NoError(t, err)
+	flavor, version, err := ParseVersionString(versionStr)
+	require.NoError(t, err)
+	features := newCapabilitySet(flavor, version)
+
 	// Initialize the fake mysql root directories
 	innodbDataDir := path.Join(root, "innodb_data")
 	innodbLogDir := path.Join(root, "innodb_log")
@@ -41,11 +52,18 @@ func TestFindFilesToBackup(t *testing.T) {
 			t.Fatalf("failed to create directory %v: %v", s, err)
 		}
 	}
+
+	innodbLogFile := "innodb_log_1"
+	if features.hasDynamicRedoLogCapacity() {
+		os.Mkdir(path.Join(innodbLogDir, mysql.DynamicRedoLogSubdir), os.ModePerm)
+		innodbLogFile = path.Join(mysql.DynamicRedoLogSubdir, "#ib_redo1")
+	}
+
 	if err := os.WriteFile(path.Join(innodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm); err != nil {
 		t.Fatalf("failed to write file innodb_data_1: %v", err)
 	}
-	if err := os.WriteFile(path.Join(innodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm); err != nil {
-		t.Fatalf("failed to write file innodb_log_1: %v", err)
+	if err := os.WriteFile(path.Join(innodbLogDir, innodbLogFile), []byte("innodb log 1 contents"), os.ModePerm); err != nil {
+		t.Fatalf("failed to write file %s: %v", innodbLogFile, err)
 	}
 	if err := os.WriteFile(path.Join(dataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm); err != nil {
 		t.Fatalf("failed to write file db.opt: %v", err)
@@ -101,7 +119,7 @@ func TestFindFilesToBackup(t *testing.T) {
 		},
 		{
 			Base: "InnoDBLog",
-			Name: "innodb_log_1",
+			Name: innodbLogFile,
 		},
 	}
 	if !reflect.DeepEqual(result, expected) {

go/vt/mysqlctl/backupengine.go

@@ -381,14 +381,32 @@ func addMySQL8DataDictionary(fes []FileEntry, base string, baseDir string) ([]Fi
 func findFilesToBackup(cnf *Mycnf) ([]FileEntry, int64, error) {
 	var err error
 	var result []FileEntry
-	var totalSize int64
+	var size, totalSize int64
+	var flavor MySQLFlavor
+	var version ServerVersion
+	var features capabilitySet
 
-	// first add inno db files
+	// get the flavor and version to deal with any behavioral differences
+	versionStr, err := GetVersionString()
+	if err != nil {
+		return nil, 0, err
+	}
+	flavor, version, err = ParseVersionString(versionStr)
+	if err != nil {
+		return nil, 0, err
+	}
+	features = newCapabilitySet(flavor, version)
+
+	// first add innodb files
 	result, totalSize, err = addDirectory(result, backupInnodbDataHomeDir, cnf.InnodbDataHomeDir, "")
 	if err != nil {
 		return nil, 0, err
 	}
-	result, size, err := addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, "")
+	if features.hasDynamicRedoLogCapacity() {
+		result, size, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, mysql.DynamicRedoLogSubdir)
+	} else {
+		result, size, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, "")
+	}
 	if err != nil {
 		return nil, 0, err
 	}
@@ -416,5 +434,6 @@ func findFilesToBackup(cnf *Mycnf) ([]FileEntry, int64, error) {
 			totalSize = totalSize + size
 		}
 	}
+
 	return result, totalSize, nil
 }

go/vt/mysqlctl/builtinbackupengine_test.go

@@ -6,6 +6,7 @@ package mysqlctl_test
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path"
 	"testing"
@@ -14,6 +15,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/mysql/fakesqldb"
 	"vitess.io/vitess/go/vt/logutil"
 	"vitess.io/vitess/go/vt/mysqlctl"
@@ -53,6 +55,15 @@ func TestExecuteBackup(t *testing.T) {
 
 	ctx := context.Background()
 
+	needIt, err := needInnoDBRedoLogSubdir()
+	require.NoError(t, err)
+	if needIt {
+		fpath := path.Join("log", mysql.DynamicRedoLogSubdir)
+		if err := createBackupDir(backupRoot, fpath); err != nil {
+			t.Fatalf("failed to create directory %s: %v", fpath, err)
+		}
+	}
+
 	// Set up topo
 	keyspace, shard := "mykeyspace", "-80"
 	ts := memorytopo.NewServer("cell1")
@@ -67,7 +78,7 @@ func TestExecuteBackup(t *testing.T) {
 
 	require.NoError(t, ts.CreateTablet(ctx, tablet))
 
-	_, err := ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error {
+	_, err = ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error {
 		si.PrimaryAlias = &topodata.TabletAlias{Uid: 100, Cell: "cell1"}
 
 		now := time.Now()
@@ -134,3 +145,24 @@ func TestExecuteBackup(t *testing.T) {
 	assert.Error(t, err)
 	assert.False(t, ok)
 }
+
+// needInnoDBRedoLogSubdir indicates whether we need to create a redo log subdirectory.
+// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the
+// <innodb_log_group_home_dir> (<datadir>/. by default) called "#innodb_redo". See:
+//   https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity
+func needInnoDBRedoLogSubdir() (needIt bool, err error) {
+	mysqldVersionStr, err := mysqlctl.GetVersionString()
+	if err != nil {
+		return needIt, err
+	}
+	_, sv, err := mysqlctl.ParseVersionString(mysqldVersionStr)
+	if err != nil {
+		return needIt, err
+	}
+	versionStr := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch)
+	_, capableOf, _ := mysql.GetFlavor(versionStr, nil)
+	if capableOf == nil {
+		return needIt, fmt.Errorf("cannot determine database flavor details for version %s", versionStr)
+	}
+	return capableOf(mysql.DynamicRedoLogCapacityFlavorCapability)
+}

go/vt/mysqlctl/capabilityset.go

@@ -51,6 +51,16 @@ func (c *capabilitySet) hasMaria104InstallDb() bool {
 	return c.isMariaDB() && c.version.atLeast(ServerVersion{Major: 10, Minor: 4, Patch: 0})
 }
 
+// hasDynamicRedoLogCapacity tells you if the version of MySQL in use supports dynamic redo log
+// capacity.
+// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the
+// <innodb_log_group_home_dir> (<datadir>/. by default) called "#innodb_redo" and you can
+// dynamically adjust the capacity of redo log space in the running server. See:
+//   https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity
+func (c *capabilitySet) hasDynamicRedoLogCapacity() bool {
+	return c.isMySQLLike() && c.version.atLeast(ServerVersion{Major: 8, Minor: 0, Patch: 30})
+}
+
 // IsMySQLLike tests if the server is either MySQL
 // or Percona Server. At least currently, Vitess doesn't
 // make use of any specific Percona Server features.

go/vt/wrangler/testlib/backup_test.go

@@ -91,8 +91,18 @@ func TestBackupRestore(t *testing.T) {
 	for _, s := range []string{sourceInnodbDataDir, sourceInnodbLogDir, sourceDataDbDir} {
 		require.NoError(t, os.MkdirAll(s, os.ModePerm))
 	}
+
+	needIt, err := needInnoDBRedoLogSubdir()
+	require.NoError(t, err)
+	if needIt {
+		newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir)
+		require.NoError(t, os.Mkdir(newPath, os.ModePerm))
+		require.NoError(t, os.WriteFile(path.Join(newPath, "#ib_redo1"), []byte("innodb log 1 contents"), os.ModePerm))
+	} else {
+		require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm))
+	}
+
 	require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm))
-	require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm))
 	require.NoError(t, os.WriteFile(path.Join(sourceDataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm))
 
 	// create a primary tablet, set its primary position
@@ -198,7 +208,9 @@ func TestBackupRestore(t *testing.T) {
 		RelayLogInfoPath:      path.Join(root, "relay-log.info"),
 	}
 
+	// run the backup
 	require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* backupTime */))
+
 	// verify the full status
 	require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed")
 	assert.True(t, destTablet.FakeMysqlDaemon.Replicating)
@@ -304,7 +316,17 @@ func TestBackupRestoreLagged(t *testing.T) {
 		require.NoError(t, os.MkdirAll(s, os.ModePerm))
 	}
 	require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm))
-	require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm))
+
+	needIt, err := needInnoDBRedoLogSubdir()
+	require.NoError(t, err)
+	if needIt {
+		newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir)
+		require.NoError(t, os.Mkdir(newPath, os.ModePerm))
+		require.NoError(t, os.WriteFile(path.Join(newPath, "#ib_redo1"), []byte("innodb log 1 contents"), os.ModePerm))
+	} else {
+		require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm))
+	}
+
 	require.NoError(t, os.WriteFile(path.Join(sourceDataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm))
 
 	// create a primary tablet, set its position
@@ -508,7 +530,17 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
 		require.NoError(t, os.MkdirAll(s, os.ModePerm))
 	}
 	require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm))
-	require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm))
+
+	needIt, err := needInnoDBRedoLogSubdir()
+	require.NoError(t, err)
+	if needIt {
+		newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir)
+		require.NoError(t, os.Mkdir(newPath, os.ModePerm))
+		require.NoError(t, os.WriteFile(path.Join(newPath, "#ib_redo1"), []byte("innodb log 1 contents"), os.ModePerm))
+	} else {
+		require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm))
+	}
+
 	require.NoError(t, os.WriteFile(path.Join(sourceDataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm))
 
 	// create a primary tablet, set its primary position
@@ -668,7 +700,17 @@ func TestDisableActiveReparents(t *testing.T) {
 		require.NoError(t, os.MkdirAll(s, os.ModePerm))
 	}
 	require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm))
-	require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm))
+
+	needIt, err := needInnoDBRedoLogSubdir()
+	require.NoError(t, err)
+	if needIt {
+		newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir)
+		require.NoError(t, os.Mkdir(newPath, os.ModePerm))
+		require.NoError(t, os.WriteFile(path.Join(newPath, "#ib_redo1"), []byte("innodb log 1 contents"), os.ModePerm))
+	} else {
+		require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm))
+	}
+
 	require.NoError(t, os.WriteFile(path.Join(sourceDataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm))
 
 	// create a primary tablet, set its primary position
@@ -766,3 +808,24 @@ func TestDisableActiveReparents(t *testing.T) {
 	assert.False(t, destTablet.FakeMysqlDaemon.Replicating)
 	assert.True(t, destTablet.FakeMysqlDaemon.Running)
 }
+
+// needInnoDBRedoLogSubdir indicates whether we need to create a redo log subdirectory.
+// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the
+// <innodb_log_group_home_dir> (<datadir>/. by default) called "#innodb_redo". See:
+//   https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity
+func needInnoDBRedoLogSubdir() (needIt bool, err error) {
+	mysqldVersionStr, err := mysqlctl.GetVersionString()
+	if err != nil {
+		return needIt, err
+	}
+	_, sv, err := mysqlctl.ParseVersionString(mysqldVersionStr)
+	if err != nil {
+		return needIt, err
+	}
+	versionStr := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch)
+	_, capableOf, _ := mysql.GetFlavor(versionStr, nil)
+	if capableOf == nil {
+		return needIt, fmt.Errorf("cannot determine database flavor details for version %s", versionStr)
+	}
+	return capableOf(mysql.DynamicRedoLogCapacityFlavorCapability)
+}