From be132556af057c5daefe42a223e9abef7be58ad7 Mon Sep 17 00:00:00 2001 From: Deepthi Sigireddi Date: Sat, 30 Jul 2022 09:34:20 -0700 Subject: [PATCH] Merge pull request #10847 from planetscale/8.0.30_builtin_backups ... removing the temporary workflow changes on main Backups: Support InnoDB Redo Log Location With 8.0.30+ Signed-off-by: Matt Lord --- go/mysql/flavor.go | 1 + go/mysql/flavor_mysql.go | 2 + go/mysql/flavor_mysqlgr.go | 2 + go/mysql/flavor_test.go | 15 +++++ go/mysql/innodb_constants.go | 27 ++++++++ go/vt/mysqlctl/backup_test.go | 24 +++++++- go/vt/mysqlctl/backupengine.go | 25 +++++++- go/vt/mysqlctl/builtinbackupengine_test.go | 34 ++++++++++- go/vt/mysqlctl/capabilityset.go | 10 +++ go/vt/wrangler/testlib/backup_test.go | 71 ++++++++++++++++++++-- 10 files changed, 200 insertions(+), 11 deletions(-) create mode 100644 go/mysql/innodb_constants.go diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index c12ce52cdfa..85c1247e678 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -51,6 +51,7 @@ const ( InstantChangeColumnDefaultFlavorCapability MySQLJSONFlavorCapability MySQLUpgradeInServerFlavorCapability + DynamicRedoLogCapacityFlavorCapability // supported in MySQL 8.0.30 and above: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-30.html ) const ( diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 8e67b803cf8..351fb5c9ee0 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -395,6 +395,8 @@ func (mysqlFlavor80) supportsCapability(serverVersion string, capability FlavorC return true, nil case MySQLUpgradeInServerFlavorCapability: return ServerVersionAtLeast(serverVersion, 8, 0, 16) + case DynamicRedoLogCapacityFlavorCapability: + return ServerVersionAtLeast(serverVersion, 8, 0, 30) default: return false, nil } diff --git a/go/mysql/flavor_mysqlgr.go b/go/mysql/flavor_mysqlgr.go index 9a125dd4617..0094c563b7b 100644 --- a/go/mysql/flavor_mysqlgr.go +++ b/go/mysql/flavor_mysqlgr.go @@ -261,6 +261,8 @@ func (mysqlGRFlavor) supportsCapability(serverVersion string, capability FlavorC return ServerVersionAtLeast(serverVersion, 5, 7, 0) case MySQLUpgradeInServerFlavorCapability: return ServerVersionAtLeast(serverVersion, 8, 0, 16) + case DynamicRedoLogCapacityFlavorCapability: + return ServerVersionAtLeast(serverVersion, 8, 0, 30) default: return false, nil } diff --git a/go/mysql/flavor_test.go b/go/mysql/flavor_test.go index bf9cc09188d..ff3feeaab3a 100644 --- a/go/mysql/flavor_test.go +++ b/go/mysql/flavor_test.go @@ -135,6 +135,21 @@ func TestGetFlavor(t *testing.T) { capability: MySQLJSONFlavorCapability, isCapable: true, }, + { + version: "8.0.30", + capability: DynamicRedoLogCapacityFlavorCapability, + isCapable: true, + }, + { + version: "8.0.29", + capability: DynamicRedoLogCapacityFlavorCapability, + isCapable: false, + }, + { + version: "5.7.38", + capability: DynamicRedoLogCapacityFlavorCapability, + isCapable: false, + }, } for _, tc := range testcases { name := fmt.Sprintf("%s %v", tc.version, tc.capability) diff --git a/go/mysql/innodb_constants.go b/go/mysql/innodb_constants.go new file mode 100644 index 00000000000..8f8fef0673b --- /dev/null +++ b/go/mysql/innodb_constants.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +// This file contains the constant definitions for this package. + +const ( + // The directory used for redo logs within innodb_log_group_home_dir + // in MySQL 8.0.30 and later. + // You would check to see if this is relevant using the Flavor's + // capability interface to check for DynamicRedoLogCapacityFlavorCapability. + DynamicRedoLogSubdir = "#innodb_redo" +) diff --git a/go/vt/mysqlctl/backup_test.go b/go/vt/mysqlctl/backup_test.go index 20cd1492be8..16db1a72f8a 100644 --- a/go/vt/mysqlctl/backup_test.go +++ b/go/vt/mysqlctl/backup_test.go @@ -22,11 +22,22 @@ import ( "reflect" "sort" "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" ) func TestFindFilesToBackup(t *testing.T) { root := t.TempDir() + // get the flavor and version to deal with any behavioral differences + versionStr, err := GetVersionString() + require.NoError(t, err) + flavor, version, err := ParseVersionString(versionStr) + require.NoError(t, err) + features := newCapabilitySet(flavor, version) + // Initialize the fake mysql root directories innodbDataDir := path.Join(root, "innodb_data") innodbLogDir := path.Join(root, "innodb_log") @@ -41,11 +52,18 @@ func TestFindFilesToBackup(t *testing.T) { t.Fatalf("failed to create directory %v: %v", s, err) } } + + innodbLogFile := "innodb_log_1" + if features.hasDynamicRedoLogCapacity() { + os.Mkdir(path.Join(innodbLogDir, mysql.DynamicRedoLogSubdir), os.ModePerm) + innodbLogFile = path.Join(mysql.DynamicRedoLogSubdir, "#ib_redo1") + } + if err := os.WriteFile(path.Join(innodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm); err != nil { t.Fatalf("failed to write file innodb_data_1: %v", err) } - if err := os.WriteFile(path.Join(innodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm); err != nil { - t.Fatalf("failed to write file innodb_log_1: %v", err) + if err := os.WriteFile(path.Join(innodbLogDir, innodbLogFile), []byte("innodb log 1 contents"), os.ModePerm); err != nil { + t.Fatalf("failed to write file %s: %v", innodbLogFile, err) } if err := os.WriteFile(path.Join(dataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm); err != nil { t.Fatalf("failed to write file db.opt: %v", err) @@ -101,7 +119,7 @@ func TestFindFilesToBackup(t *testing.T) { }, { Base: "InnoDBLog", - Name: "innodb_log_1", + Name: innodbLogFile, }, } if !reflect.DeepEqual(result, expected) { diff --git a/go/vt/mysqlctl/backupengine.go b/go/vt/mysqlctl/backupengine.go index 7546aec73c3..d297a05d764 100644 --- a/go/vt/mysqlctl/backupengine.go +++ b/go/vt/mysqlctl/backupengine.go @@ -381,14 +381,32 @@ func addMySQL8DataDictionary(fes []FileEntry, base string, baseDir string) ([]Fi func findFilesToBackup(cnf *Mycnf) ([]FileEntry, int64, error) { var err error var result []FileEntry - var totalSize int64 + var size, totalSize int64 + var flavor MySQLFlavor + var version ServerVersion + var features capabilitySet - // first add inno db files + // get the flavor and version to deal with any behavioral differences + versionStr, err := GetVersionString() + if err != nil { + return nil, 0, err + } + flavor, version, err = ParseVersionString(versionStr) + if err != nil { + return nil, 0, err + } + features = newCapabilitySet(flavor, version) + + // first add innodb files result, totalSize, err = addDirectory(result, backupInnodbDataHomeDir, cnf.InnodbDataHomeDir, "") if err != nil { return nil, 0, err } - result, size, err := addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, "") + if features.hasDynamicRedoLogCapacity() { + result, size, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, mysql.DynamicRedoLogSubdir) + } else { + result, size, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, "") + } if err != nil { return nil, 0, err } @@ -416,5 +434,6 @@ func findFilesToBackup(cnf *Mycnf) ([]FileEntry, int64, error) { totalSize = totalSize + size } } + return result, totalSize, nil } diff --git a/go/vt/mysqlctl/builtinbackupengine_test.go b/go/vt/mysqlctl/builtinbackupengine_test.go index 79d5357df06..92f6cc7feaf 100644 --- a/go/vt/mysqlctl/builtinbackupengine_test.go +++ b/go/vt/mysqlctl/builtinbackupengine_test.go @@ -6,6 +6,7 @@ package mysqlctl_test import ( "context" + "fmt" "os" "path" "testing" @@ -14,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl" @@ -53,6 +55,15 @@ func TestExecuteBackup(t *testing.T) { ctx := context.Background() + needIt, err := needInnoDBRedoLogSubdir() + require.NoError(t, err) + if needIt { + fpath := path.Join("log", mysql.DynamicRedoLogSubdir) + if err := createBackupDir(backupRoot, fpath); err != nil { + t.Fatalf("failed to create directory %s: %v", fpath, err) + } + } + // Set up topo keyspace, shard := "mykeyspace", "-80" ts := memorytopo.NewServer("cell1") @@ -67,7 +78,7 @@ func TestExecuteBackup(t *testing.T) { require.NoError(t, ts.CreateTablet(ctx, tablet)) - _, err := ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { + _, err = ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { si.PrimaryAlias = &topodata.TabletAlias{Uid: 100, Cell: "cell1"} now := time.Now() @@ -134,3 +145,24 @@ func TestExecuteBackup(t *testing.T) { assert.Error(t, err) assert.False(t, ok) } + +// needInnoDBRedoLogSubdir indicates whether we need to create a redo log subdirectory. +// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the +// (/. by default) called "#innodb_redo". See: +// https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity +func needInnoDBRedoLogSubdir() (needIt bool, err error) { + mysqldVersionStr, err := mysqlctl.GetVersionString() + if err != nil { + return needIt, err + } + _, sv, err := mysqlctl.ParseVersionString(mysqldVersionStr) + if err != nil { + return needIt, err + } + versionStr := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch) + _, capableOf, _ := mysql.GetFlavor(versionStr, nil) + if capableOf == nil { + return needIt, fmt.Errorf("cannot determine database flavor details for version %s", versionStr) + } + return capableOf(mysql.DynamicRedoLogCapacityFlavorCapability) +} diff --git a/go/vt/mysqlctl/capabilityset.go b/go/vt/mysqlctl/capabilityset.go index 88b17e4e455..9beb98f01d9 100644 --- a/go/vt/mysqlctl/capabilityset.go +++ b/go/vt/mysqlctl/capabilityset.go @@ -51,6 +51,16 @@ func (c *capabilitySet) hasMaria104InstallDb() bool { return c.isMariaDB() && c.version.atLeast(ServerVersion{Major: 10, Minor: 4, Patch: 0}) } +// hasDynamicRedoLogCapacity tells you if the version of MySQL in use supports dynamic redo log +// capacity. +// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the +// (/. by default) called "#innodb_redo" and you can +// dynamically adjust the capacity of redo log space in the running server. See: +// https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity +func (c *capabilitySet) hasDynamicRedoLogCapacity() bool { + return c.isMySQLLike() && c.version.atLeast(ServerVersion{Major: 8, Minor: 0, Patch: 30}) +} + // IsMySQLLike tests if the server is either MySQL // or Percona Server. At least currently, Vitess doesn't // make use of any specific Percona Server features. diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index e77e2353d98..02e2de66f40 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -91,8 +91,18 @@ func TestBackupRestore(t *testing.T) { for _, s := range []string{sourceInnodbDataDir, sourceInnodbLogDir, sourceDataDbDir} { require.NoError(t, os.MkdirAll(s, os.ModePerm)) } + + needIt, err := needInnoDBRedoLogSubdir() + require.NoError(t, err) + if needIt { + newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir) + require.NoError(t, os.Mkdir(newPath, os.ModePerm)) + require.NoError(t, os.WriteFile(path.Join(newPath, "#ib_redo1"), []byte("innodb log 1 contents"), os.ModePerm)) + } else { + require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm)) + } + require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm)) - require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm)) require.NoError(t, os.WriteFile(path.Join(sourceDataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm)) // create a primary tablet, set its primary position @@ -198,7 +208,9 @@ func TestBackupRestore(t *testing.T) { RelayLogInfoPath: path.Join(root, "relay-log.info"), } + // run the backup require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* backupTime */)) + // verify the full status require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed") assert.True(t, destTablet.FakeMysqlDaemon.Replicating) @@ -304,7 +316,17 @@ func TestBackupRestoreLagged(t *testing.T) { require.NoError(t, os.MkdirAll(s, os.ModePerm)) } require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm)) - require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm)) + + needIt, err := needInnoDBRedoLogSubdir() + require.NoError(t, err) + if needIt { + newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir) + require.NoError(t, os.Mkdir(newPath, os.ModePerm)) + require.NoError(t, os.WriteFile(path.Join(newPath, "#ib_redo1"), []byte("innodb log 1 contents"), os.ModePerm)) + } else { + require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm)) + } + require.NoError(t, os.WriteFile(path.Join(sourceDataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm)) // create a primary tablet, set its position @@ -508,7 +530,17 @@ func TestRestoreUnreachablePrimary(t *testing.T) { require.NoError(t, os.MkdirAll(s, os.ModePerm)) } require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm)) - require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm)) + + needIt, err := needInnoDBRedoLogSubdir() + require.NoError(t, err) + if needIt { + newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir) + require.NoError(t, os.Mkdir(newPath, os.ModePerm)) + require.NoError(t, os.WriteFile(path.Join(newPath, "#ib_redo1"), []byte("innodb log 1 contents"), os.ModePerm)) + } else { + require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm)) + } + require.NoError(t, os.WriteFile(path.Join(sourceDataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm)) // create a primary tablet, set its primary position @@ -668,7 +700,17 @@ func TestDisableActiveReparents(t *testing.T) { require.NoError(t, os.MkdirAll(s, os.ModePerm)) } require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm)) - require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm)) + + needIt, err := needInnoDBRedoLogSubdir() + require.NoError(t, err) + if needIt { + newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir) + require.NoError(t, os.Mkdir(newPath, os.ModePerm)) + require.NoError(t, os.WriteFile(path.Join(newPath, "#ib_redo1"), []byte("innodb log 1 contents"), os.ModePerm)) + } else { + require.NoError(t, os.WriteFile(path.Join(sourceInnodbLogDir, "innodb_log_1"), []byte("innodb log 1 contents"), os.ModePerm)) + } + require.NoError(t, os.WriteFile(path.Join(sourceDataDbDir, "db.opt"), []byte("db opt file"), os.ModePerm)) // create a primary tablet, set its primary position @@ -766,3 +808,24 @@ func TestDisableActiveReparents(t *testing.T) { assert.False(t, destTablet.FakeMysqlDaemon.Replicating) assert.True(t, destTablet.FakeMysqlDaemon.Running) } + +// needInnoDBRedoLogSubdir indicates whether we need to create a redo log subdirectory. +// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the +// (/. by default) called "#innodb_redo". See: +// https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity +func needInnoDBRedoLogSubdir() (needIt bool, err error) { + mysqldVersionStr, err := mysqlctl.GetVersionString() + if err != nil { + return needIt, err + } + _, sv, err := mysqlctl.ParseVersionString(mysqldVersionStr) + if err != nil { + return needIt, err + } + versionStr := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch) + _, capableOf, _ := mysql.GetFlavor(versionStr, nil) + if capableOf == nil { + return needIt, fmt.Errorf("cannot determine database flavor details for version %s", versionStr) + } + return capableOf(mysql.DynamicRedoLogCapacityFlavorCapability) +}