From 006f532c2819cee30df69516ba4b439af3a0b18b Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Fri, 8 Dec 2023 08:20:53 +0800 Subject: [PATCH] ddl: Support `FLASHBACK DATABASE` (release-6.5) (#8449) (#8479) close pingcap/tiflash#8450 --- dbms/src/Databases/DatabaseTiFlash.cpp | 44 +++++++++-- dbms/src/Databases/DatabaseTiFlash.h | 2 +- dbms/src/Databases/IDatabase.h | 11 ++- dbms/src/Databases/test/gtest_database.cpp | 32 +++++++- dbms/src/TiDB/Schema/SchemaBuilder.cpp | 66 +++++++++++++++- dbms/src/TiDB/Schema/SchemaBuilder.h | 2 + dbms/src/TiDB/Schema/SchemaGetter.h | 4 +- .../ddl/flashback_database.test | 77 +++++++++++++++++++ 8 files changed, 220 insertions(+), 18 deletions(-) create mode 100644 tests/fullstack-test2/ddl/flashback_database.test diff --git a/dbms/src/Databases/DatabaseTiFlash.cpp b/dbms/src/Databases/DatabaseTiFlash.cpp index 518094a3cf8..50516b222bb 100644 --- a/dbms/src/Databases/DatabaseTiFlash.cpp +++ b/dbms/src/Databases/DatabaseTiFlash.cpp @@ -508,7 +508,8 @@ void DatabaseTiFlash::shutdown() tables.clear(); } -void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombstone_) + +void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombstone_, const TiDB::DBInfoPtr & new_db_info) { const auto database_metadata_path = getDatabaseMetadataPath(metadata_path); const auto database_metadata_tmp_path = database_metadata_path + ".tmp"; @@ -520,7 +521,18 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston { // Alter the attach statement in metadata. - auto dbinfo_literal = std::make_shared(Field(db_info == nullptr ? "" : (db_info->serialize()))); + std::shared_ptr dbinfo_literal = [&]() { + String seri_info; + if (new_db_info != nullptr) + { + seri_info = new_db_info->serialize(); + } + else if (db_info != nullptr) + { + seri_info = db_info->serialize(); + } + return std::make_shared(Field(seri_info)); + }(); Field format_version_field(static_cast(DatabaseTiFlash::CURRENT_VERSION)); auto version_literal = std::make_shared(format_version_field); auto tombstone_literal = std::make_shared(Field(tombstone_)); @@ -549,6 +561,9 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston } else { + // update the seri dbinfo + args.children[0] = dbinfo_literal; + args.children[1] = version_literal; // udpate the tombstone mark args.children[2] = tombstone_literal; } @@ -566,10 +581,17 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston // Atomic replace database metadata file and its encryption info auto provider = context.getFileProvider(); bool reuse_encrypt_info = provider->isFileEncrypted(EncryptionPath(database_metadata_path, "")); - EncryptionPath encryption_path - = reuse_encrypt_info ? EncryptionPath(database_metadata_path, "") : EncryptionPath(database_metadata_tmp_path, ""); + EncryptionPath encryption_path = reuse_encrypt_info ? EncryptionPath(database_metadata_path, "") + : EncryptionPath(database_metadata_tmp_path, ""); { - WriteBufferFromFileProvider out(provider, database_metadata_tmp_path, encryption_path, !reuse_encrypt_info, nullptr, statement.size(), O_WRONLY | O_CREAT | O_TRUNC); + WriteBufferFromFileProvider out( + provider, + database_metadata_tmp_path, + encryption_path, + !reuse_encrypt_info, + nullptr, + statement.size(), + O_WRONLY | O_CREAT | O_TRUNC); writeString(statement, out); out.next(); if (context.getSettingsRef().fsync_metadata) @@ -579,7 +601,12 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston try { - provider->renameFile(database_metadata_tmp_path, encryption_path, database_metadata_path, EncryptionPath(database_metadata_path, ""), !reuse_encrypt_info); + provider->renameFile( + database_metadata_tmp_path, + encryption_path, + database_metadata_path, + EncryptionPath(database_metadata_path, ""), + !reuse_encrypt_info); } catch (...) { @@ -590,6 +617,11 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston // After all done, set the tombstone tombstone = tombstone_; + // Overwrite db_info if not null + if (new_db_info) + { + db_info = new_db_info; + } } void DatabaseTiFlash::drop(const Context & context) diff --git a/dbms/src/Databases/DatabaseTiFlash.h b/dbms/src/Databases/DatabaseTiFlash.h index a61d30cfeb4..e44aaf443f3 100644 --- a/dbms/src/Databases/DatabaseTiFlash.h +++ b/dbms/src/Databases/DatabaseTiFlash.h @@ -73,7 +73,7 @@ class DatabaseTiFlash : public DatabaseWithOwnTablesBase bool isTombstone() const override { return tombstone != 0; } Timestamp getTombstone() const override { return tombstone; } - void alterTombstone(const Context & context, Timestamp tombstone_) override; + void alterTombstone(const Context & context, Timestamp tombstone_, const TiDB::DBInfoPtr & new_db_info) override; void drop(const Context & context) override; diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index ecc418c63dc..b68f7d8e70b 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -23,6 +23,11 @@ #include #include +namespace TiDB +{ +struct DBInfo; +using DBInfoPtr = std::shared_ptr; +} // namespace TiDB class ThreadPool; @@ -51,7 +56,7 @@ class IDatabaseIterator virtual const String & name() const = 0; virtual StoragePtr & table() const = 0; - virtual ~IDatabaseIterator() {} + virtual ~IDatabaseIterator() = default; }; using DatabaseIteratorPtr = std::unique_ptr; @@ -138,12 +143,12 @@ class IDatabase : public std::enable_shared_from_this virtual bool isTombstone() const { return false; } virtual Timestamp getTombstone() const { return 0; } - virtual void alterTombstone(const Context & /*context*/, Timestamp /*tombstone_*/) {} + virtual void alterTombstone(const Context & /*context*/, Timestamp /*tombstone_*/, const TiDB::DBInfoPtr & /*new_db_info*/) {} /// Delete metadata, the deletion of which differs from the recursive deletion of the directory, if any. virtual void drop(const Context & context) = 0; - virtual ~IDatabase() {} + virtual ~IDatabase() = default; }; using DatabasePtr = std::shared_ptr; diff --git a/dbms/src/Databases/test/gtest_database.cpp b/dbms/src/Databases/test/gtest_database.cpp index e30b6dd07c0..4757fea0df6 100644 --- a/dbms/src/Databases/test/gtest_database.cpp +++ b/dbms/src/Databases/test/gtest_database.cpp @@ -943,6 +943,7 @@ try )", }; + size_t case_no = 0; for (const auto & statement : statements) { { @@ -969,22 +970,47 @@ try LOG_DEBUG(log, "After create [meta={}]", meta); DB::Timestamp tso = 1000; - db->alterTombstone(ctx, tso); + db->alterTombstone(ctx, tso, nullptr); EXPECT_TRUE(db->isTombstone()); EXPECT_EQ(db->getTombstone(), tso); + if (case_no != 0) + { + auto db_tiflash = std::dynamic_pointer_cast(db); + ASSERT_NE(db_tiflash, nullptr); + auto db_info = db_tiflash->getDatabaseInfo(); + ASSERT_EQ(db_info.name, "test_db"); // not changed + } // Try restore from disk db = detachThenAttach(ctx, db_name, std::move(db), log); EXPECT_TRUE(db->isTombstone()); EXPECT_EQ(db->getTombstone(), tso); - // Recover - db->alterTombstone(ctx, 0); + // Recover, usually recover with a new database name + auto new_db_info = std::make_shared( + R"json({"charset":"utf8mb4","collate":"utf8mb4_bin","db_name":{"L":"test_new_db","O":"test_db"},"id":1010,"state":5})json"); + db->alterTombstone(ctx, 0, new_db_info); EXPECT_FALSE(db->isTombstone()); + if (case_no != 0) + { + auto db_tiflash = std::dynamic_pointer_cast(db); + ASSERT_NE(db_tiflash, nullptr); + auto db_info = db_tiflash->getDatabaseInfo(); + ASSERT_EQ(db_info.name, "test_new_db"); // changed by the `new_db_info` + } // Try restore from disk db = detachThenAttach(ctx, db_name, std::move(db), log); EXPECT_FALSE(db->isTombstone()); + if (case_no != 0) + { + auto db_tiflash = std::dynamic_pointer_cast(db); + ASSERT_NE(db_tiflash, nullptr); + auto db_info = db_tiflash->getDatabaseInfo(); + ASSERT_EQ(db_info.name, "test_new_db"); // changed by the `new_db_info` + } + + case_no += 1; } } CATCH diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index d69b6c49a98..2b76ff065b0 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -38,8 +38,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -458,6 +460,12 @@ void SchemaBuilder::applyDiff(const SchemaDiff & diff) return; } + if (diff.type == SchemaActionType::ActionRecoverSchema) + { + applyRecoverSchema(diff.schema_id); + return; + } + if (diff.type == SchemaActionType::CreateTables) { for (auto && opt : diff.affected_opts) @@ -999,9 +1007,58 @@ void SchemaBuilder::applyDropSchema(const String & db_name) // In such way our database (and its belonging tables) will be GC-ed later than TiDB, which is safe and correct. auto & tmt_context = context.getTMTContext(); auto tombstone = tmt_context.getPDClient()->getTS(); - db->alterTombstone(context, tombstone); + db->alterTombstone(context, tombstone, nullptr); - LOG_INFO(log, "Tombstoned database {}", db_name); + LOG_INFO(log, "Tombstoned database {}, tombstone={}", db_name, tombstone); +} + +template +void SchemaBuilder::applyRecoverSchema(DatabaseID database_id) +{ + auto db_info = getter.getDatabase(database_id); + if (db_info == nullptr) + { + LOG_INFO( + log, + "Recover database is ignored because database is not exist in TiKV," + " database_id={}", + database_id); + return; + } + LOG_INFO(log, "Recover database begin, database_id={}", database_id); + auto db_name = name_mapper.mapDatabaseName(*db_info); + auto db = context.tryGetDatabase(db_name); + if (!db) + { + LOG_ERROR( + log, + "Recover database is ignored because instance is not exists, may have been physically dropped, " + "database_id={}", + db_name, + database_id); + return; + } + + { + for (auto table_iter = db->getIterator(context); table_iter->isValid(); table_iter->next()) + { + auto & storage = table_iter->table(); + auto managed_storage = std::dynamic_pointer_cast(storage); + if (!managed_storage) + { + LOG_WARNING(log, "Recover database ignore non-manageable storage, name={} engine={}", storage->getTableName(), storage->getName()); + continue; + } + LOG_WARNING(log, "Recover database on storage begin, name={}", storage->getTableName()); + auto table_id = managed_storage->getTableInfo().id; + applyCreateTable(db_info, table_id); + } + } + + // Usually `FLASHBACK DATABASE ... TO ...` will rename the database + db->alterTombstone(context, 0, db_info); + databases[db_info->id] = db_info; + LOG_INFO(log, "Recover database end, database_id={}", database_id); } std::tuple @@ -1185,6 +1242,7 @@ void SchemaBuilder::applyDropPhysicalTable(const String & db } GET_METRIC(tiflash_schema_internal_ddl_count, type_drop_table).Increment(); LOG_INFO(log, "Tombstoning table {}.{}", db_name, name_mapper.debugTableName(storage->getTableInfo())); + const UInt64 tombstone_ts = tmt_context.getPDClient()->getTS(); AlterCommands commands; { AlterCommand command; @@ -1194,12 +1252,12 @@ void SchemaBuilder::applyDropPhysicalTable(const String & db // 1. Use current timestamp, which is after TiDB's drop time, to be the tombstone of this table; // 2. Use the same GC safe point as TiDB. // In such way our table will be GC-ed later than TiDB, which is safe and correct. - command.tombstone = tmt_context.getPDClient()->getTS(); + command.tombstone = tombstone_ts; commands.emplace_back(std::move(command)); } auto alter_lock = storage->lockForAlter(getThreadName()); storage->alterFromTiDB(alter_lock, commands, db_name, storage->getTableInfo(), name_mapper, context); - LOG_INFO(log, "Tombstoned table {}.{}", db_name, name_mapper.debugTableName(storage->getTableInfo())); + LOG_INFO(log, "Tombstoned table {}.{}, tombstone={}", db_name, name_mapper.debugTableName(storage->getTableInfo()), tombstone_ts); } template diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.h b/dbms/src/TiDB/Schema/SchemaBuilder.h index 8a84efd7afd..717e4bfcd09 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.h +++ b/dbms/src/TiDB/Schema/SchemaBuilder.h @@ -53,6 +53,8 @@ struct SchemaBuilder /// Parameter db_name should be mapped. void applyDropSchema(const String & db_name); + void applyRecoverSchema(DatabaseID database_id); + bool applyCreateSchema(DatabaseID schema_id); void applyCreateSchema(const TiDB::DBInfoPtr & db_info); diff --git a/dbms/src/TiDB/Schema/SchemaGetter.h b/dbms/src/TiDB/Schema/SchemaGetter.h index fe251cc8882..02f36692da7 100644 --- a/dbms/src/TiDB/Schema/SchemaGetter.h +++ b/dbms/src/TiDB/Schema/SchemaGetter.h @@ -95,8 +95,10 @@ enum class SchemaActionType : Int8 AlterNoCacheTable = 59, CreateTables = 60, ActionMultiSchemaChange = 61, + ActionFlashbackCluster = 62, // not supported on release-6.5 + ActionRecoverSchema = 63, - // If we supporte new type from TiDB. + // If we support new type from TiDB. // MaxRecognizedType also needs to be changed. // It should always be equal to the maximum supported type + 1 MaxRecognizedType = 62, diff --git a/tests/fullstack-test2/ddl/flashback_database.test b/tests/fullstack-test2/ddl/flashback_database.test new file mode 100644 index 00000000000..38285435a77 --- /dev/null +++ b/tests/fullstack-test2/ddl/flashback_database.test @@ -0,0 +1,77 @@ +# Copyright 2023 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## case 1, normal flashback without failpoints + +mysql> drop database if exists d1; +mysql> drop database if exists d1_new; + +# non-partition table +mysql> create database d1; +mysql> create table d1.t3 (a int); +mysql> insert into d1.t3 values(1); +# partition table +mysql> create table d1.t4(id INT NOT NULL,name VARCHAR(30)) PARTITION BY RANGE (id) ( PARTITION p0 VALUES LESS THAN (50),PARTITION p1 VALUES LESS THAN (100)); +mysql> insert into d1.t4 values(1, 'abc'),(2, 'cde'),(53, 'efg'); + +mysql> alter table d1.t3 set tiflash replica 1; +mysql> alter table d1.t4 set tiflash replica 1; +func> wait_table d1 t3 t4 + +mysql> alter table d1.t3 add column b int; +mysql> insert into d1.t3 values(2,2); +mysql> alter table d1.t4 add column b int; + +mysql> drop database d1; + +mysql> flashback database d1 to d1_new +mysql> set session tidb_isolation_read_engines='tiflash'; select * from d1_new.t3 order by a; ++------+------+ +| a | b | ++------+------+ +| 1 | NULL | +| 2 | 2 | ++------+------+ +mysql> set session tidb_isolation_read_engines='tiflash'; select * from d1_new.t4 order by id; ++----+------+------+ +| id | name | b | ++----+------+------+ +| 1 | abc | NULL | +| 2 | cde | NULL | +| 53 | efg | NULL | ++----+------+------+ + +# ensure the flashbacked table and database is not mark as tombstone +>> DBGInvoke __enable_schema_sync_service('true') +>> DBGInvoke __gc_schemas(18446744073709551615) + +mysql> set session tidb_isolation_read_engines='tiflash'; select * from d1_new.t3 order by a; ++------+------+ +| a | b | ++------+------+ +| 1 | NULL | +| 2 | 2 | ++------+------+ +mysql> set session tidb_isolation_read_engines='tiflash'; select * from d1_new.t4 order by id; ++----+------+------+ +| id | name | b | ++----+------+------+ +| 1 | abc | NULL | +| 2 | cde | NULL | +| 53 | efg | NULL | ++----+------+------+ + +# cleanup +mysql> drop database if exists d1; +mysql> drop database if exists d1_new;