ddl: Support FLASHBACK DATABASE (release-6.5) (#8449) (#8479)

close #8450

dbms/src/Databases/DatabaseTiFlash.cpp

@@ -508,7 +508,8 @@ void DatabaseTiFlash::shutdown()
     tables.clear();
 }
 
-void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombstone_)
+
+void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombstone_, const TiDB::DBInfoPtr & new_db_info)
 {
     const auto database_metadata_path = getDatabaseMetadataPath(metadata_path);
     const auto database_metadata_tmp_path = database_metadata_path + ".tmp";
@@ -520,7 +521,18 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston
 
     {
         // Alter the attach statement in metadata.
-        auto dbinfo_literal = std::make_shared<ASTLiteral>(Field(db_info == nullptr ? "" : (db_info->serialize())));
+        std::shared_ptr<ASTLiteral> dbinfo_literal = [&]() {
+            String seri_info;
+            if (new_db_info != nullptr)
+            {
+                seri_info = new_db_info->serialize();
+            }
+            else if (db_info != nullptr)
+            {
+                seri_info = db_info->serialize();
+            }
+            return std::make_shared<ASTLiteral>(Field(seri_info));
+        }();
         Field format_version_field(static_cast<UInt64>(DatabaseTiFlash::CURRENT_VERSION));
         auto version_literal = std::make_shared<ASTLiteral>(format_version_field);
         auto tombstone_literal = std::make_shared<ASTLiteral>(Field(tombstone_));
@@ -549,6 +561,9 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston
         }
         else
         {
+            // update the seri dbinfo
+            args.children[0] = dbinfo_literal;
+            args.children[1] = version_literal;
             // udpate the tombstone mark
             args.children[2] = tombstone_literal;
         }
@@ -566,10 +581,17 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston
         // Atomic replace database metadata file and its encryption info
         auto provider = context.getFileProvider();
         bool reuse_encrypt_info = provider->isFileEncrypted(EncryptionPath(database_metadata_path, ""));
-        EncryptionPath encryption_path
-            = reuse_encrypt_info ? EncryptionPath(database_metadata_path, "") : EncryptionPath(database_metadata_tmp_path, "");
+        EncryptionPath encryption_path = reuse_encrypt_info ? EncryptionPath(database_metadata_path, "")
+                                                            : EncryptionPath(database_metadata_tmp_path, "");
         {
-            WriteBufferFromFileProvider out(provider, database_metadata_tmp_path, encryption_path, !reuse_encrypt_info, nullptr, statement.size(), O_WRONLY | O_CREAT | O_TRUNC);
+            WriteBufferFromFileProvider out(
+                provider,
+                database_metadata_tmp_path,
+                encryption_path,
+                !reuse_encrypt_info,
+                nullptr,
+                statement.size(),
+                O_WRONLY | O_CREAT | O_TRUNC);
             writeString(statement, out);
             out.next();
             if (context.getSettingsRef().fsync_metadata)
@@ -579,7 +601,12 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston
 
         try
         {
-            provider->renameFile(database_metadata_tmp_path, encryption_path, database_metadata_path, EncryptionPath(database_metadata_path, ""), !reuse_encrypt_info);
+            provider->renameFile(
+                database_metadata_tmp_path,
+                encryption_path,
+                database_metadata_path,
+                EncryptionPath(database_metadata_path, ""),
+                !reuse_encrypt_info);
         }
         catch (...)
         {
@@ -590,6 +617,11 @@ void DatabaseTiFlash::alterTombstone(const Context & context, Timestamp tombston
 
     // After all done, set the tombstone
     tombstone = tombstone_;
+    // Overwrite db_info if not null
+    if (new_db_info)
+    {
+        db_info = new_db_info;
+    }
 }
 
 void DatabaseTiFlash::drop(const Context & context)

dbms/src/Databases/DatabaseTiFlash.h

@@ -73,7 +73,7 @@ class DatabaseTiFlash : public DatabaseWithOwnTablesBase
 
     bool isTombstone() const override { return tombstone != 0; }
     Timestamp getTombstone() const override { return tombstone; }
-    void alterTombstone(const Context & context, Timestamp tombstone_) override;
+    void alterTombstone(const Context & context, Timestamp tombstone_, const TiDB::DBInfoPtr & new_db_info) override;
 
     void drop(const Context & context) override;
 

dbms/src/Databases/IDatabase.h

@@ -23,6 +23,11 @@
 #include <functional>
 #include <memory>
 
+namespace TiDB
+{
+struct DBInfo;
+using DBInfoPtr = std::shared_ptr<DBInfo>;
+} // namespace TiDB
 
 class ThreadPool;
 
@@ -51,7 +56,7 @@ class IDatabaseIterator
     virtual const String & name() const = 0;
     virtual StoragePtr & table() const = 0;
 
-    virtual ~IDatabaseIterator() {}
+    virtual ~IDatabaseIterator() = default;
 };
 
 using DatabaseIteratorPtr = std::unique_ptr<IDatabaseIterator>;
@@ -138,12 +143,12 @@ class IDatabase : public std::enable_shared_from_this<IDatabase>
 
     virtual bool isTombstone() const { return false; }
     virtual Timestamp getTombstone() const { return 0; }
-    virtual void alterTombstone(const Context & /*context*/, Timestamp /*tombstone_*/) {}
+    virtual void alterTombstone(const Context & /*context*/, Timestamp /*tombstone_*/, const TiDB::DBInfoPtr & /*new_db_info*/) {}
 
     /// Delete metadata, the deletion of which differs from the recursive deletion of the directory, if any.
     virtual void drop(const Context & context) = 0;
 
-    virtual ~IDatabase() {}
+    virtual ~IDatabase() = default;
 };
 
 using DatabasePtr = std::shared_ptr<IDatabase>;

dbms/src/Databases/test/gtest_database.cpp

@@ -943,6 +943,7 @@ try
 )",
     };
 
+    size_t case_no = 0;
     for (const auto & statement : statements)
     {
         {
@@ -969,22 +970,47 @@ try
         LOG_DEBUG(log, "After create [meta={}]", meta);
 
         DB::Timestamp tso = 1000;
-        db->alterTombstone(ctx, tso);
+        db->alterTombstone(ctx, tso, nullptr);
         EXPECT_TRUE(db->isTombstone());
         EXPECT_EQ(db->getTombstone(), tso);
+        if (case_no != 0)
+        {
+            auto db_tiflash = std::dynamic_pointer_cast<DatabaseTiFlash>(db);
+            ASSERT_NE(db_tiflash, nullptr);
+            auto db_info = db_tiflash->getDatabaseInfo();
+            ASSERT_EQ(db_info.name, "test_db"); // not changed
+        }
 
         // Try restore from disk
         db = detachThenAttach(ctx, db_name, std::move(db), log);
         EXPECT_TRUE(db->isTombstone());
         EXPECT_EQ(db->getTombstone(), tso);
 
-        // Recover
-        db->alterTombstone(ctx, 0);
+        // Recover, usually recover with a new database name
+        auto new_db_info = std::make_shared<TiDB::DBInfo>(
+            R"json({"charset":"utf8mb4","collate":"utf8mb4_bin","db_name":{"L":"test_new_db","O":"test_db"},"id":1010,"state":5})json");
+        db->alterTombstone(ctx, 0, new_db_info);
         EXPECT_FALSE(db->isTombstone());
+        if (case_no != 0)
+        {
+            auto db_tiflash = std::dynamic_pointer_cast<DatabaseTiFlash>(db);
+            ASSERT_NE(db_tiflash, nullptr);
+            auto db_info = db_tiflash->getDatabaseInfo();
+            ASSERT_EQ(db_info.name, "test_new_db"); // changed by the `new_db_info`
+        }
 
         // Try restore from disk
         db = detachThenAttach(ctx, db_name, std::move(db), log);
         EXPECT_FALSE(db->isTombstone());
+        if (case_no != 0)
+        {
+            auto db_tiflash = std::dynamic_pointer_cast<DatabaseTiFlash>(db);
+            ASSERT_NE(db_tiflash, nullptr);
+            auto db_info = db_tiflash->getDatabaseInfo();
+            ASSERT_EQ(db_info.name, "test_new_db"); // changed by the `new_db_info`
+        }
+
+        case_no += 1;
     }
 }
 CATCH

dbms/src/TiDB/Schema/SchemaBuilder.cpp

@@ -38,8 +38,10 @@
 #include <Storages/Transaction/TMTContext.h>
 #include <Storages/Transaction/TiDB.h>
 #include <Storages/Transaction/TypeMapping.h>
+#include <Storages/Transaction/Types.h>
 #include <TiDB/Schema/SchemaBuilder-internal.h>
 #include <TiDB/Schema/SchemaBuilder.h>
+#include <TiDB/Schema/SchemaGetter.h>
 #include <TiDB/Schema/SchemaNameMapper.h>
 #include <common/logger_useful.h>
 
@@ -458,6 +460,12 @@ void SchemaBuilder<Getter, NameMapper>::applyDiff(const SchemaDiff & diff)
         return;
     }
 
+    if (diff.type == SchemaActionType::ActionRecoverSchema)
+    {
+        applyRecoverSchema(diff.schema_id);
+        return;
+    }
+
     if (diff.type == SchemaActionType::CreateTables)
     {
         for (auto && opt : diff.affected_opts)
@@ -999,9 +1007,58 @@ void SchemaBuilder<Getter, NameMapper>::applyDropSchema(const String & db_name)
     // In such way our database (and its belonging tables) will be GC-ed later than TiDB, which is safe and correct.
     auto & tmt_context = context.getTMTContext();
     auto tombstone = tmt_context.getPDClient()->getTS();
-    db->alterTombstone(context, tombstone);
+    db->alterTombstone(context, tombstone, nullptr);
 
-    LOG_INFO(log, "Tombstoned database {}", db_name);
+    LOG_INFO(log, "Tombstoned database {}, tombstone={}", db_name, tombstone);
+}
+
+template <typename Getter, typename NameMapper>
+void SchemaBuilder<Getter, NameMapper>::applyRecoverSchema(DatabaseID database_id)
+{
+    auto db_info = getter.getDatabase(database_id);
+    if (db_info == nullptr)
+    {
+        LOG_INFO(
+            log,
+            "Recover database is ignored because database is not exist in TiKV,"
+            " database_id={}",
+            database_id);
+        return;
+    }
+    LOG_INFO(log, "Recover database begin, database_id={}", database_id);
+    auto db_name = name_mapper.mapDatabaseName(*db_info);
+    auto db = context.tryGetDatabase(db_name);
+    if (!db)
+    {
+        LOG_ERROR(
+            log,
+            "Recover database is ignored because instance is not exists, may have been physically dropped, "
+            "database_id={}",
+            db_name,
+            database_id);
+        return;
+    }
+
+    {
+        for (auto table_iter = db->getIterator(context); table_iter->isValid(); table_iter->next())
+        {
+            auto & storage = table_iter->table();
+            auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
+            if (!managed_storage)
+            {
+                LOG_WARNING(log, "Recover database ignore non-manageable storage, name={} engine={}", storage->getTableName(), storage->getName());
+                continue;
+            }
+            LOG_WARNING(log, "Recover database on storage begin, name={}", storage->getTableName());
+            auto table_id = managed_storage->getTableInfo().id;
+            applyCreateTable(db_info, table_id);
+        }
+    }
+
+    // Usually `FLASHBACK DATABASE ... TO ...` will rename the database
+    db->alterTombstone(context, 0, db_info);
+    databases[db_info->id] = db_info;
+    LOG_INFO(log, "Recover database end, database_id={}", database_id);
 }
 
 std::tuple<NamesAndTypes, Strings>
@@ -1185,6 +1242,7 @@ void SchemaBuilder<Getter, NameMapper>::applyDropPhysicalTable(const String & db
     }
     GET_METRIC(tiflash_schema_internal_ddl_count, type_drop_table).Increment();
     LOG_INFO(log, "Tombstoning table {}.{}", db_name, name_mapper.debugTableName(storage->getTableInfo()));
+    const UInt64 tombstone_ts = tmt_context.getPDClient()->getTS();
     AlterCommands commands;
     {
         AlterCommand command;
@@ -1194,12 +1252,12 @@ void SchemaBuilder<Getter, NameMapper>::applyDropPhysicalTable(const String & db
         // 1. Use current timestamp, which is after TiDB's drop time, to be the tombstone of this table;
         // 2. Use the same GC safe point as TiDB.
         // In such way our table will be GC-ed later than TiDB, which is safe and correct.
-        command.tombstone = tmt_context.getPDClient()->getTS();
+        command.tombstone = tombstone_ts;
         commands.emplace_back(std::move(command));
     }
     auto alter_lock = storage->lockForAlter(getThreadName());
     storage->alterFromTiDB(alter_lock, commands, db_name, storage->getTableInfo(), name_mapper, context);
-    LOG_INFO(log, "Tombstoned table {}.{}", db_name, name_mapper.debugTableName(storage->getTableInfo()));
+    LOG_INFO(log, "Tombstoned table {}.{}, tombstone={}", db_name, name_mapper.debugTableName(storage->getTableInfo()), tombstone_ts);
 }
 
 template <typename Getter, typename NameMapper>

dbms/src/TiDB/Schema/SchemaBuilder.h

@@ -53,6 +53,8 @@ struct SchemaBuilder
     /// Parameter db_name should be mapped.
     void applyDropSchema(const String & db_name);
 
+    void applyRecoverSchema(DatabaseID database_id);
+
     bool applyCreateSchema(DatabaseID schema_id);
 
     void applyCreateSchema(const TiDB::DBInfoPtr & db_info);

dbms/src/TiDB/Schema/SchemaGetter.h

@@ -95,8 +95,10 @@ enum class SchemaActionType : Int8
     AlterNoCacheTable = 59,
     CreateTables = 60,
     ActionMultiSchemaChange = 61,
+    ActionFlashbackCluster = 62, // not supported on release-6.5
+    ActionRecoverSchema = 63,
 
-    // If we supporte new type from TiDB.
+    // If we support new type from TiDB.
     // MaxRecognizedType also needs to be changed.
     // It should always be equal to the maximum supported type + 1
     MaxRecognizedType = 62,

tests/fullstack-test2/ddl/flashback_database.test

@@ -0,0 +1,77 @@
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## case 1, normal flashback without failpoints
+
+mysql> drop database if exists d1;
+mysql> drop database if exists d1_new;
+
+# non-partition table
+mysql> create database d1;
+mysql> create table d1.t3 (a int);
+mysql> insert into d1.t3 values(1);
+# partition table
+mysql> create table d1.t4(id INT NOT NULL,name VARCHAR(30)) PARTITION BY RANGE (id) ( PARTITION p0 VALUES LESS THAN (50),PARTITION p1 VALUES LESS THAN (100));
+mysql> insert into d1.t4 values(1, 'abc'),(2, 'cde'),(53, 'efg');
+
+mysql> alter table d1.t3 set tiflash replica 1;
+mysql> alter table d1.t4 set tiflash replica 1;
+func> wait_table d1 t3 t4
+
+mysql> alter table d1.t3 add column b int;
+mysql> insert into d1.t3 values(2,2);
+mysql> alter table d1.t4 add column b int;
+
+mysql> drop database d1;
+
+mysql> flashback database d1 to d1_new
+mysql> set session tidb_isolation_read_engines='tiflash'; select * from d1_new.t3 order by a;
++------+------+
+| a    | b    |
++------+------+
+|    1 | NULL |
+|    2 |    2 |
++------+------+
+mysql> set session tidb_isolation_read_engines='tiflash'; select * from d1_new.t4 order by id;
++----+------+------+
+| id | name | b    |
++----+------+------+
+|  1 | abc  | NULL |
+|  2 | cde  | NULL |
+| 53 | efg  | NULL |
++----+------+------+
+
+# ensure the flashbacked table and database is not mark as tombstone
+>> DBGInvoke __enable_schema_sync_service('true')
+>> DBGInvoke __gc_schemas(18446744073709551615)
+
+mysql> set session tidb_isolation_read_engines='tiflash'; select * from d1_new.t3 order by a;
++------+------+
+| a    | b    |
++------+------+
+|    1 | NULL |
+|    2 |    2 |
++------+------+
+mysql> set session tidb_isolation_read_engines='tiflash'; select * from d1_new.t4 order by id;
++----+------+------+
+| id | name | b    |
++----+------+------+
+|  1 | abc  | NULL |
+|  2 | cde  | NULL |
+| 53 | efg  | NULL |
++----+------+------+
+
+# cleanup
+mysql> drop database if exists d1;
+mysql> drop database if exists d1_new;