#448 Backfill Index Table

Summary:
High level design doc : https://github.com/yugabyte/yugabyte-db/blob/master/architecture/design/docdb-index-backfill.md

This diff implements the master side changes to be done for a create index. Populating the index tablet at the appropriate time is implemented in D7104.

    - Implement a 4 phase index creation at the master to safely create an index and backfill the data in an online manner. Index is created with the INDEX_PERM_DELETE_ONLY permission, then updated to INDEX_PERM_WRITE_AND_DELETE, then INDEX_PERM_BACKFILLING; and finally, after the backfill is complete, set to INDEX_PERM_READ_WRITE_AND_DELETE.
   - At each phase, the master waits for all the tablets of the indexed table to be updated before moving on to the next phase.
   - Ensure that the CQL proxy/Tablet Server respects the IndexPermission and only performs the operations that are permitted.
   - Ensure that master failover in the middle of a index backfill is handled.

A new flag `disable_index_backfill` is introduced to fence the new 4-stage index create process. This defaults to `true` right now -- until all the remaining parts needed for the backfill land.

Test Plan:
./yb_build.sh --cxx-test cassandra_cpp_driver-test --gtest_filter CppCassandraDriverTest.*TestCreateIndex*
./yb_build.sh --cxx-test master_failover-itest --gtest_filter MasterFailoverTestIndexCreation.*Index*

Reviewers: mihnea, sergei, hector, bogdan, rahuldesirazu, mikhail

Reviewed By: rahuldesirazu, mikhail

Subscribers: mikhail, ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D7077

src/yb/client/client.cc

@@ -498,20 +498,27 @@ Status YBClient::IsAlterTableInProgress(const YBTableName& table_name,
   return data_->IsAlterTableInProgress(this, table_name, table_id, deadline, alter_in_progress);
 }
 
-Status YBClient::GetTableSchema(const YBTableName& table_name,
-                                YBSchema* schema,
-                                PartitionSchema* partition_schema) {
-  auto deadline = CoarseMonoClock::Now() + default_admin_operation_timeout();
+Result<YBTableInfo> YBClient::GetYBTableInfo(const YBTableName& table_name) {
   YBTableInfo info;
+  auto deadline = CoarseMonoClock::Now() + default_admin_operation_timeout();
   RETURN_NOT_OK(data_->GetTableSchema(this, table_name, deadline, &info));
+  return info;
+}
 
+Status YBClient::GetTableSchema(const YBTableName& table_name,
+                                YBSchema* schema,
+                                PartitionSchema* partition_schema) {
+  Result<YBTableInfo> info = GetYBTableInfo(table_name);
+  if (!info.ok()) {
+    return info.status();
+  }
   // Verify it is not an index table.
-  if (info.index_info) {
+  if (info->index_info) {
     return STATUS(NotFound, "The table does not exist");
   }
 
-  *schema = std::move(info.schema);
-  *partition_schema = std::move(info.partition_schema);
+  *schema = std::move(info->schema);
+  *partition_schema = std::move(info->partition_schema);
   return Status::OK();
 }
 

src/yb/client/client.h

@@ -294,6 +294,7 @@ class YBClient {
   CHECKED_STATUS GetTableSchema(const YBTableName& table_name,
                                 YBSchema* schema,
                                 PartitionSchema* partition_schema);
+  Result<YBTableInfo> GetYBTableInfo(const YBTableName& table_name);
 
   CHECKED_STATUS GetTableSchemaById(const TableId& table_id, std::shared_ptr<YBTableInfo> info,
                                     StatusCallback callback);
@@ -625,6 +626,7 @@ class YBClient {
   FRIEND_TEST(ClientTest, TestScanTimeout);
   FRIEND_TEST(ClientTest, TestWriteWithDeadMaster);
   FRIEND_TEST(MasterFailoverTest, DISABLED_TestPauseAfterCreateTableIssued);
+  FRIEND_TEST(MasterFailoverTestIndexCreation, TestPauseAfterCreateIndexIssued);
 
   friend std::future<Result<internal::RemoteTabletPtr>> LookupFirstTabletFuture(
       const YBTable* table);

src/yb/client/table.cc

@@ -114,9 +114,22 @@ bool YBTable::IsIndex() const {
   return info_.index_info != boost::none;
 }
 
+bool YBTable::IsUniqueIndex() const {
+  return info_.index_info.is_initialized() && info_.index_info->is_unique();
+}
+
 const IndexInfo& YBTable::index_info() const {
-  CHECK(info_.index_info);
-  return *info_.index_info;
+  static IndexInfo kEmptyIndexInfo;
+  if (info_.index_info) {
+    return *info_.index_info;
+  }
+  return kEmptyIndexInfo;
+}
+
+std::string YBTable::ToString() const {
+  return strings::Substitute(
+      "$0 $1 IndexInfo: $2 IndexMap $3", (IsIndex() ? "Index Table" : "Normal Table"), id(),
+      yb::ToString(index_info()), yb::ToString(index_map()));
 }
 
 const PartitionSchema& YBTable::partition_schema() const {

src/yb/client/table.h

@@ -86,9 +86,12 @@ class YBTable : public std::enable_shared_from_this<YBTable> {
   // Is this an index?
   bool IsIndex() const;
 
+  bool IsUniqueIndex() const;
+
   // For index table: information about this index.
   const IndexInfo& index_info() const;
 
+  std::string ToString() const;
   //------------------------------------------------------------------------------------------------
   // CQL support
   // Create a new QL operation for this table.

src/yb/client/table_creator.cc

@@ -160,6 +160,11 @@ YBTableCreator& YBTableCreator::wait(bool wait) {
   return *this;
 }
 
+YBTableCreator& YBTableCreator::TEST_use_old_style_create_request() {
+  TEST_use_old_style_create_request_ = true;
+  return *this;
+}
+
 Status YBTableCreator::Create() {
   const char *object_type = index_info_.has_indexed_table_id() ? "index" : "table";
   if (table_name_.table_name().empty()) {
@@ -234,7 +239,9 @@ Status YBTableCreator::Create() {
 
   // Index mapping with data-table being indexed.
   if (index_info_.has_indexed_table_id()) {
-    req.mutable_index_info()->CopyFrom(index_info_);
+    if (!TEST_use_old_style_create_request_) {
+      req.mutable_index_info()->CopyFrom(index_info_);
+    }
 
     // For compatibility reasons, set the old fields just in case we have new clients talking to
     // old master server during rolling upgrade.

src/yb/client/table_creator.h

@@ -87,6 +87,9 @@ class YBTableCreator {
   // For index table: sets the indexed table id of this index.
   YBTableCreator& indexed_table_id(const std::string& id);
 
+  // For index table: uses the old style request without index_info.
+  YBTableCreator& TEST_use_old_style_create_request();
+
   // For index table: sets whether this is a local index.
   YBTableCreator& is_local_index(bool is_local_index);
 
@@ -161,6 +164,8 @@ class YBTableCreator {
   // the data-table being indexed.
   IndexInfoPB index_info_;
 
+  bool TEST_use_old_style_create_request_ = false;
+
   MonoDelta timeout_;
   bool wait_ = true;
 

src/yb/common/common.proto

@@ -147,6 +147,14 @@ enum PermissionType {
   ALL_PERMISSION = 999999999;
 }
 
+enum IndexPermissions {
+  INDEX_PERM_DELETE_ONLY = 0;
+  INDEX_PERM_WRITE_AND_DELETE = 2;
+  INDEX_PERM_DO_BACKFILL = 4;
+  INDEX_PERM_READ_WRITE_AND_DELETE = 6;
+  INDEX_PERM_BACKFILL_FAILED = 8;
+}
+
 // The type used in column schemas, which may have type parameters (i.e. for collections)
 // If types have parameters, they are stored in the params field. Otherwise params is empty.
 // e.g. (using lower case of QLTypes and upper case for DataType):
@@ -225,6 +233,7 @@ message TablePropertiesPB {
   optional bool use_mangled_column_name =  6 [ default = false ];
   optional int32 num_tablets = 7 [ default = 0 ];
   optional bool is_ysql_catalog_table = 8 [ default = false ];
+  optional bool is_backfilling = 9 [ default = false ];
 }
 
 message SchemaPB {
@@ -248,6 +257,9 @@ message IndexInfoPB {
   optional uint32 version = 2 [ default = 0]; // Index table's schema version.
   optional bool is_local = 3 [ default = false ];  // Whether the index is a local index
   optional bool is_unique = 7 [ default = false ]; // Whether the index is a unique index
+  // We should only have this in the elements of "repeated IndexInfoPB indexes" of the
+  // SysTableEntryPB of the main table.
+  optional IndexPermissions index_permissions = 12 [ default = INDEX_PERM_READ_WRITE_AND_DELETE ];
 
   // Index column mapping.
   // "colexpr" is used to compute the value of this column in an INDEX.

src/yb/common/index.cc

@@ -75,6 +75,7 @@ IndexInfo::IndexInfo(const IndexInfoPB& pb)
       range_column_count_(pb.range_column_count()),
       indexed_hash_column_ids_(ColumnIdsFromPB(pb.indexed_hash_column_ids())),
       indexed_range_column_ids_(ColumnIdsFromPB(pb.indexed_range_column_ids())),
+      index_permissions_(pb.index_permissions()),
       use_mangled_column_name_(pb.use_mangled_column_name()) {
   for (const IndexInfo::IndexColumn &index_col : columns_) {
     covered_column_ids_.insert(index_col.indexed_column_id);
@@ -99,6 +100,7 @@ void IndexInfo::ToPB(IndexInfoPB* pb) const {
     pb->add_indexed_range_column_ids(id);
   }
   pb->set_use_mangled_column_name(use_mangled_column_name_);
+  pb->set_index_permissions(index_permissions_);
 }
 
 vector<ColumnId> IndexInfo::index_key_column_ids() const {

src/yb/common/index.h

@@ -32,7 +32,7 @@ class IndexInfo {
   // Index column mapping.
   struct IndexColumn {
     ColumnId column_id;         // Column id in the index table.
-    string column_name;         // Column name in the index table - colexpr.MangledName().
+    std::string column_name;    // Column name in the index table - colexpr.MangledName().
     ColumnId indexed_column_id; // Corresponding column id in indexed table.
     QLExpressionPB colexpr;     // Index expression.
 
@@ -78,10 +78,35 @@ class IndexInfo {
   // Check if this INDEX contain the column being referenced by the given selected expression.
   // - If found, return the location of the column (columns_[loc]).
   // - Otherwise, return -1.
-  int32_t IsExprCovered(const string& expr_content) const;
+  int32_t IsExprCovered(const std::string& expr_content) const;
+
+  // Are read operations allowed to use the index? Reads are not allowed until
+  // the index backfill is successfully completed.
+  bool AllowReads() const { return index_permissions_ == INDEX_PERM_READ_WRITE_AND_DELETE; }
+
+  // Should write operations to the index be allowed to update it.
+  bool AllowWrites() const {
+    return index_permissions_ >= INDEX_PERM_WRITE_AND_DELETE &&
+           index_permissions_ <= INDEX_PERM_READ_WRITE_AND_DELETE;
+  }
+
+  // Should delete operations to the index be allowed to update it.
+  bool AllowDelete() const {
+    return index_permissions_ >= INDEX_PERM_DELETE_ONLY &&
+           index_permissions_ <= INDEX_PERM_READ_WRITE_AND_DELETE;
+  }
+
+  // Is the index being backfilled.
+  bool Backfilling() const { return index_permissions_ == INDEX_PERM_DO_BACKFILL; }
+
+  std::string ToString() const {
+    IndexInfoPB pb;
+    ToPB(&pb);
+    return yb::ToString(pb);
+  }
 
   // Same as "IsExprCovered" but only search the key columns.
-  int32_t FindKeyIndex(const string& key_name) const;
+  int32_t FindKeyIndex(const std::string& key_name) const;
 
   bool use_mangled_column_name() const {
     return use_mangled_column_name_;
@@ -98,6 +123,7 @@ class IndexInfo {
   const size_t range_column_count_ = 0;    // Number of range columns in the index.
   const std::vector<ColumnId> indexed_hash_column_ids_;  // Hash column ids in the indexed table.
   const std::vector<ColumnId> indexed_range_column_ids_; // Range column ids in the indexed table.
+  const IndexPermissions index_permissions_ = INDEX_PERM_READ_WRITE_AND_DELETE;
 
   // Column ids covered by the index (include indexed columns).
   std::unordered_set<ColumnId> covered_column_ids_;

src/yb/common/pgsql_resultset.h

@@ -44,7 +44,7 @@ class PgsqlRSRowDesc {
       return ql_type_->main();
     }
    private:
-    const string name_;
+    const std::string name_;
     QLType::SharedPtr ql_type_;
   };
 

src/yb/common/ql_type.h

@@ -37,23 +37,23 @@ class UDTypeInfo {
       : keyspace_name_(keyspace_name), name_(name) {
   }
 
-  const string& keyspace_name() const {
+  const std::string& keyspace_name() const {
     return keyspace_name_;
   }
 
-  const string& name() const {
+  const std::string& name() const {
     return name_;
   }
 
-  const string& id() const {
+  const std::string& id() const {
     return id_;
   }
 
   const std::vector<string>& field_names() const {
     return field_names_;
   }
 
-  const string& field_name(int index) const {
+  const std::string& field_name(int index) const {
     return field_names_[index];
   }
 
@@ -150,7 +150,7 @@ class QLType {
   }
 
   // Constructor for user-defined types
-  QLType(const string& keyspace_name, const string& type_name)
+  QLType(const std::string& keyspace_name, const std::string& type_name)
       : id_(USER_DEFINED_TYPE), params_(0) {
     udtype_info_ = std::make_shared<UDTypeInfo>(keyspace_name, type_name);
   }
@@ -231,19 +231,19 @@ class QLType {
     return udtype_info_->field_names();
   }
 
-  const string& udtype_field_name(int index) const {
+  const std::string& udtype_field_name(int index) const {
     return udtype_info_->field_name(index);
   }
 
-  const string& udtype_keyspace_name() const {
+  const std::string& udtype_keyspace_name() const {
     return udtype_info_->keyspace_name();
   }
 
-  const string& udtype_name() const {
+  const std::string& udtype_name() const {
     return udtype_info_->name();
   }
 
-  const string& udtype_id() const {
+  const std::string& udtype_id() const {
     return udtype_info_->id();
   }
 
@@ -255,7 +255,7 @@ class QLType {
   }
 
   // returns position of "field_name" in udtype_field_names() vector if found, otherwise -1
-  const int GetUDTypeFieldIdxByName(const string &field_name) const {
+  const int GetUDTypeFieldIdxByName(const std::string &field_name) const {
     const std::vector<string>& field_names = udtype_field_names();
     int i = 0;
     while (i != field_names.size()) {
@@ -410,9 +410,9 @@ class QLType {
 
   //------------------------------------------------------------------------------------------------
   // Logging supports.
-  const string ToString() const;
+  const std::string ToString() const;
   void ToString(std::stringstream& os) const;
-  static const string ToCQLString(const DataType& datatype);
+  static const std::string ToCQLString(const DataType& datatype);
 
   //------------------------------------------------------------------------------------------------
   // static methods

src/yb/common/schema-test.cc

@@ -98,7 +98,8 @@ TEST(TestSchema, TestSchema) {
             "]\nproperties: contain_counters: false is_transactional: false "
             "consistency_level: STRONG "
             "use_mangled_column_name: false "
-            "is_ysql_catalog_table: false",
+            "is_ysql_catalog_table: false "
+            "is_backfilling: false",
             schema.ToString());
   EXPECT_EQ("key[string NOT NULL NOT A PARTITION KEY]", schema.column(0).ToString());
   EXPECT_EQ("uint32 NULLABLE NOT A PARTITION KEY", schema.column(1).TypeToString());
@@ -371,7 +372,8 @@ TEST(TestSchema, TestCreateProjection) {
             "]\nproperties: contain_counters: false is_transactional: false "
             "consistency_level: STRONG "
             "use_mangled_column_name: false "
-            "is_ysql_catalog_table: false",
+            "is_ysql_catalog_table: false "
+            "is_backfilling: false",
             partial_schema.ToString());
 
   // By names, with IDS
@@ -383,7 +385,8 @@ TEST(TestSchema, TestCreateProjection) {
                        "]\nproperties: contain_counters: false is_transactional: false "
                        "consistency_level: STRONG "
                        "use_mangled_column_name: false "
-                       "is_ysql_catalog_table: false",
+                       "is_ysql_catalog_table: false "
+                       "is_backfilling: false",
                        schema_with_ids.column_id(0),
                        schema_with_ids.column_id(1),
                        schema_with_ids.column_id(3)),
@@ -406,7 +409,8 @@ TEST(TestSchema, TestCreateProjection) {
                        "]\nproperties: contain_counters: false is_transactional: false "
                        "consistency_level: STRONG "
                        "use_mangled_column_name: false "
-                       "is_ysql_catalog_table: false",
+                       "is_ysql_catalog_table: false "
+                       "is_backfilling: false",
                        schema_with_ids.column_id(0),
                        schema_with_ids.column_id(1),
                        schema_with_ids.column_id(3)),