PS-6125: Server crashes when querying ROCKSDB_GLOBAL_INFO table

https://jira.percona.com/browse/PS-6125

When we set 'rocksdb_update_cf_options' and provided CF that does not exist, its in memory descriptor was created + entries were added to .rocksdb/OPTIONS-ABC and Manifest file, however its CF_DEFINITION_VERSION was not stored in DB.
After that, when we query information schema for CFs, we validated all in memory CF descriptors against what we have in DB and abort if information is not found in DB.

As the 'rocksdb_update_cf_options' is intended to change parameters of CFs (not create new ones), it was added detection of nonexisting CFs passed as the value of 'rocksdb_update_cf_options'. In case of any CF does not exist, error is returned and no parameters are changed.

mysql-test/suite/rocksdb.sys_vars/r-native-partitioning/rocksdb_update_cf_options_basic.result

@@ -96,18 +96,20 @@ cf3={target_file_size_base=24m};
 SELECT * FROM INFORMATION_SCHEMA.ROCKSDB_CF_OPTIONS WHERE CF_NAME='cf3' AND OPTION_TYPE='TARGET_FILE_SIZE_BASE';
 CF_NAME	OPTION_TYPE	VALUE
 cf3	TARGET_FILE_SIZE_BASE	25165824
+call mtr.add_suppression("Column family 'foo' not found.");
 SET @@global.rocksdb_update_cf_options = 'cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8};';
+ERROR 42000: Variable 'rocksdb_update_cf_options' can't be set to the value of 'cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8}; Unknown CF: foo'
 SELECT @@global.rocksdb_update_cf_options;
 @@global.rocksdb_update_cf_options
-cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8};
+cf3={target_file_size_base=24m};
 SELECT * FROM INFORMATION_SCHEMA.ROCKSDB_CF_OPTIONS WHERE CF_NAME='cf1' AND OPTION_TYPE='TARGET_FILE_SIZE_BASE';
 CF_NAME	OPTION_TYPE	VALUE
-cf1	TARGET_FILE_SIZE_BASE	25165824
+cf1	TARGET_FILE_SIZE_BASE	2097152
 SET @@global.rocksdb_update_cf_options = 'default={foo=bar};';;
 ERROR 42000: Variable 'rocksdb_update_cf_options' can't be set to the value of 'default={foo=bar};'
 SELECT @@global.rocksdb_update_cf_options;
 @@global.rocksdb_update_cf_options
-cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8};
+cf3={target_file_size_base=24m};
 SET @@global.rocksdb_update_cf_options = 'default={write_buffer_size=67108864;target_file_size_base=67108864};';
 SET @@global.rocksdb_update_cf_options = 'cf1={write_buffer_size=67108864;target_file_size_base=67108864};';
 SET @@global.rocksdb_update_cf_options = 'cf2={write_buffer_size=67108864;target_file_size_base=67108864;max_bytes_for_level_multiplier=10.000000};';

mysql-test/suite/rocksdb.sys_vars/r/rocksdb_update_cf_options_basic.result

@@ -98,18 +98,20 @@ cf3={target_file_size_base=24m};
 SELECT * FROM INFORMATION_SCHEMA.ROCKSDB_CF_OPTIONS WHERE CF_NAME='cf3' AND OPTION_TYPE='TARGET_FILE_SIZE_BASE';
 CF_NAME	OPTION_TYPE	VALUE
 cf3	TARGET_FILE_SIZE_BASE	25165824
+call mtr.add_suppression("Column family 'foo' not found.");
 SET @@global.rocksdb_update_cf_options = 'cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8};';
+ERROR 42000: Variable 'rocksdb_update_cf_options' can't be set to the value of 'cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8}; Unknown CF: foo'
 SELECT @@global.rocksdb_update_cf_options;
 @@global.rocksdb_update_cf_options
-cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8};
+cf3={target_file_size_base=24m};
 SELECT * FROM INFORMATION_SCHEMA.ROCKSDB_CF_OPTIONS WHERE CF_NAME='cf1' AND OPTION_TYPE='TARGET_FILE_SIZE_BASE';
 CF_NAME	OPTION_TYPE	VALUE
-cf1	TARGET_FILE_SIZE_BASE	25165824
+cf1	TARGET_FILE_SIZE_BASE	2097152
 SET @@global.rocksdb_update_cf_options = 'default={foo=bar};';;
 ERROR 42000: Variable 'rocksdb_update_cf_options' can't be set to the value of 'default={foo=bar};'
 SELECT @@global.rocksdb_update_cf_options;
 @@global.rocksdb_update_cf_options
-cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8};
+cf3={target_file_size_base=24m};
 SET @@global.rocksdb_update_cf_options = 'default={write_buffer_size=67108864;target_file_size_base=67108864};';
 SET @@global.rocksdb_update_cf_options = 'cf1={write_buffer_size=67108864;target_file_size_base=67108864};';
 SET @@global.rocksdb_update_cf_options = 'cf2={write_buffer_size=67108864;target_file_size_base=67108864;max_bytes_for_level_multiplier=10.000000};';

mysql-test/suite/rocksdb.sys_vars/t/rocksdb_update_cf_options_basic.test

@@ -86,7 +86,11 @@ SELECT @@global.rocksdb_update_cf_options;
 
 SELECT * FROM INFORMATION_SCHEMA.ROCKSDB_CF_OPTIONS WHERE CF_NAME='cf3' AND OPTION_TYPE='TARGET_FILE_SIZE_BASE';
 
-# Some parts are good. Value still updated.
+# 'foo' column family does not exist, so changing its option is not possible.
+# See https://jira.percona.com/browse/PS-6125
+# Before above fix below SET created 'foo' column family
+call mtr.add_suppression("Column family 'foo' not found.");
+--error ER_WRONG_VALUE_FOR_VAR 
 SET @@global.rocksdb_update_cf_options = 'cf1={target_file_size_base=24m};foo={max_bytes_for_level_multiplier=8};';
 SELECT @@global.rocksdb_update_cf_options;
 

mysql-test/suite/rocksdb/r/rocksdb_cf_options.result

@@ -7,7 +7,17 @@ primary key (a) comment 'z') engine=rocksdb;
 insert into t1 values (1);
 insert into t2 values (2);
 insert into t3 values (2);
+call mtr.add_suppression("Column family 'cf5' not found.");
 SET @@global.rocksdb_update_cf_options = 'cf5={prefix_extractor=capped:28};';
+ERROR 42000: Variable 'rocksdb_update_cf_options' can't be set to the value of 'cf5={prefix_extractor=capped:28}; Unknown CF: cf5'
+select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO ORDER BY VALUE ASC;
+TYPE	NAME	VALUE
+MAX_INDEX_ID	MAX_INDEX_ID	258
+CF_FLAGS	2	cf1 [0]
+CF_FLAGS	3	cf2 [0]
+CF_FLAGS	0	default [0]
+CF_FLAGS	4	z [0]
+CF_FLAGS	1	__system__ [0]
 # restart
 
 Default options for all column families:
@@ -25,9 +35,6 @@ cf1	WRITE_BUFFER_SIZE	12582912
 cf2	MAX_BYTES_FOR_LEVEL_MULTIPLIER	10.000000
 cf2	TARGET_FILE_SIZE_BASE	1048576
 cf2	WRITE_BUFFER_SIZE	12582912
-cf5	MAX_BYTES_FOR_LEVEL_MULTIPLIER	10.000000
-cf5	TARGET_FILE_SIZE_BASE	1048576
-cf5	WRITE_BUFFER_SIZE	12582912
 default	MAX_BYTES_FOR_LEVEL_MULTIPLIER	10.000000
 default	TARGET_FILE_SIZE_BASE	1048576
 default	WRITE_BUFFER_SIZE	12582912
@@ -54,9 +61,6 @@ cf1	WRITE_BUFFER_SIZE	8388608
 cf2	MAX_BYTES_FOR_LEVEL_MULTIPLIER	8.000000
 cf2	TARGET_FILE_SIZE_BASE	1048576
 cf2	WRITE_BUFFER_SIZE	16777216
-cf5	MAX_BYTES_FOR_LEVEL_MULTIPLIER	10.000000
-cf5	TARGET_FILE_SIZE_BASE	1048576
-cf5	WRITE_BUFFER_SIZE	12582912
 default	MAX_BYTES_FOR_LEVEL_MULTIPLIER	10.000000
 default	TARGET_FILE_SIZE_BASE	1048576
 default	WRITE_BUFFER_SIZE	12582912
@@ -73,10 +77,20 @@ primary key (a) comment 'cf3') engine=rocksdb;
 include/assert.inc ["Expected cf3 to exist"]
 create table t5 (a int,
 primary key (a) comment 'nobodyknows') engine=rocksdb;
-ERROR HY000: Incorrect arguments to CREATE | ALTER | SET rocksdb_update_cf_options - can not find column family for storing index data and creation is not allowed.
+ERROR HY000: Incorrect arguments to CREATE | ALTER - can not find column family for storing index data and creation is not allowed.
 include/assert.inc ["Expected cf4 to NOT exist"]
+call mtr.add_suppression("Column family 'cf6' not found.");
 SET @@global.rocksdb_update_cf_options = 'cf6={prefix_extractor=capped:28};';
-ERROR HY000: Incorrect arguments to CREATE | ALTER | SET rocksdb_update_cf_options - can not find column family for storing index data and creation is not allowed.
+ERROR 42000: Variable 'rocksdb_update_cf_options' can't be set to the value of 'cf6={prefix_extractor=capped:28}; Unknown CF: cf6'
+select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO ORDER BY VALUE ASC;
+TYPE	NAME	VALUE
+MAX_INDEX_ID	MAX_INDEX_ID	259
+CF_FLAGS	2	cf1 [0]
+CF_FLAGS	3	cf2 [0]
+CF_FLAGS	5	cf3 [0]
+CF_FLAGS	0	default [0]
+CF_FLAGS	4	z [0]
+CF_FLAGS	1	__system__ [0]
 # restart
 
 drop table t1,t2,t3,t4;

mysql-test/suite/rocksdb/t/rocksdb_cf_options.test

@@ -16,10 +16,14 @@ insert into t1 values (1);
 insert into t2 values (2);
 insert into t3 values (2);
 
-# set cf_options for non-existent cf5, cf5 should be created automatically
+# set cf_options for non-existent cf5, cf5 should not be created. Error should be reported.
+call mtr.add_suppression("Column family 'cf5' not found.");
+--error ER_WRONG_VALUE_FOR_VAR 
 SET @@global.rocksdb_update_cf_options = 'cf5={prefix_extractor=capped:28};';
---let $assert_text="Expected cf5 to exist"
---let $assert_cond="[select count(distinct cf_name) from information_schema.rocksdb_cf_options where cf_name in (\'cf5\')]" = 1
+--let $assert_text="Expected cf5 to not exist"
+--let $assert_cond="[select count(distinct cf_name) from information_schema.rocksdb_cf_options where cf_name in (\'cf5\')]" = 0
+
+select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO ORDER BY VALUE ASC;
 
 # restart with new column families in DB
 
@@ -79,11 +83,14 @@ create table t5 (a int,
 --source include/assert.inc
 
 # set cf_options for non-existent cf6, cf6 should NOT be created automatically
---error ER_WRONG_ARGUMENTS
+call mtr.add_suppression("Column family 'cf6' not found.");
+--error ER_WRONG_VALUE_FOR_VAR
 SET @@global.rocksdb_update_cf_options = 'cf6={prefix_extractor=capped:28};';
 --let $assert_text="Expected cf6 to NOT exist"
 --let $assert_cond="[select count(distinct cf_name) from information_schema.rocksdb_cf_options where cf_name in (\'cf6\')]" = 0
 
+select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO ORDER BY VALUE ASC;
+
 # syntax error in options (no equal sign)
 
 --exec echo "restart:--rocksdb_override_cf_options=cf1" > $_expect_file_name

storage/rocksdb/ha_rocksdb.cc

@@ -13081,13 +13081,28 @@ static int rocksdb_validate_update_cf_options(
     my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), "rocksdb_update_cf_options", str);
     return HA_EXIT_FAILURE;
   }
-  // Loop through option_map and create missing column families
-  for (Rdb_cf_options::Name_to_config_t::iterator it = option_map.begin();
-       it != option_map.end(); ++it) {
-    if (!cf_manager.get_or_create_cf(rdb, it->first,
-                                     !rocksdb_no_create_column_family)) {
-      return HA_EXIT_FAILURE;
+  // Loop through option_map and check if all specified CFs exist.
+  std::vector<const std::string *> unknown_cfs;
+  for (const auto &option : option_map) {
+    if (!cf_manager.get_cf(option.first)) {
+      unknown_cfs.push_back(&(option.first));
+    }
+  }
+
+  if (!unknown_cfs.empty()) {
+    std::string err(str);
+    err.append(" Unknown CF: ");
+    bool first = true;
+    for (const auto cf : unknown_cfs) {
+      if (first)
+        first = false;
+      else
+        err.append(", ");
+      err.append(*cf);
     }
+    my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), "rocksdb_update_cf_options",
+             err.c_str());
+    return HA_EXIT_FAILURE;
   }
   return HA_EXIT_SUCCESS;
 }

storage/rocksdb/rdb_cf_manager.cc

@@ -120,7 +120,7 @@ Rdb_cf_manager::get_or_create_cf(rocksdb::DB *const rdb,
     } else {
       RDB_MUTEX_UNLOCK_CHECK(m_mutex);
       my_error(ER_WRONG_ARGUMENTS, MYF(0),
-               "CREATE | ALTER | SET rocksdb_update_cf_options - can not find "
+               "CREATE | ALTER - can not find "
                "column family for storing index data and creation is not "
                "allowed.");
       return nullptr;