diff --git a/dbms/src/Server/StorageConfigParser.cpp b/dbms/src/Server/StorageConfigParser.cpp index fa4326e50b3..b55681cf608 100644 --- a/dbms/src/Server/StorageConfigParser.cpp +++ b/dbms/src/Server/StorageConfigParser.cpp @@ -44,7 +44,7 @@ static std::string getCanonicalPath(std::string path) static String getNormalizedPath(const String & s) { return getCanonicalPath(Poco::Path{s}.toString()); } -void TiFlashStorageConfig::parse(const String & storage, Poco::Logger * log) +void TiFlashStorageConfig::parseStoragePath(const String & storage, Poco::Logger * log) { std::istringstream ss(storage); cpptoml::parser p(ss); @@ -130,16 +130,30 @@ void TiFlashStorageConfig::parse(const String & storage, Poco::Logger * log) kvstore_data_path[i] = getNormalizedPath(kvstore_data_path[i]); LOG_INFO(log, "Raft data candidate path: " << kvstore_data_path[i]); } +} + +void TiFlashStorageConfig::parseMisc(const String & storage_section, Poco::Logger * log) +{ + std::istringstream ss(storage_section); + cpptoml::parser p(ss); + auto table = p.parse(); - // rate limiter - if (auto rate_limit = table->get_qualified_as("bg_task_io_rate_limit"); rate_limit) - bg_task_io_rate_limit = *rate_limit; + if (table->contains("bg_task_io_rate_limit")) + { + LOG_WARNING(log, "The configuration \"bg_task_io_rate_limit\" is deprecated. Check [storage.io_rate_limit] section for new style."); + } if (auto version = table->get_qualified_as("format_version"); version) + { format_version = *version; + } if (auto lazily_init = table->get_qualified_as("lazily_init_store"); lazily_init) + { lazily_init_store = (*lazily_init != 0); + } + + LOG_INFO(log, fmt::format("format_version {} lazily_init_store {}", format_version, lazily_init_store)); } Strings TiFlashStorageConfig::getAllNormalPaths() const @@ -231,15 +245,21 @@ std::tuple TiFlashStorageConfig::parseSettings(Poc size_t global_capacity_quota = 0; // "0" by default, means no quota, use the whole disk capacity. TiFlashStorageConfig storage_config; + // Always try to parse storage miscellaneous configuration when [storage] section exist. if (config.has("storage")) { - storage_config.parse(config.getString("storage"), log); + storage_config.parseMisc(config.getString("storage"), log); + } + if (config.has("storage.main")) + { if (config.has("path")) LOG_WARNING(log, "The configuration \"path\" is ignored when \"storage\" is defined."); if (config.has("capacity")) LOG_WARNING(log, "The configuration \"capacity\" is ignored when \"storage\" is defined."); + storage_config.parseStoragePath(config.getString("storage"), log); + if (config.has("raft.kvstore_path")) { Strings & kvstore_paths = storage_config.kvstore_data_path; diff --git a/dbms/src/Server/StorageConfigParser.h b/dbms/src/Server/StorageConfigParser.h index 27434658994..5299910f551 100644 --- a/dbms/src/Server/StorageConfigParser.h +++ b/dbms/src/Server/StorageConfigParser.h @@ -16,7 +16,6 @@ class LayeredConfiguration; namespace DB { - struct StorageIORateLimitConfig { public: @@ -46,20 +45,20 @@ struct StorageIORateLimitConfig Int32 auto_tune_sec; StorageIORateLimitConfig() - : max_bytes_per_sec(0), - max_read_bytes_per_sec(0), - max_write_bytes_per_sec(0), - use_max_bytes_per_sec(true), - fg_write_weight(25), - bg_write_weight(25), - fg_read_weight(25), - bg_read_weight(25), - emergency_pct(96), - high_pct(85), - medium_pct(60), - tune_base(2), - min_bytes_per_sec(2 * 1024 * 1024), - auto_tune_sec(5) + : max_bytes_per_sec(0) + , max_read_bytes_per_sec(0) + , max_write_bytes_per_sec(0) + , use_max_bytes_per_sec(true) + , fg_write_weight(25) + , bg_write_weight(25) + , fg_read_weight(25) + , bg_read_weight(25) + , emergency_pct(96) + , high_pct(85) + , medium_pct(60) + , tune_base(2) + , min_bytes_per_sec(2 * 1024 * 1024) + , auto_tune_sec(5) {} void parse(const String & storage_io_rate_limit, Poco::Logger * log); @@ -88,9 +87,9 @@ struct TiFlashStorageConfig std::vector latest_capacity_quota; Strings kvstore_data_path; - UInt64 bg_task_io_rate_limit = 0; UInt64 format_version = 0; bool lazily_init_store = true; + public: TiFlashStorageConfig() {} @@ -99,9 +98,11 @@ struct TiFlashStorageConfig static std::tuple parseSettings(Poco::Util::LayeredConfiguration & config, Poco::Logger * log); private: - void parse(const String & storage_section, Poco::Logger * log); + void parseStoragePath(const String & storage_section, Poco::Logger * log); bool parseFromDeprecatedConfiguration(Poco::Util::LayeredConfiguration & config, Poco::Logger * log); + + void parseMisc(const String & config, Poco::Logger * log); }; diff --git a/dbms/src/Server/tests/gtest_storage_config.cpp b/dbms/src/Server/tests/gtest_storage_config.cpp index 700098b657b..e15d96bc086 100644 --- a/dbms/src/Server/tests/gtest_storage_config.cpp +++ b/dbms/src/Server/tests/gtest_storage_config.cpp @@ -392,6 +392,61 @@ dt_enable_rough_set_filter = false } CATCH +TEST_F(StorageConfig_test, CompatibilityWithIORateLimitConfig) +try +{ + Strings tests = { + R"( +path = "/tmp/tiflash/data/db0/,/tmp/tiflash/data/db1/" +[storage] +format_version = 123 +lazily_init_store = 1 +)", + R"( +path = "/tmp/tiflash/data/db0/,/tmp/tiflash/data/db1/" +[storage] +format_version = 123 +lazily_init_store = 1 +[storage.main] +dir = [ "/data0/tiflash/", "/data1/tiflash/" ] +)", + R"( +path = "/data0/tiflash/,/data1/tiflash/" +[storage] +format_version = 123 +lazily_init_store = 1 +[storage.io_rate_limit] +max_bytes_per_sec=1024000 +)", + }; + + for (size_t i = 0; i < tests.size(); ++i) + { + const auto & test_case = tests[i]; + auto config = loadConfigFromString(test_case); + LOG_INFO(log, "parsing [index=" << i << "] [content=" << test_case << "]"); + auto [global_capacity_quota, storage] = TiFlashStorageConfig::parseSettings(*config, log); + std::ignore = global_capacity_quota; + Strings paths; + if (i == 0) + { + paths = Strings{"/tmp/tiflash/data/db0/", "/tmp/tiflash/data/db1/"}; + } + else if (i == 1) + { + paths = Strings{"/data0/tiflash/", "/data1/tiflash/"}; + } + else if (i == 2) + { + paths = Strings{"/data0/tiflash/", "/data1/tiflash/"}; + } + ASSERT_EQ(storage.main_data_paths, paths); + ASSERT_EQ(storage.format_version, 123); + ASSERT_EQ(storage.lazily_init_store, 1); + } +} +CATCH + TEST(StorageIORateLimitConfig_test, StorageIORateLimitConfig) try { diff --git a/tests/docker/config/tiflash_dt.toml b/tests/docker/config/tiflash_dt.toml index 8423fadb76d..d581137c703 100644 --- a/tests/docker/config/tiflash_dt.toml +++ b/tests/docker/config/tiflash_dt.toml @@ -17,57 +17,55 @@ http_port = 8123 ## Storage paths settings. # [storage] - ## The storage format version in storage engine. Valid values: 1, 2 (experimental). - ## format_version = 1 +## The storage format version in storage engine. Valid values: 1, 2 (experimental). +## format_version = 1 - ## Controls the total write rate of background tasks in bytes per second, 0 means no limit - ## bg_task_io_rate_limit = 104857600 +## If there are multiple SSD disks on the machine, +## specify the path list on `storage.main.dir` can improve TiFlash performance. - ## If there are multiple SSD disks on the machine, - ## specify the path list on `storage.main.dir` can improve TiFlash performance. +## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs) +## on the machine, +## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics) +## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics) +## can improve TiFlash performance. - ## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs) - ## on the machine, - ## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics) - ## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics) - ## can improve TiFlash performance. +# [storage.main] +## The path to store main data. +# e.g. +# dir = [ "/data0/tiflash" ] +# or +# dir = [ "/data0/tiflash", "/data1/tiflash" ] - # [storage.main] - ## The path to store main data. - # e.g. - # dir = [ "/data0/tiflash" ] - # or - # dir = [ "/data0/tiflash", "/data1/tiflash" ] +## Store capacity of each path, i.e. max data size allowed. +## If it is not set, or is set to 0s, the actual disk capacity is used. +## Note that we don't support human-readable big numbers(like "10GB") yet. +## Please set in the specified number of bytes. +# e.g. +# capacity = [ 10737418240, 10737418240 ] - ## Store capacity of each path, i.e. max data size allowed. - ## If it is not set, or is set to 0s, the actual disk capacity is used. - ## Note that we don't support human-readable big numbers(like "10GB") yet. - ## Please set in the specified number of bytes. - # e.g. - # capacity = [ 10737418240, 10737418240 ] +# [storage.latest] +## The path(s) to store latest data. +## If not set, it will be the same with `storage.main.dir`. +# dir = [ ] - # [storage.latest] - ## The path(s) to store latest data. - ## If not set, it will be the same with `storage.main.dir`. - # dir = [ ] +## Store capacity of each path, i.e. max data size allowed. +## If it is not set, or is set to 0s, the actual disk capacity is used. +# e.g. +# capacity = [ 10737418240, 10737418240 ] - ## Store capacity of each path, i.e. max data size allowed. - ## If it is not set, or is set to 0s, the actual disk capacity is used. - # e.g. - # capacity = [ 10737418240, 10737418240 ] +# [storage.raft] +## The path(s) to store Raft data. +## If not set, it will be the paths in `storage.latest.dir` appended with "/kvstore". +# dir = [ ] - # [storage.raft] - ## The path(s) to store Raft data. - ## If not set, it will be the paths in `storage.latest.dir` appended with "/kvstore". - # dir = [ ] +# [storage.io_rate_limit] +## The max I/O bandwith. Default value is 0 and I/O rate limit is disabled. +# max_bytes_per_sec = 268435456 +## max_read_bytes_per_sec and max_write_bytes_per_sec are the same meaning as max_bytes_per_sec, +## but for disk that read bandwidth and write bandwith are calculated separatly, such as GCP's persistent disks. +# max_read_bytes_per_sec = 0 +# max_write_bytes_per_sec = 0 - # [storage.io_rate_limit] - ## The max I/O bandwith. Default value is 0 and I/O rate limit is disabled. - # max_bytes_per_sec = 268435456 - ## max_read_bytes_per_sec and max_write_bytes_per_sec are the same meaning as max_bytes_per_sec, - ## but for disk that read bandwidth and write bandwith are calculated separatly, such as GCP's persistent disks. - # max_read_bytes_per_sec = 0 - # max_write_bytes_per_sec = 0 [flash] tidb_status_addr = "tidb0:10080" service_addr = "0.0.0.0:3930"