Skip to content

Commit

Permalink
Fix storage configuration compatibility. (#3019) (#3049)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-chi-bot
ti-chi-bot authored Oct 18, 2021
1 parent 88a80ad commit a990a1b
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 64 deletions.
30 changes: 25 additions & 5 deletions dbms/src/Server/StorageConfigParser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ static std::string getCanonicalPath(std::string path)

static String getNormalizedPath(const String & s) { return getCanonicalPath(Poco::Path{s}.toString()); }

void TiFlashStorageConfig::parse(const String & storage, Poco::Logger * log)
void TiFlashStorageConfig::parseStoragePath(const String & storage, Poco::Logger * log)
{
std::istringstream ss(storage);
cpptoml::parser p(ss);
Expand Down Expand Up @@ -130,16 +130,30 @@ void TiFlashStorageConfig::parse(const String & storage, Poco::Logger * log)
kvstore_data_path[i] = getNormalizedPath(kvstore_data_path[i]);
LOG_INFO(log, "Raft data candidate path: " << kvstore_data_path[i]);
}
}

void TiFlashStorageConfig::parseMisc(const String & storage_section, Poco::Logger * log)
{
std::istringstream ss(storage_section);
cpptoml::parser p(ss);
auto table = p.parse();

// rate limiter
if (auto rate_limit = table->get_qualified_as<UInt64>("bg_task_io_rate_limit"); rate_limit)
bg_task_io_rate_limit = *rate_limit;
if (table->contains("bg_task_io_rate_limit"))
{
LOG_WARNING(log, "The configuration \"bg_task_io_rate_limit\" is deprecated. Check [storage.io_rate_limit] section for new style.");
}

if (auto version = table->get_qualified_as<UInt64>("format_version"); version)
{
format_version = *version;
}

if (auto lazily_init = table->get_qualified_as<Int32>("lazily_init_store"); lazily_init)
{
lazily_init_store = (*lazily_init != 0);
}

LOG_INFO(log, fmt::format("format_version {} lazily_init_store {}", format_version, lazily_init_store));
}

Strings TiFlashStorageConfig::getAllNormalPaths() const
Expand Down Expand Up @@ -231,15 +245,21 @@ std::tuple<size_t, TiFlashStorageConfig> TiFlashStorageConfig::parseSettings(Poc
size_t global_capacity_quota = 0; // "0" by default, means no quota, use the whole disk capacity.
TiFlashStorageConfig storage_config;

// Always try to parse storage miscellaneous configuration when [storage] section exist.
if (config.has("storage"))
{
storage_config.parse(config.getString("storage"), log);
storage_config.parseMisc(config.getString("storage"), log);
}

if (config.has("storage.main"))
{
if (config.has("path"))
LOG_WARNING(log, "The configuration \"path\" is ignored when \"storage\" is defined.");
if (config.has("capacity"))
LOG_WARNING(log, "The configuration \"capacity\" is ignored when \"storage\" is defined.");

storage_config.parseStoragePath(config.getString("storage"), log);

if (config.has("raft.kvstore_path"))
{
Strings & kvstore_paths = storage_config.kvstore_data_path;
Expand Down
35 changes: 18 additions & 17 deletions dbms/src/Server/StorageConfigParser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ class LayeredConfiguration;

namespace DB
{

struct StorageIORateLimitConfig
{
public:
Expand Down Expand Up @@ -46,20 +45,20 @@ struct StorageIORateLimitConfig
Int32 auto_tune_sec;

StorageIORateLimitConfig()
: max_bytes_per_sec(0),
max_read_bytes_per_sec(0),
max_write_bytes_per_sec(0),
use_max_bytes_per_sec(true),
fg_write_weight(25),
bg_write_weight(25),
fg_read_weight(25),
bg_read_weight(25),
emergency_pct(96),
high_pct(85),
medium_pct(60),
tune_base(2),
min_bytes_per_sec(2 * 1024 * 1024),
auto_tune_sec(5)
: max_bytes_per_sec(0)
, max_read_bytes_per_sec(0)
, max_write_bytes_per_sec(0)
, use_max_bytes_per_sec(true)
, fg_write_weight(25)
, bg_write_weight(25)
, fg_read_weight(25)
, bg_read_weight(25)
, emergency_pct(96)
, high_pct(85)
, medium_pct(60)
, tune_base(2)
, min_bytes_per_sec(2 * 1024 * 1024)
, auto_tune_sec(5)
{}

void parse(const String & storage_io_rate_limit, Poco::Logger * log);
Expand Down Expand Up @@ -88,9 +87,9 @@ struct TiFlashStorageConfig
std::vector<size_t> latest_capacity_quota;
Strings kvstore_data_path;

UInt64 bg_task_io_rate_limit = 0;
UInt64 format_version = 0;
bool lazily_init_store = true;

public:
TiFlashStorageConfig() {}

Expand All @@ -99,9 +98,11 @@ struct TiFlashStorageConfig
static std::tuple<size_t, TiFlashStorageConfig> parseSettings(Poco::Util::LayeredConfiguration & config, Poco::Logger * log);

private:
void parse(const String & storage_section, Poco::Logger * log);
void parseStoragePath(const String & storage_section, Poco::Logger * log);

bool parseFromDeprecatedConfiguration(Poco::Util::LayeredConfiguration & config, Poco::Logger * log);

void parseMisc(const String & config, Poco::Logger * log);
};


Expand Down
55 changes: 55 additions & 0 deletions dbms/src/Server/tests/gtest_storage_config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,61 @@ dt_enable_rough_set_filter = false
}
CATCH

TEST_F(StorageConfig_test, CompatibilityWithIORateLimitConfig)
try
{
Strings tests = {
R"(
path = "/tmp/tiflash/data/db0/,/tmp/tiflash/data/db1/"
[storage]
format_version = 123
lazily_init_store = 1
)",
R"(
path = "/tmp/tiflash/data/db0/,/tmp/tiflash/data/db1/"
[storage]
format_version = 123
lazily_init_store = 1
[storage.main]
dir = [ "/data0/tiflash/", "/data1/tiflash/" ]
)",
R"(
path = "/data0/tiflash/,/data1/tiflash/"
[storage]
format_version = 123
lazily_init_store = 1
[storage.io_rate_limit]
max_bytes_per_sec=1024000
)",
};

for (size_t i = 0; i < tests.size(); ++i)
{
const auto & test_case = tests[i];
auto config = loadConfigFromString(test_case);
LOG_INFO(log, "parsing [index=" << i << "] [content=" << test_case << "]");
auto [global_capacity_quota, storage] = TiFlashStorageConfig::parseSettings(*config, log);
std::ignore = global_capacity_quota;
Strings paths;
if (i == 0)
{
paths = Strings{"/tmp/tiflash/data/db0/", "/tmp/tiflash/data/db1/"};
}
else if (i == 1)
{
paths = Strings{"/data0/tiflash/", "/data1/tiflash/"};
}
else if (i == 2)
{
paths = Strings{"/data0/tiflash/", "/data1/tiflash/"};
}
ASSERT_EQ(storage.main_data_paths, paths);
ASSERT_EQ(storage.format_version, 123);
ASSERT_EQ(storage.lazily_init_store, 1);
}
}
CATCH

TEST(StorageIORateLimitConfig_test, StorageIORateLimitConfig)
try
{
Expand Down
82 changes: 40 additions & 42 deletions tests/docker/config/tiflash_dt.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,57 +17,55 @@ http_port = 8123

## Storage paths settings.
# [storage]
## The storage format version in storage engine. Valid values: 1, 2 (experimental).
## format_version = 1
## The storage format version in storage engine. Valid values: 1, 2 (experimental).
## format_version = 1

## Controls the total write rate of background tasks in bytes per second, 0 means no limit
## bg_task_io_rate_limit = 104857600
## If there are multiple SSD disks on the machine,
## specify the path list on `storage.main.dir` can improve TiFlash performance.

## If there are multiple SSD disks on the machine,
## specify the path list on `storage.main.dir` can improve TiFlash performance.
## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs)
## on the machine,
## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics)
## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics)
## can improve TiFlash performance.

## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs)
## on the machine,
## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics)
## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics)
## can improve TiFlash performance.
# [storage.main]
## The path to store main data.
# e.g.
# dir = [ "/data0/tiflash" ]
# or
# dir = [ "/data0/tiflash", "/data1/tiflash" ]

# [storage.main]
## The path to store main data.
# e.g.
# dir = [ "/data0/tiflash" ]
# or
# dir = [ "/data0/tiflash", "/data1/tiflash" ]
## Store capacity of each path, i.e. max data size allowed.
## If it is not set, or is set to 0s, the actual disk capacity is used.
## Note that we don't support human-readable big numbers(like "10GB") yet.
## Please set in the specified number of bytes.
# e.g.
# capacity = [ 10737418240, 10737418240 ]

## Store capacity of each path, i.e. max data size allowed.
## If it is not set, or is set to 0s, the actual disk capacity is used.
## Note that we don't support human-readable big numbers(like "10GB") yet.
## Please set in the specified number of bytes.
# e.g.
# capacity = [ 10737418240, 10737418240 ]
# [storage.latest]
## The path(s) to store latest data.
## If not set, it will be the same with `storage.main.dir`.
# dir = [ ]

# [storage.latest]
## The path(s) to store latest data.
## If not set, it will be the same with `storage.main.dir`.
# dir = [ ]
## Store capacity of each path, i.e. max data size allowed.
## If it is not set, or is set to 0s, the actual disk capacity is used.
# e.g.
# capacity = [ 10737418240, 10737418240 ]

## Store capacity of each path, i.e. max data size allowed.
## If it is not set, or is set to 0s, the actual disk capacity is used.
# e.g.
# capacity = [ 10737418240, 10737418240 ]
# [storage.raft]
## The path(s) to store Raft data.
## If not set, it will be the paths in `storage.latest.dir` appended with "/kvstore".
# dir = [ ]

# [storage.raft]
## The path(s) to store Raft data.
## If not set, it will be the paths in `storage.latest.dir` appended with "/kvstore".
# dir = [ ]
# [storage.io_rate_limit]
## The max I/O bandwith. Default value is 0 and I/O rate limit is disabled.
# max_bytes_per_sec = 268435456
## max_read_bytes_per_sec and max_write_bytes_per_sec are the same meaning as max_bytes_per_sec,
## but for disk that read bandwidth and write bandwith are calculated separatly, such as GCP's persistent disks.
# max_read_bytes_per_sec = 0
# max_write_bytes_per_sec = 0

# [storage.io_rate_limit]
## The max I/O bandwith. Default value is 0 and I/O rate limit is disabled.
# max_bytes_per_sec = 268435456
## max_read_bytes_per_sec and max_write_bytes_per_sec are the same meaning as max_bytes_per_sec,
## but for disk that read bandwidth and write bandwith are calculated separatly, such as GCP's persistent disks.
# max_read_bytes_per_sec = 0
# max_write_bytes_per_sec = 0
[flash]
tidb_status_addr = "tidb0:10080"
service_addr = "0.0.0.0:3930"
Expand Down

0 comments on commit a990a1b

Please sign in to comment.