diff --git a/dbms/src/Server/CLIService.h b/dbms/src/Server/CLIService.h new file mode 100644 index 00000000000..18c9d61260f --- /dev/null +++ b/dbms/src/Server/CLIService.h @@ -0,0 +1,237 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +using RaftStoreFFIFunc = void (*)(int argc, const char * const * argv, const DB::EngineStoreServerHelper *); + +template +struct CLIService : public BaseDaemon +{ + struct TiFlashProxyConfig + { + static const std::string config_prefix; + std::vector args; + std::unordered_map val_map; + bool is_proxy_runnable = false; + + static constexpr char ENGINE_STORE_VERSION[] = "engine-version"; + static constexpr char ENGINE_STORE_GIT_HASH[] = "engine-git-hash"; + static constexpr char ENGINE_STORE_ADDRESS[] = "engine-addr"; + static constexpr char ENGINE_STORE_ADVERTISE_ADDRESS[] = "advertise-engine-addr"; + static constexpr char PD_ENDPOINTS[] = "pd-endpoints"; + static constexpr char ENGINE_LABEL[] = "engine-label"; + static constexpr char ENGINE_LABEL_VALUE[] = "tiflash"; + + explicit TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config); + }; + + struct RaftStoreProxyRunner : boost::noncopyable + { + struct RunRaftStoreProxyParms + { + const DB::EngineStoreServerHelper * helper; + const TiFlashProxyConfig & conf; + const RaftStoreFFIFunc ffi_function; + + /// set big enough stack size to avoid runtime error like stack-overflow. + size_t stack_size = 1024 * 1024 * 20; + }; + + explicit RaftStoreProxyRunner(RunRaftStoreProxyParms && parms_); + + void join(); + + void run(); + + private: + static void * runRaftStoreProxyFfi(void * pv); + + private: + RunRaftStoreProxyParms parms; + pthread_t thread; + }; + + Func func; + RaftStoreFFIFunc ffi_function; + const Args & args; + std::unique_ptr global_context; + + explicit CLIService(Func func_, const Args & args_, const std::string & config_file, RaftStoreFFIFunc ffi_function = nullptr); + + int main(const std::vector &) override; +}; + +template +CLIService::TiFlashProxyConfig::TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config) +{ + if (!config.has(config_prefix)) + return; + + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(config_prefix, keys); + { + std::unordered_map args_map; + for (const auto & key : keys) + { + const auto k = config_prefix + "." + key; + args_map[key] = config.getString(k); + } + args_map[PD_ENDPOINTS] = config.getString("raft.pd_addr"); + args_map[ENGINE_STORE_VERSION] = TiFlashBuildInfo::getReleaseVersion(); + args_map[ENGINE_STORE_GIT_HASH] = TiFlashBuildInfo::getGitHash(); + if (!args_map.count(ENGINE_STORE_ADDRESS)) + args_map[ENGINE_STORE_ADDRESS] = config.getString("flash.service_addr"); + else + args_map[ENGINE_STORE_ADVERTISE_ADDRESS] = args_map[ENGINE_STORE_ADDRESS]; + args_map[ENGINE_LABEL] = ENGINE_LABEL_VALUE; + + for (auto && [k, v] : args_map) + { + val_map.emplace("--" + k, std::move(v)); + } + } + + args.push_back("TiFlash Proxy"); + for (const auto & v : val_map) + { + args.push_back(v.first.data()); + args.push_back(v.second.data()); + } + is_proxy_runnable = true; +} +template +CLIService::RaftStoreProxyRunner::RaftStoreProxyRunner(CLIService::RaftStoreProxyRunner::RunRaftStoreProxyParms && parms_) + : parms(std::move(parms_)) +{} +template +void CLIService::RaftStoreProxyRunner::join() +{ + if (!parms.conf.is_proxy_runnable) + return; + pthread_join(thread, nullptr); +} +template +void CLIService::RaftStoreProxyRunner::run() +{ + if (!parms.conf.is_proxy_runnable) + return; + pthread_attr_t attribute; + pthread_attr_init(&attribute); + pthread_attr_setstacksize(&attribute, parms.stack_size); + pthread_create(&thread, &attribute, runRaftStoreProxyFfi, &parms); + pthread_attr_destroy(&attribute); +} +template +void * CLIService::RaftStoreProxyRunner::runRaftStoreProxyFfi(void * pv) +{ + auto & parms = *static_cast(pv); + if (nullptr == parms.ffi_function) + { + throw DB::Exception("proxy is not available"); + } + parms.ffi_function(static_cast(parms.conf.args.size()), parms.conf.args.data(), parms.helper); + return nullptr; +} + +template +CLIService::CLIService(Func func_, const Args & args_, const std::string & config_file, RaftStoreFFIFunc ffi_function) + : func(std::move(func_)) + , ffi_function(ffi_function) + , args(args_) +{ + config_path = config_file; + ConfigProcessor config_processor(config_file); + auto loaded_config = config_processor.loadConfig(); + BaseDaemon::config().add(loaded_config.configuration); + BaseDaemon::config().setString("config-file", config_file); +} + +template +int CLIService::main(const std::vector &) +{ + using namespace DB; + TiFlashProxyConfig proxy_conf(config()); + EngineStoreServerWrap tiflash_instance_wrap{}; + auto helper = GetEngineStoreServerHelper( + &tiflash_instance_wrap); + + typename RaftStoreProxyRunner::RunRaftStoreProxyParms parms{&helper, proxy_conf, ffi_function}; + RaftStoreProxyRunner proxy_runner(std::move(parms)); + + proxy_runner.run(); + + if (proxy_conf.is_proxy_runnable) + { + while (!tiflash_instance_wrap.proxy_helper) + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + SCOPE_EXIT({ + if (!proxy_conf.is_proxy_runnable) + { + proxy_runner.join(); + return; + } + tiflash_instance_wrap.status = EngineStoreServerStatus::Terminated; + tiflash_instance_wrap.tmt = nullptr; + proxy_runner.join(); + }); + + global_context = std::make_unique(Context::createGlobal()); + global_context->setGlobalContext(*global_context); + global_context->setApplicationType(Context::ApplicationType::SERVER); + + /// Init File Provider + if (proxy_conf.is_proxy_runnable) + { + bool enable_encryption = tiflash_instance_wrap.proxy_helper->checkEncryptionEnabled(); + if (enable_encryption) + { + auto method = tiflash_instance_wrap.proxy_helper->getEncryptionMethod(); + enable_encryption = (method != EncryptionMethod::Plaintext); + } + KeyManagerPtr key_manager = std::make_shared(&tiflash_instance_wrap); + global_context->initializeFileProvider(key_manager, enable_encryption); + } + else + { + KeyManagerPtr key_manager = std::make_shared(false); + global_context->initializeFileProvider(key_manager, false); + } + + return func(*global_context, args); +} + + +template +inline const std::string CLIService::TiFlashProxyConfig::config_prefix = "flash.proxy"; diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index 6c3d289dea6..63cf6d0e1f9 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -22,6 +22,7 @@ option(ENABLE_CLICKHOUSE_SERVER "Enable server" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_CLICKHOUSE_CLIENT "Enable client" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_TIFLASH_DTTOOL "Enable dttool: tools to manage dmfile" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_TIFLASH_DTWORKLOAD "Enable dtworkload: tools to test and stress DeltaTree" ${ENABLE_CLICKHOUSE_ALL}) +option(ENABLE_TIFLASH_PAGECTL "Enable pagectl: tools to debug page storage" ${ENABLE_CLICKHOUSE_ALL}) configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) @@ -135,6 +136,9 @@ endif () if (ENABLE_TIFLASH_DTWORKLOAD) target_link_libraries(tiflash dt-workload-lib) endif () +if (ENABLE_TIFLASH_PAGECTL) + target_link_libraries(tiflash page-ctl-lib) +endif () # install always because depian package want this files: install (TARGETS tiflash RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT tiflash) diff --git a/dbms/src/Server/DTTool/DTTool.h b/dbms/src/Server/DTTool/DTTool.h index 911c29bf98b..6236bd6cdb9 100644 --- a/dbms/src/Server/DTTool/DTTool.h +++ b/dbms/src/Server/DTTool/DTTool.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -76,205 +77,6 @@ int migrateEntry(const std::vector & opts, RaftStoreFFIFunc ffi_fun namespace DTTool { -template -struct CLIService : public BaseDaemon -{ - struct TiFlashProxyConfig - { - static const std::string config_prefix; - std::vector args; - std::unordered_map val_map; - bool is_proxy_runnable = false; - - static constexpr char ENGINE_STORE_VERSION[] = "engine-version"; - static constexpr char ENGINE_STORE_GIT_HASH[] = "engine-git-hash"; - static constexpr char ENGINE_STORE_ADDRESS[] = "engine-addr"; - static constexpr char ENGINE_STORE_ADVERTISE_ADDRESS[] = "advertise-engine-addr"; - static constexpr char PD_ENDPOINTS[] = "pd-endpoints"; - - explicit TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config); - }; - - struct RaftStoreProxyRunner : boost::noncopyable - { - struct RunRaftStoreProxyParms - { - const DB::EngineStoreServerHelper * helper; - const TiFlashProxyConfig & conf; - const RaftStoreFFIFunc ffi_function; - - /// set big enough stack size to avoid runtime error like stack-overflow. - size_t stack_size = 1024 * 1024 * 20; - }; - - explicit RaftStoreProxyRunner(RunRaftStoreProxyParms && parms_); - - void join(); - - void run(); - - private: - static void * runRaftStoreProxyFfi(void * pv); - - private: - RunRaftStoreProxyParms parms; - pthread_t thread; - }; - - Func func; - RaftStoreFFIFunc ffi_function; - const Args & args; - std::unique_ptr global_context; - - explicit CLIService(Func func_, const Args & args_, const std::string & config_file, RaftStoreFFIFunc ffi_function = nullptr); - - int main(const std::vector &) override; -}; - -template -CLIService::TiFlashProxyConfig::TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config) -{ - if (!config.has(config_prefix)) - return; - - Poco::Util::AbstractConfiguration::Keys keys; - config.keys(config_prefix, keys); - { - std::unordered_map args_map; - for (const auto & key : keys) - { - const auto k = config_prefix + "." + key; - args_map[key] = config.getString(k); - } - args_map[PD_ENDPOINTS] = config.getString("raft.pd_addr"); - args_map[ENGINE_STORE_VERSION] = TiFlashBuildInfo::getReleaseVersion(); - args_map[ENGINE_STORE_GIT_HASH] = TiFlashBuildInfo::getGitHash(); - if (!args_map.count(ENGINE_STORE_ADDRESS)) - args_map[ENGINE_STORE_ADDRESS] = config.getString("flash.service_addr"); - else - args_map[ENGINE_STORE_ADVERTISE_ADDRESS] = args_map[ENGINE_STORE_ADDRESS]; - - for (auto && [k, v] : args_map) - { - val_map.emplace("--" + k, std::move(v)); - } - } - - args.push_back("TiFlash Proxy"); - for (const auto & v : val_map) - { - args.push_back(v.first.data()); - args.push_back(v.second.data()); - } - is_proxy_runnable = true; -} -template -CLIService::RaftStoreProxyRunner::RaftStoreProxyRunner(CLIService::RaftStoreProxyRunner::RunRaftStoreProxyParms && parms_) - : parms(std::move(parms_)) -{} -template -void CLIService::RaftStoreProxyRunner::join() -{ - if (!parms.conf.is_proxy_runnable) - return; - pthread_join(thread, nullptr); -} -template -void CLIService::RaftStoreProxyRunner::run() -{ - if (!parms.conf.is_proxy_runnable) - return; - pthread_attr_t attribute; - pthread_attr_init(&attribute); - pthread_attr_setstacksize(&attribute, parms.stack_size); - pthread_create(&thread, &attribute, runRaftStoreProxyFfi, &parms); - pthread_attr_destroy(&attribute); -} -template -void * CLIService::RaftStoreProxyRunner::runRaftStoreProxyFfi(void * pv) -{ - auto & parms = *static_cast(pv); - if (nullptr == parms.ffi_function) - { - throw DB::Exception("proxy is not available"); - } - parms.ffi_function(static_cast(parms.conf.args.size()), parms.conf.args.data(), parms.helper); - return nullptr; -} - -template -CLIService::CLIService(Func func_, const Args & args_, const std::string & config_file, RaftStoreFFIFunc ffi_function) - : func(std::move(func_)) - , ffi_function(ffi_function) - , args(args_) -{ - config_path = config_file; - ConfigProcessor config_processor(config_file); - auto loaded_config = config_processor.loadConfig(); - BaseDaemon::config().add(loaded_config.configuration); - BaseDaemon::config().setString("config-file", config_file); -} - -template -int CLIService::main(const std::vector &) -{ - using namespace DB; - TiFlashProxyConfig proxy_conf(config()); - EngineStoreServerWrap tiflash_instance_wrap{}; - auto helper = GetEngineStoreServerHelper( - &tiflash_instance_wrap); - - typename RaftStoreProxyRunner::RunRaftStoreProxyParms parms{&helper, proxy_conf, ffi_function}; - RaftStoreProxyRunner proxy_runner(std::move(parms)); - - proxy_runner.run(); - - if (proxy_conf.is_proxy_runnable) - { - while (!tiflash_instance_wrap.proxy_helper) - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - } - - SCOPE_EXIT({ - if (!proxy_conf.is_proxy_runnable) - { - proxy_runner.join(); - return; - } - tiflash_instance_wrap.status = EngineStoreServerStatus::Terminated; - tiflash_instance_wrap.tmt = nullptr; - proxy_runner.join(); - }); - - global_context = std::make_unique(Context::createGlobal()); - global_context->setGlobalContext(*global_context); - global_context->setApplicationType(Context::ApplicationType::SERVER); - - /// Init File Provider - if (proxy_conf.is_proxy_runnable) - { - bool enable_encryption = tiflash_instance_wrap.proxy_helper->checkEncryptionEnabled(); - if (enable_encryption) - { - auto method = tiflash_instance_wrap.proxy_helper->getEncryptionMethod(); - enable_encryption = (method != EncryptionMethod::Plaintext); - } - KeyManagerPtr key_manager = std::make_shared(&tiflash_instance_wrap); - global_context->initializeFileProvider(key_manager, enable_encryption); - } - else - { - KeyManagerPtr key_manager = std::make_shared(false); - global_context->initializeFileProvider(key_manager, false); - } - - return func(*global_context, args); -} - - -template -inline const std::string CLIService::TiFlashProxyConfig::config_prefix = "flash.proxy"; - namespace detail { using namespace DB; diff --git a/dbms/src/Server/config_tools.h.in b/dbms/src/Server/config_tools.h.in index 03df94cc8e1..61aa3f41591 100644 --- a/dbms/src/Server/config_tools.h.in +++ b/dbms/src/Server/config_tools.h.in @@ -6,3 +6,4 @@ #cmakedefine01 ENABLE_CLICKHOUSE_CLIENT #cmakedefine01 ENABLE_TIFLASH_DTTOOL #cmakedefine01 ENABLE_TIFLASH_DTWORKLOAD +#cmakedefine01 ENABLE_TIFLASH_PAGECTL diff --git a/dbms/src/Server/main.cpp b/dbms/src/Server/main.cpp index ace9dfc80ee..11cccf84729 100644 --- a/dbms/src/Server/main.cpp +++ b/dbms/src/Server/main.cpp @@ -38,6 +38,9 @@ #if ENABLE_TIFLASH_DTWORKLOAD #include #endif +#if ENABLE_TIFLASH_PAGECTL +#include +#endif #include #include @@ -103,6 +106,9 @@ std::pair clickhouse_applications[] = { #endif #if ENABLE_TIFLASH_DTWORKLOAD {"dtworkload", DB::DM::tests::DTWorkload::mainEntry}, +#endif +#if ENABLE_TIFLASH_PAGECTL + {"pagectl", DB::PageStorageCtl::mainEntry}, #endif {"version", mainEntryVersion}, {"errgen", mainExportError}}; diff --git a/dbms/src/Storages/Page/CMakeLists.txt b/dbms/src/Storages/Page/CMakeLists.txt index 1883f0bc0aa..cead83fa126 100644 --- a/dbms/src/Storages/Page/CMakeLists.txt +++ b/dbms/src/Storages/Page/CMakeLists.txt @@ -13,6 +13,7 @@ # limitations under the License. add_subdirectory(V2) +add_subdirectory(tools) # PageStorage Stress test if (ENABLE_V3_PAGESTORAGE) diff --git a/dbms/src/Storages/Page/V2/CMakeLists.txt b/dbms/src/Storages/Page/V2/CMakeLists.txt index e840960b653..f994358f2b1 100644 --- a/dbms/src/Storages/Page/V2/CMakeLists.txt +++ b/dbms/src/Storages/Page/V2/CMakeLists.txt @@ -42,16 +42,4 @@ add_library(page_storage_v2 EXCLUDE_FROM_ALL ) target_include_directories(page_storage_v2 PUBLIC ${TiFlash_SOURCE_DIR}/contrib/tiflash-proxy/raftstore-proxy/ffi/src) target_link_libraries(page_storage_v2 clickhouse_common_io cpptoml - kv_client tipb) # TODO: remove dependency on these libs. Now we need them for DB::Context - -### Build a control binary for PageStorage -## For `page_ctl`, we need to define `PAGE_STORAGE_UTIL_DEBUGGGING` -add_executable(page_ctl EXCLUDE_FROM_ALL - tests/page_storage_ctl.cpp - ${page_storage_v2_headers} ${page_storage_v2_sources} - ${io_base_headers} ${io_base_sources} -) -target_include_directories(page_ctl PUBLIC ${TiFlash_SOURCE_DIR}/contrib/tiflash-proxy/raftstore-proxy/ffi/src) -target_link_libraries(page_ctl clickhouse_common_io cpptoml) -target_compile_options(page_ctl PRIVATE -Wno-format) -target_compile_definitions(page_ctl PRIVATE PAGE_STORAGE_UTIL_DEBUGGGING DBMS_PUBLIC_GTEST) + kv_client tipb) # TODO: remove dependency on these libs. Now we need them for DB::Context \ No newline at end of file diff --git a/dbms/src/Storages/Page/V2/gc/DataCompactor.h b/dbms/src/Storages/Page/V2/gc/DataCompactor.h index 1ddd36f87dd..eede1775cdf 100644 --- a/dbms/src/Storages/Page/V2/gc/DataCompactor.h +++ b/dbms/src/Storages/Page/V2/gc/DataCompactor.h @@ -60,10 +60,6 @@ class DataCompactor : private boost::noncopyable std::tuple tryMigrate(const PageFileSet & page_files, SnapshotPtr && snapshot, const WritingFilesSnapshot & writing_files); -#ifndef DBMS_PUBLIC_GTEST -private: -#endif - /** * Collect valid page of snapshot. * Return { @@ -72,6 +68,9 @@ class DataCompactor : private boost::noncopyable * } */ static ValidPages collectValidPagesInPageFile(const SnapshotPtr & snapshot); +#ifndef DBMS_PUBLIC_GTEST +private: +#endif struct CompactCandidates { diff --git a/dbms/src/Storages/Page/V3/BlobStore.h b/dbms/src/Storages/Page/V3/BlobStore.h index 5a3e98400d1..24bf4652123 100644 --- a/dbms/src/Storages/Page/V3/BlobStore.h +++ b/dbms/src/Storages/Page/V3/BlobStore.h @@ -316,7 +316,7 @@ class BlobStore : private Allocator BlobFilePtr getBlobFile(BlobFileId blob_id); friend class PageDirectoryFactory; - friend class PageStorageControl; + friend class PageStorageControlV3; #ifndef DBMS_PUBLIC_GTEST private: diff --git a/dbms/src/Storages/Page/V3/PageDirectory.h b/dbms/src/Storages/Page/V3/PageDirectory.h index 39b5a05a40a..bd7c433022f 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.h +++ b/dbms/src/Storages/Page/V3/PageDirectory.h @@ -255,7 +255,7 @@ class VersionedPageEntries being_ref_count, entries.size()); } - friend class PageStorageControl; + friend class PageStorageControlV3; private: mutable std::mutex m; @@ -376,7 +376,7 @@ class PageDirectory DISALLOW_COPY_AND_MOVE(PageDirectory); friend class PageDirectoryFactory; - friend class PageStorageControl; + friend class PageStorageControlV3; private: // Only `std::map` is allow for `MVCCMap`. Cause `std::map::insert` ensure that diff --git a/dbms/src/Storages/Page/V3/PageStorageImpl.h b/dbms/src/Storages/Page/V3/PageStorageImpl.h index f49601ce2ad..b4cdd425e59 100644 --- a/dbms/src/Storages/Page/V3/PageStorageImpl.h +++ b/dbms/src/Storages/Page/V3/PageStorageImpl.h @@ -115,7 +115,7 @@ class PageStorageImpl : public DB::PageStorage #endif friend class PageDirectoryFactory; - friend class PageStorageControl; + friend class PageStorageControlV3; #ifndef DBMS_PUBLIC_GTEST private: #endif diff --git a/dbms/src/Storages/Page/V3/tests/CMakeLists.txt b/dbms/src/Storages/Page/V3/tests/CMakeLists.txt index 8bab6afcded..355247c9eba 100644 --- a/dbms/src/Storages/Page/V3/tests/CMakeLists.txt +++ b/dbms/src/Storages/Page/V3/tests/CMakeLists.txt @@ -26,10 +26,4 @@ add_executable(gtests_page_storage_v3 ${ps_v3_gtest_sources} ${TiFlash_SOURCE_DI target_link_libraries(gtests_page_storage_v3 page_storage_v3 gtest_main) target_compile_options(gtests_page_storage_v3 PRIVATE -Wno-unknown-pragmas) target_compile_definitions(gtests_page_storage_v3 PRIVATE DBMS_PUBLIC_GTEST) -add_check(gtests_page_storage_v3) - - -add_executable(page_storage_ctl EXCLUDE_FROM_ALL page_storage_ctl.cpp) -target_compile_definitions(page_storage_ctl PUBLIC DBMS_PUBLIC_GTEST) -target_link_libraries(page_storage_ctl dbms page_storage_v3) -target_compile_options(page_storage_ctl PRIVATE -Wno-format -lc++) # turn off printf format check +add_check(gtests_page_storage_v3) \ No newline at end of file diff --git a/dbms/src/Storages/Page/tools/CMakeLists.txt b/dbms/src/Storages/Page/tools/CMakeLists.txt new file mode 100644 index 00000000000..629dedd01fc --- /dev/null +++ b/dbms/src/Storages/Page/tools/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include_directories (${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory (PageCtl EXCLUDE_FROM_ALL) diff --git a/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt b/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt new file mode 100644 index 00000000000..576b5e07a0f --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include_directories (${CMAKE_CURRENT_BINARY_DIR}) + +add_library(page-ctl-lib MainEntry.cpp PageStorageCtlV3.cpp PageStorageCtlV2.cpp ${page-ctl-src}) +target_include_directories(page-ctl-lib PUBLIC ${TiFlash_SOURCE_DIR}/libs/libdaemon/include) +target_link_libraries(page-ctl-lib dbms daemon tiflash-dttool-entry-object clickhouse-server-lib) +target_compile_options(page-ctl-lib PRIVATE -Wno-format) + +add_executable(page-ctl Main.cpp) +target_link_libraries(page-ctl page-ctl-lib dbms clickhouse_functions clickhouse-server-lib) +target_compile_options(page-ctl PRIVATE -Wno-format) diff --git a/dbms/src/Storages/Page/tools/PageCtl/Main.cpp b/dbms/src/Storages/Page/tools/PageCtl/Main.cpp new file mode 100644 index 00000000000..ae9901ec864 --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/Main.cpp @@ -0,0 +1,20 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +int main(int argc, char ** argv) +{ + return DB::PageStorageCtl::mainEntry(argc, argv); +} diff --git a/dbms/src/Storages/Page/tools/PageCtl/MainEntry.cpp b/dbms/src/Storages/Page/tools/PageCtl/MainEntry.cpp new file mode 100644 index 00000000000..69b41435c34 --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/MainEntry.cpp @@ -0,0 +1,69 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include +#include +namespace DB +{ +int PageStorageCtl::mainEntry(int argc, char ** argv) +{ + namespace po = boost::program_options; + using po::value; + po::options_description desc("Allowed commands"); + desc.add_options()("help,h", "produce help message") // + ("page_storage_version,V", value(), "PageStorage Version: 2 means PageStorage V2, 3 means PageStorage V3.\n"); + po::variables_map options; + po::parsed_options parsed = po::command_line_parser(argc, argv) + .options(desc) + .allow_unregistered() + .run(); + po::store(parsed, options); + po::notify(options); + if (options.count("page_storage_version") == 0 && options.count("help") == 0) + { + std::cerr << "Invalid arg page_storage_version." << std::endl; + std::cerr << desc << std::endl; + exit(0); + } + if (options.count("page_storage_version") > 0) + { + int ps_version = options["page_storage_version"].as(); + if (ps_version == 3) + { + pageStorageV3CtlEntry(argc - 2, argv + 2); + return 0; + } + else if (ps_version == 2) + { + return pageStorageV2CtlEntry(argc - 2, argv + 2); + } + else + { + std::cerr << "Invalid arg page_storage_version." << std::endl; + std::cerr << desc << std::endl; + exit(0); + } + } + if (options.count("help") > 0) + { + std::cerr << desc << std::endl; + exit(0); + } + return 0; +} +} // namespace DB diff --git a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtl.h b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtl.h new file mode 100644 index 00000000000..c8f35a7750a --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtl.h @@ -0,0 +1,24 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +namespace DB +{ +class PageStorageCtl +{ +public: + static int mainEntry(int argc, char ** argv); +}; +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Storages/Page/V2/tests/page_storage_ctl.cpp b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.cpp similarity index 93% rename from dbms/src/Storages/Page/V2/tests/page_storage_ctl.cpp rename to dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.cpp index 2fb7e31d70e..f9488c4dfd9 100644 --- a/dbms/src/Storages/Page/V2/tests/page_storage_ctl.cpp +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.cpp @@ -28,34 +28,21 @@ using namespace DB::PS::V2; DB::WriteBatch::SequenceID debugging_recover_stop_sequence = 0; -/* some exported global vars */ -namespace DB -{ -#if __APPLE__ && __clang__ -__thread bool is_background_thread = false; -#else -thread_local bool is_background_thread = false; -#endif -} // namespace DB -/* some exported global vars */ -void Usage(const char * prog) +void Usage() { fprintf(stderr, R"HELP( -Usage: %s +Usage: mode == 1 -> dump all page entries 2 -> dump valid page entries - param: %s 2 [max-recover-sequence] + param: 2 [max-recover-sequence] 3 -> check all page entries and page data checksum 4 -> list capacity of all page files 5 -> list all page files 1000 -> gc files - param: %s 1000 [run-gc-times=1] [min-gc-file-num=10] [min-gc-bytes=134217728] [max-gc-valid-rate=0.35] -)HELP", - prog, - prog, - prog); + param: 1000 [run-gc-times=1] [min-gc-file-num=10] [min-gc-bytes=134217728] [max-gc-valid-rate=0.35] + )HELP"); } void printPageEntry(const DB::PageId pid, const DB::PageEntry & entry) @@ -117,7 +104,7 @@ PageStorage::Config parse_storage_config(int argc, char ** argv, Poco::Logger * return config; } -int main(int argc, char ** argv) +int pageStorageV2CtlEntry(int argc, char ** argv) try { (void)argc; @@ -125,7 +112,7 @@ try if (argc < 3) { - Usage(argv[0]); + Usage(); return 1; } @@ -153,7 +140,7 @@ try LOG_FMT_INFO(logger, "Running with [mode={}]", mode); break; default: - Usage(argv[0]); + Usage(); return 1; } @@ -271,13 +258,13 @@ void dump_all_entries(PageFileSet & page_files, int32_t mode) id_and_caches.emplace_back(std::make_pair(record.page_id, record.entry)); break; case DB::WriteBatch::WriteType::DEL: - printf("DEL\t%lld\n", // + printf("DEL\t%lld\t%llu\t%u\n", // record.page_id, page_file.getFileId(), page_file.getLevel()); break; case DB::WriteBatch::WriteType::REF: - printf("REF\t%lld\t%lld\t\n", // + printf("REF\t%lld\t%lld\t\t%llu\t%u\n", // record.page_id, record.ori_page_id, page_file.getFileId(), diff --git a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.h b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.h new file mode 100644 index 00000000000..6d573ffaba7 --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.h @@ -0,0 +1,17 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +int pageStorageV2CtlEntry(int argc, char ** argv); diff --git a/dbms/src/Storages/Page/V3/tests/page_storage_ctl.cpp b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp similarity index 87% rename from dbms/src/Storages/Page/V3/tests/page_storage_ctl.cpp rename to dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp index 7ea8da6892a..c9b871c67d2 100644 --- a/dbms/src/Storages/Page/V3/tests/page_storage_ctl.cpp +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp @@ -12,19 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include +#include #include -#include -#include #include -#include -#include -#include +#include #include #include #include -#include #include #include @@ -32,6 +27,9 @@ namespace DB::PS::V3 { +extern "C" { +void run_raftstore_proxy_ffi(int argc, const char * const * argv, const DB::EngineStoreServerHelper *); +} struct ControlOptions { enum DisplayType @@ -49,6 +47,8 @@ struct ControlOptions UInt64 query_ns_id = DB::TEST_NAMESPACE_ID; UInt64 check_page_id = UINT64_MAX; bool enable_fo_check = true; + bool is_imitative = true; + String config_file_path; static ControlOptions parse(int argc, char ** argv); }; @@ -65,12 +65,12 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) ("display_mode,D", value()->default_value(1), "Display Mode: 1 is summary information,\n 2 is display all of stored page and version chaim(will be very long),\n 3 is display all blobs(in disk) data distribution. \n 4 is check every data is valid.") // ("enable_fo_check,E", value()->default_value(true), "Also check the evert field offsets. This options only works when `display_mode` is 4.") // ("query_ns_id,N", value()->default_value(DB::TEST_NAMESPACE_ID), "When used `check_page_id`/`query_page_id`/`query_blob_id` to query results. You can specify a namespace id.")("check_page_id,C", value()->default_value(UINT64_MAX), "Check a single Page id, display the exception if meet. And also will check the field offsets.") // - ("query_page_id,W", value()->default_value(UINT64_MAX), "Quert a single Page id, and print its version chaim.") // - ("query_blob_id,B", value()->default_value(UINT32_MAX), "Quert a single Blob id, and print its data distribution."); + ("query_page_id,W", value()->default_value(UINT64_MAX), "Query a single Page id, and print its version chaim.") // + ("query_blob_id,B", value()->default_value(UINT32_MAX), "Query a single Blob id, and print its data distribution.")("imitative,I", value()->default_value(true), "Use imitative context instead. (encryption is not supported in this mode so that no need to set config_file_path)")("config_file_path", value(), "Path to TiFlash config (tiflash.toml)."); static_assert(sizeof(DB::PageId) == sizeof(UInt64)); - static_assert(sizeof(DB::BlobFileId) == sizeof(UInt32)); + static_assert(sizeof(DB::BlobFileId) == sizeof(UInt64)); po::variables_map options; po::store(po::parse_command_line(argc, argv, desc), options); @@ -97,6 +97,21 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) opt.enable_fo_check = options["enable_fo_check"].as(); opt.check_page_id = options["check_page_id"].as(); opt.query_ns_id = options["query_ns_id"].as(); + opt.is_imitative = options["imitative"].as(); + if (opt.is_imitative && options.count("config_file_path") != 0) + { + std::cerr << "config_file_path is not allowed in imitative mode" << std::endl; + exit(0); + } + else if (!opt.is_imitative && options.count("config_file_path") == 0) + { + std::cerr << "config_file_path is required in proxy mode" << std::endl; + exit(0); + } + if (options.count("config_file_path") != 0) + { + opt.config_file_path = options["config_file_path"].as(); + } if (opt.display_mode < DisplayType::DISPLAY_SUMMARY_INFO || opt.display_mode > DisplayType::CHECK_ALL_DATA_CRC) { @@ -108,15 +123,39 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) return opt; } -class PageStorageControl +class PageStorageControlV3 { public: - explicit PageStorageControl(const ControlOptions & options_) + explicit PageStorageControlV3(const ControlOptions & options_) : options(options_) { } void run() + { + try + { + if (options.is_imitative) + { + Context context = Context::createGlobal(); + getPageStorageV3Info(context, options); + } + else + { + PageDirectory::MVCCMapType type; + CLIService service(getPageStorageV3Info, options, options.config_file_path, run_raftstore_proxy_ffi); + service.run({""}); + } + } + catch (...) + { + DB::tryLogCurrentException("exception thrown"); + std::abort(); // Finish testing if some error happened. + } + } + +private: + static int getPageStorageV3Info(Context & context, const ControlOptions & options) { DB::PSDiskDelegatorPtr delegator; if (options.paths.size() == 1) @@ -128,13 +167,20 @@ class PageStorageControl delegator = std::make_shared(options.paths); } - auto key_manager = std::make_shared(false); - auto file_provider = std::make_shared(key_manager, false); - + FileProviderPtr file_provider_ptr; + if (options.is_imitative) + { + auto key_manager = std::make_shared(false); + file_provider_ptr = std::make_shared(key_manager, false); + } + else + { + file_provider_ptr = context.getFileProvider(); + } BlobStore::Config blob_config; PageStorage::Config config; - PageStorageImpl ps_v3("PageStorageControl", delegator, config, file_provider); + PageStorageImpl ps_v3("PageStorageControlV3", delegator, config, file_provider_ptr); ps_v3.restore(); PageDirectory::MVCCMapType & mvcc_table_directory = ps_v3.page_directory->mvcc_table_directory; @@ -171,9 +217,9 @@ class PageStorageControl std::cout << "Invalid display mode." << std::endl; break; } + return 0; } -private: static String getBlobsInfo(BlobStore & blob_store, UInt32 blob_id) { auto stat_info = [](const BlobStore::BlobStats::BlobStatPtr & stat, const String & path) { @@ -469,9 +515,9 @@ class PageStorageControl } // namespace DB::PS::V3 using namespace DB::PS::V3; -int main(int argc, char ** argv) + +void pageStorageV3CtlEntry(int argc, char ** argv) { const auto & options = ControlOptions::parse(argc, argv); - PageStorageControl(options).run(); - return 0; + PageStorageControlV3(options).run(); } diff --git a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.h b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.h new file mode 100644 index 00000000000..21a929ee599 --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.h @@ -0,0 +1,17 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +void pageStorageV3CtlEntry(int argc, char ** argv);