feat: add page_stress_testing as a subcommand of tiflash

dbms/CMakeLists.txt

@@ -311,6 +311,9 @@ if (ENABLE_TESTS)
     if (ENABLE_TIFLASH_DTWORKLOAD)
         target_link_libraries(bench_dbms dt-workload-lib)
     endif ()
+    if (ENABLE_TIFLASH_PAGEWORKLOAD)
+        target_link_libraries(bench_dbms page-workload-lib)
+    endif ()
 
     add_check(bench_dbms)
 endif ()

dbms/src/Server/CMakeLists.txt

@@ -22,6 +22,7 @@ option(ENABLE_CLICKHOUSE_SERVER "Enable server" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_CLICKHOUSE_CLIENT "Enable client" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_DTTOOL "Enable dttool: tools to manage dmfile" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_DTWORKLOAD "Enable dtworkload: tools to test and stress DeltaTree" ${ENABLE_CLICKHOUSE_ALL})
+option(ENABLE_TIFLASH_PAGEWORKLOAD "Enable pageworkload: tools to test and stress PageStorage" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_PAGECTL "Enable pagectl: tools to debug page storage" ${ENABLE_CLICKHOUSE_ALL})
 
 configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
@@ -136,6 +137,9 @@ endif ()
 if (ENABLE_TIFLASH_DTWORKLOAD)
     target_link_libraries(tiflash dt-workload-lib)
 endif ()
+if (ENABLE_TIFLASH_PAGEWORKLOAD)
+    target_link_libraries(tiflash page-workload-lib)
+endif()
 if (ENABLE_TIFLASH_PAGECTL)
     target_link_libraries(tiflash page-ctl-lib)
 endif ()

dbms/src/Server/config_tools.h.in

@@ -6,4 +6,5 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_CLIENT
 #cmakedefine01 ENABLE_TIFLASH_DTTOOL
 #cmakedefine01 ENABLE_TIFLASH_DTWORKLOAD
+#cmakedefine01 ENABLE_TIFLASH_PAGEWORKLOAD
 #cmakedefine01 ENABLE_TIFLASH_PAGECTL

dbms/src/Server/main.cpp

@@ -38,6 +38,9 @@
 #if ENABLE_TIFLASH_DTWORKLOAD
 #include <Storages/DeltaMerge/tools/workload/DTWorkload.h>
 #endif
+#if ENABLE_TIFLASH_PAGEWORKLOAD
+#include <Storages/Page/stress/workload/PSWorkload.h>
+#endif
 #if ENABLE_TIFLASH_PAGECTL
 #include <Storages/Page/tools/PageCtl/PageStorageCtl.h>
 #endif
@@ -107,6 +110,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] = {
 #if ENABLE_TIFLASH_DTWORKLOAD
     {"dtworkload", DB::DM::tests::DTWorkload::mainEntry},
 #endif
+#if ENABLE_TIFLASH_PAGEWORKLOAD
+    {"pageworkload", DB::PS::tests::StressWorkload::mainEntry},
+#endif
 #if ENABLE_TIFLASH_PAGECTL
     {"pagectl", DB::PageStorageCtl::mainEntry},
 #endif

dbms/src/Storages/CMakeLists.txt

@@ -17,6 +17,10 @@ add_subdirectory (Page)
 add_subdirectory (DeltaMerge/File/dtpb)
 add_subdirectory (DeltaMerge/tools)
 
+if (ENABLE_V3_PAGESTORAGE)
+    add_subdirectory (Page/stress)
+endif ()
+
 if (ENABLE_TESTS)
     add_subdirectory (tests EXCLUDE_FROM_ALL)
     add_subdirectory (Transaction/tests EXCLUDE_FROM_ALL)

dbms/src/Storages/Page/CMakeLists.txt

@@ -14,13 +14,3 @@
 
 add_subdirectory(V2)
 add_subdirectory(tools)
-
-# PageStorage Stress test
-if (ENABLE_V3_PAGESTORAGE)
-    add_headers_and_sources(page_stress_testing stress)
-    add_headers_and_sources(page_stress_testing stress/workload)
-    add_executable(page_stress_testing EXCLUDE_FROM_ALL ${page_stress_testing_sources})
-    target_link_libraries(page_stress_testing dbms page_storage_v3)
-    target_include_directories(page_stress_testing PRIVATE stress)
-    target_compile_options(page_stress_testing PRIVATE -Wno-format -lc++) # turn off printf format check
-endif()

dbms/src/Storages/Page/stress/CMakeLists.txt

@@ -0,0 +1,17 @@
+# Copyright 2022 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_directories (${CMAKE_CURRENT_BINARY_DIR})
+
+add_subdirectory (workload EXCLUDE_FROM_ALL)

dbms/src/Storages/Page/stress/workload/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Copyright 2022 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_directories (${CMAKE_CURRENT_BINARY_DIR})
+
+set (page-workload-src HeavyMemoryCostInGC.cpp HeavyRead.cpp HeavySkewWriteRead.cpp HeavyWrite.cpp HighValidBigFileGC.cpp HoldSnapshotsLongTime.cpp Normal.cpp 
+     PageStorageInMemoryCapacity.cpp ThousandsOfOffset.cpp MainEntry.cpp Normal.cpp PageStorageInMemoryCapacity.cpp PSBackground.cpp PSRunnable.cpp PSStressEnv.cpp PSWorkload.cpp)
+
+add_library (page-workload-lib ${page-workload-src})
+target_link_libraries (page-workload-lib dbms clickhouse_functions clickhouse-server-lib)
+target_compile_options (page-workload-lib PRIVATE -Wno-format -lc++)
+
+add_executable (page-workload Main.cpp ${page-workload-src})
+target_link_libraries (page-workload dbms clickhouse_functions clickhouse-server-lib)
+target_compile_options (page-workload PRIVATE -Wno-format -lc++)

dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HeavyMemoryCostInGC
     : public StressWorkload
     , public StressWorkloadFunc<HeavyMemoryCostInGC>
@@ -81,3 +83,4 @@ class HeavyMemoryCostInGC
 };
 
 REGISTER_WORKLOAD(HeavyMemoryCostInGC)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HeavyRead.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HeavyRead : public StressWorkload
     , public StressWorkloadFunc<HeavyRead>
 {
@@ -69,4 +71,5 @@ class HeavyRead : public StressWorkload
     }
 };
 
-REGISTER_WORKLOAD(HeavyRead)
+REGISTER_WORKLOAD(HeavyRead)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HeavySkewWriteRead : public StressWorkload
     , public StressWorkloadFunc<HeavySkewWriteRead>
 {
@@ -84,4 +86,5 @@ class HeavySkewWriteRead : public StressWorkload
     }
 };
 
-REGISTER_WORKLOAD(HeavySkewWriteRead)
+REGISTER_WORKLOAD(HeavySkewWriteRead)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HeavyWrite : public StressWorkload
     , public StressWorkloadFunc<HeavyWrite>
 {
@@ -71,4 +73,5 @@ class HeavyWrite : public StressWorkload
     }
 };
 
-REGISTER_WORKLOAD(HeavyWrite)
+REGISTER_WORKLOAD(HeavyWrite)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HighValidBigFileGCWorkload
     : public StressWorkload
     , public StressWorkloadFunc<HighValidBigFileGCWorkload>
@@ -129,3 +131,4 @@ class HighValidBigFileGCWorkload
 };
 
 REGISTER_WORKLOAD(HighValidBigFileGCWorkload)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HoldSnapshotsLongTime : public StressWorkload
     , public StressWorkloadFunc<HoldSnapshotsLongTime>
 {
@@ -93,4 +95,5 @@ class HoldSnapshotsLongTime : public StressWorkload
     }
 };
 
-REGISTER_WORKLOAD(HoldSnapshotsLongTime)
+REGISTER_WORKLOAD(HoldSnapshotsLongTime)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/Main.cpp

@@ -0,0 +1,20 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <Storages/Page/stress/workload/PSWorkload.h>
+
+int main(int argc, char ** argv)
+{
+    return DB::PS::tests::StressWorkload::mainEntry(argc, argv);
+}

dbms/src/Storages/Page/stress/workload/MainEntry.cpp

@@ -0,0 +1,70 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <Storages/Page/stress/workload/PSStressEnv.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
+
+using namespace DB::PS::tests;
+
+int StressWorkload::mainEntry(int argc, char ** argv)
+{
+    {
+        // maybe due to sequence of linking, REGISTER_WORKLOAD is not visible to main function in dbms/src/Server/main.cpp
+        // cause that REGISTER_WORKLOAD will not be triggered before mainEntry
+        // we do this to trigger REGISTER_WORKLOAD explicitly.
+        void _work_load_register_named_HeavyMemoryCostInGC();
+        void (*f)() = _work_load_register_named_HeavyMemoryCostInGC;
+        (void)f;
+        void _work_load_register_named_HeavyRead();
+        f = _work_load_register_named_HeavyRead;
+        (void)f;
+        void _work_load_register_named_HeavySkewWriteRead();
+        f = _work_load_register_named_HeavySkewWriteRead;
+        (void)f;
+        void _work_load_register_named_HeavyWrite();
+        f = _work_load_register_named_HeavyWrite;
+        (void)f;
+        void _work_load_register_named_HighValidBigFileGCWorkload();
+        f = _work_load_register_named_HighValidBigFileGCWorkload;
+        (void)f;
+        void _work_load_register_named_HoldSnapshotsLongTime();
+        f = _work_load_register_named_HoldSnapshotsLongTime;
+        (void)f;
+        void _work_load_register_named_PageStorageInMemoryCapacity();
+        f = _work_load_register_named_PageStorageInMemoryCapacity;
+        (void)f;
+        void _work_load_register_named_NormalWorkload();
+        f = _work_load_register_named_NormalWorkload;
+        (void)f;
+        void _work_load_register_named_ThousandsOfOffset();
+        f = _work_load_register_named_ThousandsOfOffset;
+        (void)f;
+    }
+    try
+    {
+        StressEnv::initGlobalLogger();
+        auto env = StressEnv::parse(argc, argv);
+        env.setup();
+
+        auto & mamager = StressWorkloadManger::getInstance();
+        mamager.setEnv(env);
+        mamager.runWorkload();
+
+        return StressEnvStatus::getInstance().isSuccess();
+    }
+    catch (...)
+    {
+        DB::tryLogCurrentException("");
+        exit(-1);
+    }
+}