Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add page_stress_testing as a subcommand of tiflash #5038

Merged
merged 23 commits into from
Jun 10, 2022
Merged

feat: add page_stress_testing as a subcommand of tiflash #5038

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
5f8096d
move stree workload under namespace DB::PS::tests
Lloyd-Pottiger May 31, 2022
e7cf388
fix clang-tidy error
Lloyd-Pottiger May 31, 2022
75c0c86
fix clang-tidy error
Lloyd-Pottiger May 31, 2022
d3be4aa
add mainEntry.cpp
Lloyd-Pottiger May 31, 2022
5a36b3f
change directory layout and add CMakeLists.txt
Lloyd-Pottiger May 31, 2022
8ba638c
add it to main.cpp
Lloyd-Pottiger May 31, 2022
f266aaa
fix some known errors
Lloyd-Pottiger Jun 1, 2022
79e1dee
remove page_storage_v3 in workload CMakeLists
Lloyd-Pottiger Jun 1, 2022
e80288d
explicit trigger REGISTER_WORKLOAD
Lloyd-Pottiger Jun 1, 2022
3a3f46a
fix tidy error
Lloyd-Pottiger Jun 1, 2022
28cc416
fix tidy error
Lloyd-Pottiger Jun 1, 2022
9124d5d
change CMakeLists
Lloyd-Pottiger Jun 1, 2022
cb478ef
add some comments about trigger register
Lloyd-Pottiger Jun 2, 2022
9473123
Merge branch 'master' into add-page-stress-as-a-command
Lloyd-Pottiger Jun 2, 2022
5e5e06c
reslove conflicts
Lloyd-Pottiger Jun 2, 2022
c406056
rename stressworkload to pageworkload
Lloyd-Pottiger Jun 7, 2022
09c139c
rollback irrelevant changes
Lloyd-Pottiger Jun 7, 2022
201ddbf
remove ENABLE_V3_PAGESTORAGE
Lloyd-Pottiger Jun 9, 2022
5f8afb2
remove dt-workload & page-workload executable target
Lloyd-Pottiger Jun 9, 2022
fdee8de
remove useless code
Lloyd-Pottiger Jun 9, 2022
5394c4b
remove stress & tools dir
Lloyd-Pottiger Jun 9, 2022
247ae77
clear clang-tidy error
Lloyd-Pottiger Jun 9, 2022
88fb2d6
Merge branch 'master' into add-page-stress-as-a-command
ti-chi-bot Jun 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,9 +424,6 @@ else (ENABLE_FAILPOINTS)
message (STATUS "Failpoints are disabled")
endif (ENABLE_FAILPOINTS)

# Enable PageStorage V3 test.
option (ENABLE_V3_PAGESTORAGE "Enables V3 PageStorage" ON)

# Flags for test coverage
option (TEST_COVERAGE "Enables flags for test coverage" OFF)
option (TEST_COVERAGE_XML "Output XML report for test coverage" OFF)
Expand Down
13 changes: 7 additions & 6 deletions dbms/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,12 +91,10 @@ add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet)
add_headers_and_sources(dbms src/Storages/Page/V2/gc)
add_headers_and_sources(dbms src/WindowFunctions)
add_headers_and_sources(dbms src/TiDB/Schema)
if (ENABLE_V3_PAGESTORAGE)
add_headers_and_sources(dbms src/Storages/Page/V3)
add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
endif()
add_headers_and_sources(dbms src/Storages/Page/V3)
add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
add_headers_and_sources(dbms src/Storages/Page/)
add_headers_and_sources(dbms src/TiDB)
add_headers_and_sources(dbms src/Client)
Expand Down Expand Up @@ -311,6 +309,9 @@ if (ENABLE_TESTS)
if (ENABLE_TIFLASH_DTWORKLOAD)
target_link_libraries(bench_dbms dt-workload-lib)
endif ()
if (ENABLE_TIFLASH_PAGEWORKLOAD)
target_link_libraries(bench_dbms page-workload-lib)
endif ()

add_check(bench_dbms)
endif ()
Expand Down
4 changes: 4 additions & 0 deletions dbms/src/Server/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ option(ENABLE_CLICKHOUSE_SERVER "Enable server" ${ENABLE_CLICKHOUSE_ALL})
option(ENABLE_CLICKHOUSE_CLIENT "Enable client" ${ENABLE_CLICKHOUSE_ALL})
option(ENABLE_TIFLASH_DTTOOL "Enable dttool: tools to manage dmfile" ${ENABLE_CLICKHOUSE_ALL})
option(ENABLE_TIFLASH_DTWORKLOAD "Enable dtworkload: tools to test and stress DeltaTree" ${ENABLE_CLICKHOUSE_ALL})
option(ENABLE_TIFLASH_PAGEWORKLOAD "Enable pageworkload: tools to test and stress PageStorage" ${ENABLE_CLICKHOUSE_ALL})
option(ENABLE_TIFLASH_PAGECTL "Enable pagectl: tools to debug page storage" ${ENABLE_CLICKHOUSE_ALL})

configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
Expand Down Expand Up @@ -136,6 +137,9 @@ endif ()
if (ENABLE_TIFLASH_DTWORKLOAD)
target_link_libraries(tiflash dt-workload-lib)
endif ()
if (ENABLE_TIFLASH_PAGEWORKLOAD)
target_link_libraries(tiflash page-workload-lib)
endif()
if (ENABLE_TIFLASH_PAGECTL)
target_link_libraries(tiflash page-ctl-lib)
endif ()
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Server/config_tools.h.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@
#cmakedefine01 ENABLE_CLICKHOUSE_CLIENT
#cmakedefine01 ENABLE_TIFLASH_DTTOOL
#cmakedefine01 ENABLE_TIFLASH_DTWORKLOAD
#cmakedefine01 ENABLE_TIFLASH_PAGEWORKLOAD
#cmakedefine01 ENABLE_TIFLASH_PAGECTL
6 changes: 6 additions & 0 deletions dbms/src/Server/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@
#if ENABLE_TIFLASH_DTWORKLOAD
#include <Storages/DeltaMerge/tools/workload/DTWorkload.h>
#endif
#if ENABLE_TIFLASH_PAGEWORKLOAD
#include <Storages/Page/stress/workload/PSWorkload.h>
#endif
#if ENABLE_TIFLASH_PAGECTL
#include <Storages/Page/tools/PageCtl/PageStorageCtl.h>
#endif
Expand Down Expand Up @@ -107,6 +110,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] = {
#if ENABLE_TIFLASH_DTWORKLOAD
{"dtworkload", DB::DM::tests::DTWorkload::mainEntry},
#endif
#if ENABLE_TIFLASH_PAGEWORKLOAD
{"pageworkload", DB::PS::tests::StressWorkload::mainEntry},
#endif
#if ENABLE_TIFLASH_PAGECTL
{"pagectl", DB::PageStorageCtl::mainEntry},
#endif
Expand Down
7 changes: 3 additions & 4 deletions dbms/src/Storages/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,14 @@ add_subdirectory (System)
add_subdirectory (Page)
add_subdirectory (DeltaMerge/File/dtpb)
add_subdirectory (DeltaMerge/tools)
add_subdirectory (Page/stress)

if (ENABLE_TESTS)
add_subdirectory (tests EXCLUDE_FROM_ALL)
add_subdirectory (Transaction/tests EXCLUDE_FROM_ALL)
add_subdirectory (Page/V2/tests EXCLUDE_FROM_ALL)
if (ENABLE_V3_PAGESTORAGE)
add_subdirectory (Page/V3 EXCLUDE_FROM_ALL)
add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL)
endif ()
add_subdirectory (Page/V3 EXCLUDE_FROM_ALL)
add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL)
add_subdirectory (DeltaMerge/tests EXCLUDE_FROM_ALL)
endif ()

10 changes: 0 additions & 10 deletions dbms/src/Storages/Page/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,3 @@

add_subdirectory(V2)
add_subdirectory(tools)

# PageStorage Stress test
if (ENABLE_V3_PAGESTORAGE)
add_headers_and_sources(page_stress_testing stress)
add_headers_and_sources(page_stress_testing stress/workload)
add_executable(page_stress_testing EXCLUDE_FROM_ALL ${page_stress_testing_sources})
target_link_libraries(page_stress_testing dbms page_storage_v3)
target_include_directories(page_stress_testing PRIVATE stress)
target_compile_options(page_stress_testing PRIVATE -Wno-format -lc++) # turn off printf format check
endif()
17 changes: 17 additions & 0 deletions dbms/src/Storages/Page/stress/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Copyright 2022 PingCAP, Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

include_directories (${CMAKE_CURRENT_BINARY_DIR})

add_subdirectory (workload EXCLUDE_FROM_ALL)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we still need to keep this "stress" directory? Maybe move "dbms/src/Storages/Page/stress/workload" -> "dbms/src/Storages/Page/workload"?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed, the same as dbms/src/Storages/DeltaMerge/workload

46 changes: 0 additions & 46 deletions dbms/src/Storages/Page/stress/stress_page_storage.cpp
View file
Open in desktop

This file was deleted.

26 changes: 26 additions & 0 deletions dbms/src/Storages/Page/stress/workload/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Copyright 2022 PingCAP, Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

include_directories (${CMAKE_CURRENT_BINARY_DIR})

set (page-workload-src HeavyMemoryCostInGC.cpp HeavyRead.cpp HeavySkewWriteRead.cpp HeavyWrite.cpp HighValidBigFileGC.cpp HoldSnapshotsLongTime.cpp Normal.cpp
PageStorageInMemoryCapacity.cpp ThousandsOfOffset.cpp MainEntry.cpp Normal.cpp PageStorageInMemoryCapacity.cpp PSBackground.cpp PSRunnable.cpp PSStressEnv.cpp PSWorkload.cpp)

add_library (page-workload-lib ${page-workload-src})
target_link_libraries (page-workload-lib dbms clickhouse_functions clickhouse-server-lib)
target_compile_options (page-workload-lib PRIVATE -Wno-format -lc++)

add_executable (page-workload Main.cpp ${page-workload-src})
target_link_libraries (page-workload dbms clickhouse_functions clickhouse-server-lib)
target_compile_options (page-workload PRIVATE -Wno-format -lc++)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems the binary page_workload is useless. Do you think we still need this standalone binary? @jiaqizho

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would increase the compile time of tiflash but can decrease the compile when we just want to run page-workload only. So maybe we can keep it just like dt-workload?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In fact, I want to remove the binary of dt-workload...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, I will remove both of them.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok for remove it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please delete useless code files as well： https://github.com/pingcap/tiflash/blob/master/dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

5 changes: 4 additions & 1 deletion dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <PSWorkload.h>
#include <Storages/Page/stress/workload/PSWorkload.h>

namespace DB::PS::tests
{
class HeavyMemoryCostInGC
: public StressWorkload
, public StressWorkloadFunc<HeavyMemoryCostInGC>
Expand Down Expand Up @@ -81,3 +83,4 @@ class HeavyMemoryCostInGC
};

REGISTER_WORKLOAD(HeavyMemoryCostInGC)
} // namespace DB::PS::tests
7 changes: 5 additions & 2 deletions dbms/src/Storages/Page/stress/workload/HeavyRead.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <PSWorkload.h>
#include <Storages/Page/stress/workload/PSWorkload.h>

namespace DB::PS::tests
{
class HeavyRead : public StressWorkload
, public StressWorkloadFunc<HeavyRead>
{
Expand Down Expand Up @@ -69,4 +71,5 @@ class HeavyRead : public StressWorkload
}
};

REGISTER_WORKLOAD(HeavyRead)
REGISTER_WORKLOAD(HeavyRead)
} // namespace DB::PS::tests
7 changes: 5 additions & 2 deletions dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <PSWorkload.h>
#include <Storages/Page/stress/workload/PSWorkload.h>

namespace DB::PS::tests
{
class HeavySkewWriteRead : public StressWorkload
, public StressWorkloadFunc<HeavySkewWriteRead>
{
Expand Down Expand Up @@ -84,4 +86,5 @@ class HeavySkewWriteRead : public StressWorkload
}
};

REGISTER_WORKLOAD(HeavySkewWriteRead)
REGISTER_WORKLOAD(HeavySkewWriteRead)
} // namespace DB::PS::tests
7 changes: 5 additions & 2 deletions dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <PSWorkload.h>
#include <Storages/Page/stress/workload/PSWorkload.h>

namespace DB::PS::tests
{
class HeavyWrite : public StressWorkload
, public StressWorkloadFunc<HeavyWrite>
{
Expand Down Expand Up @@ -71,4 +73,5 @@ class HeavyWrite : public StressWorkload
}
};

REGISTER_WORKLOAD(HeavyWrite)
REGISTER_WORKLOAD(HeavyWrite)
} // namespace DB::PS::tests
5 changes: 4 additions & 1 deletion dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <PSWorkload.h>
#include <Storages/Page/stress/workload/PSWorkload.h>

namespace DB::PS::tests
{
class HighValidBigFileGCWorkload
: public StressWorkload
, public StressWorkloadFunc<HighValidBigFileGCWorkload>
Expand Down Expand Up @@ -129,3 +131,4 @@ class HighValidBigFileGCWorkload
};

REGISTER_WORKLOAD(HighValidBigFileGCWorkload)
} // namespace DB::PS::tests
7 changes: 5 additions & 2 deletions dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <PSWorkload.h>
#include <Storages/Page/stress/workload/PSWorkload.h>

namespace DB::PS::tests
{
class HoldSnapshotsLongTime : public StressWorkload
, public StressWorkloadFunc<HoldSnapshotsLongTime>
{
Expand Down Expand Up @@ -93,4 +95,5 @@ class HoldSnapshotsLongTime : public StressWorkload
}
};

REGISTER_WORKLOAD(HoldSnapshotsLongTime)
REGISTER_WORKLOAD(HoldSnapshotsLongTime)
} // namespace DB::PS::tests
20 changes: 20 additions & 0 deletions dbms/src/Storages/Page/stress/workload/Main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright 2022 PingCAP, Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <Storages/Page/stress/workload/PSWorkload.h>

int main(int argc, char ** argv)
{
return DB::PS::tests::StressWorkload::mainEntry(argc, argv);
}
Loading