From 01b8754509796d80fa4e3849b4463ae726ae53b9 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 14 Aug 2024 17:13:35 +0800 Subject: [PATCH] refactor(scripts): Separate admin and build scripts to independent directories (#2096) Separate the `scripts` directory to `admin_tools` and `build_tools` directories. --- .../rebuild_thirdparty_if_needed/action.yaml | 4 +-- .github/workflows/lint_and_test_cpp.yaml | 8 ++--- .github/workflows/module_labeler_conf.yml | 3 +- .github/workflows/regular-build.yml | 2 +- .licenserc.yaml | 6 ++-- LICENSE | 2 +- {scripts => admin_tools}/config_hdfs.sh | 0 {scripts => admin_tools}/downgrade_node.sh | 0 {scripts => admin_tools}/learn_stat.py | 0 {scripts => admin_tools}/migrate_node.sh | 0 {scripts => admin_tools}/minos_common.sh | 0 .../pegasus_add_node_list.sh | 4 +-- {scripts => admin_tools}/pegasus_bench_run.sh | 0 .../pegasus_check_arguments.sh | 2 +- {scripts => admin_tools}/pegasus_kill_test.sh | 0 .../pegasus_manual_compact.sh | 0 .../pegasus_migrate_zookeeper.sh | 2 +- .../pegasus_offline_node.sh | 2 +- .../pegasus_offline_node_list.sh | 4 +-- .../pegasus_rebalance_cluster.sh | 0 .../pegasus_replica_thread.sh | 0 .../pegasus_restart_node.sh | 2 +- .../pegasus_rolling_update.sh | 4 +-- .../pegasus_set_usage_scenario.sh | 0 .../pegasus_stat_available.sh | 0 .../pegasus_update_ingest_behind.sh | 2 +- {scripts => admin_tools}/sendmail.sh | 0 {scripts => build_tools}/bump_version.sh | 0 {scripts => build_tools}/check_license.py | 0 {scripts => build_tools}/clang_tidy.py | 3 +- {scripts => build_tools}/clear_zk.sh | 0 .../collector_table_counter_gen.py | 2 +- {scripts => build_tools}/compile_thrift.py | 0 {scripts => build_tools}/download_hadoop.sh | 0 {scripts => build_tools}/download_package.sh | 0 {scripts => build_tools}/download_zk.sh | 0 {scripts => build_tools}/format_files.sh | 0 {scripts => build_tools}/pack_client.sh | 0 {scripts => build_tools}/pack_common.sh | 0 {scripts => build_tools}/pack_server.sh | 4 +-- {scripts => build_tools}/pack_tools.sh | 11 +++++-- {scripts => build_tools}/recompile_thrift.sh | 0 {scripts => build_tools}/redis_proto_check.py | 0 {scripts => build_tools}/run-clang-format.py | 0 {scripts => build_tools}/start_zk.sh | 0 {scripts => build_tools}/stop_zk.sh | 0 docker/thirdparties-bin/Dockerfile | 4 +-- java-client/README.md | 2 +- run.sh | 30 +++++++++---------- src/replica/replica.h | 4 +-- 50 files changed, 57 insertions(+), 50 deletions(-) rename {scripts => admin_tools}/config_hdfs.sh (100%) rename {scripts => admin_tools}/downgrade_node.sh (100%) rename {scripts => admin_tools}/learn_stat.py (100%) rename {scripts => admin_tools}/migrate_node.sh (100%) rename {scripts => admin_tools}/minos_common.sh (100%) rename {scripts => admin_tools}/pegasus_add_node_list.sh (92%) rename {scripts => admin_tools}/pegasus_bench_run.sh (100%) rename {scripts => admin_tools}/pegasus_check_arguments.sh (98%) rename {scripts => admin_tools}/pegasus_kill_test.sh (100%) rename {scripts => admin_tools}/pegasus_manual_compact.sh (100%) rename {scripts => admin_tools}/pegasus_migrate_zookeeper.sh (99%) rename {scripts => admin_tools}/pegasus_offline_node.sh (99%) rename {scripts => admin_tools}/pegasus_offline_node_list.sh (96%) rename {scripts => admin_tools}/pegasus_rebalance_cluster.sh (100%) rename {scripts => admin_tools}/pegasus_replica_thread.sh (100%) rename {scripts => admin_tools}/pegasus_restart_node.sh (99%) rename {scripts => admin_tools}/pegasus_rolling_update.sh (98%) rename {scripts => admin_tools}/pegasus_set_usage_scenario.sh (100%) rename {scripts => admin_tools}/pegasus_stat_available.sh (100%) rename {scripts => admin_tools}/pegasus_update_ingest_behind.sh (99%) rename {scripts => admin_tools}/sendmail.sh (100%) rename {scripts => build_tools}/bump_version.sh (100%) rename {scripts => build_tools}/check_license.py (100%) rename {scripts => build_tools}/clang_tidy.py (88%) rename {scripts => build_tools}/clear_zk.sh (100%) rename {scripts => build_tools}/collector_table_counter_gen.py (98%) rename {scripts => build_tools}/compile_thrift.py (100%) rename {scripts => build_tools}/download_hadoop.sh (100%) rename {scripts => build_tools}/download_package.sh (100%) rename {scripts => build_tools}/download_zk.sh (100%) rename {scripts => build_tools}/format_files.sh (100%) rename {scripts => build_tools}/pack_client.sh (100%) rename {scripts => build_tools}/pack_common.sh (100%) rename {scripts => build_tools}/pack_server.sh (98%) rename {scripts => build_tools}/pack_tools.sh (94%) rename {scripts => build_tools}/recompile_thrift.sh (100%) rename {scripts => build_tools}/redis_proto_check.py (100%) rename {scripts => build_tools}/run-clang-format.py (100%) rename {scripts => build_tools}/start_zk.sh (100%) rename {scripts => build_tools}/stop_zk.sh (100%) diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml b/.github/actions/rebuild_thirdparty_if_needed/action.yaml index 6e64f7db46..838ac3f615 100644 --- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml +++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml @@ -42,8 +42,8 @@ runs: cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin + ../build_tools/download_hadoop.sh hadoop-bin + ../build_tools/download_zk.sh zookeeper-bin rm -rf hadoop-bin/share/doc rm -rf zookeeper-bin/docs shell: bash diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 5e4fb7d468..1ae74b268f 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -30,13 +30,13 @@ on: - .github/actions - .github/workflows/lint_and_test_cpp.yaml - .github/workflows/thirdparty-regular-push.yml + - build_tools/pack_server.sh + - build_tools/pack_tools.sh - docker/thirdparties-bin/** - docker/thirdparties-src/** - CMakeLists.txt - cmake_modules/** - run.sh - - scripts/pack_server.sh - - scripts/pack_tools.sh - src/** - thirdparty/** @@ -55,7 +55,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: clang-format - run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . + run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . cpp_clang_tidy_linter: name: Tidy @@ -76,7 +76,7 @@ jobs: run: | git config --global --add safe.directory $(pwd) ./run.sh build --test --compiler clang-14,clang++-14 -t debug --skip_thirdparty -c --cmake_only - ./scripts/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H") + ./build_tools/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H") shell: bash iwyu: diff --git a/.github/workflows/module_labeler_conf.yml b/.github/workflows/module_labeler_conf.yml index ad23e4bf08..b002b2e3cd 100644 --- a/.github/workflows/module_labeler_conf.yml +++ b/.github/workflows/module_labeler_conf.yml @@ -42,7 +42,8 @@ thrift: docs: - '**/*.md' scripts: - - 'scripts/**/*' + - 'admin_tools/**/*' + - 'build_tools/**/*' - '**/*.sh' build: - 'cmake_modules/**/*' diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml index 04468e4d39..01c17f4fc7 100644 --- a/.github/workflows/regular-build.yml +++ b/.github/workflows/regular-build.yml @@ -41,7 +41,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: clang-format - run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . + run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . build_cpp: name: Build Cpp diff --git a/.licenserc.yaml b/.licenserc.yaml index b8e45b5ec4..71376cd101 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -110,8 +110,10 @@ header: - 'cmake_modules/FindRT.cmake' - 'cmake_modules/FindDL.cmake' # Copyright (c) 2017 Guillaume Papin - - 'scripts/run-clang-format.py' + - 'build_tools/run-clang-format.py' # The MIT License (MIT), Copyright (c) 2015 Microsoft Corporation + - 'admin_tools/learn_stat.py' + - 'build_tools/compile_thrift.py' - 'cmake_modules/BaseFunctions.cmake' - 'docs/rdsn-README.md' - 'idl/command.thrift' @@ -120,8 +122,6 @@ header: - 'idl/metadata.thrift' - 'idl/meta_admin.thrift' - 'idl/replica_admin.thrift' - - 'scripts/compile_thrift.py' - - 'scripts/learn_stat.py' - 'src/runtime/api_layer1.h' - 'src/runtime/api_task.h' - 'src/utils/api_utilities.h' diff --git a/LICENSE b/LICENSE index a8e975f469..eb22d54de3 100644 --- a/LICENSE +++ b/LICENSE @@ -514,7 +514,7 @@ RESULTING FROM THE USE OF THIS SOFTWARE. -------------------------------------------------------------------------------- -scripts/run-clang-format.py - MIT License +build_tools/run-clang-format.py - MIT License MIT License diff --git a/scripts/config_hdfs.sh b/admin_tools/config_hdfs.sh similarity index 100% rename from scripts/config_hdfs.sh rename to admin_tools/config_hdfs.sh diff --git a/scripts/downgrade_node.sh b/admin_tools/downgrade_node.sh similarity index 100% rename from scripts/downgrade_node.sh rename to admin_tools/downgrade_node.sh diff --git a/scripts/learn_stat.py b/admin_tools/learn_stat.py similarity index 100% rename from scripts/learn_stat.py rename to admin_tools/learn_stat.py diff --git a/scripts/migrate_node.sh b/admin_tools/migrate_node.sh similarity index 100% rename from scripts/migrate_node.sh rename to admin_tools/migrate_node.sh diff --git a/scripts/minos_common.sh b/admin_tools/minos_common.sh similarity index 100% rename from scripts/minos_common.sh rename to admin_tools/minos_common.sh diff --git a/scripts/pegasus_add_node_list.sh b/admin_tools/pegasus_add_node_list.sh similarity index 92% rename from scripts/pegasus_add_node_list.sh rename to admin_tools/pegasus_add_node_list.sh index ed40a8b6d2..1665d5a6df 100755 --- a/scripts/pegasus_add_node_list.sh +++ b/admin_tools/pegasus_add_node_list.sh @@ -52,7 +52,7 @@ shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir echo "Check the argument..." -source ./scripts/pegasus_check_arguments.sh add_node_list $cluster $meta_list $replica_task_id_list +source ./admin_tools/pegasus_check_arguments.sh add_node_list $cluster $meta_list $replica_task_id_list if [ $? -ne 0 ]; then echo "ERROR: the argument check failed" @@ -78,7 +78,7 @@ do echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" done -./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list true $nfs_rate_megabytes_per_disk +./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list true $nfs_rate_megabytes_per_disk echo "Finish time: `date`" add_node_finish_time=$((`date +%s`)) diff --git a/scripts/pegasus_bench_run.sh b/admin_tools/pegasus_bench_run.sh similarity index 100% rename from scripts/pegasus_bench_run.sh rename to admin_tools/pegasus_bench_run.sh diff --git a/scripts/pegasus_check_arguments.sh b/admin_tools/pegasus_check_arguments.sh similarity index 98% rename from scripts/pegasus_check_arguments.sh rename to admin_tools/pegasus_check_arguments.sh index 4b0231d87c..9574051c58 100755 --- a/scripts/pegasus_check_arguments.sh +++ b/admin_tools/pegasus_check_arguments.sh @@ -41,7 +41,7 @@ if [ "$check_type" != "add_node_list" -a "$check_type" != "offline_node_list" ]; exit 1 fi -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" diff --git a/scripts/pegasus_kill_test.sh b/admin_tools/pegasus_kill_test.sh similarity index 100% rename from scripts/pegasus_kill_test.sh rename to admin_tools/pegasus_kill_test.sh diff --git a/scripts/pegasus_manual_compact.sh b/admin_tools/pegasus_manual_compact.sh similarity index 100% rename from scripts/pegasus_manual_compact.sh rename to admin_tools/pegasus_manual_compact.sh diff --git a/scripts/pegasus_migrate_zookeeper.sh b/admin_tools/pegasus_migrate_zookeeper.sh similarity index 99% rename from scripts/pegasus_migrate_zookeeper.sh rename to admin_tools/pegasus_migrate_zookeeper.sh index 52e7c0952f..d4b78f5e24 100755 --- a/scripts/pegasus_migrate_zookeeper.sh +++ b/admin_tools/pegasus_migrate_zookeeper.sh @@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0" )" && pwd )" shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" diff --git a/scripts/pegasus_offline_node.sh b/admin_tools/pegasus_offline_node.sh similarity index 99% rename from scripts/pegasus_offline_node.sh rename to admin_tools/pegasus_offline_node.sh index d16f368f1a..fee4107af4 100755 --- a/scripts/pegasus_offline_node.sh +++ b/admin_tools/pegasus_offline_node.sh @@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0" )" && pwd )" shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" diff --git a/scripts/pegasus_offline_node_list.sh b/admin_tools/pegasus_offline_node_list.sh similarity index 96% rename from scripts/pegasus_offline_node_list.sh rename to admin_tools/pegasus_offline_node_list.sh index 7a2483f491..4da739c698 100755 --- a/scripts/pegasus_offline_node_list.sh +++ b/admin_tools/pegasus_offline_node_list.sh @@ -52,7 +52,7 @@ shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir echo "Check the argument..." -source ./scripts/pegasus_check_arguments.sh offline_node_list $cluster $meta_list $replica_task_id_list +source ./admin_tools/pegasus_check_arguments.sh offline_node_list $cluster $meta_list $replica_task_id_list if [ $? -ne 0 ]; then echo "ERROR: the argument check failed" @@ -93,7 +93,7 @@ echo for id in $id_list do echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - ./scripts/pegasus_offline_node.sh $cluster $meta_list $id + ./admin_tools/pegasus_offline_node.sh $cluster $meta_list $id if [ $? -ne 0 ]; then echo "ERROR: offline replica task $id failed" exit 1 diff --git a/scripts/pegasus_rebalance_cluster.sh b/admin_tools/pegasus_rebalance_cluster.sh similarity index 100% rename from scripts/pegasus_rebalance_cluster.sh rename to admin_tools/pegasus_rebalance_cluster.sh diff --git a/scripts/pegasus_replica_thread.sh b/admin_tools/pegasus_replica_thread.sh similarity index 100% rename from scripts/pegasus_replica_thread.sh rename to admin_tools/pegasus_replica_thread.sh diff --git a/scripts/pegasus_restart_node.sh b/admin_tools/pegasus_restart_node.sh similarity index 99% rename from scripts/pegasus_restart_node.sh rename to admin_tools/pegasus_restart_node.sh index b65e6ec18a..27d3588c1c 100755 --- a/scripts/pegasus_restart_node.sh +++ b/admin_tools/pegasus_restart_node.sh @@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0" )" && pwd )" shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" diff --git a/scripts/pegasus_rolling_update.sh b/admin_tools/pegasus_rolling_update.sh similarity index 98% rename from scripts/pegasus_rolling_update.sh rename to admin_tools/pegasus_rolling_update.sh index 9e207fc303..c805426db4 100755 --- a/scripts/pegasus_rolling_update.sh +++ b/admin_tools/pegasus_rolling_update.sh @@ -72,7 +72,7 @@ pwd="$( cd "$( dirname "$0" )" && pwd )" shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" @@ -338,7 +338,7 @@ fi if [ "$rebalance_cluster_after_rolling" == "true" ]; then echo "Start to rebalance cluster..." - ./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary + ./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary fi echo "Finish time: `date`" diff --git a/scripts/pegasus_set_usage_scenario.sh b/admin_tools/pegasus_set_usage_scenario.sh similarity index 100% rename from scripts/pegasus_set_usage_scenario.sh rename to admin_tools/pegasus_set_usage_scenario.sh diff --git a/scripts/pegasus_stat_available.sh b/admin_tools/pegasus_stat_available.sh similarity index 100% rename from scripts/pegasus_stat_available.sh rename to admin_tools/pegasus_stat_available.sh diff --git a/scripts/pegasus_update_ingest_behind.sh b/admin_tools/pegasus_update_ingest_behind.sh similarity index 99% rename from scripts/pegasus_update_ingest_behind.sh rename to admin_tools/pegasus_update_ingest_behind.sh index 8932f667e8..1000a058a5 100755 --- a/scripts/pegasus_update_ingest_behind.sh +++ b/admin_tools/pegasus_update_ingest_behind.sh @@ -327,7 +327,7 @@ echo if [ "$rebalance_cluster_after_rolling" == "true" ]; then echo "Start to rebalance cluster..." - ./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary + ./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary fi echo "Finish time: `date`" diff --git a/scripts/sendmail.sh b/admin_tools/sendmail.sh similarity index 100% rename from scripts/sendmail.sh rename to admin_tools/sendmail.sh diff --git a/scripts/bump_version.sh b/build_tools/bump_version.sh similarity index 100% rename from scripts/bump_version.sh rename to build_tools/bump_version.sh diff --git a/scripts/check_license.py b/build_tools/check_license.py similarity index 100% rename from scripts/check_license.py rename to build_tools/check_license.py diff --git a/scripts/clang_tidy.py b/build_tools/clang_tidy.py similarity index 88% rename from scripts/clang_tidy.py rename to build_tools/clang_tidy.py index 2a24eb8129..4071497659 100755 --- a/scripts/clang_tidy.py +++ b/build_tools/clang_tidy.py @@ -54,12 +54,13 @@ def tidy_on_path(path): path] subprocess.check_call(cmd, stdout=patch_file, cwd=ROOT) # TODO(yingchun): some checks could be disabled before we fix them. - # "-checks=-llvm-include-order,-modernize-concat-nested-namespaces,-cppcoreguidelines-pro-type-union-access,-cppcoreguidelines-macro-usage,-cppcoreguidelines-special-member-functions,-hicpp-special-member-functions,-modernize-use-trailing-return-type,-bugprone-easily-swappable-parameters,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-llvm-header-guard,-cppcoreguidelines-pro-bounds-pointer-arithmetic", + # "-checks=-llvm-include-order,-modernize-concat-nested-namespaces,-cppcoreguidelines-macro-usage,-cppcoreguidelines-special-member-functions,-hicpp-special-member-functions,-bugprone-easily-swappable-parameters,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-llvm-header-guard,-cppcoreguidelines-pro-bounds-pointer-arithmetic", cmdline = ["clang-tidy-diff", "-clang-tidy-binary", "clang-tidy", "-p0", "-path", BUILD_PATH, + "-checks=-cppcoreguidelines-pro-type-union-access,-modernize-use-trailing-return-type", "-extra-arg=-language=c++", "-extra-arg=-std=c++17", "-extra-arg=-Ithirdparty/output/include"] diff --git a/scripts/clear_zk.sh b/build_tools/clear_zk.sh similarity index 100% rename from scripts/clear_zk.sh rename to build_tools/clear_zk.sh diff --git a/scripts/collector_table_counter_gen.py b/build_tools/collector_table_counter_gen.py similarity index 98% rename from scripts/collector_table_counter_gen.py rename to build_tools/collector_table_counter_gen.py index 03a3f9f297..0cb96b66c1 100755 --- a/scripts/collector_table_counter_gen.py +++ b/build_tools/collector_table_counter_gen.py @@ -118,7 +118,7 @@ def generate_code_in_command_helper_header(replica_counter): # python3 ./collector_table_counter_gen.py counter1,counter2 -# please use `./scripts/format_files.sh` to format after generate code +# please use `./build_tools/format_files.sh` to format after generate code if __name__ == '__main__': if len(sys.argv) != 2: print("python3 ./collector_table_counter_gen.py {counter1,counter2..}") diff --git a/scripts/compile_thrift.py b/build_tools/compile_thrift.py similarity index 100% rename from scripts/compile_thrift.py rename to build_tools/compile_thrift.py diff --git a/scripts/download_hadoop.sh b/build_tools/download_hadoop.sh similarity index 100% rename from scripts/download_hadoop.sh rename to build_tools/download_hadoop.sh diff --git a/scripts/download_package.sh b/build_tools/download_package.sh similarity index 100% rename from scripts/download_package.sh rename to build_tools/download_package.sh diff --git a/scripts/download_zk.sh b/build_tools/download_zk.sh similarity index 100% rename from scripts/download_zk.sh rename to build_tools/download_zk.sh diff --git a/scripts/format_files.sh b/build_tools/format_files.sh similarity index 100% rename from scripts/format_files.sh rename to build_tools/format_files.sh diff --git a/scripts/pack_client.sh b/build_tools/pack_client.sh similarity index 100% rename from scripts/pack_client.sh rename to build_tools/pack_client.sh diff --git a/scripts/pack_common.sh b/build_tools/pack_common.sh similarity index 100% rename from scripts/pack_common.sh rename to build_tools/pack_common.sh diff --git a/scripts/pack_server.sh b/build_tools/pack_server.sh similarity index 98% rename from scripts/pack_server.sh rename to build_tools/pack_server.sh index 5f1172e918..1c27ac32fb 100755 --- a/scripts/pack_server.sh +++ b/build_tools/pack_server.sh @@ -130,10 +130,10 @@ fi copy_file ${THIRDPARTY_ROOT}/output/lib/libboost*.so.1.69.0 ${pack}/bin copy_file ${THIRDPARTY_ROOT}/output/lib/libhdfs* ${pack}/bin copy_file ${THIRDPARTY_ROOT}/output/lib/librocksdb.so.8 ${pack}/bin -copy_file ./scripts/sendmail.sh ${pack}/bin +copy_file ./admin_tools/config_hdfs.sh ${pack}/bin +copy_file ./admin_tools/sendmail.sh ${pack}/bin copy_file ./src/server/config.ini ${pack}/bin copy_file ./src/server/config.min.ini ${pack}/bin -copy_file ./scripts/config_hdfs.sh ${pack}/bin copy_file "$(get_stdcpp_lib $custom_gcc $separate_servers)" "${pack}/bin" diff --git a/scripts/pack_tools.sh b/build_tools/pack_tools.sh similarity index 94% rename from scripts/pack_tools.sh rename to build_tools/pack_tools.sh index de1d6cdf2c..24d78f2efe 100755 --- a/scripts/pack_tools.sh +++ b/build_tools/pack_tools.sh @@ -155,9 +155,14 @@ pack_tools_lib ssl $separate_servers chmod -x ${pack}/lib/* -mkdir -p ${pack}/scripts -copy_file ./scripts/* ${pack}/scripts/ -chmod +x ${pack}/scripts/*.sh +mkdir -p ${pack}/admin_tools +copy_file ./admin_tools/* ${pack}/admin_tools/ +chmod +x ${pack}/admin_tools/*.sh + +mkdir -p ${pack}/build_tools +copy_file ./build_tools/download_*.sh ${pack}/build_tools/ +copy_file ./build_tools/*_zk.sh ${pack}/build_tools/ +chmod +x ${pack}/build_tools/*.sh mkdir -p ${pack}/src/server copy_file ./src/server/*.ini ${pack}/src/server/ diff --git a/scripts/recompile_thrift.sh b/build_tools/recompile_thrift.sh similarity index 100% rename from scripts/recompile_thrift.sh rename to build_tools/recompile_thrift.sh diff --git a/scripts/redis_proto_check.py b/build_tools/redis_proto_check.py similarity index 100% rename from scripts/redis_proto_check.py rename to build_tools/redis_proto_check.py diff --git a/scripts/run-clang-format.py b/build_tools/run-clang-format.py similarity index 100% rename from scripts/run-clang-format.py rename to build_tools/run-clang-format.py diff --git a/scripts/start_zk.sh b/build_tools/start_zk.sh similarity index 100% rename from scripts/start_zk.sh rename to build_tools/start_zk.sh diff --git a/scripts/stop_zk.sh b/build_tools/stop_zk.sh similarity index 100% rename from scripts/stop_zk.sh rename to build_tools/stop_zk.sh diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index 85137791bc..f080da3f03 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -35,8 +35,8 @@ RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \ && unzip /root/thirdparties-src.zip -d . \ && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=${ROCKSDB_PORTABLE} -DUSE_JEMALLOC=${USE_JEMALLOC} -B build/ . \ && cmake --build build/ -j $(($(nproc)/2+1)) \ - && ../scripts/download_hadoop.sh ${HADOOP_BIN_PATH} \ - && ../scripts/download_zk.sh ${ZOOKEEPER_BIN_PATH} \ + && ../build_tools/download_hadoop.sh ${HADOOP_BIN_PATH} \ + && ../build_tools/download_zk.sh ${ZOOKEEPER_BIN_PATH} \ && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper ${HADOOP_BIN_PATH} ${ZOOKEEPER_BIN_PATH} \ && cd ~ \ && rm -rf incubator-pegasus; diff --git a/java-client/README.md b/java-client/README.md index 9cfdb9a703..c03a93541c 100644 --- a/java-client/README.md +++ b/java-client/README.md @@ -60,7 +60,7 @@ mvn clean package -Dtest=TestPing ### Install ``` -cd scripts && bash recompile_thrift.sh && cd - +cd scripts && bash download_thrift.sh && cd - mvn clean install -DskipTests ``` diff --git a/run.sh b/run.sh index 8c6048b019..f01b9c022f 100755 --- a/run.sh +++ b/run.sh @@ -324,8 +324,8 @@ function run_build() if [ ! -f "${ROOT}/src/common/serialization_helper/dsn.layer2_types.h" ]; then echo "Gen thrift" # TODO(yingchun): should be optimized - python3 $ROOT/scripts/compile_thrift.py - sh ${ROOT}/scripts/recompile_thrift.sh + python3 $ROOT/build_tools/compile_thrift.py + sh ${ROOT}/build_tools/recompile_thrift.sh fi if [ ! -d "$BUILD_DIR" ]; then @@ -656,7 +656,7 @@ function run_start_zk() fi fi - INSTALL_DIR="$INSTALL_DIR" PORT="$PORT" $ROOT/scripts/start_zk.sh + INSTALL_DIR="$INSTALL_DIR" PORT="$PORT" $ROOT/build_tools/start_zk.sh } ##################### @@ -693,7 +693,7 @@ function run_stop_zk() esac shift done - INSTALL_DIR="$INSTALL_DIR" $ROOT/scripts/stop_zk.sh + INSTALL_DIR="$INSTALL_DIR" $ROOT/build_tools/stop_zk.sh } ##################### @@ -730,7 +730,7 @@ function run_clear_zk() esac shift done - INSTALL_DIR="$INSTALL_DIR" $ROOT/scripts/clear_zk.sh + INSTALL_DIR="$INSTALL_DIR" $ROOT/build_tools/clear_zk.sh } ##################### @@ -853,7 +853,7 @@ function run_start_onebox() exit 1 fi - source "${ROOT}"/scripts/config_hdfs.sh + source "${ROOT}"/admin_tools/config_hdfs.sh if [ $USE_PRODUCT_CONFIG == "true" ]; then [ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini [ ! -f "${CONFIG_FILE}" ] && { echo "${CONFIG_FILE} is not exist"; exit 1; } @@ -1097,7 +1097,7 @@ function run_start_onebox_instance() esac shift done - source "${ROOT}"/scripts/config_hdfs.sh + source "${ROOT}"/admin_tools/config_hdfs.sh if [ $META_ID = "0" -a $REPLICA_ID = "0" -a $COLLECTOR_ID = "0" ]; then echo "ERROR: no meta_id or replica_id or collector set" exit 1 @@ -1887,9 +1887,9 @@ function run_migrate_node() cd ${ROOT} echo "------------------------------" if [ "$CLUSTER" != "" ]; then - ./scripts/migrate_node.sh $CLUSTER $NODE "$APP" $TYPE + ./admin_tools/migrate_node.sh $CLUSTER $NODE "$APP" $TYPE else - ./scripts/migrate_node.sh $CONFIG $NODE "$APP" $TYPE -f + ./admin_tools/migrate_node.sh $CONFIG $NODE "$APP" $TYPE -f fi echo "------------------------------" echo @@ -1995,9 +1995,9 @@ function run_downgrade_node() cd ${ROOT} echo "------------------------------" if [ "$CLUSTER" != "" ]; then - ./scripts/downgrade_node.sh $CLUSTER $NODE "$APP" $TYPE + ./admin_tools/downgrade_node.sh $CLUSTER $NODE "$APP" $TYPE else - ./scripts/downgrade_node.sh $CONFIG $NODE "$APP" $TYPE -f + ./admin_tools/downgrade_node.sh $CONFIG $NODE "$APP" $TYPE -f fi echo "------------------------------" echo @@ -2105,19 +2105,19 @@ case $cmd in ;; pack_server) shift - PEGASUS_ROOT=$ROOT ./scripts/pack_server.sh $* + PEGASUS_ROOT=$ROOT ./build_tools/pack_server.sh $* ;; pack_client) shift - PEGASUS_ROOT=$ROOT ./scripts/pack_client.sh $* + PEGASUS_ROOT=$ROOT ./build_tools/pack_client.sh $* ;; pack_tools) shift - PEGASUS_ROOT=$ROOT ./scripts/pack_tools.sh $* + PEGASUS_ROOT=$ROOT ./build_tools/pack_tools.sh $* ;; bump_version) shift - ./scripts/bump_version.sh $* + ./build_tools/bump_version.sh $* ;; *) echo "ERROR: unknown command $cmd" diff --git a/src/replica/replica.h b/src/replica/replica.h index 4bb2ca5892..eb177f5454 100644 --- a/src/replica/replica.h +++ b/src/replica/replica.h @@ -532,8 +532,8 @@ class replica : public serverlet, public ref_counter, public replica_ba void update_restore_progress(uint64_t f_size); // Used for remote command - // TODO: remove this interface and only expose the http interface - // now this remote commend will be used by `scripts/pegasus_manual_compact.sh` + // TODO(clang-tidy): remove this interface and only expose the http interface + // now this remote commend will be used by `admin_tools/pegasus_manual_compact.sh` std::string query_manual_compact_state() const; manual_compaction_status::type get_manual_compact_status() const;