refactor(scripts): Separate admin and build scripts to independent di…

…rectories (#2096)

Separate the `scripts` directory to `admin_tools` and `build_tools` directories.

.github/actions/rebuild_thirdparty_if_needed/action.yaml

@@ -42,8 +42,8 @@ runs:
         cmake --build build/ -j $(nproc)
         rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
         find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-        ../scripts/download_hadoop.sh hadoop-bin
-        ../scripts/download_zk.sh zookeeper-bin
+        ../build_tools/download_hadoop.sh hadoop-bin
+        ../build_tools/download_zk.sh zookeeper-bin
         rm -rf hadoop-bin/share/doc
         rm -rf zookeeper-bin/docs
       shell: bash

.github/workflows/lint_and_test_cpp.yaml

@@ -30,13 +30,13 @@ on:
       - .github/actions
       - .github/workflows/lint_and_test_cpp.yaml
       - .github/workflows/thirdparty-regular-push.yml
+      - build_tools/pack_server.sh
+      - build_tools/pack_tools.sh
       - docker/thirdparties-bin/**
       - docker/thirdparties-src/**
       - CMakeLists.txt
       - cmake_modules/**
       - run.sh
-      - scripts/pack_server.sh
-      - scripts/pack_tools.sh
       - src/**
       - thirdparty/**
 
@@ -55,7 +55,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: clang-format
-        run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
+        run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
 
   cpp_clang_tidy_linter:
     name: Tidy
@@ -76,7 +76,7 @@ jobs:
         run: |
           git config --global --add safe.directory $(pwd)
           ./run.sh build --test --compiler clang-14,clang++-14 -t debug --skip_thirdparty -c --cmake_only
-          ./scripts/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H")
+          ./build_tools/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H")
         shell: bash
 
   iwyu:

.github/workflows/module_labeler_conf.yml

@@ -42,7 +42,8 @@ thrift:
 docs:
   - '**/*.md'
 scripts:
-  - 'scripts/**/*'
+  - 'admin_tools/**/*'
+  - 'build_tools/**/*'
   - '**/*.sh'
 build:
   - 'cmake_modules/**/*'

.github/workflows/regular-build.yml

@@ -41,7 +41,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: clang-format
-        run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
+        run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
 
   build_cpp:
     name: Build Cpp

.licenserc.yaml

@@ -110,8 +110,10 @@ header:
     - 'cmake_modules/FindRT.cmake'
     - 'cmake_modules/FindDL.cmake'
     # Copyright (c) 2017 Guillaume Papin
-    - 'scripts/run-clang-format.py'
+    - 'build_tools/run-clang-format.py'
     # The MIT License (MIT), Copyright (c) 2015 Microsoft Corporation
+    - 'admin_tools/learn_stat.py'
+    - 'build_tools/compile_thrift.py'
     - 'cmake_modules/BaseFunctions.cmake'
     - 'docs/rdsn-README.md'
     - 'idl/command.thrift'
@@ -120,8 +122,6 @@ header:
     - 'idl/metadata.thrift'
     - 'idl/meta_admin.thrift'
     - 'idl/replica_admin.thrift'
-    - 'scripts/compile_thrift.py'
-    - 'scripts/learn_stat.py'
     - 'src/runtime/api_layer1.h'
     - 'src/runtime/api_task.h'
     - 'src/utils/api_utilities.h'

LICENSE

@@ -514,7 +514,7 @@ RESULTING FROM THE USE OF THIS SOFTWARE.
 
 --------------------------------------------------------------------------------
 
-scripts/run-clang-format.py - MIT License
+build_tools/run-clang-format.py - MIT License
 
 MIT License
 

scripts/pegasus_add_node_list.sh → admin_tools/pegasus_add_node_list.sh

@@ -52,7 +52,7 @@ shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
 echo "Check the argument..."
-source ./scripts/pegasus_check_arguments.sh add_node_list $cluster $meta_list $replica_task_id_list
+source ./admin_tools/pegasus_check_arguments.sh add_node_list $cluster $meta_list $replica_task_id_list
 
 if [ $? -ne 0 ]; then
     echo "ERROR: the argument check failed"
@@ -78,7 +78,7 @@ do
   echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 done
 
-./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list true $nfs_rate_megabytes_per_disk
+./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list true $nfs_rate_megabytes_per_disk
 
 echo "Finish time: `date`"
 add_node_finish_time=$((`date +%s`))

scripts/pegasus_check_arguments.sh → admin_tools/pegasus_check_arguments.sh

@@ -41,7 +41,7 @@ if [ "$check_type" != "add_node_list" -a "$check_type" != "offline_node_list" ];
   exit 1
 fi
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"

scripts/pegasus_migrate_zookeeper.sh → admin_tools/pegasus_migrate_zookeeper.sh

@@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0"  )" && pwd )"
 shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"

scripts/pegasus_offline_node.sh → admin_tools/pegasus_offline_node.sh

@@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0"  )" && pwd )"
 shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"

scripts/pegasus_offline_node_list.sh → admin_tools/pegasus_offline_node_list.sh

@@ -52,7 +52,7 @@ shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
 echo "Check the argument..."
-source ./scripts/pegasus_check_arguments.sh offline_node_list $cluster $meta_list $replica_task_id_list
+source ./admin_tools/pegasus_check_arguments.sh offline_node_list $cluster $meta_list $replica_task_id_list
 
 if [ $? -ne 0 ]; then
     echo "ERROR: the argument check failed"
@@ -93,7 +93,7 @@ echo
 for id in $id_list
 do
   echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-  ./scripts/pegasus_offline_node.sh $cluster $meta_list $id
+  ./admin_tools/pegasus_offline_node.sh $cluster $meta_list $id
   if [ $? -ne 0 ]; then
     echo "ERROR: offline replica task $id failed"
     exit 1

scripts/pegasus_restart_node.sh → admin_tools/pegasus_restart_node.sh

@@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0"  )" && pwd )"
 shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"

scripts/pegasus_rolling_update.sh → admin_tools/pegasus_rolling_update.sh

@@ -72,7 +72,7 @@ pwd="$( cd "$( dirname "$0"  )" && pwd )"
 shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"
@@ -338,7 +338,7 @@ fi
 
 if [ "$rebalance_cluster_after_rolling" == "true" ]; then
   echo "Start to rebalance cluster..."
-  ./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary
+  ./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary
 fi
 
 echo "Finish time: `date`"

scripts/pegasus_update_ingest_behind.sh → admin_tools/pegasus_update_ingest_behind.sh

@@ -327,7 +327,7 @@ echo
 
 if [ "$rebalance_cluster_after_rolling" == "true" ]; then
   echo "Start to rebalance cluster..."
-  ./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary
+  ./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary
 fi
 
 echo "Finish time: `date`"

scripts/clang_tidy.py → build_tools/clang_tidy.py

@@ -54,12 +54,13 @@ def tidy_on_path(path):
             path]
         subprocess.check_call(cmd, stdout=patch_file, cwd=ROOT)
         # TODO(yingchun): some checks could be disabled before we fix them.
-        #  "-checks=-llvm-include-order,-modernize-concat-nested-namespaces,-cppcoreguidelines-pro-type-union-access,-cppcoreguidelines-macro-usage,-cppcoreguidelines-special-member-functions,-hicpp-special-member-functions,-modernize-use-trailing-return-type,-bugprone-easily-swappable-parameters,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-llvm-header-guard,-cppcoreguidelines-pro-bounds-pointer-arithmetic",
+        #  "-checks=-llvm-include-order,-modernize-concat-nested-namespaces,-cppcoreguidelines-macro-usage,-cppcoreguidelines-special-member-functions,-hicpp-special-member-functions,-bugprone-easily-swappable-parameters,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-llvm-header-guard,-cppcoreguidelines-pro-bounds-pointer-arithmetic",
         cmdline = ["clang-tidy-diff",
                    "-clang-tidy-binary",
                    "clang-tidy",
                    "-p0",
                    "-path", BUILD_PATH,
+                   "-checks=-cppcoreguidelines-pro-type-union-access,-modernize-use-trailing-return-type",
                    "-extra-arg=-language=c++",
                    "-extra-arg=-std=c++17",
                    "-extra-arg=-Ithirdparty/output/include"]

scripts/collector_table_counter_gen.py → build_tools/collector_table_counter_gen.py

@@ -118,7 +118,7 @@ def generate_code_in_command_helper_header(replica_counter):
 
 
 # python3 ./collector_table_counter_gen.py counter1,counter2
-# please use `./scripts/format_files.sh` to format after generate code
+# please use `./build_tools/format_files.sh` to format after generate code
 if __name__ == '__main__':
     if len(sys.argv) != 2:
         print("python3 ./collector_table_counter_gen.py {counter1,counter2..}")

scripts/pack_server.sh → build_tools/pack_server.sh

@@ -130,10 +130,10 @@ fi
 copy_file ${THIRDPARTY_ROOT}/output/lib/libboost*.so.1.69.0 ${pack}/bin
 copy_file ${THIRDPARTY_ROOT}/output/lib/libhdfs* ${pack}/bin
 copy_file ${THIRDPARTY_ROOT}/output/lib/librocksdb.so.8 ${pack}/bin
-copy_file ./scripts/sendmail.sh ${pack}/bin
+copy_file ./admin_tools/config_hdfs.sh ${pack}/bin
+copy_file ./admin_tools/sendmail.sh ${pack}/bin
 copy_file ./src/server/config.ini ${pack}/bin
 copy_file ./src/server/config.min.ini ${pack}/bin
-copy_file ./scripts/config_hdfs.sh ${pack}/bin
 
 copy_file "$(get_stdcpp_lib $custom_gcc $separate_servers)" "${pack}/bin"
 

scripts/pack_tools.sh → build_tools/pack_tools.sh

@@ -155,9 +155,14 @@ pack_tools_lib ssl $separate_servers
 
 chmod -x ${pack}/lib/*
 
-mkdir -p ${pack}/scripts
-copy_file ./scripts/* ${pack}/scripts/
-chmod +x ${pack}/scripts/*.sh
+mkdir -p ${pack}/admin_tools
+copy_file ./admin_tools/* ${pack}/admin_tools/
+chmod +x ${pack}/admin_tools/*.sh
+
+mkdir -p ${pack}/build_tools
+copy_file ./build_tools/download_*.sh ${pack}/build_tools/
+copy_file ./build_tools/*_zk.sh ${pack}/build_tools/
+chmod +x ${pack}/build_tools/*.sh
 
 mkdir -p ${pack}/src/server
 copy_file ./src/server/*.ini ${pack}/src/server/

docker/thirdparties-bin/Dockerfile

@@ -35,8 +35,8 @@ RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \
     && unzip /root/thirdparties-src.zip -d . \
     && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=${ROCKSDB_PORTABLE} -DUSE_JEMALLOC=${USE_JEMALLOC} -B build/ . \
     && cmake --build build/ -j $(($(nproc)/2+1)) \
-    && ../scripts/download_hadoop.sh ${HADOOP_BIN_PATH} \
-    && ../scripts/download_zk.sh ${ZOOKEEPER_BIN_PATH} \
+    && ../build_tools/download_hadoop.sh ${HADOOP_BIN_PATH} \
+    && ../build_tools/download_zk.sh ${ZOOKEEPER_BIN_PATH} \
     && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper ${HADOOP_BIN_PATH} ${ZOOKEEPER_BIN_PATH} \
     && cd ~ \
     && rm -rf incubator-pegasus;

java-client/README.md

@@ -60,7 +60,7 @@ mvn clean package -Dtest=TestPing
 ### Install
 
 ```
-cd scripts && bash recompile_thrift.sh && cd -
+cd scripts && bash download_thrift.sh && cd -
 mvn clean install -DskipTests
 ```
 

run.sh

@@ -324,8 +324,8 @@ function run_build()
     if [ ! -f "${ROOT}/src/common/serialization_helper/dsn.layer2_types.h" ]; then
         echo "Gen thrift"
         # TODO(yingchun): should be optimized
-        python3 $ROOT/scripts/compile_thrift.py
-        sh ${ROOT}/scripts/recompile_thrift.sh
+        python3 $ROOT/build_tools/compile_thrift.py
+        sh ${ROOT}/build_tools/recompile_thrift.sh
     fi
 
     if [ ! -d "$BUILD_DIR" ]; then
@@ -656,7 +656,7 @@ function run_start_zk()
         fi
     fi
 
-    INSTALL_DIR="$INSTALL_DIR" PORT="$PORT" $ROOT/scripts/start_zk.sh
+    INSTALL_DIR="$INSTALL_DIR" PORT="$PORT" $ROOT/build_tools/start_zk.sh
 }
 
 #####################
@@ -693,7 +693,7 @@ function run_stop_zk()
         esac
         shift
     done
-    INSTALL_DIR="$INSTALL_DIR" $ROOT/scripts/stop_zk.sh
+    INSTALL_DIR="$INSTALL_DIR" $ROOT/build_tools/stop_zk.sh
 }
 
 #####################
@@ -730,7 +730,7 @@ function run_clear_zk()
         esac
         shift
     done
-    INSTALL_DIR="$INSTALL_DIR" $ROOT/scripts/clear_zk.sh
+    INSTALL_DIR="$INSTALL_DIR" $ROOT/build_tools/clear_zk.sh
 }
 
 #####################
@@ -853,7 +853,7 @@ function run_start_onebox()
         exit 1
     fi
 
-    source "${ROOT}"/scripts/config_hdfs.sh
+    source "${ROOT}"/admin_tools/config_hdfs.sh
     if [ $USE_PRODUCT_CONFIG == "true" ]; then
         [ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini
         [ ! -f "${CONFIG_FILE}" ] && { echo "${CONFIG_FILE} is not exist"; exit 1; }
@@ -1097,7 +1097,7 @@ function run_start_onebox_instance()
         esac
         shift
     done
-    source "${ROOT}"/scripts/config_hdfs.sh
+    source "${ROOT}"/admin_tools/config_hdfs.sh
     if [ $META_ID = "0" -a $REPLICA_ID = "0" -a $COLLECTOR_ID = "0" ]; then
         echo "ERROR: no meta_id or replica_id or collector set"
         exit 1
@@ -1887,9 +1887,9 @@ function run_migrate_node()
     cd ${ROOT}
     echo "------------------------------"
     if [ "$CLUSTER" != "" ]; then
-        ./scripts/migrate_node.sh $CLUSTER $NODE "$APP" $TYPE
+        ./admin_tools/migrate_node.sh $CLUSTER $NODE "$APP" $TYPE
     else
-        ./scripts/migrate_node.sh $CONFIG $NODE "$APP" $TYPE -f
+        ./admin_tools/migrate_node.sh $CONFIG $NODE "$APP" $TYPE -f
     fi
     echo "------------------------------"
     echo
@@ -1995,9 +1995,9 @@ function run_downgrade_node()
     cd ${ROOT}
     echo "------------------------------"
     if [ "$CLUSTER" != "" ]; then
-        ./scripts/downgrade_node.sh $CLUSTER $NODE "$APP" $TYPE
+        ./admin_tools/downgrade_node.sh $CLUSTER $NODE "$APP" $TYPE
     else
-        ./scripts/downgrade_node.sh $CONFIG $NODE "$APP" $TYPE -f
+        ./admin_tools/downgrade_node.sh $CONFIG $NODE "$APP" $TYPE -f
     fi
     echo "------------------------------"
     echo
@@ -2105,19 +2105,19 @@ case $cmd in
         ;;
     pack_server)
         shift
-        PEGASUS_ROOT=$ROOT ./scripts/pack_server.sh $*
+        PEGASUS_ROOT=$ROOT ./build_tools/pack_server.sh $*
         ;;
     pack_client)
         shift
-        PEGASUS_ROOT=$ROOT ./scripts/pack_client.sh $*
+        PEGASUS_ROOT=$ROOT ./build_tools/pack_client.sh $*
         ;;
     pack_tools)
         shift
-        PEGASUS_ROOT=$ROOT ./scripts/pack_tools.sh $*
+        PEGASUS_ROOT=$ROOT ./build_tools/pack_tools.sh $*
         ;;
     bump_version)
         shift
-        ./scripts/bump_version.sh $*
+        ./build_tools/bump_version.sh $*
         ;;
     *)
         echo "ERROR: unknown command $cmd"

src/replica/replica.h

@@ -532,8 +532,8 @@ class replica : public serverlet<replica>, public ref_counter, public replica_ba
     void update_restore_progress(uint64_t f_size);
 
     // Used for remote command
-    // TODO: remove this interface and only expose the http interface
-    // now this remote commend will be used by `scripts/pegasus_manual_compact.sh`
+    // TODO(clang-tidy): remove this interface and only expose the http interface
+    // now this remote commend will be used by `admin_tools/pegasus_manual_compact.sh`
     std::string query_manual_compact_state() const;
 
     manual_compaction_status::type get_manual_compact_status() const;