feat: Support FQDN #103
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| --- | |
| # Developer Notes: | |
| # | |
| # This config is for github actions. Before merging your changes of this file, | |
| # it's recommended to create a PR against the ci-test branch to test if it works | |
| # as expected. | |
| name: Cpp CI | |
| on: | |
| # run on each pull request | |
| pull_request: | |
| types: [ synchronize, opened, reopened ] | |
| branches: | |
| - master | |
| - 'v[0-9]+.*' # release branch | |
| - ci-test # testing branch for github action | |
| - '*dev' | |
| paths: | |
| - '.github/workflows/lint_and_test_cpp.yaml' | |
| - '.github/workflows/thirdparty-regular-push.yml' | |
| - 'docker/thirdparties-src/**' | |
| - 'docker/thirdparties-bin/**' | |
| - 'CMakeLists.txt' | |
| - 'run.sh' | |
| - 'cmake_modules/**' | |
| - 'src/**' | |
| - 'thirdparty/**' | |
| # for manually triggering workflow | |
| workflow_dispatch: | |
| env: | |
| # Update the options to reduce the consumption of the disk space | |
| ONEBOX_OPTS: disk_min_available_space_ratio=5 | |
| TEST_OPTS: disk_min_available_space_ratio=5,throttle_test_medium_value_kb=10,throttle_test_large_value_kb=25 | |
| jobs: | |
| cpp_clang_format_linter: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/pegasus:clang-format-3.9 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: clang-format | |
| run: ./scripts/run-clang-format.py --clang-format-executable clang-format-3.9 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . | |
| iwyu: | |
| name: IWYU | |
| needs: cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Free Disk Space (Ubuntu) | |
| run: | | |
| .github/workflows/free_disk_space.sh | |
| - uses: dorny/paths-filter@v2 | |
| id: changes | |
| with: | |
| filters: | | |
| thirdparty: | |
| - '.github/workflows/thirdparty-regular-push.yml' | |
| - 'docker/thirdparties-src/**' | |
| - 'docker/thirdparties-bin/**' | |
| - 'thirdparty/**' | |
| - name: Unpack prebuilt third-parties | |
| if: steps.changes.outputs.thirdparty == 'false' | |
| run: unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| - name: Rebuild third-parties | |
| if: steps.changes.outputs.thirdparty == 'true' | |
| working-directory: thirdparty | |
| # Build thirdparties and leave some necessary libraries and source | |
| run: | | |
| mkdir build | |
| cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ | |
| cmake --build build/ -j $(nproc) | |
| rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* | |
| ../scripts/download_hadoop.sh hadoop-bin | |
| ../scripts/download_zk.sh zookeeper-bin | |
| - name: Build IWYU binary | |
| run: | | |
| mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git | |
| cd include-what-you-use && git checkout clang_14 | |
| cd .. && mkdir build && cd build && cmake -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use | |
| make -j $(nproc) | |
| - name: Check IWYU | |
| run: | | |
| PWD=`pwd` | |
| ./run.sh build --test --compiler clang-14,clang++-14 -j $(nproc) -t release --iwyu ${PWD}/iwyu/build/bin/include-what-you-use --skip_thirdparty | |
| python3 iwyu/include-what-you-use/fix_includes.py --blank_lines --reorder --nocomments --nosafe_headers --separate_project_includes="<tld>" --ignore_re="thrift-gen|rrdb_types|layer2_types|metrics_test.cpp" --dry_run < build/latest/iwyu.out | |
| python3 iwyu/include-what-you-use/fix_includes.py --blank_lines --reorder --nocomments --nosafe_headers --separate_project_includes="<tld>" --ignore_re="thrift-gen|rrdb_types|layer2_types|metrics_test.cpp" --dry_run < build/latest/iwyu.out | egrep "IWYU edited 0 files on your behalf" | |
| build_Release: | |
| name: Build Release | |
| needs: cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| /github/home/.ccache | |
| key: release_ccache | |
| - name: Free Disk Space (Ubuntu) | |
| run: | | |
| .github/workflows/free_disk_space.sh | |
| - uses: dorny/paths-filter@v2 | |
| id: changes | |
| with: | |
| filters: | | |
| thirdparty: | |
| - '.github/workflows/thirdparty-regular-push.yml' | |
| - 'docker/thirdparties-src/**' | |
| - 'docker/thirdparties-bin/**' | |
| - 'thirdparty/**' | |
| - name: Unpack prebuilt third-parties | |
| if: steps.changes.outputs.thirdparty == 'false' | |
| run: | | |
| rm -f /root/thirdparties-src.zip | |
| unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| rm -f /root/thirdparties-bin.zip | |
| find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| rm -rf ./thirdparty/hadoop-bin/share/doc | |
| rm -rf ./thirdparty/zookeeper-bin/docs | |
| - name: Rebuild third-parties | |
| if: steps.changes.outputs.thirdparty == 'true' | |
| working-directory: thirdparty | |
| # Build thirdparties and leave some necessary libraries and source | |
| run: | | |
| rm -f /root/thirdparties-src.zip | |
| mkdir build | |
| cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ | |
| cmake --build build/ -j $(nproc) | |
| rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* | |
| find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| ../scripts/download_hadoop.sh hadoop-bin | |
| ../scripts/download_zk.sh zookeeper-bin | |
| rm -rf hadoop-bin/share/doc | |
| rm -rf zookeeper-bin/docs | |
| - name: Compilation | |
| run: | | |
| ccache -p | |
| ccache -z | |
| ./run.sh build --test --skip_thirdparty -j $(nproc) -t release | |
| ccache -s | |
| - name: Clear Build Files | |
| run: | | |
| find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| - name: Pack Server | |
| run: | | |
| ./run.sh pack_server | |
| rm -rf pegasus-server-* | |
| - name: Pack Tools | |
| run: | | |
| ./run.sh pack_tools | |
| rm -rf pegasus-tools-* | |
| - name: Tar files | |
| run: | | |
| mv thirdparty/hadoop-bin ./ | |
| mv thirdparty/zookeeper-bin ./ | |
| rm -rf thirdparty | |
| tar -zcvhf release__builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: release_artifact_${{ github.sha }} | |
| path: release__builder.tar | |
| test_Release: | |
| name: Test Release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_module: | |
| - backup_restore_test | |
| - base_api_test | |
| - base_test | |
| - bulk_load_test | |
| - detect_hotspot_test | |
| - dsn_aio_test | |
| - dsn_block_service_test | |
| - dsn_client_test | |
| - dsn.failure_detector.tests | |
| - dsn_http_test | |
| - dsn_meta_state_tests | |
| - dsn.meta.test | |
| - dsn_nfs_test | |
| - dsn_perf_counter_test | |
| - dsn_replica_backup_test | |
| - dsn_replica_bulk_load_test | |
| - dsn_replica_dup_test | |
| - dsn_replica_split_test | |
| - dsn.replica.test | |
| - dsn_replication_common_test | |
| - dsn.replication.simple_kv | |
| - dsn.rep_tests.simple_kv | |
| - dsn_runtime_tests | |
| - dsn_utils_tests | |
| - dsn.zookeeper.tests | |
| # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after | |
| # it has been optimized. | |
| # - partition_split_test | |
| - pegasus_geo_test | |
| - pegasus_rproxy_test | |
| - pegasus_unit_test | |
| - recovery_test | |
| - restore_test | |
| - throttle_test | |
| needs: build_Release | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| options: --cap-add=SYS_PTRACE | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Unpack prebuilt third-parties | |
| run: | | |
| unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| rm -f /root/thirdparties-bin.zip | |
| - name: Download Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: release_artifact_${{ github.sha }} | |
| path: . | |
| - name: Tar files | |
| run: | | |
| tar -zxvf release__builder.tar | |
| - name: Unit Testing | |
| run: | | |
| export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server | |
| ulimit -s unlimited | |
| . ./scripts/config_hdfs.sh | |
| ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} | |
| build_ASAN: | |
| name: Build ASAN | |
| needs: cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| /github/home/.ccache | |
| key: asan_ccache | |
| - name: Free Disk Space (Ubuntu) | |
| run: | | |
| .github/workflows/free_disk_space.sh | |
| - uses: dorny/paths-filter@v2 | |
| id: changes | |
| with: | |
| filters: | | |
| thirdparty: | |
| - '.github/workflows/thirdparty-regular-push.yml' | |
| - 'docker/thirdparties-src/**' | |
| - 'docker/thirdparties-bin/**' | |
| - 'thirdparty/**' | |
| - name: Unpack prebuilt third-parties | |
| if: steps.changes.outputs.thirdparty == 'false' | |
| run: | | |
| rm -f /root/thirdparties-src.zip | |
| unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| rm -f /root/thirdparties-bin.zip | |
| find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| rm -rf ./thirdparty/hadoop-bin/share/doc | |
| rm -rf ./thirdparty/zookeeper-bin/docs | |
| - name: Rebuild third-parties | |
| if: steps.changes.outputs.thirdparty == 'true' | |
| working-directory: thirdparty | |
| # Build thirdparties and leave some necessary libraries and source | |
| run: | | |
| rm -f /root/thirdparties-src.zip | |
| mkdir build | |
| cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ | |
| cmake --build build/ -j $(nproc) | |
| rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* | |
| find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| ../scripts/download_hadoop.sh hadoop-bin | |
| ../scripts/download_zk.sh zookeeper-bin | |
| rm -rf hadoop-bin/share/doc | |
| rm -rf zookeeper-bin/docs | |
| - name: Compilation | |
| run: | | |
| ccache -p | |
| ccache -z | |
| ./run.sh build --test --sanitizer address --skip_thirdparty --disable_gperf -j $(nproc) | |
| ccache -s | |
| - name: Clear Build Files | |
| run: | | |
| find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| - name: Tar files | |
| run: | | |
| mv thirdparty/hadoop-bin ./ | |
| mv thirdparty/zookeeper-bin ./ | |
| rm -rf thirdparty | |
| tar -zcvhf release_address_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: release_address_artifact_${{ github.sha }} | |
| path: release_address_builder.tar | |
| test_ASAN: | |
| name: Test ASAN | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_module: | |
| - backup_restore_test | |
| - base_api_test | |
| - base_test | |
| - bulk_load_test | |
| - detect_hotspot_test | |
| - dsn_aio_test | |
| - dsn_block_service_test | |
| - dsn_client_test | |
| - dsn.failure_detector.tests | |
| - dsn_http_test | |
| - dsn_meta_state_tests | |
| - dsn.meta.test | |
| - dsn_nfs_test | |
| - dsn_perf_counter_test | |
| - dsn_replica_backup_test | |
| - dsn_replica_bulk_load_test | |
| - dsn_replica_dup_test | |
| - dsn_replica_split_test | |
| - dsn.replica.test | |
| - dsn_replication_common_test | |
| - dsn.replication.simple_kv | |
| - dsn.rep_tests.simple_kv | |
| - dsn_runtime_tests | |
| - dsn_utils_tests | |
| - dsn.zookeeper.tests | |
| # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after | |
| # it has been optimized. | |
| # - partition_split_test | |
| - pegasus_geo_test | |
| - pegasus_rproxy_test | |
| - pegasus_unit_test | |
| - recovery_test | |
| - restore_test | |
| # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after | |
| # it has been optimized. | |
| # - throttle_test | |
| needs: build_ASAN | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| options: --cap-add=SYS_PTRACE | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Unpack prebuilt third-parties | |
| run: | | |
| unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| rm -f /root/thirdparties-bin.zip | |
| - name: Download Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: release_address_artifact_${{ github.sha }} | |
| path: . | |
| - name: Tar files | |
| run: | | |
| tar -zxvf release_address_builder.tar | |
| - name: Unit Testing | |
| run: | | |
| export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server | |
| ulimit -s unlimited | |
| . ./scripts/config_hdfs.sh | |
| ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} | |
| # TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests | |
| # when we are going to release a stable version. So we disable them in regular CI | |
| # before we find any way to reduce the time cost. | |
| # build_UBSAN: | |
| # name: Build UBSAN | |
| # needs: cpp_clang_format_linter | |
| # runs-on: ubuntu-latest | |
| # container: | |
| # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - name: Setup cache | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: | | |
| # /github/home/.ccache | |
| # key: ubsan_ccache | |
| # - name: Free Disk Space (Ubuntu) | |
| # run: | | |
| # .github/workflows/free_disk_space.sh | |
| # - uses: dorny/paths-filter@v2 | |
| # id: changes | |
| # with: | |
| # filters: | | |
| # thirdparty: | |
| # - '.github/workflows/thirdparty-regular-push.yml' | |
| # - 'docker/thirdparties-src/**' | |
| # - 'docker/thirdparties-bin/**' | |
| # - 'thirdparty/**' | |
| # - name: Unpack prebuilt third-parties | |
| # if: steps.changes.outputs.thirdparty == 'false' | |
| # run: | | |
| # rm -f /root/thirdparties-src.zip | |
| # unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| # rm -f /root/thirdparties-bin.zip | |
| # find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| # rm -rf ./thirdparty/hadoop-bin/share/doc | |
| # rm -rf ./thirdparty/zookeeper-bin/docs | |
| # - name: Rebuild third-parties | |
| # if: steps.changes.outputs.thirdparty == 'true' | |
| # working-directory: thirdparty | |
| # # Build thirdparties and leave some necessary libraries and source | |
| # run: | | |
| # rm -f /root/thirdparties-src.zip | |
| # mkdir build | |
| # cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ | |
| # cmake --build build/ -j $(nproc) | |
| # rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* | |
| # find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| # ../scripts/download_hadoop.sh hadoop-bin | |
| # ../scripts/download_zk.sh zookeeper-bin | |
| # rm -rf hadoop-bin/share/doc | |
| # rm -rf zookeeper-bin/docs | |
| # - name: Compilation | |
| # run: | | |
| # ccache -p | |
| # ccache -z | |
| # ./run.sh build --test --sanitizer undefined --skip_thirdparty --disable_gperf -j $(nproc) | |
| # ccache -s | |
| # - name: Clear Build Files | |
| # run: | | |
| # find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| # - name: Tar files | |
| # run: | | |
| # mv thirdparty/hadoop-bin ./ | |
| # mv thirdparty/zookeeper-bin ./ | |
| # rm -rf thirdparty | |
| # tar -zcvhf release_undefined_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin | |
| # - name: Upload Artifact | |
| # uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: release_undefined_artifact_${{ github.sha }} | |
| # path: release_undefined_builder.tar | |
| # | |
| # test_UBSAN: | |
| # name: Test UBSAN | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # test_module: | |
| # - backup_restore_test | |
| # - base_api_test | |
| # - base_test | |
| # - bulk_load_test | |
| # - detect_hotspot_test | |
| # - dsn_aio_test | |
| # - dsn_block_service_test | |
| # - dsn_client_test | |
| # - dsn.failure_detector.tests | |
| # - dsn_http_test | |
| # - dsn_meta_state_tests | |
| # - dsn.meta.test | |
| # - dsn_nfs_test | |
| # - dsn_perf_counter_test | |
| # - dsn_replica_backup_test | |
| # - dsn_replica_bulk_load_test | |
| # - dsn_replica_dup_test | |
| # - dsn_replica_split_test | |
| # - dsn.replica.test | |
| # - dsn_replication_common_test | |
| # - dsn.replication.simple_kv | |
| # - dsn.rep_tests.simple_kv | |
| # - dsn_runtime_tests | |
| # - dsn_utils_tests | |
| # - dsn.zookeeper.tests | |
| # - partition_split_test | |
| # - pegasus_geo_test | |
| # - pegasus_rproxy_test | |
| # - pegasus_unit_test | |
| # - recovery_test | |
| # - restore_test | |
| # - throttle_test | |
| # needs: build_UBSAN | |
| # runs-on: ubuntu-latest | |
| # container: | |
| # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} | |
| # options: --cap-add=SYS_PTRACE | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - name: Free Disk Space (Ubuntu) | |
| # run: | | |
| # .github/workflows/free_disk_space.sh | |
| # - name: Unpack prebuilt third-parties | |
| # run: | | |
| # unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| # rm -f /root/thirdparties-bin.zip | |
| # - name: Download Artifact | |
| # uses: actions/download-artifact@v3 | |
| # with: | |
| # name: release_undefined_artifact_${{ github.sha }} | |
| # path: . | |
| # - name: Tar files | |
| # run: | | |
| # tar -zxvf release_undefined_builder.tar | |
| # - name: Unit Testing | |
| # run: | | |
| # export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server | |
| # ulimit -s unlimited | |
| # . ./scripts/config_hdfs.sh | |
| # ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} | |
| build_with_jemalloc: | |
| name: Build with jemalloc | |
| needs: cpp_clang_format_linter | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| /github/home/.ccache | |
| key: jemalloc_ccache | |
| - name: Free Disk Space (Ubuntu) | |
| run: | | |
| .github/workflows/free_disk_space.sh | |
| - uses: dorny/paths-filter@v2 | |
| id: changes | |
| with: | |
| filters: | | |
| thirdparty: | |
| - '.github/workflows/thirdparty-regular-push.yml' | |
| - 'docker/thirdparties-src/**' | |
| - 'docker/thirdparties-bin/**' | |
| - 'thirdparty/**' | |
| - name: Unpack prebuilt third-parties | |
| if: steps.changes.outputs.thirdparty == 'false' | |
| run: | | |
| rm -f /root/thirdparties-src.zip | |
| unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| rm -f /root/thirdparties-bin.zip | |
| find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| rm -rf ./thirdparty/hadoop-bin/share/doc | |
| rm -rf ./thirdparty/zookeeper-bin/docs | |
| - name: Rebuild third-parties | |
| if: steps.changes.outputs.thirdparty == 'true' | |
| working-directory: thirdparty | |
| # Build thirdparties and leave some necessary libraries and source | |
| run: | | |
| rm -f /root/thirdparties-src.zip | |
| mkdir build | |
| cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/ | |
| cmake --build build/ -j $(nproc) | |
| rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* | |
| find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| ../scripts/download_hadoop.sh hadoop-bin | |
| ../scripts/download_zk.sh zookeeper-bin | |
| rm -rf hadoop-bin/share/doc | |
| rm -rf zookeeper-bin/docs | |
| - name: Compilation | |
| run: | | |
| ccache -p | |
| ccache -z | |
| ./run.sh build --test --skip_thirdparty -j $(nproc) -t release --use_jemalloc | |
| ccache -s | |
| - name: Clear Build Files | |
| run: | | |
| find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + | |
| - name: Pack Server | |
| run: | | |
| ./run.sh pack_server -j | |
| rm -rf pegasus-server-* | |
| - name: Pack Tools | |
| run: | | |
| ./run.sh pack_tools -j | |
| rm -rf pegasus-tools-* | |
| - name: Tar files | |
| run: | | |
| mv thirdparty/hadoop-bin ./ | |
| mv thirdparty/zookeeper-bin ./ | |
| rm -rf thirdparty | |
| tar -zcvhf release_jemalloc_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: release_jemalloc_artifact_${{ github.sha }} | |
| path: release_jemalloc_builder.tar | |
| test_with_jemalloc: | |
| name: Test with jemallc | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_module: | |
| - dsn_utils_tests | |
| needs: build_with_jemalloc | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} | |
| options: --cap-add=SYS_PTRACE | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Unpack prebuilt third-parties | |
| run: | | |
| unzip /root/thirdparties-bin.zip -d ./thirdparty | |
| rm -f /root/thirdparties-bin.zip | |
| - name: Download Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: release_jemalloc_artifact_${{ github.sha }} | |
| path: . | |
| - name: Tar files | |
| run: | | |
| tar -zxvf release_jemalloc_builder.tar | |
| - name: Unit Testing | |
| run: | | |
| export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server | |
| ulimit -s unlimited | |
| . ./scripts/config_hdfs.sh | |
| ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} | |
| build_pegasus_on_macos: | |
| name: macOS | |
| needs: cpp_clang_format_linter | |
| runs-on: macos-12 | |
| env: | |
| OPENSSL_ROOT_DIR: /usr/local/opt/openssl | |
| steps: | |
| - name: Install Softwares | |
| run: | | |
| # Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md | |
| brew install ccache | |
| brew install snappy | |
| brew install lz4 | |
| brew install zstd | |
| brew install [email protected] | |
| - uses: actions/checkout@v3 | |
| - name: Setup cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| /Users/runner/Library/Caches/ccache | |
| key: macos_build_ccache | |
| restore-keys: | | |
| macos_build_ccache | |
| macos-12-thirdparty-bins | |
| - name: Build thirdparty | |
| working-directory: thirdparty | |
| run: | | |
| export JAVA_HOME="${JAVA_HOME_8_X64}" | |
| mkdir -p build | |
| cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DROCKSDB_PORTABLE=1 | |
| cmake --build build/ -j $(sysctl -n hw.physicalcpu) | |
| - name: Compilation | |
| run: | | |
| ccache -p | |
| ccache -z | |
| ./run.sh build --test --skip_thirdparty -j $(sysctl -n hw.physicalcpu) | |
| ccache -s |