lint_and_test_cpp.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Cpp CI

on:
  # run on each pull request
  pull_request:
    types: [ synchronize, opened, reopened ]
    branches:
      - master
      - 'v[0-9]+.*' # release branch
      - ci-test # testing branch for github action
      - '*dev'
    paths:
      - .github/actions
      - .github/workflows/lint_and_test_cpp.yaml
      - .github/workflows/thirdparty-regular-push.yml
      - docker/thirdparties-bin/**
      - docker/thirdparties-src/**
      - CMakeLists.txt
      - cmake_modules/**
      - run.sh
      - scripts/pack_server.sh
      - scripts/pack_tools.sh
      - src/**
      - thirdparty/**

  # for manually triggering workflow
  workflow_dispatch:

env:
  # Update the options to reduce the consumption of the disk space
  ONEBOX_OPTS: disk_min_available_space_ratio=5
  TEST_OPTS: disk_min_available_space_ratio=5;throttle_test_medium_value_kb=10;throttle_test_large_value_kb=25

jobs:
  cpp_clang_format_linter:
    name: Lint
    runs-on: ubuntu-latest
    container:
      image: apache/pegasus:clang-format-3.9
    steps:
      - uses: actions/checkout@v3
      - name: clang-format
        run: ./scripts/run-clang-format.py --clang-format-executable clang-format-3.9 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .

  iwyu:
    name: IWYU
    needs: cpp_clang_format_linter
    runs-on: ubuntu-latest
    env:
      USE_JEMALLOC: OFF
    container:
      image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }}
    steps:
      - uses: actions/checkout@v3
      - name: Free Disk Space (Ubuntu)
        run: |
          .github/workflows/free_disk_space.sh
      - name: Rebuild thirdparty if needed
        uses: "./.github/actions/rebuild_thirdparty_if_needed"
      - name: Build IWYU binary
        run: |
          mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git
          cd include-what-you-use && git checkout clang_14
          cd .. && mkdir build && cd build && cmake -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use
          make -j $(nproc)
      - name: Check IWYU
        run: |
          PWD=`pwd`
          ./run.sh build --test --compiler clang-14,clang++-14 -j $(nproc) -t release --iwyu ${PWD}/iwyu/build/bin/include-what-you-use --skip_thirdparty
          python3 iwyu/include-what-you-use/fix_includes.py --blank_lines --reorder --nocomments --nosafe_headers --separate_project_includes="<tld>" --ignore_re="thrift-gen|rrdb_types|layer2_types|metrics_test.cpp" --dry_run < build/latest/iwyu.out
          python3 iwyu/include-what-you-use/fix_includes.py --blank_lines --reorder --nocomments --nosafe_headers --separate_project_includes="<tld>" --ignore_re="thrift-gen|rrdb_types|layer2_types|metrics_test.cpp" --dry_run < build/latest/iwyu.out | egrep "IWYU edited 0 files on your behalf"

  build_Release:
    name: Build Release
    needs: cpp_clang_format_linter
    runs-on: ubuntu-latest
    env:
      USE_JEMALLOC: OFF
      ARTIFACT_NAME: release
      BUILD_OPTIONS: -t release --test
    container:
      image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
    steps:
      - name: Clone code
        uses: actions/checkout@v3
      - name: Rebuild thirdparty if needed
        uses: "./.github/actions/rebuild_thirdparty_if_needed"
      - name: Build Pegasus
        uses: "./.github/actions/build_pegasus"
      - name: Upload artifact
        uses: "./.github/actions/upload_artifact"

  test_Release:
    name: Test Release
    strategy:
      fail-fast: false
      matrix:
        test_module:
          - backup_restore_test
          - base_api_test
          - base_test
          - bulk_load_test
          # TODO(wangdan): Since the hotspot detection depends on the perf-counters system which
          # is being replaced with the new metrics system, its test will fail. Temporarily disable
          # the test and re-enable it after the hotspot detection is migrated to the new metrics
          # system.
          # - detect_hotspot_test
          - dsn_aio_test
          - dsn_block_service_test
          - dsn_client_test
          - dsn.failure_detector.tests
          - dsn_http_test
          - dsn_meta_state_tests
          - dsn.meta.test
          - dsn_nfs_test
          # TODO(wangdan): Since builtin_counters (memused.virt and memused.res) for perf-counters
          # have been removed and dsn_perf_counter_test depends on them, disable it.
          # - dsn_perf_counter_test
          - dsn_ranger_tests
          - dsn_replica_backup_test
          - dsn_replica_bulk_load_test
          - dsn_replica_dup_test
          - dsn_replica_split_test
          - dsn.replica.test
          - dsn_replication_common_test
          - dsn.replication.simple_kv
          - dsn.rep_tests.simple_kv
          - dsn_runtime_tests
          - dsn_security_tests
          - dsn_utils_tests
          - dsn.zookeeper.tests
          # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after
          # it has been optimized.
#          - partition_split_test
          - pegasus_geo_test
          - pegasus_rproxy_test
          - pegasus_unit_test
          - recovery_test
          - restore_test
          - security_test
          - throttle_test
    needs: build_Release
    runs-on: ubuntu-latest
    env:
      ARTIFACT_NAME: release
    container:
      image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
      options: --cap-add=SYS_PTRACE
    steps:
      - uses: actions/checkout@v3
      - name: Download artifact
        uses: "./.github/actions/download_artifact"
      - name: Run server tests
        uses: "./.github/actions/run_server_tests"

  build_ASAN:
    name: Build ASAN
    needs: cpp_clang_format_linter
    runs-on: ubuntu-latest
    env:
      USE_JEMALLOC: OFF
      ARTIFACT_NAME: release_address
      BUILD_OPTIONS: --sanitizer address --disable_gperf --test
    container:
      image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
    steps:
      - uses: actions/checkout@v3
      - name: Rebuild thirdparty if needed
        uses: "./.github/actions/rebuild_thirdparty_if_needed"
      - name: Build Pegasus
        uses: "./.github/actions/build_pegasus"
      - name: Upload artifact
        uses: "./.github/actions/upload_artifact"

  test_ASAN:
    name: Test ASAN
    strategy:
      fail-fast: false
      matrix:
        test_module:
          - backup_restore_test
          - base_api_test
          - base_test
          - bulk_load_test
          # TODO(wangdan): Since the hotspot detection depends on the perf-counters system which
          # is being replaced with the new metrics system, its test will fail. Temporarily disable
          # the test and re-enable it after the hotspot detection is migrated to the new metrics
          # system.
          # - detect_hotspot_test
          - dsn_aio_test
          - dsn_block_service_test
          - dsn_client_test
          - dsn.failure_detector.tests
          - dsn_http_test
          - dsn_meta_state_tests
          - dsn.meta.test
          - dsn_nfs_test
          # TODO(wangdan): Since builtin_counters (memused.virt and memused.res) for perf-counters
          # have been removed and dsn_perf_counter_test depends on them, disable it.
          # - dsn_perf_counter_test
          - dsn_ranger_tests
          - dsn_replica_backup_test
          - dsn_replica_bulk_load_test
          - dsn_replica_dup_test
          - dsn_replica_split_test
          - dsn.replica.test
          - dsn_replication_common_test
          - dsn.replication.simple_kv
          - dsn.rep_tests.simple_kv
          - dsn_runtime_tests
          - dsn_security_tests
          - dsn_utils_tests
          - dsn.zookeeper.tests
          # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after
          # it has been optimized.
#          - partition_split_test
          - pegasus_geo_test
          - pegasus_rproxy_test
          - pegasus_unit_test
          - recovery_test
          - restore_test
          - security_test
          # TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after
          # it has been optimized.
#          - throttle_test
    needs: build_ASAN
    runs-on: ubuntu-latest
    env:
      ARTIFACT_NAME: release_address
    container:
      image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
      options: --cap-add=SYS_PTRACE
    steps:
      - uses: actions/checkout@v3
      - name: Download artifact
        uses: "./.github/actions/download_artifact"
      - name: Run server tests
        uses: "./.github/actions/run_server_tests"

# TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests
#                 when we are going to release a stable version. So we disable them in regular CI
#                 before we find any way to reduce the time cost.
#  build_UBSAN:
#    name: Build UBSAN
#    needs: cpp_clang_format_linter
#    runs-on: ubuntu-latest
#    env:
#      USE_JEMALLOC: OFF
#      ARTIFACT_NAME: release_undefined
#      BUILD_OPTIONS: --sanitizer undefined --disable_gperf --test
#    container:
#      image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
#    steps:
#      - uses: actions/checkout@v3
#      - name: Rebuild thirdparty if needed
#        uses: "./.github/actions/rebuild_thirdparty_if_needed"
#      - name: Build Pegasus
#        uses: "./.github/actions/build_pegasus"
#      - name: Upload artifact
#        uses: "./.github/actions/upload_artifact"
#
#  test_UBSAN:
#    name: Test UBSAN
#    strategy:
#      fail-fast: false
#      matrix:
#        test_module:
#          - backup_restore_test
#          - base_api_test
#          - base_test
#          - bulk_load_test
#          # TODO(wangdan): Since the hotspot detection depends on the perf-counters system which
#          # is being replaced with the new metrics system, its test will fail. Temporarily disable
#          # the test and re-enable it after the hotspot detection is migrated to the new metrics
#          # system.
#          # - detect_hotspot_test
#          - dsn_aio_test
#          - dsn_block_service_test
#          - dsn_client_test
#          - dsn.failure_detector.tests
#          - dsn_http_test
#          - dsn_meta_state_tests
#          - dsn.meta.test
#          - dsn_nfs_test
#          # TODO(wangdan): Since builtin_counters (memused.virt and memused.res) for perf-counters
#          # have been removed and dsn_perf_counter_test depends on them, disable it.
#          # - dsn_perf_counter_test
#          - dsn_ranger_tests
#          - dsn_replica_backup_test
#          - dsn_replica_bulk_load_test
#          - dsn_replica_dup_test
#          - dsn_replica_split_test
#          - dsn.replica.test
#          - dsn_replication_common_test
#          - dsn.replication.simple_kv
#          - dsn.rep_tests.simple_kv
#          - dsn_runtime_tests
#          - dsn_security_tests
#          - dsn_utils_tests
#          - dsn.zookeeper.tests
#          - partition_split_test
#          - pegasus_geo_test
#          - pegasus_rproxy_test
#          - pegasus_unit_test
#          - recovery_test
#          - restore_test
#          - security_test
#          - throttle_test
#    needs: build_UBSAN
#    runs-on: ubuntu-latest
#    env:
#      ARTIFACT_NAME: release_undefined
#    container:
#      image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
#      options: --cap-add=SYS_PTRACE
#    steps:
#      - uses: actions/checkout@v3
#      - name: Download artifact
#        uses: "./.github/actions/download_artifact"
#      - name: Run server tests
#        uses: "./.github/actions/run_server_tests"

  build_with_jemalloc:
    name: Build with jemalloc
    needs: cpp_clang_format_linter
    runs-on: ubuntu-latest
    env:
      USE_JEMALLOC: ON
      ARTIFACT_NAME: release_jemalloc
      BUILD_OPTIONS: -t release --use_jemalloc --test
    container:
      image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
    steps:
      - uses: actions/checkout@v3
      - name: Rebuild thirdparty if needed
        uses: "./.github/actions/rebuild_thirdparty_if_needed"
      # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked.
      - name: Build Pegasus
        uses: "./.github/actions/build_pegasus"
      - name: Upload artifact
        uses: "./.github/actions/upload_artifact"

  test_with_jemalloc:
    name: Test with jemallc
    strategy:
      fail-fast: false
      matrix:
        test_module:
          - dsn_utils_tests
    needs: build_with_jemalloc
    runs-on: ubuntu-latest
    env:
      ARTIFACT_NAME: release_jemalloc
    container:
      image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
      options: --cap-add=SYS_PTRACE
    steps:
      - uses: actions/checkout@v3
      - name: Download artifact
        uses: "./.github/actions/download_artifact"
      - name: Run server tests
        uses: "./.github/actions/run_server_tests"

  build_pegasus_on_macos:
    name: macOS
    needs: cpp_clang_format_linter
    runs-on: macos-12
    steps:
      - name: Install Softwares
        run: |
          # Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md
          brew install ccache
          brew install openssl@1.1
      - uses: actions/checkout@v3
      - name: Setup cache
        uses: actions/cache@v3
        with:
          path: |
            /Users/runner/Library/Caches/ccache
          key: macos_build_ccache
          restore-keys: |
            macos_build_ccache
            macos-12-thirdparty-bins
      - name: Build thirdparty
        working-directory: thirdparty
        run: |
          export JAVA_HOME="${JAVA_HOME_8_X64}"
          mkdir -p build
          cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DROCKSDB_PORTABLE=1
          cmake --build build/ -j $(sysctl -n hw.physicalcpu)
      - name: Compilation
        run: |
          ccache -p
          ccache -z
          ./run.sh build --test --skip_thirdparty -j $(sysctl -n hw.physicalcpu)
          ccache -s