Initial, partial CircleCI -> GitHub Actions migration

Summary: Largely based on facebook#12085 but grouped into one large workflow because of bad GHA UI design (see comments). Test Plan: TODO
pdillinger · Dec 19, 2023 · f8893f2 · f8893f2
1 parent f7486ff
commit f8893f2
Show file tree

Hide file tree

Showing 21 changed files with 883 additions and 95 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -877,60 +877,11 @@ jobs:
       - perform-benchmarks
       - post-benchmarks
 
-workflows:
+workflows: # Only jobs that haven't been successfully migrated to GitHub Actions
   version: 2
-  jobs-linux-run-tests:
-    jobs:
-      - build-linux
-      - build-linux-cmake-with-folly
-      - build-linux-cmake-with-folly-lite-no-test
-      - build-linux-gcc-7-with-folly
-      - build-linux-gcc-7-with-folly-lite-no-test
-      - build-linux-cmake-with-folly-coroutines
-      - build-linux-cmake-with-benchmark
-      - build-linux-encrypted_env-no_compression
-  jobs-linux-run-tests-san:
-    jobs:
-      - build-linux-clang10-asan
-      - build-linux-clang10-ubsan
-      - build-linux-clang10-mini-tsan
-      - build-linux-static_lib-alt_namespace-status_checked
-  jobs-linux-no-test-run:
-    jobs:
-      - build-linux-release
-      - build-linux-release-rtti
-      - build-examples
-      - build-fuzzers
-      - build-linux-clang-no_test_run
-      - build-linux-clang-13-no_test_run
-      - build-linux-gcc-8-no_test_run
-      - build-linux-gcc-10-cxx20-no_test_run
-      - build-linux-gcc-11-no_test_run
-      - build-linux-arm-cmake-no_test_run
-  jobs-linux-other-checks:
-    jobs:
-      - build-linux-clang10-clang-analyze
-      - build-linux-unity-and-headers
-      - build-linux-mini-crashtest
-  jobs-windows:
-    jobs:
-      - build-windows-vs2019
-      - build-cmake-mingw
   jobs-java:
     jobs:
-      - build-linux-java
       - build-linux-java-static
-      - build-macos-java
-      - build-macos-java-static
-      - build-macos-java-static-universal
-      - build-linux-java-pmd
-  jobs-macos:
-    jobs:
-      - build-macos
-      - build-macos-cmake:
-          run_even_tests: true
-      - build-macos-cmake:
-          run_even_tests: false
   jobs-linux-arm:
     jobs:
       - build-linux-arm

diff --git a/.github/actions/build-folly/action.yml b/.github/actions/build-folly/action.yml
@@ -0,0 +1,7 @@
+name: build-folly
+runs:
+  using: composite
+  steps:
+  - name: Build folly and dependencies
+    run: make build_folly
+    shell: bash
diff --git a/.github/actions/build-for-benchmarks/action.yml b/.github/actions/build-for-benchmarks/action.yml
@@ -0,0 +1,8 @@
+name: build-for-benchmarks
+runs:
+  using: composite
+  steps:
+  - uses: "./.github/actions/pre-steps"
+  - name: Linux build for benchmarks
+    run: make V=1 J=8 -j8 release
+    shell: bash
diff --git a/.github/actions/increase-max-open-files-on-macos/action.yml b/.github/actions/increase-max-open-files-on-macos/action.yml
@@ -0,0 +1,10 @@
+name: increase-max-open-files-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Increase max open files
+    run: |-
+      sudo sysctl -w kern.maxfiles=1048576
+      sudo sysctl -w kern.maxfilesperproc=1048576
+      sudo launchctl limit maxfiles 1048576
+    shell: bash
diff --git a/.github/actions/install-gflags-on-macos/action.yml b/.github/actions/install-gflags-on-macos/action.yml
@@ -0,0 +1,7 @@
+name: install-gflags-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install gflags on macos
+    run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
+    shell: bash
diff --git a/.github/actions/install-gflags/action.yml b/.github/actions/install-gflags/action.yml
@@ -0,0 +1,7 @@
+name: install-gflags
+runs:
+  using: composite
+  steps:
+  - name: Install gflags
+    run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+    shell: bash
diff --git a/.github/actions/install-jdk8-on-macos/action.yml b/.github/actions/install-jdk8-on-macos/action.yml
@@ -0,0 +1,9 @@
+name: install-jdk8-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install JDK 8 on macos
+    run: |-
+      HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica
+      HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8
+    shell: bash
diff --git a/.github/actions/install-maven/action.yml b/.github/actions/install-maven/action.yml
@@ -0,0 +1,7 @@
+name: install-maven
+runs:
+  using: composite
+  steps:
+  - name: Install maven
+    run: sudo apt-get update -y && sudo apt-get install -y maven
+    shell: bash
diff --git a/.github/actions/perform-benchmarks/action.yml b/.github/actions/perform-benchmarks/action.yml
@@ -0,0 +1,22 @@
+name: perform-benchmarks
+runs:
+  using: composite
+  steps:
+  - name: Test low-variance benchmarks
+    run: "./tools/benchmark_ci.py --db_dir ${{ runner.temp }}/rocksdb-benchmark-datadir --output_dir ${{ runner.temp }}/benchmark-results --num_keys 20000000"
+    env:
+      LD_LIBRARY_PATH: "/usr/local/lib"
+      DURATION_RO: 300
+      DURATION_RW: 500
+      NUM_THREADS: 1
+      MAX_BACKGROUND_JOBS: 4
+      CI_TESTS_ONLY: 'true'
+      WRITE_BUFFER_SIZE_MB: 16
+      TARGET_FILE_SIZE_BASE_MB: 16
+      MAX_BYTES_FOR_LEVEL_BASE_MB: 64
+      COMPRESSION_TYPE: none
+      CACHE_INDEX_AND_FILTER_BLOCKS: 1
+      MIN_LEVEL_TO_COMPRESS: 3
+      CACHE_SIZE_MB: 10240
+      MB_WRITE_PER_SEC: 2
+    shell: bash
diff --git a/.github/actions/post-benchmarks/action.yml b/.github/actions/post-benchmarks/action.yml
@@ -0,0 +1,14 @@
+name: post-benchmarks
+runs:
+  using: composite
+  steps:
+  - uses: actions/[email protected]
+    with:
+      path: "${{ runner.temp }}/benchmark-results"
+  - name: Send benchmark report to visualisation
+    run: |-
+      set +e
+      set +o pipefail
+      ./build_tools/benchmark_log_tool.py --tsvfile ${{ runner.temp }}/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc
+      true
+    shell: bash
diff --git a/.github/actions/post-pmd-steps/action.yml b/.github/actions/post-pmd-steps/action.yml
@@ -0,0 +1,10 @@
+name: post-pmd-steps
+runs:
+  using: composite
+  steps:
+  - uses: actions/[email protected]
+    with:
+      path: "${{ github.workspace }}/java/target/pmd.xml"
+  - uses: actions/[email protected]
+    with:
+      path: "${{ github.workspace }}/java/target/site"
diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml
@@ -0,0 +1,31 @@
+name: post-steps
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v3
+  - name: Setup tmate session #TEMPORARY!
+    if: ${{ failure() }}
+    uses: mxschmitt/action-tmate@v3
+    with:
+      limit-access-to-actor: true
+  - uses: actions/[email protected]
+    with:
+      path: "${{ runner.temp }}/test-results"
+  - uses: actions/[email protected]
+    with:
+      path: LOG
+  - name: Compress Test Logs
+    run: tar -cvzf t.tar.gz t
+    if: ${{ failure() }}
+    shell: bash
+  - uses: actions/[email protected]
+    with:
+      path: t.tar.gz
+  - run: |-
+      mkdir -p ${{ runner.temp }}/core_dumps
+      cp core.* ${{ runner.temp }}/core_dumps
+    if: ${{ failure() }}
+    shell: bash
+  - uses: actions/[email protected]
+    with:
+      path: "${{ runner.temp }}/core_dumps"
diff --git a/.github/actions/pre-steps-macos/action.yml b/.github/actions/pre-steps-macos/action.yml
@@ -0,0 +1,5 @@
+name: pre-steps-macos
+runs:
+  using: composite
+  steps:
+  - uses: "./.github/actions/pre-steps"
diff --git a/.github/actions/pre-steps/action.yml b/.github/actions/pre-steps/action.yml
@@ -0,0 +1,18 @@
+name: pre-steps
+runs:
+  using: composite
+  steps:
+  - name: Setup Environment Variables
+    run: |-
+      echo "GTEST_THROW_ON_FAILURE=0" >> "$GITHUB_ENV"
+      echo "GTEST_OUTPUT=\"xml:${{ runner.temp }}/test-results/\"" >> "$GITHUB_ENV"
+      echo "SKIP_FORMAT_BUCK_CHECKS=1" >> "$GITHUB_ENV"
+      echo "GTEST_COLOR=1" >> "$GITHUB_ENV"
+      echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV"
+      echo "CTEST_TEST_TIMEOUT=300" >> "$GITHUB_ENV"
+      echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> "$GITHUB_ENV"
+      echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> "$GITHUB_ENV"
+      echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> "$GITHUB_ENV"
+      echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> "$GITHUB_ENV"
+      echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> "$GITHUB_ENV"
+    shell: bash
diff --git a/.github/actions/setup-folly/action.yml b/.github/actions/setup-folly/action.yml
@@ -0,0 +1,7 @@
+name: setup-folly
+runs:
+  using: composite
+  steps:
+  - name: Checkout folly sources
+    run: make checkout_folly
+    shell: bash
diff --git a/.github/actions/windows-build-steps/action.yml b/.github/actions/windows-build-steps/action.yml
@@ -0,0 +1,44 @@
+name: windows-build-steps
+runs:
+  using: composite
+  steps:
+  - name: Add msbuild to PATH
+    uses: microsoft/[email protected]
+  - name: Custom steps
+    env:
+      THIRDPARTY_HOME: ${{ github.workspace }}/thirdparty
+      CMAKE_HOME: C:/Program Files/CMake
+      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
+      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
+      SNAPPY_HOME: ${{ github.workspace }}/thirdparty/snappy-1.1.8
+      SNAPPY_INCLUDE: ${{ github.workspace }}/thirdparty/snappy-1.1.8;${{ github.workspace }}/thirdparty/snappy-1.1.8/build
+      SNAPPY_LIB_DEBUG: ${{ github.workspace }}/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
+    run: |-
+      echo ===================== Install Dependencies =====================
+      choco install liberica8jdk -y
+      mkdir $Env:THIRDPARTY_HOME
+      cd $Env:THIRDPARTY_HOME
+      echo "Building Snappy dependency..."
+      curl -Lo snappy-1.1.8.zip https://github.com/google/snappy/archive/refs/tags/1.1.8.zip
+      unzip -q snappy-1.1.8.zip
+      cd snappy-1.1.8
+      mkdir build
+      cd build
+      & cmake -G "$Env:CMAKE_GENERATOR" ..
+      msbuild Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+      echo ======================== Build RocksDB =========================
+      cd ${{ github.workspace }}
+      $env:Path = $env:JAVA_HOME + ";" + $env:Path
+      mkdir build
+      cd build
+      & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 ..
+      cd ..
+      echo "Building with VS version: $Env:CMAKE_GENERATOR"
+      msbuild build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+      echo ========================= Test RocksDB =========================
+      build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+      echo ======================== Test RocksJava ========================
+      cd build\java
+      & ctest -C Debug -j 16
+    shell: pwsh
diff --git a/.github/workflows/benchmark-linux.yml b/.github/workflows/benchmark-linux.yml
@@ -0,0 +1,10 @@
+name: facebook/rocksdb/benchmark-linux
+on: [push, pull_request] # TEMPORARY: test on PR
+jobs:
+  benchmark-linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/build-for-benchmarks"
+    - uses: "./.github/actions/perform-benchmarks"
+    - uses: "./.github/actions/post-benchmarks"
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -0,0 +1,96 @@
+name: facebook/rocksdb/nightly
+on: [push, pull_request] # TEMPORARY: test on PR
+jobs:
+  build-format-compatible:
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/pre-steps"
+    - name: test
+      run: |-
+        export TEST_TMPDIR=/dev/shm/rocksdb
+        rm -rf /dev/shm/rocksdb
+        mkdir /dev/shm/rocksdb
+        git config --global --add safe.directory /__w/rocksdb/rocksdb
+        tools/check_format_compatible.sh
+    - uses: "./.github/actions/post-steps"
+  build-linux-arm-test-full:
+    runs-on:
+      labels: arm64large
+    container:
+      image: ubuntu-2004:202111-02
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/pre-steps"
+    - uses: "./.github/actions/install-gflags"
+    - run: make V=1 J=4 -j4 check
+    - uses: "./.github/actions/post-steps"
+  build-linux-run-microbench:
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/pre-steps"
+    - run: DEBUG_LEVEL=0 make -j32 run_microbench
+    - uses: "./.github/actions/post-steps"
+  build-linux-non-shm:
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    env:
+      TEST_TMPDIR: "/tmp/rocksdb_test_tmp"
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/pre-steps"
+    - run: make V=1 -j32 check
+    - uses: "./.github/actions/post-steps"
+  build-linux-clang-13-asan-ubsan-with-folly:
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/pre-steps"
+    - uses: "./.github/actions/setup-folly"
+    - uses: "./.github/actions/build-folly"
+    - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check
+    - uses: "./.github/actions/post-steps"
+  build-linux-valgrind:
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/pre-steps"
+    - run: PORTABLE=1 make V=1 -j32 valgrind_test
+    - uses: "./.github/actions/post-steps"
+  build-windows-vs2022-avx2:
+    runs-on: windows-2022
+    env:
+      CMAKE_GENERATOR: Visual Studio 17 2022
+      CMAKE_PORTABLE: AVX2
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/windows-build-steps"
+  build-windows-vs2022:
+    runs-on: windows-2022
+    env:
+      CMAKE_GENERATOR: Visual Studio 17 2022
+      CMAKE_PORTABLE: 1
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/windows-build-steps"