From 36a340c2f8bf834772d02fb147a4fe9a0cc9d377 Mon Sep 17 00:00:00 2001 From: Rob Anderson Date: Mon, 20 Nov 2023 17:06:55 -0800 Subject: [PATCH 01/13] GitHub actions and workflows --- .github/actions/build-folly/action.yml | 7 ++ .../actions/build-for-benchmarks/action.yml | 8 ++ .../action.yml | 10 ++ .../install-gflags-on-macos/action.yml | 7 ++ .github/actions/install-gflags/action.yml | 7 ++ .../actions/install-jdk8-on-macos/action.yml | 9 ++ .github/actions/install-maven/action.yml | 7 ++ .github/actions/perform-benchmarks/action.yml | 22 ++++ .github/actions/post-benchmarks/action.yml | 14 +++ .github/actions/post-pmd-steps/action.yml | 10 ++ .github/actions/post-steps/action.yml | 25 ++++ .github/actions/pre-steps-macos/action.yml | 5 + .github/actions/pre-steps/action.yml | 18 +++ .github/actions/setup-folly/action.yml | 7 ++ .../actions/windows-build-steps/action.yml | 38 ++++++ .github/workflows/benchmark-linux.yml | 13 ++ .github/workflows/jobs-java.yml | 117 ++++++++++++++++++ .github/workflows/jobs-linux-arm.yml | 44 +++++++ .github/workflows/jobs-linux-no-test-run.yml | 117 ++++++++++++++++++ .github/workflows/jobs-linux-other-checks.yml | 46 +++++++ .../workflows/jobs-linux-run-tests-san.yml | 46 +++++++ .github/workflows/jobs-linux-run-tests.yml | 115 +++++++++++++++++ .github/workflows/jobs-macos.yml | 46 +++++++ .github/workflows/jobs-windows.yml | 40 ++++++ .github/workflows/nightly.yml | 77 ++++++++++++ 25 files changed, 855 insertions(+) create mode 100644 .github/actions/build-folly/action.yml create mode 100644 .github/actions/build-for-benchmarks/action.yml create mode 100644 .github/actions/increase-max-open-files-on-macos/action.yml create mode 100644 .github/actions/install-gflags-on-macos/action.yml create mode 100644 .github/actions/install-gflags/action.yml create mode 100644 .github/actions/install-jdk8-on-macos/action.yml create mode 100644 .github/actions/install-maven/action.yml create mode 100644 .github/actions/perform-benchmarks/action.yml create mode 100644 .github/actions/post-benchmarks/action.yml create mode 100644 .github/actions/post-pmd-steps/action.yml create mode 100644 .github/actions/post-steps/action.yml create mode 100644 .github/actions/pre-steps-macos/action.yml create mode 100644 .github/actions/pre-steps/action.yml create mode 100644 .github/actions/setup-folly/action.yml create mode 100644 .github/actions/windows-build-steps/action.yml create mode 100644 .github/workflows/benchmark-linux.yml create mode 100644 .github/workflows/jobs-java.yml create mode 100644 .github/workflows/jobs-linux-arm.yml create mode 100644 .github/workflows/jobs-linux-no-test-run.yml create mode 100644 .github/workflows/jobs-linux-other-checks.yml create mode 100644 .github/workflows/jobs-linux-run-tests-san.yml create mode 100644 .github/workflows/jobs-linux-run-tests.yml create mode 100644 .github/workflows/jobs-macos.yml create mode 100644 .github/workflows/jobs-windows.yml create mode 100644 .github/workflows/nightly.yml diff --git a/.github/actions/build-folly/action.yml b/.github/actions/build-folly/action.yml new file mode 100644 index 00000000000..909ca549b69 --- /dev/null +++ b/.github/actions/build-folly/action.yml @@ -0,0 +1,7 @@ +name: build-folly +runs: + using: composite + steps: + - name: Build folly and dependencies + run: make build_folly + shell: bash \ No newline at end of file diff --git a/.github/actions/build-for-benchmarks/action.yml b/.github/actions/build-for-benchmarks/action.yml new file mode 100644 index 00000000000..da78fdae43e --- /dev/null +++ b/.github/actions/build-for-benchmarks/action.yml @@ -0,0 +1,8 @@ +name: build-for-benchmarks +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" + - name: Linux build for benchmarks + run: make V=1 J=8 -j8 release + shell: bash \ No newline at end of file diff --git a/.github/actions/increase-max-open-files-on-macos/action.yml b/.github/actions/increase-max-open-files-on-macos/action.yml new file mode 100644 index 00000000000..0a217ff5516 --- /dev/null +++ b/.github/actions/increase-max-open-files-on-macos/action.yml @@ -0,0 +1,10 @@ +name: increase-max-open-files-on-macos +runs: + using: composite + steps: + - name: Increase max open files + run: |- + sudo sysctl -w kern.maxfiles=1048576 + sudo sysctl -w kern.maxfilesperproc=1048576 + sudo launchctl limit maxfiles 1048576 + shell: bash \ No newline at end of file diff --git a/.github/actions/install-gflags-on-macos/action.yml b/.github/actions/install-gflags-on-macos/action.yml new file mode 100644 index 00000000000..4ad14474065 --- /dev/null +++ b/.github/actions/install-gflags-on-macos/action.yml @@ -0,0 +1,7 @@ +name: install-gflags-on-macos +runs: + using: composite + steps: + - name: Install gflags on macos + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags + shell: bash \ No newline at end of file diff --git a/.github/actions/install-gflags/action.yml b/.github/actions/install-gflags/action.yml new file mode 100644 index 00000000000..b3695aed5da --- /dev/null +++ b/.github/actions/install-gflags/action.yml @@ -0,0 +1,7 @@ +name: install-gflags +runs: + using: composite + steps: + - name: Install gflags + run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + shell: bash \ No newline at end of file diff --git a/.github/actions/install-jdk8-on-macos/action.yml b/.github/actions/install-jdk8-on-macos/action.yml new file mode 100644 index 00000000000..c6cd991ddc6 --- /dev/null +++ b/.github/actions/install-jdk8-on-macos/action.yml @@ -0,0 +1,9 @@ +name: install-jdk8-on-macos +runs: + using: composite + steps: + - name: Install JDK 8 on macos + run: |- + HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8 + shell: bash \ No newline at end of file diff --git a/.github/actions/install-maven/action.yml b/.github/actions/install-maven/action.yml new file mode 100644 index 00000000000..0924f5058bb --- /dev/null +++ b/.github/actions/install-maven/action.yml @@ -0,0 +1,7 @@ +name: install-maven +runs: + using: composite + steps: + - name: Install maven + run: sudo apt-get update -y && sudo apt-get install -y maven + shell: bash \ No newline at end of file diff --git a/.github/actions/perform-benchmarks/action.yml b/.github/actions/perform-benchmarks/action.yml new file mode 100644 index 00000000000..3eac7c7b3be --- /dev/null +++ b/.github/actions/perform-benchmarks/action.yml @@ -0,0 +1,22 @@ +name: perform-benchmarks +runs: + using: composite + steps: + - name: Test low-variance benchmarks + run: "./tools/benchmark_ci.py --db_dir ${{ runner.temp }}/rocksdb-benchmark-datadir --output_dir ${{ runner.temp }}/benchmark-results --num_keys 20000000" + env: + LD_LIBRARY_PATH: "/usr/local/lib" + DURATION_RO: 300 + DURATION_RW: 500 + NUM_THREADS: 1 + MAX_BACKGROUND_JOBS: 4 + CI_TESTS_ONLY: 'true' + WRITE_BUFFER_SIZE_MB: 16 + TARGET_FILE_SIZE_BASE_MB: 16 + MAX_BYTES_FOR_LEVEL_BASE_MB: 64 + COMPRESSION_TYPE: none + CACHE_INDEX_AND_FILTER_BLOCKS: 1 + MIN_LEVEL_TO_COMPRESS: 3 + CACHE_SIZE_MB: 10240 + MB_WRITE_PER_SEC: 2 + shell: bash \ No newline at end of file diff --git a/.github/actions/post-benchmarks/action.yml b/.github/actions/post-benchmarks/action.yml new file mode 100644 index 00000000000..9d9b07827d7 --- /dev/null +++ b/.github/actions/post-benchmarks/action.yml @@ -0,0 +1,14 @@ +name: post-benchmarks +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/benchmark-results" + - name: Send benchmark report to visualisation + run: |- + set +e + set +o pipefail + ./build_tools/benchmark_log_tool.py --tsvfile ${{ runner.temp }}/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc + true + shell: bash \ No newline at end of file diff --git a/.github/actions/post-pmd-steps/action.yml b/.github/actions/post-pmd-steps/action.yml new file mode 100644 index 00000000000..a9313895b9d --- /dev/null +++ b/.github/actions/post-pmd-steps/action.yml @@ -0,0 +1,10 @@ +name: post-pmd-steps +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ github.workspace }}/java/target/pmd.xml" + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ github.workspace }}/java/target/site" \ No newline at end of file diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml new file mode 100644 index 00000000000..20aa8326484 --- /dev/null +++ b/.github/actions/post-steps/action.yml @@ -0,0 +1,25 @@ +name: post-steps +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/test-results" + - uses: actions/upload-artifact@v3.1.3 + with: + path: LOG + - name: Compress Test Logs + run: tar -cvzf t.tar.gz t + if: ${{ failure() }} + shell: bash + - uses: actions/upload-artifact@v3.1.3 + with: + path: t.tar.gz + - run: |- + mkdir -p ${{ runner.temp }}/core_dumps + cp core.* ${{ runner.temp }}/core_dumps + if: ${{ failure() }} + shell: bash + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/core_dumps" \ No newline at end of file diff --git a/.github/actions/pre-steps-macos/action.yml b/.github/actions/pre-steps-macos/action.yml new file mode 100644 index 00000000000..2cc9195c884 --- /dev/null +++ b/.github/actions/pre-steps-macos/action.yml @@ -0,0 +1,5 @@ +name: pre-steps-macos +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" \ No newline at end of file diff --git a/.github/actions/pre-steps/action.yml b/.github/actions/pre-steps/action.yml new file mode 100644 index 00000000000..b192c87ab92 --- /dev/null +++ b/.github/actions/pre-steps/action.yml @@ -0,0 +1,18 @@ +name: pre-steps +runs: + using: composite + steps: + - name: Setup Environment Variables + run: |- + echo "GTEST_THROW_ON_FAILURE=0" >> "$GITHUB_ENV" + echo "GTEST_OUTPUT=\"xml:${{ runner.temp }}/test-results/\"" >> "$GITHUB_ENV" + echo "SKIP_FORMAT_BUCK_CHECKS=1" >> "$GITHUB_ENV" + echo "GTEST_COLOR=1" >> "$GITHUB_ENV" + echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV" + echo "CTEST_TEST_TIMEOUT=300" >> "$GITHUB_ENV" + echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> "$GITHUB_ENV" + echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> "$GITHUB_ENV" + echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> "$GITHUB_ENV" + echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> "$GITHUB_ENV" + echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> "$GITHUB_ENV" + shell: bash \ No newline at end of file diff --git a/.github/actions/setup-folly/action.yml b/.github/actions/setup-folly/action.yml new file mode 100644 index 00000000000..dd19977e3aa --- /dev/null +++ b/.github/actions/setup-folly/action.yml @@ -0,0 +1,7 @@ +name: setup-folly +runs: + using: composite + steps: + - name: Checkout folly sources + run: make checkout_folly + shell: bash \ No newline at end of file diff --git a/.github/actions/windows-build-steps/action.yml b/.github/actions/windows-build-steps/action.yml new file mode 100644 index 00000000000..5923a54bc30 --- /dev/null +++ b/.github/actions/windows-build-steps/action.yml @@ -0,0 +1,38 @@ +name: windows-build-steps +runs: + using: composite + steps: + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@v1.3.1 + - name: Install thirdparty dependencies + run: |- + choco install liberica8jdk -y + mkdir $Env:THIRDPARTY_HOME + cd $Env:THIRDPARTY_HOME + echo "Building Snappy dependency..." + curl -Lo snappy-1.1.8.zip https://github.com/google/snappy/archive/refs/tags/1.1.8.zip + unzip -q snappy-1.1.8.zip + cd snappy-1.1.8 + mkdir build + cd build + & cmake -G "$Env:CMAKE_GENERATOR" .. + msbuild Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + shell: pwsh + - name: Build RocksDB + run: |- + $env:Path = $env:JAVA_HOME + ";" + $env:Path + mkdir build + cd build + & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 .. + cd .. + echo "Building with VS version: $Env:CMAKE_GENERATOR" + msbuild build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + shell: pwsh + - name: Test RocksDB + run: build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16 + shell: pwsh + - name: Test RocksJava + run: |- + cd build\java + & ctest -C Debug -j 16 + shell: pwsh \ No newline at end of file diff --git a/.github/workflows/benchmark-linux.yml b/.github/workflows/benchmark-linux.yml new file mode 100644 index 00000000000..8e190eed94e --- /dev/null +++ b/.github/workflows/benchmark-linux.yml @@ -0,0 +1,13 @@ +name: facebook/rocksdb/benchmark-linux +on: + schedule: + - cron: 0 * * * * + workflow_dispatch: +jobs: + benchmark-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/build-for-benchmarks" + - uses: "./.github/actions/perform-benchmarks" + - uses: "./.github/actions/post-benchmarks" diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml new file mode 100644 index 00000000000..1097caa5a6c --- /dev/null +++ b/.github/workflows/jobs-java.yml @@ -0,0 +1,117 @@ +name: facebook/rocksdb/jobs-java +on: + push: + branches: + - main +jobs: + build-linux-java: + runs-on: + group: large + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: make V=1 J=8 -j8 jtest + - uses: "./.github/actions/post-steps" + build-linux-java-static: + runs-on: + group: large + container: + image: evolvedbinary/rocksjava:centos6_x64-be + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-macos-java: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: make V=1 J=16 -j16 jtest + - uses: "./.github/actions/post-steps" + build-macos-java-static: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava x86 and ARM Static Libraries + run: make V=1 J=16 -j16 rocksdbjavastaticosx + - uses: "./.github/actions/post-steps" + build-macos-java-static-universal: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Universal Binary Static Library + run: make V=1 J=16 -j16 rocksdbjavastaticosx_ub + - uses: "./.github/actions/post-steps" + build-linux-java-pmd: + runs-on: + group: large + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/install-maven" + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: PMD RocksDBJava + run: make V=1 J=8 -j8 jpmd + - uses: "./.github/actions/post-pmd-steps" diff --git a/.github/workflows/jobs-linux-arm.yml b/.github/workflows/jobs-linux-arm.yml new file mode 100644 index 00000000000..4a955aa165a --- /dev/null +++ b/.github/workflows/jobs-linux-arm.yml @@ -0,0 +1,44 @@ +name: facebook/rocksdb/jobs-linux-arm +on: + push: + branches: + - main +jobs: + build-linux-arm: + runs-on: + group: arm64large # GitHub hosted ARM runners do not yet exist + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some + - uses: "./.github/actions/post-steps" + build-linux-arm-cmake-no_test_run: + runs-on: + group: arm64large # GitHub hosted ARM runners do not yet exist + env: + JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-arm64" + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - name: Build with cmake + run: |- + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. + make -j4 + - name: Build Java with cmake + run: |- + rm -rf build + mkdir build + cd build + cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 .. + make -j4 rocksdb rocksdbjni + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml new file mode 100644 index 00000000000..65e58a36868 --- /dev/null +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -0,0 +1,117 @@ +name: facebook/rocksdb/jobs-linux-no-test-run +on: + push: + branches: + - main +jobs: + build-linux-release: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: "./db_stress --version" + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: "./db_stress --version" + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: if ./db_stress --version; then false; else true; fi + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: if ./db_stress --version; then false; else true; fi + - uses: "./.github/actions/post-steps" + build-linux-release-rtti: + runs-on: + group: xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: "./db_stress --version" + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: if ./db_stress --version; then false; else true; fi + build-examples: + runs-on: + group: large + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Build examples + run: make V=1 -j4 static_lib && cd examples && make V=1 -j4 + - uses: "./.github/actions/post-steps" + build-fuzzers: + runs-on: + group: large + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Build rocksdb lib + run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib + - name: Build fuzzers + run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer + - uses: "./.github/actions/post-steps" + build-linux-clang-no_test_run: + runs-on: + group: xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all + - uses: "./.github/actions/post-steps" + build-linux-clang-13-no_test_run: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench + - uses: "./.github/actions/post-steps" + build-linux-gcc-8-no_test_run: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-gcc-10-cxx20-no_test_run: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-gcc-11-no_test_run: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench + - uses: "./.github/actions/post-steps" + \ No newline at end of file diff --git a/.github/workflows/jobs-linux-other-checks.yml b/.github/workflows/jobs-linux-other-checks.yml new file mode 100644 index 00000000000..2b804097e8b --- /dev/null +++ b/.github/workflows/jobs-linux-other-checks.yml @@ -0,0 +1,46 @@ +name: facebook/rocksdb/jobs-linux-other-checks +on: + push: + branches: + - main +jobs: + build-linux-clang10-clang-analyze: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze + - uses: "./.github/actions/post-steps" + - name: compress test report + run: tar -cvzf scan_build_report.tar.gz scan_build_report + if: failure() + - uses: actions/upload-artifact@v3.1.3 + with: + path: scan_build_report.tar.gz + build-linux-unity-and-headers: + runs-on: + group: large + container: + image: gcc:latest + env: + EXTRA_CXXFLAGS: "-mno-avx512f" + steps: + - uses: actions/checkout@v4.1.0 + - run: apt-get update -y && apt-get install -y libgflags-dev + - name: Unity build + run: make V=1 -j8 unity_test + - run: make V=1 -j8 -k check-headers + - uses: "./.github/actions/post-steps" + build-linux-mini-crashtest: + runs-on: + group: large + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml new file mode 100644 index 00000000000..90018b5150e --- /dev/null +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -0,0 +1,46 @@ +name: facebook/rocksdb/jobs-linux-run-tests-san +on: + push: + branches: + - main +jobs: + build-linux-clang10-asan: + runs-on: + group: 2xlargeplus + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-clang10-ubsan: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check + - uses: "./.github/actions/post-steps" + build-linux-clang10-mini-tsan: + runs-on: + group: 2xlargeplus + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-static_lib-alt_namespace-status_checked: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-linux-run-tests.yml b/.github/workflows/jobs-linux-run-tests.yml new file mode 100644 index 00000000000..d88beb4d999 --- /dev/null +++ b/.github/workflows/jobs-linux-run-tests.yml @@ -0,0 +1,115 @@ +name: facebook/rocksdb/jobs-linux-run-tests +on: + push: + branches: + - main +jobs: + build-linux: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: make V=1 J=32 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-cmake-mingw: + runs-on: + group: large + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix + - name: Build cmake-mingw + run: |- + export PATH=$JAVA_HOME/bin:$PATH + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly-lite-no-test: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)" + - uses: "./.github/actions/post-steps" + build-linux-gcc-7-with-folly: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check + - uses: "./.github/actions/post-steps" + build-linux-gcc-7-with-folly-lite-no-test: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly-coroutines: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + env: + CC: gcc-10 + CXX: g++-10 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-benchmark: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 + - uses: "./.github/actions/post-steps" + build-linux-encrypted_env-no_compression: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check + - run: "./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression\n" + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/jobs-macos.yml b/.github/workflows/jobs-macos.yml new file mode 100644 index 00000000000..7f02634a507 --- /dev/null +++ b/.github/workflows/jobs-macos.yml @@ -0,0 +1,46 @@ +name: facebook/rocksdb/jobs-macos +on: + push: + branches: + - main +jobs: + build-macos: + runs-on: macos-13 + env: + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Run all tests + run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=16 -j16 all + - uses: "./.github/actions/post-steps" + build-macos-cmake: + runs-on: macos-13 + strategy: + matrix: + run_even_tests: [true, false] + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: cmake generate project file + run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. + - name: Build tests + run: cd build && make V=1 -j16 + - name: Run even tests + run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2 + if: ${{ matrix.run_even_tests }} + - name: Run odd tests + run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2 + if: ${{ ! matrix.run_even_tests }} + - uses: "./.github/actions/post-steps" + diff --git a/.github/workflows/jobs-windows.yml b/.github/workflows/jobs-windows.yml new file mode 100644 index 00000000000..59fc27fc3e2 --- /dev/null +++ b/.github/workflows/jobs-windows.yml @@ -0,0 +1,40 @@ +name: facebook/rocksdb/jobs-windows +on: + push: + branches: + - main +env: + THIRDPARTY_HOME: ${{ github.workspace }}/thirdparty + CMAKE_HOME: C:/Program Files/CMake + CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe + JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8 + SNAPPY_HOME: ${{ github.workspace }}/thirdparty/snappy-1.1.8 + SNAPPY_INCLUDE: ${{ github.workspace }}/thirdparty/snappy-1.1.8;${{ github.workspace }}/thirdparty/snappy-1.1.8/build + SNAPPY_LIB_DEBUG: ${{ github.workspace }}/thirdparty/snappy-1.1.8/build/Debug/snappy.lib +jobs: + build-windows-vs2022-avx2: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: AVX2 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + build-windows-vs2022: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + build-windows-vs2019: + runs-on: windows-2019 + env: + CMAKE_GENERATOR: Visual Studio 16 2019 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + \ No newline at end of file diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000000..52697f4591b --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,77 @@ +name: facebook/rocksdb/nightly +on: + schedule: + - cron: 0 9 * * * + workflow_dispatch: +jobs: + build-format-compatible: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: test + run: |- + export TEST_TMPDIR=/dev/shm/rocksdb + rm -rf /dev/shm/rocksdb + mkdir /dev/shm/rocksdb + git config --global --add safe.directory /__w/rocksdb/rocksdb + tools/check_format_compatible.sh + - uses: "./.github/actions/post-steps" + build-linux-arm-test-full: + runs-on: + group: arm64large + container: + image: ubuntu-2004:202111-02 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: make V=1 J=4 -j4 check + - uses: "./.github/actions/post-steps" + build-linux-run-microbench: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: DEBUG_LEVEL=0 make -j32 run_microbench + - uses: "./.github/actions/post-steps" + build-linux-non-shm: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + env: + TEST_TMPDIR: "/tmp/rocksdb_test_tmp" + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-clang-13-asan-ubsan-with-folly: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check + - uses: "./.github/actions/post-steps" + build-linux-valgrind: + runs-on: + group: 2xlarge + container: + image: zjay437/rocksdb:0.6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: PORTABLE=1 make V=1 -j32 valgrind_test + - uses: "./.github/actions/post-steps" From 0ce2d2d38bb10afc3a5cc15c58d385e42ac1991d Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Wed, 29 Nov 2023 14:40:37 -0800 Subject: [PATCH 02/13] Pull request trigger --- .github/workflows/jobs-java.yml | 5 +---- .github/workflows/jobs-linux-arm.yml | 5 +---- .github/workflows/jobs-linux-no-test-run.yml | 7 ++----- .github/workflows/jobs-linux-other-checks.yml | 5 +---- .github/workflows/jobs-linux-run-tests-san.yml | 5 +---- .github/workflows/jobs-linux-run-tests.yml | 5 +---- .github/workflows/jobs-macos.yml | 5 +---- .github/workflows/jobs-windows.yml | 7 ++----- 8 files changed, 10 insertions(+), 34 deletions(-) diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml index 1097caa5a6c..984b92734ec 100644 --- a/.github/workflows/jobs-java.yml +++ b/.github/workflows/jobs-java.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/jobs-java -on: - push: - branches: - - main +on: [push, pull_request] jobs: build-linux-java: runs-on: diff --git a/.github/workflows/jobs-linux-arm.yml b/.github/workflows/jobs-linux-arm.yml index 4a955aa165a..e406a08d2c8 100644 --- a/.github/workflows/jobs-linux-arm.yml +++ b/.github/workflows/jobs-linux-arm.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/jobs-linux-arm -on: - push: - branches: - - main +on: [push, pull_request] jobs: build-linux-arm: runs-on: diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml index 65e58a36868..be989a783a0 100644 --- a/.github/workflows/jobs-linux-no-test-run.yml +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/jobs-linux-no-test-run -on: - push: - branches: - - main +on: [push, pull_request] jobs: build-linux-release: runs-on: @@ -114,4 +111,4 @@ jobs: - uses: "./.github/actions/pre-steps" - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench - uses: "./.github/actions/post-steps" - \ No newline at end of file + diff --git a/.github/workflows/jobs-linux-other-checks.yml b/.github/workflows/jobs-linux-other-checks.yml index 2b804097e8b..49dbddd5fff 100644 --- a/.github/workflows/jobs-linux-other-checks.yml +++ b/.github/workflows/jobs-linux-other-checks.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/jobs-linux-other-checks -on: - push: - branches: - - main +on: [push, pull_request] jobs: build-linux-clang10-clang-analyze: runs-on: diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index 90018b5150e..fe85232ae4c 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/jobs-linux-run-tests-san -on: - push: - branches: - - main +on: [push, pull_request] jobs: build-linux-clang10-asan: runs-on: diff --git a/.github/workflows/jobs-linux-run-tests.yml b/.github/workflows/jobs-linux-run-tests.yml index d88beb4d999..462a432fd4f 100644 --- a/.github/workflows/jobs-linux-run-tests.yml +++ b/.github/workflows/jobs-linux-run-tests.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/jobs-linux-run-tests -on: - push: - branches: - - main +on: [push, pull_request] jobs: build-linux: runs-on: diff --git a/.github/workflows/jobs-macos.yml b/.github/workflows/jobs-macos.yml index 7f02634a507..68bc839060f 100644 --- a/.github/workflows/jobs-macos.yml +++ b/.github/workflows/jobs-macos.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/jobs-macos -on: - push: - branches: - - main +on: [push, pull_request] jobs: build-macos: runs-on: macos-13 diff --git a/.github/workflows/jobs-windows.yml b/.github/workflows/jobs-windows.yml index 59fc27fc3e2..7b6fb71040b 100644 --- a/.github/workflows/jobs-windows.yml +++ b/.github/workflows/jobs-windows.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/jobs-windows -on: - push: - branches: - - main +on: [push, pull_request] env: THIRDPARTY_HOME: ${{ github.workspace }}/thirdparty CMAKE_HOME: C:/Program Files/CMake @@ -37,4 +34,4 @@ jobs: steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/windows-build-steps" - \ No newline at end of file + From d890102e0fa6431ca9c5122cc59d8d84719eb604 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Wed, 29 Nov 2023 14:45:55 -0800 Subject: [PATCH 03/13] Comment --- .github/workflows/jobs-windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/jobs-windows.yml b/.github/workflows/jobs-windows.yml index 7b6fb71040b..d71150d238a 100644 --- a/.github/workflows/jobs-windows.yml +++ b/.github/workflows/jobs-windows.yml @@ -10,6 +10,7 @@ env: SNAPPY_INCLUDE: ${{ github.workspace }}/thirdparty/snappy-1.1.8;${{ github.workspace }}/thirdparty/snappy-1.1.8/build SNAPPY_LIB_DEBUG: ${{ github.workspace }}/thirdparty/snappy-1.1.8/build/Debug/snappy.lib jobs: +# TODO: move some to nightly as in a140b51 build-windows-vs2022-avx2: runs-on: windows-2022 env: From fe49d8e8248b2ac07518ee9a7708319c1983aa33 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 1 Dec 2023 10:47:25 -0800 Subject: [PATCH 04/13] Use facebook GHA runners --- .github/workflows/jobs-java.yml | 6 +++--- .github/workflows/jobs-linux-no-test-run.yml | 18 +++++++++--------- .github/workflows/jobs-linux-other-checks.yml | 6 +++--- .github/workflows/jobs-linux-run-tests-san.yml | 8 ++++---- .github/workflows/jobs-linux-run-tests.yml | 18 +++++++++--------- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml index 984b92734ec..e8d365b2304 100644 --- a/.github/workflows/jobs-java.yml +++ b/.github/workflows/jobs-java.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-java: runs-on: - group: large + group: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -19,7 +19,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-java-static: runs-on: - group: large + group: 4-core-ubuntu container: image: evolvedbinary/rocksjava:centos6_x64-be steps: @@ -99,7 +99,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-java-pmd: runs-on: - group: large + group: 4-core-ubuntu steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/install-maven" diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml index be989a783a0..a6b70055606 100644 --- a/.github/workflows/jobs-linux-no-test-run.yml +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-release: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -27,7 +27,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-release-rtti: runs-on: - group: xlarge + group: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -40,7 +40,7 @@ jobs: - run: if ./db_stress --version; then false; else true; fi build-examples: runs-on: - group: large + group: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -51,7 +51,7 @@ jobs: - uses: "./.github/actions/post-steps" build-fuzzers: runs-on: - group: large + group: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -64,7 +64,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-clang-no_test_run: runs-on: - group: xlarge + group: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -73,7 +73,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-clang-13-no_test_run: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -83,7 +83,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-8-no_test_run: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -93,7 +93,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-10-cxx20-no_test_run: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -103,7 +103,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-11-no_test_run: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: diff --git a/.github/workflows/jobs-linux-other-checks.yml b/.github/workflows/jobs-linux-other-checks.yml index 49dbddd5fff..eb1f3f0fd04 100644 --- a/.github/workflows/jobs-linux-other-checks.yml +++ b/.github/workflows/jobs-linux-other-checks.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-clang10-clang-analyze: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -19,7 +19,7 @@ jobs: path: scan_build_report.tar.gz build-linux-unity-and-headers: runs-on: - group: large + group: 4-core-ubuntu container: image: gcc:latest env: @@ -33,7 +33,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-mini-crashtest: runs-on: - group: large + group: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index fe85232ae4c..b8ef69f52af 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-clang10-asan: runs-on: - group: 2xlargeplus + group: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -13,7 +13,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-clang10-ubsan: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -23,7 +23,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-clang10-mini-tsan: runs-on: - group: 2xlargeplus + group: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -33,7 +33,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-static_lib-alt_namespace-status_checked: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: diff --git a/.github/workflows/jobs-linux-run-tests.yml b/.github/workflows/jobs-linux-run-tests.yml index 462a432fd4f..a4853bbddbb 100644 --- a/.github/workflows/jobs-linux-run-tests.yml +++ b/.github/workflows/jobs-linux-run-tests.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -13,7 +13,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-mingw: runs-on: - group: large + group: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -30,7 +30,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -42,7 +42,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-lite-no-test: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -53,7 +53,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-7-with-folly: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -65,7 +65,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-7-with-folly-lite-no-test: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -76,7 +76,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-coroutines: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 env: @@ -91,7 +91,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-with-benchmark: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -101,7 +101,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-encrypted_env-no_compression: runs-on: - group: 2xlarge + group: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: From ee5b9eb5c775b3e0f1a5845a9d58a077b9bfa96e Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 1 Dec 2023 12:10:57 -0800 Subject: [PATCH 05/13] s/group:/labels:/ --- .github/workflows/jobs-java.yml | 6 +++--- .github/workflows/jobs-linux-arm.yml | 4 ++-- .github/workflows/jobs-linux-no-test-run.yml | 18 +++++++++--------- .github/workflows/jobs-linux-other-checks.yml | 6 +++--- .github/workflows/jobs-linux-run-tests-san.yml | 8 ++++---- .github/workflows/jobs-linux-run-tests.yml | 18 +++++++++--------- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml index e8d365b2304..5ef779f85c7 100644 --- a/.github/workflows/jobs-java.yml +++ b/.github/workflows/jobs-java.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-java: runs-on: - group: 4-core-ubuntu + labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -19,7 +19,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-java-static: runs-on: - group: 4-core-ubuntu + labels: 4-core-ubuntu container: image: evolvedbinary/rocksjava:centos6_x64-be steps: @@ -99,7 +99,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-java-pmd: runs-on: - group: 4-core-ubuntu + labels: 4-core-ubuntu steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/install-maven" diff --git a/.github/workflows/jobs-linux-arm.yml b/.github/workflows/jobs-linux-arm.yml index e406a08d2c8..665cd3eaf08 100644 --- a/.github/workflows/jobs-linux-arm.yml +++ b/.github/workflows/jobs-linux-arm.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-arm: runs-on: - group: arm64large # GitHub hosted ARM runners do not yet exist + labels: arm64large # GitHub hosted ARM runners do not yet exist steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -12,7 +12,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-arm-cmake-no_test_run: runs-on: - group: arm64large # GitHub hosted ARM runners do not yet exist + labels: arm64large # GitHub hosted ARM runners do not yet exist env: JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-arm64" steps: diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml index a6b70055606..963408d1e34 100644 --- a/.github/workflows/jobs-linux-no-test-run.yml +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-release: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -27,7 +27,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-release-rtti: runs-on: - group: 8-core-ubuntu + labels: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -40,7 +40,7 @@ jobs: - run: if ./db_stress --version; then false; else true; fi build-examples: runs-on: - group: 4-core-ubuntu + labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -51,7 +51,7 @@ jobs: - uses: "./.github/actions/post-steps" build-fuzzers: runs-on: - group: 4-core-ubuntu + labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -64,7 +64,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-clang-no_test_run: runs-on: - group: 8-core-ubuntu + labels: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -73,7 +73,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-clang-13-no_test_run: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -83,7 +83,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-8-no_test_run: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -93,7 +93,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-10-cxx20-no_test_run: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -103,7 +103,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-11-no_test_run: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: diff --git a/.github/workflows/jobs-linux-other-checks.yml b/.github/workflows/jobs-linux-other-checks.yml index eb1f3f0fd04..fbaf386304c 100644 --- a/.github/workflows/jobs-linux-other-checks.yml +++ b/.github/workflows/jobs-linux-other-checks.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-clang10-clang-analyze: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -19,7 +19,7 @@ jobs: path: scan_build_report.tar.gz build-linux-unity-and-headers: runs-on: - group: 4-core-ubuntu + labels: 4-core-ubuntu container: image: gcc:latest env: @@ -33,7 +33,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-mini-crashtest: runs-on: - group: 4-core-ubuntu + labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index b8ef69f52af..6df77d4b40d 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux-clang10-asan: runs-on: - group: 32-core-ubuntu + labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -13,7 +13,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-clang10-ubsan: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -23,7 +23,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-clang10-mini-tsan: runs-on: - group: 32-core-ubuntu + labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -33,7 +33,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-static_lib-alt_namespace-status_checked: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: diff --git a/.github/workflows/jobs-linux-run-tests.yml b/.github/workflows/jobs-linux-run-tests.yml index a4853bbddbb..723ac310077 100644 --- a/.github/workflows/jobs-linux-run-tests.yml +++ b/.github/workflows/jobs-linux-run-tests.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: build-linux: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -13,7 +13,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-mingw: runs-on: - group: 4-core-ubuntu + labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -30,7 +30,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -42,7 +42,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-lite-no-test: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -53,7 +53,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-7-with-folly: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -65,7 +65,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-gcc-7-with-folly-lite-no-test: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -76,7 +76,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-coroutines: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 env: @@ -91,7 +91,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-cmake-with-benchmark: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: @@ -101,7 +101,7 @@ jobs: - uses: "./.github/actions/post-steps" build-linux-encrypted_env-no_compression: runs-on: - group: 16-core-ubuntu + labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: From d197c5732882093e1af744d8ed8cfc6f060373fc Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 15 Dec 2023 11:59:18 -0800 Subject: [PATCH 06/13] Attempt fix build-linux-java-static checkout, temporary ssh --- .github/actions/post-steps/action.yml | 9 ++++++++- .github/workflows/jobs-java.yml | 2 +- .github/workflows/jobs-linux-run-tests-san.yml | 11 ++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml index 20aa8326484..a7a9f2faff4 100644 --- a/.github/actions/post-steps/action.yml +++ b/.github/actions/post-steps/action.yml @@ -2,6 +2,13 @@ name: post-steps runs: using: composite steps: + - uses: actions/checkout@v3 + - name: Setup tmate session #TEMPORARY! + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout: 60 + with: + limit-access-to-actor: true - uses: actions/upload-artifact@v3.1.3 with: path: "${{ runner.temp }}/test-results" @@ -22,4 +29,4 @@ runs: shell: bash - uses: actions/upload-artifact@v3.1.3 with: - path: "${{ runner.temp }}/core_dumps" \ No newline at end of file + path: "${{ runner.temp }}/core_dumps" diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml index 5ef779f85c7..ca4dc24b0ae 100644 --- a/.github/workflows/jobs-java.yml +++ b/.github/workflows/jobs-java.yml @@ -23,7 +23,7 @@ jobs: container: image: evolvedbinary/rocksjava:centos6_x64-be steps: - - uses: actions/checkout@v4.1.0 + - uses: actions/checkout@v3 # see https://github.com/actions/checkout/issues/1474 - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index 6df77d4b40d..7fd68ff3805 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -2,17 +2,22 @@ name: facebook/rocksdb/jobs-linux-run-tests-san on: [push, pull_request] jobs: build-linux-clang10-asan: - runs-on: + runs-on: labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" + - name: Setup tmate session #TEMPORARY! + uses: mxschmitt/action-tmate@v3 + timeout: 20 + with: + limit-access-to-actor: true - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check - uses: "./.github/actions/post-steps" build-linux-clang10-ubsan: - runs-on: + runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 @@ -22,7 +27,7 @@ jobs: - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check - uses: "./.github/actions/post-steps" build-linux-clang10-mini-tsan: - runs-on: + runs-on: labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 From a4b0bd9ee0259bac964cd8f0bb8ad91be691b0d8 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 15 Dec 2023 12:07:42 -0800 Subject: [PATCH 07/13] Fix timeout syntax, disable circleci on push --- .circleci/config.yml | 52 ------------------- .github/actions/post-steps/action.yml | 2 +- .../workflows/jobs-linux-run-tests-san.yml | 2 +- 3 files changed, 2 insertions(+), 54 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 370ad301a8d..2206f1f1b8b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -879,58 +879,6 @@ jobs: workflows: version: 2 - jobs-linux-run-tests: - jobs: - - build-linux - - build-linux-cmake-with-folly - - build-linux-cmake-with-folly-lite-no-test - - build-linux-gcc-7-with-folly - - build-linux-gcc-7-with-folly-lite-no-test - - build-linux-cmake-with-folly-coroutines - - build-linux-cmake-with-benchmark - - build-linux-encrypted_env-no_compression - jobs-linux-run-tests-san: - jobs: - - build-linux-clang10-asan - - build-linux-clang10-ubsan - - build-linux-clang10-mini-tsan - - build-linux-static_lib-alt_namespace-status_checked - jobs-linux-no-test-run: - jobs: - - build-linux-release - - build-linux-release-rtti - - build-examples - - build-fuzzers - - build-linux-clang-no_test_run - - build-linux-clang-13-no_test_run - - build-linux-gcc-8-no_test_run - - build-linux-gcc-10-cxx20-no_test_run - - build-linux-gcc-11-no_test_run - - build-linux-arm-cmake-no_test_run - jobs-linux-other-checks: - jobs: - - build-linux-clang10-clang-analyze - - build-linux-unity-and-headers - - build-linux-mini-crashtest - jobs-windows: - jobs: - - build-windows-vs2019 - - build-cmake-mingw - jobs-java: - jobs: - - build-linux-java - - build-linux-java-static - - build-macos-java - - build-macos-java-static - - build-macos-java-static-universal - - build-linux-java-pmd - jobs-macos: - jobs: - - build-macos - - build-macos-cmake: - run_even_tests: true - - build-macos-cmake: - run_even_tests: false jobs-linux-arm: jobs: - build-linux-arm diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml index a7a9f2faff4..17535e4a73d 100644 --- a/.github/actions/post-steps/action.yml +++ b/.github/actions/post-steps/action.yml @@ -6,7 +6,7 @@ runs: - name: Setup tmate session #TEMPORARY! if: ${{ failure() }} uses: mxschmitt/action-tmate@v3 - timeout: 60 + timeout-minutes: 60 with: limit-access-to-actor: true - uses: actions/upload-artifact@v3.1.3 diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index 7fd68ff3805..0217cebb890 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -11,7 +11,7 @@ jobs: - uses: "./.github/actions/pre-steps" - name: Setup tmate session #TEMPORARY! uses: mxschmitt/action-tmate@v3 - timeout: 20 + timeout-minutes: 20 with: limit-access-to-actor: true - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check From e25d443c1b0c766c3f2563c080254e30a2997969 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 15 Dec 2023 12:33:09 -0800 Subject: [PATCH 08/13] Try fix /dev/shm and weird error on timeout-minutes --- .github/actions/post-steps/action.yml | 1 - .github/workflows/jobs-java.yml | 2 ++ .github/workflows/jobs-linux-no-test-run.yml | 10 +++++++- .github/workflows/jobs-linux-other-checks.yml | 3 +++ .../workflows/jobs-linux-run-tests-san.yml | 4 +++ .github/workflows/jobs-linux-run-tests.yml | 25 +++++++++++++------ 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml index 17535e4a73d..b41bfc30860 100644 --- a/.github/actions/post-steps/action.yml +++ b/.github/actions/post-steps/action.yml @@ -6,7 +6,6 @@ runs: - name: Setup tmate session #TEMPORARY! if: ${{ failure() }} uses: mxschmitt/action-tmate@v3 - timeout-minutes: 60 with: limit-access-to-actor: true - uses: actions/upload-artifact@v3.1.3 diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml index ca4dc24b0ae..493d21db633 100644 --- a/.github/workflows/jobs-java.yml +++ b/.github/workflows/jobs-java.yml @@ -6,6 +6,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -22,6 +23,7 @@ jobs: labels: 4-core-ubuntu container: image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=2gb steps: - uses: actions/checkout@v3 # see https://github.com/actions/checkout/issues/1474 - uses: "./.github/actions/pre-steps" diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml index 963408d1e34..a90feb2a0b0 100644 --- a/.github/workflows/jobs-linux-no-test-run.yml +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -6,6 +6,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - run: make V=1 -j32 LIB_MODE=shared release @@ -30,6 +31,7 @@ jobs: labels: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench @@ -43,6 +45,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -54,6 +57,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -67,6 +71,7 @@ jobs: labels: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all @@ -76,6 +81,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -86,6 +92,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -96,6 +103,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -106,9 +114,9 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench - uses: "./.github/actions/post-steps" - diff --git a/.github/workflows/jobs-linux-other-checks.yml b/.github/workflows/jobs-linux-other-checks.yml index fbaf386304c..8c29cd470cd 100644 --- a/.github/workflows/jobs-linux-other-checks.yml +++ b/.github/workflows/jobs-linux-other-checks.yml @@ -6,6 +6,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -22,6 +23,7 @@ jobs: labels: 4-core-ubuntu container: image: gcc:latest + options: --shm-size=2gb env: EXTRA_CXXFLAGS: "-mno-avx512f" steps: @@ -36,6 +38,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index 0217cebb890..d21b946c2da 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -6,6 +6,7 @@ jobs: labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -21,6 +22,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -31,6 +33,7 @@ jobs: labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -41,6 +44,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" diff --git a/.github/workflows/jobs-linux-run-tests.yml b/.github/workflows/jobs-linux-run-tests.yml index 723ac310077..704a7131d05 100644 --- a/.github/workflows/jobs-linux-run-tests.yml +++ b/.github/workflows/jobs-linux-run-tests.yml @@ -2,20 +2,22 @@ name: facebook/rocksdb/jobs-linux-run-tests on: [push, pull_request] jobs: build-linux: - runs-on: + runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: make V=1 J=32 -j32 check - uses: "./.github/actions/post-steps" build-linux-cmake-mingw: - runs-on: + runs-on: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -29,10 +31,11 @@ jobs: mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly: - runs-on: + runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -41,10 +44,11 @@ jobs: - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-lite-no-test: - runs-on: + runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -52,10 +56,11 @@ jobs: - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)" - uses: "./.github/actions/post-steps" build-linux-gcc-7-with-folly: - runs-on: + runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -64,10 +69,11 @@ jobs: - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check - uses: "./.github/actions/post-steps" build-linux-gcc-7-with-folly-lite-no-test: - runs-on: + runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -75,10 +81,11 @@ jobs: - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-coroutines: - runs-on: + runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb env: CC: gcc-10 CXX: g++-10 @@ -90,10 +97,11 @@ jobs: - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" - uses: "./.github/actions/post-steps" build-linux-cmake-with-benchmark: - runs-on: + runs-on: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -104,6 +112,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 + options: --shm-size=2gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" From 6f1e926984aa7d53768707e1560a4d5efdabef8f Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 15 Dec 2023 13:39:19 -0800 Subject: [PATCH 09/13] Re-attempt fix shm size and checker ver --- .github/workflows/jobs-java.yml | 6 +++--- .github/workflows/jobs-linux-no-test-run.yml | 18 +++++++++--------- .github/workflows/jobs-linux-other-checks.yml | 6 +++--- .github/workflows/jobs-linux-run-tests-san.yml | 8 ++++---- .github/workflows/jobs-linux-run-tests.yml | 18 +++++++++--------- .github/workflows/nightly.yml | 6 ++++++ 6 files changed, 34 insertions(+), 28 deletions(-) diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml index 493d21db633..3c5e34a185c 100644 --- a/.github/workflows/jobs-java.yml +++ b/.github/workflows/jobs-java.yml @@ -6,7 +6,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -23,9 +23,9 @@ jobs: labels: 4-core-ubuntu container: image: evolvedbinary/rocksjava:centos6_x64-be - options: --shm-size=2gb + options: --shm-size=16gb steps: - - uses: actions/checkout@v3 # see https://github.com/actions/checkout/issues/1474 + - uses: actions/checkout@v4.0 # see https://github.com/actions/checkout/issues/1474 - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml index a90feb2a0b0..217dbcb2ea6 100644 --- a/.github/workflows/jobs-linux-no-test-run.yml +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -6,7 +6,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: make V=1 -j32 LIB_MODE=shared release @@ -31,7 +31,7 @@ jobs: labels: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench @@ -45,7 +45,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -57,7 +57,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -71,7 +71,7 @@ jobs: labels: 8-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all @@ -81,7 +81,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -92,7 +92,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -103,7 +103,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -114,7 +114,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" diff --git a/.github/workflows/jobs-linux-other-checks.yml b/.github/workflows/jobs-linux-other-checks.yml index 8c29cd470cd..61d90ea09f6 100644 --- a/.github/workflows/jobs-linux-other-checks.yml +++ b/.github/workflows/jobs-linux-other-checks.yml @@ -6,7 +6,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -23,7 +23,7 @@ jobs: labels: 4-core-ubuntu container: image: gcc:latest - options: --shm-size=2gb + options: --shm-size=16gb env: EXTRA_CXXFLAGS: "-mno-avx512f" steps: @@ -38,7 +38,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index d21b946c2da..80b9c1653f8 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -6,7 +6,7 @@ jobs: labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -22,7 +22,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -33,7 +33,7 @@ jobs: labels: 32-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -44,7 +44,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" diff --git a/.github/workflows/jobs-linux-run-tests.yml b/.github/workflows/jobs-linux-run-tests.yml index 704a7131d05..8c207f56664 100644 --- a/.github/workflows/jobs-linux-run-tests.yml +++ b/.github/workflows/jobs-linux-run-tests.yml @@ -6,7 +6,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -17,7 +17,7 @@ jobs: labels: 4-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -35,7 +35,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -48,7 +48,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -60,7 +60,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -73,7 +73,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -85,7 +85,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb env: CC: gcc-10 CXX: g++-10 @@ -101,7 +101,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -112,7 +112,7 @@ jobs: labels: 16-core-ubuntu container: image: zjay437/rocksdb:0.6 - options: --shm-size=2gb + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 52697f4591b..d1b9f01fa84 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,6 +9,7 @@ jobs: group: 2xlarge container: image: zjay437/rocksdb:0.6 + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -25,6 +26,7 @@ jobs: group: arm64large container: image: ubuntu-2004:202111-02 + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -36,6 +38,7 @@ jobs: group: 2xlarge container: image: zjay437/rocksdb:0.6 + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -46,6 +49,7 @@ jobs: group: 2xlarge container: image: zjay437/rocksdb:0.6 + options: --shm-size=16gb env: TEST_TMPDIR: "/tmp/rocksdb_test_tmp" steps: @@ -58,6 +62,7 @@ jobs: group: 2xlarge container: image: zjay437/rocksdb:0.6 + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" @@ -70,6 +75,7 @@ jobs: group: 2xlarge container: image: zjay437/rocksdb:0.6 + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" From f8ac25293c59af2b7e2e122c547c6f605888141b Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 15 Dec 2023 14:12:12 -0800 Subject: [PATCH 10/13] Try more checkout versions --- .github/workflows/jobs-java.yml | 72 ++++++++++++++++++- .../workflows/jobs-linux-run-tests-san.yml | 5 -- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml index 3c5e34a185c..e9399e405fe 100644 --- a/.github/workflows/jobs-java.yml +++ b/.github/workflows/jobs-java.yml @@ -18,14 +18,82 @@ jobs: - name: Test RocksDBJava run: make V=1 J=8 -j8 jtest - uses: "./.github/actions/post-steps" - build-linux-java-static: + build-linux-java-static27: runs-on: labels: 4-core-ubuntu container: image: evolvedbinary/rocksjava:centos6_x64-be options: --shm-size=16gb steps: - - uses: actions/checkout@v4.0 # see https://github.com/actions/checkout/issues/1474 + - uses: actions/checkout@v2.7.0 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-linux-java-static35: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v3.5.3 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-linux-java-static36: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v3.6.0 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-linux-java-static40: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.0.0 # see https://github.com/actions/checkout/issues/1474 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + build-linux-java-static41: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.1 # see https://github.com/actions/checkout/issues/1474 - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index 80b9c1653f8..e316f66ae30 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -10,11 +10,6 @@ jobs: steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - - name: Setup tmate session #TEMPORARY! - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 20 - with: - limit-access-to-actor: true - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check - uses: "./.github/actions/post-steps" build-linux-clang10-ubsan: From 9b103673f322e923c4bd1341933d7da60c89bb3e Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 15 Dec 2023 14:46:45 -0800 Subject: [PATCH 11/13] Migrate and temporary test nightly --- .github/workflows/jobs-windows.yml | 19 +------------------ .github/workflows/nightly.yml | 21 +++++++++++++++++---- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/.github/workflows/jobs-windows.yml b/.github/workflows/jobs-windows.yml index d71150d238a..cbe9d98a176 100644 --- a/.github/workflows/jobs-windows.yml +++ b/.github/workflows/jobs-windows.yml @@ -10,23 +10,7 @@ env: SNAPPY_INCLUDE: ${{ github.workspace }}/thirdparty/snappy-1.1.8;${{ github.workspace }}/thirdparty/snappy-1.1.8/build SNAPPY_LIB_DEBUG: ${{ github.workspace }}/thirdparty/snappy-1.1.8/build/Debug/snappy.lib jobs: -# TODO: move some to nightly as in a140b51 - build-windows-vs2022-avx2: - runs-on: windows-2022 - env: - CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_PORTABLE: AVX2 - steps: - - uses: actions/checkout@v4.1.0 - - uses: "./.github/actions/windows-build-steps" - build-windows-vs2022: - runs-on: windows-2022 - env: - CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_PORTABLE: 1 - steps: - - uses: actions/checkout@v4.1.0 - - uses: "./.github/actions/windows-build-steps" + # NOTE: some windows jobs are in "nightly" to save resources build-windows-vs2019: runs-on: windows-2019 env: @@ -35,4 +19,3 @@ jobs: steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/windows-build-steps" - diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d1b9f01fa84..49fa74f5d8b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,8 +1,5 @@ name: facebook/rocksdb/nightly -on: - schedule: - - cron: 0 9 * * * - workflow_dispatch: +on: [push, pull_request] # TEMPORARY: test on PR jobs: build-format-compatible: runs-on: @@ -81,3 +78,19 @@ jobs: - uses: "./.github/actions/pre-steps" - run: PORTABLE=1 make V=1 -j32 valgrind_test - uses: "./.github/actions/post-steps" + build-windows-vs2022-avx2: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: AVX2 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + build-windows-vs2022: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" From 1114c9380d9facd2abed6666cd1f741fdeb606b0 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 15 Dec 2023 15:13:53 -0800 Subject: [PATCH 12/13] Test --- .github/workflows/jobs-java.yml | 1 + .github/workflows/jobs-linux-no-test-run.yml | 1 + .github/workflows/jobs-linux-other-checks.yml | 1 + .github/workflows/jobs-windows.yml | 1 + .github/workflows/nightly.yml | 1 + 5 files changed, 5 insertions(+) diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml index e9399e405fe..a3a95565b6a 100644 --- a/.github/workflows/jobs-java.yml +++ b/.github/workflows/jobs-java.yml @@ -1,5 +1,6 @@ name: facebook/rocksdb/jobs-java on: [push, pull_request] +syntax-error: foo jobs: build-linux-java: runs-on: diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml index 217dbcb2ea6..267491b1658 100644 --- a/.github/workflows/jobs-linux-no-test-run.yml +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -2,6 +2,7 @@ name: facebook/rocksdb/jobs-linux-no-test-run on: [push, pull_request] jobs: build-linux-release: + syntax-error: foo runs-on: labels: 16-core-ubuntu container: diff --git a/.github/workflows/jobs-linux-other-checks.yml b/.github/workflows/jobs-linux-other-checks.yml index 61d90ea09f6..8f85ef52569 100644 --- a/.github/workflows/jobs-linux-other-checks.yml +++ b/.github/workflows/jobs-linux-other-checks.yml @@ -31,6 +31,7 @@ jobs: - run: apt-get update -y && apt-get install -y libgflags-dev - name: Unity build run: make V=1 -j8 unity_test + syntax-error: foo - run: make V=1 -j8 -k check-headers - uses: "./.github/actions/post-steps" build-linux-mini-crashtest: diff --git a/.github/workflows/jobs-windows.yml b/.github/workflows/jobs-windows.yml index cbe9d98a176..4f452a2c48f 100644 --- a/.github/workflows/jobs-windows.yml +++ b/.github/workflows/jobs-windows.yml @@ -12,6 +12,7 @@ env: jobs: # NOTE: some windows jobs are in "nightly" to save resources build-windows-vs2019: + syntax-error: foo runs-on: windows-2019 env: CMAKE_GENERATOR: Visual Studio 16 2019 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 49fa74f5d8b..cfdd3d85d5e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -19,6 +19,7 @@ jobs: tools/check_format_compatible.sh - uses: "./.github/actions/post-steps" build-linux-arm-test-full: + syntax-error: blah runs-on: group: arm64large container: From e35378ca25b9319f05d91c8f53310160790de8a1 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 15 Dec 2023 15:23:20 -0800 Subject: [PATCH 13/13] Tweak --- .github/workflows/jobs-linux-arm.yml | 1 + .github/workflows/jobs-linux-no-test-run.yml | 1 - .github/workflows/jobs-linux-run-tests-san.yml | 1 + .github/workflows/jobs-macos.yml | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jobs-linux-arm.yml b/.github/workflows/jobs-linux-arm.yml index 665cd3eaf08..c420e967f11 100644 --- a/.github/workflows/jobs-linux-arm.yml +++ b/.github/workflows/jobs-linux-arm.yml @@ -2,6 +2,7 @@ name: facebook/rocksdb/jobs-linux-arm on: [push, pull_request] jobs: build-linux-arm: + syntax-error: foo runs-on: labels: arm64large # GitHub hosted ARM runners do not yet exist steps: diff --git a/.github/workflows/jobs-linux-no-test-run.yml b/.github/workflows/jobs-linux-no-test-run.yml index 267491b1658..217dbcb2ea6 100644 --- a/.github/workflows/jobs-linux-no-test-run.yml +++ b/.github/workflows/jobs-linux-no-test-run.yml @@ -2,7 +2,6 @@ name: facebook/rocksdb/jobs-linux-no-test-run on: [push, pull_request] jobs: build-linux-release: - syntax-error: foo runs-on: labels: 16-core-ubuntu container: diff --git a/.github/workflows/jobs-linux-run-tests-san.yml b/.github/workflows/jobs-linux-run-tests-san.yml index e316f66ae30..ac6b757503f 100644 --- a/.github/workflows/jobs-linux-run-tests-san.yml +++ b/.github/workflows/jobs-linux-run-tests-san.yml @@ -1,6 +1,7 @@ name: facebook/rocksdb/jobs-linux-run-tests-san on: [push, pull_request] jobs: + syntax-error: foo build-linux-clang10-asan: runs-on: labels: 32-core-ubuntu diff --git a/.github/workflows/jobs-macos.yml b/.github/workflows/jobs-macos.yml index 68bc839060f..265e3d501b2 100644 --- a/.github/workflows/jobs-macos.yml +++ b/.github/workflows/jobs-macos.yml @@ -2,6 +2,7 @@ name: facebook/rocksdb/jobs-macos on: [push, pull_request] jobs: build-macos: + syntax-error: foo runs-on: macos-13 env: ROCKSDB_DISABLE_JEMALLOC: 1 @@ -40,4 +41,3 @@ jobs: run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2 if: ${{ ! matrix.run_even_tests }} - uses: "./.github/actions/post-steps" -