From 14e5c9812e9bd69fc2fd9d23fe54e649a4310a9a Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Tue, 27 Aug 2024 14:36:39 +0200 Subject: [PATCH 01/11] try to reuse external workflows --- .../workflows/test-fuzzer-ci-still-works.yml | 87 +++---------------- 1 file changed, 12 insertions(+), 75 deletions(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index 8418883..c303406 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -10,88 +10,25 @@ on: jobs: build-duckdb: name: Build DuckDB - runs-on: ubuntu-latest - timeout-minutes: 120 - outputs: - duckdb-hash: ${{ steps.find-hash.outputs.hash }} - env: - BUILD_ICU: 1 - BUILD_JSON: 1 - BUILD_TPCH: 1 - BUILD_TPCDS: 1 - BUILD_PARQUET: 1 - BUILD_JEMALLOC: 1 - CRASH_ON_ASSERT: 1 - GEN: ninja - - steps: - - name: Dependencies - shell: bash - run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build ccache - - - uses: actions/checkout@v3 - with: - repository: duckdb/duckdb - fetch-depth: 0 - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - - - id: find-hash - run: echo "::set-output name=hash::$(git rev-parse HEAD)" - - - name: create build sqlsmith extension file - shell: bash - run: | - echo "duckdb_extension_load(sqlsmith - GIT_URL https://github.com/duckdb/duckdb_sqlsmith - GIT_TAG main - )" > sqlsmith.cmake - - - name: Build - shell: bash - run: | - EXTENSION_CONFIGS="sqlsmith.cmake" make debug - - - uses: actions/upload-artifact@v3 - with: - name: duckdb - path: build/debug/duckdb + uses: duckdb/duckdb-fuzzer-ci/.github/workflows/reusable_build.yml@reusable-workflows-for-fuzzer-ci + with: + git_url: ${{ github.actor }} + git_tag: ${{ github.ref_name }} + timeout-minutes: 120 fuzzer: name: Fuzzer needs: - build-duckdb - runs-on: ubuntu-latest - timeout-minutes: 20 strategy: fail-fast: false matrix: fuzzer: [duckfuzz, sqlsmith, duckfuzz_functions] data: [emptyalltypes] - - steps: - - uses: actions/checkout@v3 - with: - path: duckdb_sqlsmith - fetch-depth: 0 - - - name: Download a single artifact - uses: actions/download-artifact@v3 - with: - name: duckdb - - - name: Fuzz - shell: bash - run: | - chmod +x duckdb - runtime="1 minute" - endtime=$(date -ud "$runtime" +%s) - - cd duckdb_sqlsmith - while [[ $(date -u +%s) -le $endtime ]] - do - echo "Time Now: `date +%H:%M:%S`" - python3 scripts/run_fuzzer.py --no_checks --${{ matrix.fuzzer }} --${{ matrix.data }} --shell=../duckdb --dry --max_queries=10 - done - + uses: duckdb/duckdb-fuzzer-ci/.github/workflows/reusable_fuzzer.yml@reusable-workflows-for-fuzzer-ci + with: + fuzzer: ${{ matrix.fuzzer }} + data: ${{ matrix.data }} + timeout-minutes: 20 + dry: dry + max_queries: 10 \ No newline at end of file From cb99129f1f038bc553c49189e6a1ce9cec36b288 Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Tue, 27 Aug 2024 14:40:00 +0200 Subject: [PATCH 02/11] fix organisation name --- .github/workflows/test-fuzzer-ci-still-works.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index c303406..a72d630 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -10,7 +10,7 @@ on: jobs: build-duckdb: name: Build DuckDB - uses: duckdb/duckdb-fuzzer-ci/.github/workflows/reusable_build.yml@reusable-workflows-for-fuzzer-ci + uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_build.yml@reusable-workflows-for-fuzzer-ci with: git_url: ${{ github.actor }} git_tag: ${{ github.ref_name }} @@ -25,7 +25,7 @@ jobs: matrix: fuzzer: [duckfuzz, sqlsmith, duckfuzz_functions] data: [emptyalltypes] - uses: duckdb/duckdb-fuzzer-ci/.github/workflows/reusable_fuzzer.yml@reusable-workflows-for-fuzzer-ci + uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_fuzzer.yml@reusable-workflows-for-fuzzer-ci with: fuzzer: ${{ matrix.fuzzer }} data: ${{ matrix.data }} From 29948b02bb7ba04be752d6bfd10405786c04af29 Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Tue, 27 Aug 2024 14:43:51 +0200 Subject: [PATCH 03/11] fix the reference to the commit --- .github/workflows/test-fuzzer-ci-still-works.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index a72d630..8a7e75e 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -10,7 +10,7 @@ on: jobs: build-duckdb: name: Build DuckDB - uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_build.yml@reusable-workflows-for-fuzzer-ci + uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_build.yml@a718b91bfe729290ad0525722cd77651416d2046 with: git_url: ${{ github.actor }} git_tag: ${{ github.ref_name }} @@ -25,7 +25,7 @@ jobs: matrix: fuzzer: [duckfuzz, sqlsmith, duckfuzz_functions] data: [emptyalltypes] - uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_fuzzer.yml@reusable-workflows-for-fuzzer-ci + uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_fuzzer.yml@a718b91bfe729290ad0525722cd77651416d2046 with: fuzzer: ${{ matrix.fuzzer }} data: ${{ matrix.data }} From 1da5236ea35f56934a1bb2128a03fd312ef9d1db Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Wed, 28 Aug 2024 15:19:54 +0200 Subject: [PATCH 04/11] pass --dry --- .github/workflows/test-fuzzer-ci-still-works.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index 8a7e75e..8498b82 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -10,6 +10,7 @@ on: jobs: build-duckdb: name: Build DuckDB + # should be a valid tag, branch name or commit. After it gets merged, we can replace it with the branch name uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_build.yml@a718b91bfe729290ad0525722cd77651416d2046 with: git_url: ${{ github.actor }} @@ -30,5 +31,5 @@ jobs: fuzzer: ${{ matrix.fuzzer }} data: ${{ matrix.data }} timeout-minutes: 20 - dry: dry + dry: --dry max_queries: 10 \ No newline at end of file From a937e1fab7877c2c112de5a5e7d08acd4d3f4b88 Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Wed, 28 Aug 2024 15:21:07 +0200 Subject: [PATCH 05/11] remove dry parameter passing to make it run as it was in main --- .github/workflows/test-fuzzer-ci-still-works.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index 8498b82..4afc149 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -31,5 +31,4 @@ jobs: fuzzer: ${{ matrix.fuzzer }} data: ${{ matrix.data }} timeout-minutes: 20 - dry: --dry max_queries: 10 \ No newline at end of file From f9af73a2d0620665d84510e5ca4beb4fba0c4bcd Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Thu, 29 Aug 2024 10:28:48 +0200 Subject: [PATCH 06/11] handle passing a boolean to --dry parameter --- scripts/run_fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index 84bb256..beb97a0 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -40,7 +40,7 @@ elif param.startswith('--max_queries='): max_queries = int(param.replace('--max_queries=', '')) elif param.startswith('--dry'): - dry = True + dry = param.replace('--dry=', '').lower() == 'true' if fuzzer is None: print("Unrecognized fuzzer to run, expected e.g. --sqlsmith or --duckfuzz") From 70632e31e748016252a41b469cf3de5645fef958 Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Thu, 29 Aug 2024 10:30:12 +0200 Subject: [PATCH 07/11] update reusable workflows paths, pass dry parameter to Fuzzer --- .github/workflows/test-fuzzer-ci-still-works.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index 4afc149..ae5afb3 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -10,8 +10,7 @@ on: jobs: build-duckdb: name: Build DuckDB - # should be a valid tag, branch name or commit. After it gets merged, we can replace it with the branch name - uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_build.yml@a718b91bfe729290ad0525722cd77651416d2046 + uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/build_fuzzer.yml@main with: git_url: ${{ github.actor }} git_tag: ${{ github.ref_name }} @@ -26,9 +25,10 @@ jobs: matrix: fuzzer: [duckfuzz, sqlsmith, duckfuzz_functions] data: [emptyalltypes] - uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_fuzzer.yml@a718b91bfe729290ad0525722cd77651416d2046 + uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/fuzz_duckdb.yml@main with: fuzzer: ${{ matrix.fuzzer }} data: ${{ matrix.data }} timeout-minutes: 20 - max_queries: 10 \ No newline at end of file + max_queries: 10 + dry: false \ No newline at end of file From a17d276da3288788c04dadb392f283b73bae634e Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Thu, 29 Aug 2024 10:53:20 +0200 Subject: [PATCH 08/11] pass enable_verification to reusable Fuzzer --- .github/workflows/test-fuzzer-ci-still-works.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index ae5afb3..33c9bec 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -25,10 +25,17 @@ jobs: matrix: fuzzer: [duckfuzz, sqlsmith, duckfuzz_functions] data: [emptyalltypes] + enable_verification: [true, false] + exclude: + - enable_verification: true + fuzzer: sqlsmith + - enable_verification: true + fuzzer: duckfuzz_functions uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/fuzz_duckdb.yml@main with: fuzzer: ${{ matrix.fuzzer }} data: ${{ matrix.data }} timeout-minutes: 20 max_queries: 10 - dry: false \ No newline at end of file + dry: false + enable_verification: ${{ matrix.enable_verification }} \ No newline at end of file From c111ecb2ca13c35e98b23041998e8fddb5799cc0 Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Thu, 29 Aug 2024 17:23:36 +0200 Subject: [PATCH 09/11] remove not required parameter --dry --- .github/workflows/test-fuzzer-ci-still-works.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index 33c9bec..f67ad99 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -37,5 +37,4 @@ jobs: data: ${{ matrix.data }} timeout-minutes: 20 max_queries: 10 - dry: false enable_verification: ${{ matrix.enable_verification }} \ No newline at end of file From 72f7baf70f2466ea18e3453d9c3def004108be2d Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Thu, 29 Aug 2024 17:24:41 +0200 Subject: [PATCH 10/11] fix path to the workflows --- .github/workflows/test-fuzzer-ci-still-works.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index f67ad99..3660000 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -10,7 +10,7 @@ on: jobs: build-duckdb: name: Build DuckDB - uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/build_fuzzer.yml@main + uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_workflows/build_fuzzer.yml@main with: git_url: ${{ github.actor }} git_tag: ${{ github.ref_name }} @@ -31,7 +31,7 @@ jobs: fuzzer: sqlsmith - enable_verification: true fuzzer: duckfuzz_functions - uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/fuzz_duckdb.yml@main + uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/reusable_workflows/fuzz_duckdb.yml@main with: fuzzer: ${{ matrix.fuzzer }} data: ${{ matrix.data }} From c67e7f9a60ea377ee9399ed6d519d668c0f1d99e Mon Sep 17 00:00:00 2001 From: Zuleykha Pavlichenkova Date: Thu, 29 Aug 2024 17:32:45 +0200 Subject: [PATCH 11/11] rename --dry to --no-git-checks --- scripts/run_fuzzer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index beb97a0..0172006 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -13,7 +13,7 @@ db = None shell = None perform_checks = True -dry = False +no-git-checks = False max_queries = 1000 verification = False for param in sys.argv: @@ -39,8 +39,8 @@ seed = int(param.replace('--seed=', '')) elif param.startswith('--max_queries='): max_queries = int(param.replace('--max_queries=', '')) - elif param.startswith('--dry'): - dry = param.replace('--dry=', '').lower() == 'true' + elif param.startswith('--no-git-checks'): + no-git-checks = param.replace('--no-git-checks=', '').lower() == 'true' if fuzzer is None: print("Unrecognized fuzzer to run, expected e.g. --sqlsmith or --duckfuzz") @@ -103,7 +103,7 @@ def run_shell_command(cmd): # first get a list of all github issues, and check if we can still reproduce them -if dry: +if no-git-checks: current_errors = [] else: current_errors = fuzzer_helper.extract_github_issues(shell, perform_checks) @@ -212,5 +212,5 @@ def run_shell_command(cmd): print(f"{error_msg}") print(f"================MARKER====================") -if not dry: +if not no-git-checks: fuzzer_helper.file_issue(cmd, error_msg, fuzzer_name, seed, git_hash)