Merge branch 'master' into asof-perf #662
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LinuxRelease | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| push: | |
| branches: | |
| - '**' | |
| - '!master' | |
| - '!feature' | |
| tags: | |
| - '**' | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/odbc/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/LinuxRelease.yml' | |
| - '.github/config/uncovered_files.csv' | |
| pull_request: | |
| types: [opened, reopened, ready_for_review] | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/odbc/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/LinuxRelease.yml' | |
| - '.github/config/uncovered_files.csv' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/master' || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| jobs: | |
| linux-release-64: | |
| # Builds binaries for linux_amd64_gcc4 | |
| name: Linux (x64) | |
| runs-on: ubuntu-latest | |
| container: quay.io/pypa/manylinux2014_x86_64 | |
| env: | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| BUILD_ICU: 1 | |
| BUILD_INET: 1 | |
| BUILD_TPCH: 1 | |
| BUILD_FTS: 1 | |
| BUILD_JSON: 1 | |
| BUILD_EXCEL: 1 | |
| BUILD_ODBC: 1 | |
| BUILD_JEMALLOC: 1 | |
| DEBUG_STACKTRACE: 1 | |
| BUILD_AUTOCOMPLETE: 1 | |
| FORCE_WARN_UNUSED: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/manylinux_2014_setup | |
| with: | |
| ninja-build: 1 | |
| ccache: 1 | |
| odbc: 1 | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Print platform | |
| shell: bash | |
| run: ./build/release/duckdb -c "PRAGMA platform;" | |
| - name: Test | |
| shell: bash | |
| run: make allunit | |
| - name: Tools Tests | |
| shell: bash | |
| run: | | |
| python3.7 tools/shell/shell-test.py build/release/duckdb | |
| - name: Examples | |
| shell: bash | |
| run: | | |
| (cd examples/embedded-c; make) | |
| (cd examples/embedded-c++; make) | |
| build/release/benchmark/benchmark_runner benchmark/tpch/sf1/q01.benchmark | |
| build/release/duckdb -c "COPY (SELECT 42) TO '/dev/stdout' (FORMAT PARQUET)" | cat | |
| - name: Deploy | |
| shell: bash | |
| run: | | |
| python3.7 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-amd64.zip build/release/duckdb | |
| zip -j libduckdb-linux-amd64.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| zip -j libduckdb-src.zip src/amalgamation/duckdb.hpp src/amalgamation/duckdb.cpp src/include/duckdb.h | |
| zip -j duckdb_odbc-linux-amd64.zip build/release/tools/odbc/libduckdb_odbc.so tools/odbc/linux_setup/unixodbc_setup.sh | |
| python3.7 scripts/asset-upload-gha.py libduckdb-src.zip libduckdb-linux-amd64.zip duckdb_cli-linux-amd64.zip duckdb_odbc-linux-amd64.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux | |
| path: | | |
| libduckdb-linux-amd64.zip | |
| duckdb_cli-linux-amd64.zip | |
| linux-release-aarch64: | |
| # Builds binaries for linux_arm64 | |
| name: Linux (aarch64) | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' | |
| runs-on: ubuntu-latest | |
| needs: linux-release-64 | |
| container: ubuntu:18.04 | |
| env: | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| BUILD_ICU: 1 | |
| BUILD_INET: 1 | |
| BUILD_TPCH: 1 | |
| BUILD_FTS: 1 | |
| BUILD_JSON: 1 | |
| BUILD_EXCEL: 1 | |
| BUILD_JEMALLOC: 1 | |
| TREAT_WARNINGS_AS_ERRORS: 1 | |
| FORCE_WARN_UNUSED: 1 | |
| BUILD_ODBC: 1 | |
| ODBC_CONFIG: ../../build/unixodbc/build/bin/odbc_config | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| ccache: 1 | |
| aarch64_cross_compile: 1 | |
| - name: Install unixODBC | |
| shell: bash | |
| run: | # we need an x86 odbc_config tool to run cmake. fun. | |
| apt-get remove -y unixodbc unixodbc-dev | |
| CC=gcc ./scripts/install_unixodbc.sh | |
| cp build/unixodbc/build/bin/odbc_config . | |
| CC=aarch64-linux-gnu-gcc ./scripts/install_unixodbc.sh --host aarch64-linux-gnu | |
| cp odbc_config build/unixodbc/build/bin/odbc_config | |
| - name: Build | |
| shell: bash | |
| run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ make | |
| - name: Deploy | |
| shell: bash | |
| run: | | |
| python3.7 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-aarch64.zip build/release/duckdb | |
| zip -j duckdb_odbc-linux-aarch64.zip build/release/tools/odbc/libduckdb_odbc.so | |
| zip -j libduckdb-linux-aarch64.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| python3.7 scripts/asset-upload-gha.py libduckdb-linux-aarch64.zip duckdb_cli-linux-aarch64.zip duckdb_odbc-linux-aarch64.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux-aarch64 | |
| path: | | |
| libduckdb-linux-aarch64.zip | |
| duckdb_odbc-linux-aarch64.zip | |
| duckdb_cli-linux-aarch64.zip | |
| # Linux extensions for builds that use C++11 ABI, currently these are all linux builds based on ubuntu >= 18 (e.g. NodeJS) | |
| # note that the linux-release-64 is based on the manylinux-based extensions, which are built in .github/workflows/Python.yml | |
| linux-extensions-64: | |
| # Builds extensions for linux_amd64 | |
| name: Linux Extensions (x64) | |
| runs-on: ubuntu-latest | |
| container: ubuntu:18.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| vcpkg: 1 | |
| openssl: 1 | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| vcpkg_target_triplet: x64-linux | |
| post_install: rm build/release/src/libduckdb* | |
| deploy_as: linux_amd64 | |
| treat_warn_as_error: 0 | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: linux-extensions-64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| linux-extensions-64-aarch64: | |
| # Builds extensions for linux_arm64 | |
| name: Linux Extensions (aarch64) | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master' | |
| runs-on: ubuntu-latest | |
| container: ubuntu:18.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| vcpkg: 1 | |
| openssl: 1 | |
| aarch64_cross_compile: 1 | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| vcpkg_target_triplet: arm64-linux | |
| deploy_as: linux_arm64 | |
| treat_warn_as_error: 0 | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| aarch64_cross_compile: 1 | |
| run_tests: 0 # Cannot run tests here due to cross-compiling | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: linux-extensions-64-aarch64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| check-load-install-extensions: | |
| name: Checks extension functions | |
| runs-on: ubuntu-20.04 | |
| needs: linux-extensions-64 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| env: | |
| DISABLE_BUILTIN_EXTENSIONS: 1 | |
| run: | | |
| make | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: linux-extensions-64 | |
| path: /tmp | |
| - name: Check if extension_functions.hpp is up to date | |
| shell: bash | |
| env: | |
| EXTENSION_CONFIGS: '.github/config/in_tree_extensions.cmake;.github/config/out_of_tree_extensions.cmake' | |
| run: | | |
| make extension_configuration | |
| python scripts/generate_extensions_function.py --validate | |
| symbol-leakage: | |
| name: Symbol Leakage | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Symbol Leakage Test | |
| shell: bash | |
| run: python3.7 scripts/exported_symbols_check.py build/release/src/libduckdb*.so | |
| linux-httpfs: | |
| name: Linux HTTPFS | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| env: | |
| BUILD_VISUALIZER: 1 | |
| BUILD_HTTPFS: 1 | |
| BUILD_TPCH: 1 | |
| BUILD_TPCDS: 1 | |
| BUILD_PARQUET: 1 | |
| BUILD_JSON: 1 | |
| S3_TEST_SERVER_AVAILABLE: 1 | |
| AWS_DEFAULT_REGION: eu-west-1 | |
| AWS_ACCESS_KEY_ID: minio_duckdb_user | |
| AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password | |
| DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000 | |
| DUCKDB_S3_USE_SSL: false | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install Ninja | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Start S3/HTTP test server | |
| shell: bash | |
| run: | | |
| sudo ./scripts/install_s3_test_server.sh | |
| ./scripts/generate_presigned_url.sh | |
| source ./scripts/run_s3_test_server.sh | |
| sleep 60 | |
| - name: Test | |
| shell: bash | |
| run: | | |
| python3 scripts/get_test_list.py --file-contains 'require-env S3_TEST_SERVER_AVAILABLE 1' --list '"*"' > test.list | |
| build/release/test/unittest -f test.list | |
| amalgamation-tests: | |
| name: Amalgamation Tests | |
| needs: linux-release-64 | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CC: clang | |
| CXX: clang++ | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install LLVM and Clang | |
| uses: KyleMayes/install-llvm-action@v1 | |
| with: | |
| version: "10.0" | |
| - name: Generate Amalgamation | |
| shell: bash | |
| run: | | |
| python scripts/amalgamation.py --extended | |
| clang++ -std=c++17 -Isrc/amalgamation src/amalgamation/duckdb.cpp -emit-llvm -S -O0 |