Merge branch 'main' into streaming-partial-lead

hawkfish · Jun 28, 2024 · 7494f36 · 7494f36
2 parents 18b943b + c89fe10
commit 7494f36
Show file tree

Hide file tree

Showing 309 changed files with 3,723 additions and 15,264 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -12,6 +12,8 @@ body:
           * APIs:
             * [duckdb-java](https://github.com/duckdb/duckdb-java/issues/new)
             * [duckdb-node](https://github.com/duckdb/duckdb-node/issues/new)
+            * [duckdb-node-neo](https://github.com/duckdb/duckdb-node-neo/issues/new)
+            * [duckdb-odbc](https://github.com/duckdb/duckdb-odbc/issues/new)
             * [duckdb-r](https://github.com/duckdb/duckdb-r/issues/new)
             * [duckdb-rs](https://github.com/duckdb/duckdb-rs/issues/new)
             * [duckdb-wasm](https://github.com/duckdb/duckdb-wasm/issues/new)

diff --git a/.github/regression/csv.csv b/.github/regression/csv.csv
@@ -1,10 +1,10 @@
-benchmark/micro/csv/sniffer.benchmark
-benchmark/micro/csv/read.benchmark
-benchmark/micro/csv/small_csv.benchmark
-benchmark/micro/csv/null_padding.benchmark
-benchmark/micro/csv/projection_pushdown.benchmark
-benchmark/micro/csv/1_byte_values.benchmark
-benchmark/micro/csv/16_byte_values.benchmark
-benchmark/micro/csv/multiple_small_files.benchmark
-benchmark/micro/csv/time_type.benchmark
-benchmark/micro/csv/ignore_errors.benchmark
+benchmark/csv/sniffer.benchmark
+benchmark/csv/read.benchmark
+benchmark/csv/small_csv.benchmark
+benchmark/csv/null_padding.benchmark
+benchmark/csv/projection_pushdown.benchmark
+benchmark/csv/1_byte_values.benchmark
+benchmark/csv/16_byte_values.benchmark
+benchmark/csv/multiple_small_files.benchmark
+benchmark/csv/time_type.benchmark
+benchmark/csv/ignore_errors.benchmark
diff --git a/.github/regression/micro_extended.csv b/.github/regression/micro_extended.csv
@@ -77,19 +77,6 @@ benchmark/micro/copy/from_parquet_medium_row_groups.benchmark
 benchmark/micro/copy/from_parquet_small_row_groups.benchmark
 benchmark/micro/copy/to_parquet_partition_by_few.benchmark
 benchmark/micro/copy/to_parquet_partition_by_many.benchmark
-benchmark/micro/csv/16_byte_values.benchmark
-benchmark/micro/csv/1_byte_values.benchmark
-benchmark/micro/csv/ignore_errors.benchmark
-benchmark/micro/csv/multiple_read.benchmark
-benchmark/micro/csv/multiple_small_files.benchmark
-benchmark/micro/csv/multiple_small_read_csv.benchmark
-benchmark/micro/csv/null_padding.benchmark
-benchmark/micro/csv/projection_pushdown.benchmark
-benchmark/micro/csv/read.benchmark
-benchmark/micro/csv/small_csv.benchmark
-benchmark/micro/csv/sniffer.benchmark
-benchmark/micro/csv/sniffer_quotes.benchmark
-benchmark/micro/csv/time_type.benchmark
 benchmark/micro/cte/cte.benchmark
 benchmark/micro/cte/materialized_cte.benchmark
 benchmark/micro/cte/stacked_materialized_cte.benchmark
@@ -181,6 +168,8 @@ benchmark/micro/optimizer/topn_optimization.benchmark
 benchmark/micro/order/orderby.benchmark
 benchmark/micro/pushdown/or_pushdown.benchmark
 benchmark/micro/pushdown/window_partition_pushdown.benchmark
+benchmark/micro/result_collection/arrow_result.benchmark
+benchmark/micro/result_collection/batched_arrow_result.benchmark
 benchmark/micro/result_collection/batched_stream_query.benchmark
 benchmark/micro/simd/auto-vectorization.benchmark
 benchmark/micro/string/bitstring.benchmark

diff --git a/.github/workflows/CodeQuality.yml b/.github/workflows/CodeQuality.yml
@@ -49,6 +49,10 @@ jobs:
         shell: bash
         run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format-11 && sudo pip3 install cmake-format black cxxheaderparser pcpp
 
+      - name: List Installed Packages
+        shell: bash
+        run: pip3 freeze
+
       - name: Format Check
         shell: bash
         run: |

diff --git a/.github/workflows/LinuxRelease.yml b/.github/workflows/LinuxRelease.yml
@@ -25,7 +25,6 @@ on:
     paths-ignore:
       - '**.md'
       - 'tools/**'
-      - '!tools/odbc/**'
       - '!tools/shell/**'
       - '.github/patches/duckdb-wasm/**'
       - '.github/workflows/**'
@@ -36,7 +35,6 @@ on:
     paths-ignore:
       - '**.md'
       - 'tools/**'
-      - '!tools/odbc/**'
       - '!tools/shell/**'
       - '.github/patches/duckdb-wasm/**'
       - '.github/workflows/**'
@@ -63,7 +61,6 @@ jobs:
       ENABLE_EXTENSION_AUTOINSTALL: 1
       GEN: ninja
       BUILD_BENCHMARK: 1
-      BUILD_ODBC: 1
       DEBUG_STACKTRACE: 1
       FORCE_WARN_UNUSED: 1
       DUCKDB_RUN_PARALLEL_CSV_TESTS: 1
@@ -78,7 +75,6 @@ jobs:
       with:
         ninja-build: 1
         ccache: 1
-        odbc: 1
         python_alias: 1
         aws-cli: 1
 
@@ -126,16 +122,14 @@ jobs:
         zip -j duckdb_cli-linux-amd64.zip build/release/duckdb
         zip -j libduckdb-linux-amd64.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h
         zip -j libduckdb-src.zip src/amalgamation/duckdb.hpp src/amalgamation/duckdb.cpp src/include/duckdb.h
-        zip -j duckdb_odbc-linux-amd64.zip build/release/tools/odbc/libduckdb_odbc.so tools/odbc/linux_setup/unixodbc_setup.sh tools/odbc/linux_setup/update_odbc_path.py
-        ./scripts/upload-assets-to-staging.sh github_release libduckdb-src.zip libduckdb-linux-amd64.zip duckdb_cli-linux-amd64.zip duckdb_odbc-linux-amd64.zip
+        ./scripts/upload-assets-to-staging.sh github_release libduckdb-src.zip libduckdb-linux-amd64.zip duckdb_cli-linux-amd64.zip
 
     - uses: actions/upload-artifact@v3
       with:
         name: duckdb-binaries-linux
         path: |
           libduckdb-linux-amd64.zip
           duckdb_cli-linux-amd64.zip
-          duckdb_odbc-linux-amd64.zip
 
  linux-release-aarch64:
    # Builds binaries for linux_arm64
@@ -152,8 +146,6 @@ jobs:
      BUILD_BENCHMARK: 1
      TREAT_WARNINGS_AS_ERRORS: 1
      FORCE_WARN_UNUSED: 1
-     BUILD_ODBC: 1
-     ODBC_CONFIG: ../../build/unixodbc/build/bin/odbc_config
      DUCKDB_PLATFORM: linux_arm64
 
    steps:
@@ -168,15 +160,6 @@ jobs:
         aarch64_cross_compile: 1
         git_ref: ${{ inputs.git_ref }}
 
-     - name: Install unixODBC
-       shell: bash
-       run: | # we need an x86 odbc_config tool to run cmake. fun.
-         apt-get remove -y unixodbc unixodbc-dev
-         CC=gcc ./scripts/install_unixodbc.sh
-         cp build/unixodbc/build/bin/odbc_config .
-         CC=aarch64-linux-gnu-gcc ./scripts/install_unixodbc.sh --host aarch64-linux-gnu
-         cp odbc_config build/unixodbc/build/bin/odbc_config
-
      - name: Build
        shell: bash
        run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ make
@@ -189,16 +172,14 @@ jobs:
        run: |
          python3 scripts/amalgamation.py
          zip -j duckdb_cli-linux-aarch64.zip build/release/duckdb
-         zip -j duckdb_odbc-linux-aarch64.zip build/release/tools/odbc/libduckdb_odbc.so
          zip -j libduckdb-linux-aarch64.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h
-         ./scripts/upload-assets-to-staging.sh github_release libduckdb-linux-aarch64.zip duckdb_cli-linux-aarch64.zip duckdb_odbc-linux-aarch64.zip
+         ./scripts/upload-assets-to-staging.sh github_release libduckdb-linux-aarch64.zip duckdb_cli-linux-aarch64.zip
 
      - uses: actions/upload-artifact@v3
        with:
          name: duckdb-binaries-linux-aarch64
          path: |
            libduckdb-linux-aarch64.zip
-           duckdb_odbc-linux-aarch64.zip
            duckdb_cli-linux-aarch64.zip
 
  # Linux extensions for builds that use C++11 ABI, currently these are all linux builds based on ubuntu >= 18 (e.g. NodeJS)

diff --git a/.github/workflows/Main.yml b/.github/workflows/Main.yml
@@ -10,7 +10,6 @@ on:
     paths-ignore:
       - '**.md'
       - 'tools/**'
-      - '!tools/odbc/**'
       - '!tools/shell/**'
       - '.github/patches/duckdb-wasm/**'
       - '.github/workflows/**'
@@ -21,7 +20,6 @@ on:
     paths-ignore:
       - '**.md'
       - 'tools/**'
-      - '!tools/odbc/**'
       - '!tools/shell/**'
       - '.github/patches/duckdb-wasm/**'
       - '.github/workflows/**'
@@ -239,80 +237,3 @@ jobs:
       run: |
         cd duckdb-web
         python3 scripts/generate_docs.py ..
-
- odbc:
-    name: ODBC
-    if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
-    runs-on: ubuntu-22.04
-    needs: linux-debug
-    env:
-      BUILD_ODBC: 1
-      GEN: ninja
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-
-    - name: Dependencies
-      shell: bash
-      run: |
-        sudo apt-get update -y -qq
-        sudo apt-get install -y -qq ninja-build unixodbc-dev
-        pip3 install pyodbc
-
-    - name: Install nanodbc
-      shell: bash
-      run: |
-        wget https://github.com/nanodbc/nanodbc/archive/refs/tags/v2.14.0.tar.gz -O nanodbc.tgz
-        mkdir nanodbc
-        tar xvf nanodbc.tgz -C nanodbc --strip-components=1
-        cd nanodbc
-        wget https://github.com/catchorg/Catch2/releases/download/v2.13.9/catch.hpp
-        cp catch.hpp test/catch/catch.hpp
-        sed -i -e "s/set(test_list/set(test_list odbc/" test/CMakeLists.txt
-        mkdir build
-        cd build
-        cmake -DNANODBC_DISABLE_TESTS=OFF ..
-        cmake --build .
-
-    - name: Setup Ccache
-      uses: hendrikmuhs/ccache-action@main
-      with:
-        key: ${{ github.job }}
-        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
-
-    - name: Build
-      shell: bash
-      run: DISABLE_SANITIZER=1 make debug
-
-    - name: Test Standard ODBC tests
-      shell: bash
-      run:  build/debug/tools/odbc/test/test_odbc
-
-    - name: Test Connection ODBC tests
-      shell: bash
-      run: |
-        chmod +x tools/odbc/scripts/create_ini_file.sh
-        tools/odbc/scripts/create_ini_file.sh
-        build/debug/tools/odbc/test/test_connection_odbc
-
-    - name: Test nanodbc
-      shell: bash
-      run: ./tools/odbc/test/run_nanodbc_tests.sh
-
-    - name: Test isql
-      shell: bash
-      run: ./tools/odbc/test/run_isql_tests.sh
-
-    - name: Test R ODBC
-      shell: bash
-      run: R -f tools/odbc/test/rodbc.R
-
-    - name: Test Python ODBC
-      shell: bash
-      run: ./tools/odbc/test/run_pyodbc_tests.sh
diff --git a/.github/workflows/OSX.yml b/.github/workflows/OSX.yml
@@ -25,7 +25,6 @@ on:
     paths-ignore:
       - '**.md'
       - 'tools/**'
-      - '!tools/odbc/**'
       - '!tools/shell/**'
       - '.github/patches/duckdb-wasm/**'
       - '.github/workflows/**'
@@ -103,10 +102,8 @@ jobs:
       EXTENSION_CONFIGS: '${GITHUB_WORKSPACE}/.github/config/bundled_extensions.cmake'
       ENABLE_EXTENSION_AUTOLOADING: 1
       ENABLE_EXTENSION_AUTOINSTALL: 1
-      BUILD_ODBC: 1
       OSX_BUILD_UNIVERSAL: 1
       GEN: ninja
-      ODBC_CONFIG: ../../build/unixodbc/build/bin/odbc_config
 
     steps:
       - uses: actions/checkout@v3
@@ -127,11 +124,6 @@ jobs:
           key: ${{ github.job }}
           save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
 
-      - name: Install UnixODBC
-        shell: bash
-        run: |
-          CFLAGS="-arch x86_64 -arch arm64" ./scripts/install_unixodbc.sh
-
       - name: Install pytest
         run: |
           python -m pip install pytest
@@ -175,7 +167,6 @@ jobs:
 
             codesign --all-architectures --force --sign "Developer ID Application: Stichting DuckDB Foundation" build/release/duckdb
             codesign --all-architectures --force --sign "Developer ID Application: Stichting DuckDB Foundation" build/release/src/libduckdb*.dylib
-            codesign --all-architectures --force --sign "Developer ID Application: Stichting DuckDB Foundation" build/release/tools/odbc/libduckdb_odbc.dylib
           fi
 
       - name: Deploy
@@ -187,16 +178,14 @@ jobs:
           python scripts/amalgamation.py
           zip -j duckdb_cli-osx-universal.zip build/release/duckdb
           zip -j libduckdb-osx-universal.zip build/release/src/libduckdb*.dylib src/amalgamation/duckdb.hpp src/include/duckdb.h
-          zip -j duckdb_odbc-osx-universal.zip build/release/tools/odbc/libduckdb_odbc.dylib
-          ./scripts/upload-assets-to-staging.sh github_release libduckdb-osx-universal.zip duckdb_cli-osx-universal.zip duckdb_odbc-osx-universal.zip
+          ./scripts/upload-assets-to-staging.sh github_release libduckdb-osx-universal.zip duckdb_cli-osx-universal.zip
 
       - uses: actions/upload-artifact@v3
         with:
           name: duckdb-binaries-osx
           path: |
             libduckdb-osx-universal.zip
             duckdb_cli-osx-universal.zip
-            duckdb_odbc-osx-universal.zip
 
   xcode-extensions:
     # Builds extensions for osx_arm64 and osx_amd64

diff --git a/.github/workflows/R_CMD_CHECK.yml b/.github/workflows/R_CMD_CHECK.yml
@@ -22,7 +22,6 @@ on:
     paths-ignore:
       - '**.md'
       - 'tools/**'
-      - '!tools/odbc/**'
       - '.github/patches/duckdb-wasm/**'
       - '.github/workflows/**'
       - '!.github/workflows/R_CMD_CHECK.yml'