Merge branch 'feature' into ns-strptime

.github/actions/ubuntu_18_setup/action.yml

@@ -16,6 +16,9 @@ inputs:
   ccache:
     description: 'Install ccache'
     default: 0
+  git_ref:
+    description: 'git reference to checkout'
+    default: ''
 
 runs:
   using: "composite"
@@ -27,7 +30,7 @@ runs:
         apt-get install -y -qq software-properties-common
         add-apt-repository ppa:git-core/ppa
         apt-get update -y -qq
-        apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config
+        apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config
 
     - name: Install
       shell: bash
@@ -48,6 +51,7 @@ runs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
+        ref: ${{ inputs.git_ref }}
 
     - name: Install CMake 3.21
       shell: bash

.github/patches/extensions/substrait/pushdown_semi_anti.patch

@@ -1,5 +1,5 @@
 diff --git a/src/to_substrait.cpp b/src/to_substrait.cpp
-index 90f7a67..f252aa7 100644
+index de0ffb3..ece0f5d 100644
 --- a/src/to_substrait.cpp
 +++ b/src/to_substrait.cpp
 @@ -864,7 +864,11 @@ substrait::Rel *DuckDBToSubstrait::TransformComparisonJoin(LogicalOperator &dop)
@@ -15,3 +15,26 @@ index 90f7a67..f252aa7 100644
  	}
  	sjoin->set_allocated_expression(
  	    CreateConjunction(djoin.conditions, [&](JoinCondition &in) { return TransformJoinCond(in, left_col_count); }));
+diff --git a/test/sql/test_substrait_types.test b/test/sql/test_substrait_types.test
+index 2bc47da..f3d2c65 100644
+--- a/test/sql/test_substrait_types.test
++++ b/test/sql/test_substrait_types.test
+@@ -148,6 +148,9 @@ INSERT INTO timestz VALUES ('2021-11-26 10:15:13.123456+00');
+ statement ok
+ CALL get_substrait('SELECT s FROM timestz WHERE s > ''2001-11-26 05:02:23.123456+00'' ');
+
++# not supported because TIMETZ comparisons generate a UBIGINT
++mode skip
++
+ statement ok
+ CREATE table times_tz (s TIMETZ);
+
+@@ -157,6 +160,8 @@ INSERT INTO times_tz VALUES ('05:40:00.000001');
+ statement ok
+ CALL get_substrait('SELECT s FROM times_tz WHERE s = ''05:40:00.000001'' ');
+
++mode unskip
++
+ statement ok
+ CREATE table times (s TIME);
+

.github/regression/micro_extended.csv

@@ -164,6 +164,7 @@ benchmark/micro/list/column_data_collection_copy/small_lists.benchmark
 benchmark/micro/list/column_data_collection_copy/unnest.benchmark
 benchmark/micro/list/list_few_rows.benchmark
 benchmark/micro/list/list_many_rows.benchmark
+benchmark/micro/list/list_min_max.benchmark
 benchmark/micro/list/list_order_by.benchmark
 benchmark/micro/list/list_value.benchmark
 benchmark/micro/list/string_split.benchmark

.github/workflows/LinuxRelease.yml

@@ -166,10 +166,7 @@ jobs:
        with:
         ccache: 1
         aarch64_cross_compile: 1
-
-     - name: Checkout (again)
-       shell: bash
-       run: git checkout ${{ inputs.git_ref }}
+        git_ref: ${{ inputs.git_ref }}
 
      - name: Install unixODBC
        shell: bash
@@ -224,10 +221,7 @@ jobs:
         vcpkg: 1
         openssl: 1
         ccache: 1
-
-    - name: Checkout (again)
-      shell: bash
-      run: git checkout ${{ inputs.git_ref }}
+        git_ref: ${{ inputs.git_ref }}
 
     - uses: ./.github/actions/build_extensions
       with:
@@ -268,10 +262,7 @@ jobs:
           openssl: 1
           aarch64_cross_compile: 1
           ccache: 1
-
-      - name: Checkout (again)
-        shell: bash
-        run: git checkout ${{ inputs.git_ref }}
+          git_ref: ${{ inputs.git_ref }}
 
       - uses: ./.github/actions/build_extensions
         with:

.github/workflows/Windows.yml

@@ -264,36 +264,36 @@ jobs:
          shell: msys2 {0}
          run: |
            tools/sqlite3_api_wrapper/test_sqlite3_api_wrapper.exe
-#
-# win-extensions-64:
-#   # Builds extensions for windows_amd64
-#   name: Windows Extensions (64-bit)
-#   runs-on: windows-latest
-#   needs: win-release-64
-#   steps:
-#     - name: Keep \n line endings
-#       shell: bash
-#       run: |
-#         git config --global core.autocrlf false
-#         git config --global core.eol lf
-#
-#     - uses: actions/checkout@v3
-#       with:
-#         fetch-depth: 0
-#         ref: ${{ inputs.git_ref }}
-#
-#     - uses: actions/setup-python@v5
-#       with:
-#         python-version: '3.12'
-#
-#     - uses: ./.github/actions/build_extensions
-#       with:
-#         vcpkg_target_triplet: x64-windows-static-md
-#         deploy_as: windows_amd64
-#         treat_warn_as_error: 0
-#         s3_id: ${{ secrets.S3_ID }}
-#         s3_key: ${{ secrets.S3_KEY }}
-#         signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
-#         run_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
-#         run_autoload_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
-#         unittest_script: python3 scripts/run_tests_one_by_one.py ./build/release/test/Release/unittest.exe
+
+ win-extensions-64:
+   # Builds extensions for windows_amd64
+   name: Windows Extensions (64-bit)
+   runs-on: windows-latest
+   needs: win-release-64
+   steps:
+     - name: Keep \n line endings
+       shell: bash
+       run: |
+         git config --global core.autocrlf false
+         git config --global core.eol lf
+
+     - uses: actions/checkout@v3
+       with:
+         fetch-depth: 0
+         ref: ${{ inputs.git_ref }}
+
+     - uses: actions/setup-python@v5
+       with:
+         python-version: '3.12'
+
+     - uses: ./.github/actions/build_extensions
+       with:
+         vcpkg_target_triplet: x64-windows-static-md
+         deploy_as: windows_amd64
+         treat_warn_as_error: 0
+         s3_id: ${{ secrets.S3_ID }}
+         s3_key: ${{ secrets.S3_KEY }}
+         signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
+         run_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
+         run_autoload_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
+         unittest_script: python3 scripts/run_tests_one_by_one.py ./build/release/test/Release/unittest.exe

CMakeLists.txt

@@ -48,19 +48,22 @@ else()
 endif()
 
 # Determine install paths
+# default to gnu standard installation directories (lib, bin, include)
+# https://cmake.org/cmake/help/latest/module/GNUInstallDirs.html
+include(GNUInstallDirs)
 set(INSTALL_LIB_DIR
-    lib
+    ${CMAKE_INSTALL_LIBDIR}
     CACHE PATH "Installation directory for libraries")
 set(INSTALL_BIN_DIR
-    bin
+    ${CMAKE_INSTALL_BINDIR}
     CACHE PATH "Installation directory for executables")
 set(INSTALL_INCLUDE_DIR
-    include
+    ${CMAKE_INSTALL_INCLUDEDIR}
     CACHE PATH "Installation directory for header files")
 if(WIN32 AND NOT CYGWIN)
   set(DEF_INSTALL_CMAKE_DIR cmake)
 else()
-  set(DEF_INSTALL_CMAKE_DIR lib/cmake/DuckDB)
+  set(DEF_INSTALL_CMAKE_DIR ${INSTALL_LIB_DIR}/cmake/DuckDB)
 endif()
 set(INSTALL_CMAKE_DIR
     ${DEF_INSTALL_CMAKE_DIR}
@@ -568,7 +571,7 @@ if(NOT MSVC)
   endif()
 else()
   set(CMAKE_CXX_WINDOWS_FLAGS
-      "/wd4244 /wd4267 /wd4200 /wd26451 /wd26495 /D_CRT_SECURE_NO_WARNINGS /utf-8 /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR")
+      "/wd4244 /wd4267 /wd4200 /wd26451 /wd26495 /D_CRT_SECURE_NO_WARNINGS /utf-8")
   if(TREAT_WARNINGS_AS_ERRORS)
     set(CMAKE_CXX_WINDOWS_FLAGS "${CMAKE_CXX_WINDOWS_FLAGS} /WX")
   endif()

Makefile

@@ -439,7 +439,7 @@ generate-files:
 	python3 scripts/generate_functions.py
 	python3 scripts/generate_serialization.py
 	python3 scripts/generate_enum_util.py
-	python3 tools/pythonpkg/scripts/generate_connection_code.py
+	-@python3 tools/pythonpkg/scripts/generate_connection_code.py || echo "Warning: generate_connection_code.py failed, libclang==16.0.6 is required to perform this step"
 	./scripts/generate_micro_extended.sh
 # Run the formatter again after (re)generating the files
 	$(MAKE) format-main

benchmark/micro/date/extract_month.benchmark

@@ -6,7 +6,7 @@ name Extract Month
 group date
 
 load
-CREATE TABLE dates AS SELECT DATE '1992-01-01' + concat(i % 10000, ' days')::interval AS d FROM range(0, 10000000) tbl(i);
+CREATE TABLE dates AS SELECT DATE '1992-01-01' + (i % 10000)::INT AS d FROM range(0, 1000000000) tbl(i);
 
 run
 SELECT MIN(EXTRACT(MONTH FROM d)) FROM dates

benchmark/micro/date/extract_year.benchmark

@@ -6,7 +6,7 @@ name Extract Year
 group date
 
 load
-CREATE TABLE dates AS SELECT DATE '1992-01-01' + concat(i % 10000, ' days')::interval AS d FROM range(0, 10000000) tbl(i);
+CREATE TABLE dates AS SELECT DATE '1992-01-01' + (i % 10000)::INT AS d FROM range(0, 1000000000) tbl(i);
 
 run
 SELECT MIN(EXTRACT(YEAR FROM d)) FROM dates

benchmark/micro/list/list_min_max.benchmark

@@ -0,0 +1,16 @@
+# name: benchmark/micro/list/list_min_max.benchmark
+# description: List min/max
+# group: [list]
+
+name List min/max
+group micro
+subgroup list
+
+load
+CREATE TABLE lists AS SELECT [i, i + 1, i + 2] l FROM range(100000001) t(i);
+
+run
+SELECT MIN(l), MAX(l) FROM lists;
+
+result II
+[0, 1, 2]	[100000000, 100000001, 100000002]

benchmark/micro/result_collection/batched_stream_query.benchmark

@@ -9,8 +9,9 @@ subgroup result_collection
 resultmode streaming
 
 load
+set streaming_buffer_size = '100gb';
 create table tbl (a varchar);
-insert into tbl select 'this is a long string' from range(100000000)
+insert into tbl select 'this is a long string' from range(500000000) t(i) where i % 20 == 0;
 
 run
 select * from tbl;

data/json/map_50_50.jsonl

@@ -0,0 +1,14 @@
+{
+  "a": {
+    "s1": {"f1": [1]},
+    "s2": {"f2": [1]},
+    "s3": {"f1": [1]},
+    "s4": {"f2": [1]},
+    "s5": {"f1": [1]},
+    "s6": {"f2": [1]},
+    "s7": {"f1": [1]},
+    "s8": {"f2": [1]},
+    "s9": {"f1": [1]},
+    "s10": {"f2": [1]}
+  }
+}

data/json/map_incompatible.jsonl

@@ -0,0 +1,14 @@
+{
+  "a": {
+    "s1": {"1": null},
+    "s2": {"1": {}},
+    "s3": {"1": "1"},
+    "s4": {"1": [1]},
+    "s5": {"1": 1},
+    "s6": {"1": "1"},
+    "s7": {"1": [1]},
+    "s8": {"1": 1},
+    "s9": {"1": "1"},
+    "s10": {"1": [1]}
+  }
+}

data/json/map_of_dates.jsonl

@@ -0,0 +1,29 @@
+{
+  "a": {
+    "1": null,
+    "2": "2024-01-01",
+    "3": "2024-01-01",
+    "4": "2024-01-01",
+    "5": "2024-01-01",
+    "6": "2024-01-01",
+    "7": "2024-01-01",
+    "8": "2024-01-01",
+    "9": null,
+    "10": "2024-01-01",
+    "11": "2024-01-01",
+    "12": "2024-01-01",
+    "13": "2024-01-01",
+    "14": "2024-01-01",
+    "15": "2024-01-01",
+    "16": "2024-01-01",
+    "17": "2024-01-01",
+    "18": "2024-01-01",
+    "19": "2024-01-01",
+    "20": "2024-01-01",
+    "21": "2024-01-01",
+    "22": "2024-01-01",
+    "23": "2024-01-01",
+    "24": "2024-01-01",
+    "25": "2024-01-01"
+  }
+}

data/json/map_of_map.jsonl

@@ -0,0 +1,52 @@
+{
+  "a": {
+    "l1k2": {
+      "l2k1": 1
+    },
+    "l1k1": {
+      "l2k1": 1,
+      "l2k2": 2,
+      "l2k3": 3,
+      "l2k4": 4,
+      "l2k5": 5,
+      "l2k6": 6,
+      "l2k7": 7,
+      "l2k8": 8,
+      "l2k9": 9,
+      "l2k10": 10
+    },
+    "l1k3": {
+      "l2k1": 1
+    },
+    "l1k4": {
+      "l2k1": 1
+    },
+    "l1k5": {
+      "l2k1": null,
+      "l2k2": null,
+      "l2k3": null,
+      "l2k4": null,
+      "l2k5": null,
+      "l2k6": null,
+      "l2k7": null,
+      "l2k8": null,
+      "l2k9": null,
+      "l2k10": null
+    },
+    "l1k6": {
+      "l2k1": 1
+    },
+    "l1k7": {
+      "l2k1": 1
+    },
+    "l1k8": {
+      "l2k1": 1
+    },
+    "l1k9": {
+      "l2k1": 1
+    },
+    "l1k10": {
+      "l2k1": 1
+    }
+  }
+}