Merge branch 'duckdb:main' into median-evens

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -70,7 +70,7 @@ body:
     attributes:
       label: "OS:"
       placeholder: e.g., iOS
-      description: Please include operating system version and architecture (e.g., aarch64, x86, x64, etc)
+      description: Please include operating system version and architecture (e.g., aarch64, x86_64, etc.).
     validations:
       required: true
   - type: input
@@ -85,7 +85,12 @@ body:
       placeholder: e.g., Python
     validations:
       required: true
-
+  - type: input
+    attributes:
+      label: "Hardware:"
+      placeholder: If your issue is performance-related, please include information on your CPU and memory.
+    validations:
+      required: false
   - type: markdown
     attributes:
       value: "# Identity Disclosure:"

.github/config/out_of_tree_extensions.cmake

@@ -93,17 +93,18 @@ if (NOT MINGW)
     duckdb_extension_load(postgres_scanner
             DONT_LINK
             GIT_URL https://github.com/duckdb/postgres_scanner
-            GIT_TAG d0e0115f29a9dbe44f026aea7290a1c6d4622a73
+            GIT_TAG 58dc3d0c7620f3978c27d4a490563fd65884d103
             )
 endif()
 
 ################# SPATIAL
 duckdb_extension_load(spatial
     DONT_LINK LOAD_TESTS
     GIT_URL https://github.com/duckdb/duckdb_spatial.git
-    GIT_TAG 0bc9dff490cff60526061e7c72dceed945d3b3a3
+    GIT_TAG 58e0fcd09f2306803da36c4b1e8a66bb1e263316
     INCLUDE_DIR spatial/include
     TEST_DIR test/sql
+    APPLY_PATCHES
     )
 
 ################# SQLITE_SCANNER
@@ -117,7 +118,7 @@ endif()
 duckdb_extension_load(sqlite_scanner
         ${STATIC_LINK_SQLITE} LOAD_TESTS
         GIT_URL https://github.com/duckdb/sqlite_scanner
-        GIT_TAG 647f1403791890b65a1419841df02bf17d634639
+        GIT_TAG 315861963c8106397af36cbda10faebc8dae485a
         )
 
 duckdb_extension_load(sqlsmith
@@ -131,7 +132,7 @@ if (NOT WIN32)
     duckdb_extension_load(substrait
             LOAD_TESTS DONT_LINK
             GIT_URL https://github.com/duckdb/substrait
-            GIT_TAG 55922a3e77756054abbe3e04dae17ccf4203ad6f
+            GIT_TAG 800be4945807b831754f6b0d1a064a3d30f9cada
             )
 endif()
 
@@ -141,7 +142,7 @@ duckdb_extension_load(vss
         LOAD_TESTS
         DONT_LINK
         GIT_URL https://github.com/duckdb/duckdb_vss
-        GIT_TAG 3e192f25de97bdd759f96eeb488c59750db73937
+        GIT_TAG 77739ea5382cce3220af83803ac0b1e98b3ab7d8
         TEST_DIR test/sql
     )
 
@@ -151,6 +152,6 @@ if (NOT MINGW)
             DONT_LINK
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_mysql
-            GIT_TAG 64cb6aec994fbe441157086599c265eb86303c84
+            GIT_TAG d0c56abf1169cca1c54f55448b68a85ae4279ea4
             )
 endif()

.github/patches/extensions/spatial/buffer_manager_allocate.patch

@@ -0,0 +1,24 @@
+diff --git a/spatial/include/spatial/core/util/managed_collection.hpp b/spatial/include/spatial/core/util/managed_collection.hpp
+index 3535aff..dbf21f1 100644
+--- a/spatial/include/spatial/core/util/managed_collection.hpp
++++ b/spatial/include/spatial/core/util/managed_collection.hpp
+@@ -95,7 +95,8 @@ void ManagedCollection<T>::InitializeAppend(ManagedCollectionAppendState &state,
+ 		state.block = &blocks.back();
+ 		state.block->item_count = 0;
+ 		state.block->item_capacity = block_capacity;
+-		state.handle = manager.Allocate(MemoryTag::EXTENSION, block_size, true, &state.block->handle);
++		state.handle = manager.Allocate(MemoryTag::EXTENSION, block_size, true);
++		state.block->handle = state.handle.GetBlockHandle();
+ 	}
+ }
+
+@@ -108,7 +109,8 @@ void ManagedCollection<T>::Append(ManagedCollectionAppendState &state, const T *
+ 			state.block = &blocks.back();
+ 			state.block->item_count = 0;
+ 			state.block->item_capacity = block_capacity;
+-			state.handle = manager.Allocate(MemoryTag::EXTENSION, block_size, true, &state.block->handle);
++			state.handle = manager.Allocate(MemoryTag::EXTENSION, block_size, true);
++			state.block->handle = state.handle.GetBlockHandle();
+ 		}
+
+ 		// Compute how much we can copy before the block is full or we run out of elements

.github/workflows/NightlyTests.yml

@@ -470,6 +470,7 @@ jobs:
       DISABLE_STRING_INLINE: 1
       DESTROY_UNPINNED_BLOCKS: 1
       ALTERNATIVE_VERIFY: 1
+      LSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-leak-suppressions.txt
 
     steps:
       - uses: actions/checkout@v3
@@ -758,32 +759,40 @@ jobs:
           python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[detailed_profiler]" --no-exit --timeout 600
           python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest test/sql/tpch/tpch_sf01.test_slow --no-exit --timeout 600
 
+
   vector-sizes:
     name: Vector Sizes
     runs-on: ubuntu-20.04
     needs: linux-memory-leaks
     env:
       CC: gcc-10
       CXX: g++-10
+      GEN: ninja
+      STANDARD_VECTOR_SIZE: 2
 
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
+      - name: Install
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
 
-    - name: Setup Ccache
-      uses: hendrikmuhs/ccache-action@main
-      with:
-        key: ${{ github.job }}
-        save: ${{ env.CCACHE_SAVE }}
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+          save: ${{ env.CCACHE_SAVE }}
 
-    - name: Test
-      shell: bash
-      run: python scripts/test_vector_sizes.py
+      - name: Build
+        shell: bash
+        run: make relassert
+
+      - name: Test
+        shell: bash
+        run: |
+          python3 scripts/run_tests_one_by_one.py build/relassert/test/unittest --no-exit --time_execution
 
   block-sizes:
     name: Block Sizes
@@ -792,36 +801,45 @@ jobs:
     env:
       CC: gcc-10
       CXX: g++-10
+      GEN: ninja
       BLOCK_ALLOC_SIZE: 16384
+      BUILD_JSON: 1
+      BUILD_PARQUET: 1
 
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
+      - name: Install
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
 
       - name: Setup Ccache
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
           save: ${{ env.CCACHE_SAVE }}
 
-      - name: Build
+      - name: Build with standard vector size
         shell: bash
         run: make relassert
 
-      - name: Test
+      - name: Fast and storage tests
         shell: bash
         run: |
           python3 scripts/run_tests_one_by_one.py ./build/relassert/test/unittest --no-exit --time_execution
           python3 scripts/run_tests_one_by_one.py ./build/relassert/test/unittest "test/sql/storage/*" --no-exit --time_execution
 
-      - name: Test
+      - name: Build with vector size of 512
+        shell: bash
+        run: rm -rf ./build && rm -rf ./duckdb_unittest_tempdir && make clean && STANDARD_VECTOR_SIZE=512 make relassert
+
+      - name: Fast and storage tests
         shell: bash
-        run: python scripts/test_block_sizes.py
+        run: |
+          python3 scripts/run_tests_one_by_one.py ./build/relassert/test/unittest --no-exit --time_execution
+          python3 scripts/run_tests_one_by_one.py ./build/relassert/test/unittest "test/sql/storage/*" --no-exit --time_execution
 
   linux-wasm-experimental:
     name: WebAssembly duckdb-wasm builds

.sanitizer-leak-suppressions.txt

@@ -0,0 +1,5 @@
+# dsdgen extension global statics
+leak:load_dist
+leak:find_dist
+leak:makePermutation
+leak:init_params

CMakeLists.txt

@@ -215,6 +215,10 @@ if(${EXPLICIT_EXCEPTIONS})
   set(CXX_EXTRA "${CXX_EXTRA} -fexceptions")
 endif()
 
+if (UNSAFE_NUMERIC_CAST)
+  message(status "UNSAFE_NUMERIC_CAST")
+  add_definitions(-DUNSAFE_NUMERIC_CAST=1)
+endif()
 if (ENABLE_EXTENSION_AUTOLOADING)
   add_definitions(-DDUCKDB_EXTENSION_AUTOLOAD_DEFAULT=1)
 endif()
@@ -1375,7 +1379,8 @@ if(NOT DUCKDB_EXPLICIT_PLATFORM)
                                          ${PROJECT_BINARY_DIR})
   add_custom_target(
           duckdb_platform ALL
-          COMMAND duckdb_platform_binary > duckdb_platform_out || ( echo "Provide explicit DUCKDB_PLATFORM=your_target_arch to avoid build-type detection of the platform" && exit 1 )
+          COMMAND duckdb_platform_binary > ${PROJECT_BINARY_DIR}/duckdb_platform_out || ( echo "Provide explicit DUCKDB_PLATFORM=your_target_arch to avoid build-type detection of the platform" && exit 1 )
+          WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
           )
   add_dependencies(duckdb_platform duckdb_platform_binary)
 else()

Makefile

@@ -94,7 +94,7 @@ endif
 ifneq (${ENABLE_EXTENSION_AUTOINSTALL}, "")
 	CMAKE_VARS:=${CMAKE_VARS} -DENABLE_EXTENSION_AUTOINSTALL=${ENABLE_EXTENSION_AUTOINSTALL}
 endif
-ifneq (${UNSAFE_NUMERIC_CAST}, "")
+ifneq (${UNSAFE_NUMERIC_CAST}, )
 	CMAKE_VARS:=${CMAKE_VARS} -DUNSAFE_NUMERIC_CAST=1
 endif
 ifeq (${BUILD_EXTENSIONS_ONLY}, 1)

data/csv/error.csv

@@ -0,0 +1,4 @@
+column1,column2
+yes,no
+yes,yes,yes
+yes

data/csv/fuzzing/0.csv

@@ -0,0 +1,3 @@
+0�� testEST13	text1
+345	TEST13	'te	xt2'
+'#67'	TESt13	tex

data/csv/fuzzing/1.csv

@@ -0,0 +1,3 @@
+   bl1|2
+1690|00i\047v|2
+4a;    bla;  b  lbla;    bla;  b  la; 1a# 1;2;3;4;5

data/csv/fuzzing/11.csv

@@ -0,0 +1,7 @@
+a
+10"10-2020
+10-10-2020
+10-1-# A few comments
+# I'm a csv file
+a;-10-2020
+10-1-# A fedd

data/csv/fuzzing/15.csv

@@ -0,0 +1,7 @@
+a
+10-10-2020
+;  ; 3b
+1;2;3;4;51/-10-2020
+10-10-# A �ew commenad
+# I'm a csv fild�`;b
+# This is also a btsd

data/csv/fuzzing/16.csv

@@ -0,0 +1,7 @@
+sv file
+a;b
+# This       o a�ba
+1O-10-2020|||||u|||||||||||-# A new c@��nt}
+# I'm a csv file
+a;b
+# This is mlso a�badd

data/csv/fuzzing/17.csv

@@ -0,0 +1,8 @@
+a
+10-10-2020
+10-10-2020
+10-:02:03;0# this Ns i file
+10-# Aew com-ents
+# I'm a csv1:02:03;0# this Ns i file
+a;b
+# This is also a badd

data/csv/fuzzing/18.csv

@@ -0,0 +1,7 @@
+a
+10-10-2020
+10-10-2020
+10-10-# A few c�mments
+# I'm a csv file
+a;b
+ aThis is# lso a badd

data/csv/fuzzing/19.csv

@@ -0,0 +1,7 @@
+a&10-10-2020
+10-134fd321
+fg50a&10-10-2020
+10-10-2020
+10-10--# A few ||1
+||||||||||||||| a csv file
+a;b	# ThiW is also a Oadd

data/csv/fuzzing/2.csv

@@ -0,0 +1,5 @@
+1  bl00000|19 4|658
+10000000|a;    bla;  b  la;  ; 3b
+1;2;3;4;5
+1;2;3;4;5
+1#2;3;4;5

data/csv/fuzzing/20.csv

@@ -0,0 +1,3 @@
+@|1few comme010-10-2020
+10 10-# A 0-150-202nus
+# J'm a csv file�a;a�# Th�s is also a badd

data/csv/fuzzing/21.csv

@@ -0,0 +1,7 @@
+a
+10 I'm a csv fe
+a;b
+#
+0-10-200YYYYYYYYYY# I'm a csv file
+a;b
+# This@is also a bad

data/csv/fuzzing/31.csv

@@ -0,0 +1,10 @@
+a
+10-10-2020
+10-10-2020
+e
+a;b
+0-# A few comments
+# I'm a -10-# A few comments
+# I'm a csv file
+a;b
+# This is also a badd

data/csv/fuzzing/32.csv

@@ -0,0 +1,14 @@
+a
+1�-2020
+10-1�-2020
+10-10-2020
+10-10-# A few commea,b
+1,2
+1,2
+)
+
+����ts
+# I'm a csv file
+a;b
+## A -2020
+10-10-# A few cowmea,s �lso a badd

data/csv/fuzzing/34.csv

@@ -0,0 +1,4 @@
+2a,b,t,d,ts
+123,TEST2,12:12:12,2000.01.01,2000.01.01 12:12:00
+345,TEST2,14:15:30,2002.02.02,2002.02.02 14:15:00
+b	# This2is al1017,2004.16:00

data/csv/fuzzing/36.csv

@@ -0,0 +1,2 @@
+2.2.2;;1.2.2,twR,t2  bla  hwee,1.2.2,twR,t22,twR,t2 0
+10-10-10-#0 bla  hwee,fru

data/csv/fuzzing/37.csv

@@ -0,0 +1,2 @@
+2.2.2;:twR,t,1.2.22    bla;3 ll wi13	text1
+345		xt2'#'5teTE 						 bla   blG;-  hwee,fru

data/csv/fuzzing/4.csv

@@ -0,0 +1,3 @@
+3b
+1;2;#
+1Y3#5: