Merge branch 'feature' into summarize-temporal

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -20,11 +20,13 @@ body:
             * [Arrow extension](https://github.com/duckdb/duckdb_arrow/issues/new)
             * [AWS extension](https://github.com/duckdb/duckdb_aws/issues/new)
             * [Azure extension](https://github.com/duckdb/duckdb_azure/issues/new)
+            * [Delta extension](https://github.com/duckdb/duckdb_delta/issues/new)
             * [Iceberg extension](https://github.com/duckdb/duckdb_iceberg/issues/new)
             * [MySQL extension](https://github.com/duckdb/duckdb_mysql/issues/new)
             * [Postgres scanner](https://github.com/duckdb/postgres_scanner/issues/new)
             * [Spatial extension](https://github.com/duckdb/duckdb_spatial/issues/new)
             * [SQLite scanner](https://github.com/duckdb/sqlite_scanner/issues/new)
+            * [VSS extension](https://github.com/duckdb/duckdb_vss/issues/new)
           * Connectors:
             * [dbt-duckdb](https://github.com/duckdb/dbt-duckdb)
 

.github/workflows/CodeQuality.yml

@@ -47,7 +47,7 @@ jobs:
 
       - name: Install
         shell: bash
-        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format-11 && sudo pip3 install cmake-format black
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format-11 && sudo pip3 install cmake-format black libclang==16.0.6
 
       - name: Format Check
         shell: bash

.github/workflows/InvokeCI.yml

@@ -45,6 +45,15 @@ jobs:
       override_git_describe: ${{ inputs.override_git_describe }}
       git_ref: ${{ inputs.git_ref }}
       skip_tests: ${{ inputs.skip_tests }}
+
+  pyodide:
+    uses: ./.github/workflows/Pyodide.yml
+    secrets: inherit
+    with:
+      override_git_describe: ${{ inputs.override_git_describe }}
+      git_ref: ${{ inputs.git_ref }}
+      skip_tests: ${{ inputs.skip_tests }}
+
   R:
     uses: ./.github/workflows/R.yml
     secrets: inherit

.github/workflows/Pyodide.yml

@@ -53,6 +53,21 @@ jobs:
           # fetch everything so that the version on the built wheel path is
           # correct
           fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
+
+      - name: Check/Act on inputs.override_git_describe
+        shell: bash
+        run: |
+          if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then
+            echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string"
+            exit 1
+          elif [[ -z "${{ inputs.override_git_describe }}" ]]; then
+            echo "No override_git_describe provided"
+          else
+            echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV"
+            echo "override_git_describe ${{ inputs.override_git_describe }}: add tag"
+            git tag ${{ inputs.override_git_describe }}
+          fi
 
       - uses: actions/setup-python@v5
         with:
@@ -96,6 +111,7 @@ jobs:
           pip install ./tools/pythonpkg/dist/*.whl
 
       - name: run tests using pyodide
+        if: ${{ inputs.skip_tests != 'true' }}
         run: |
           source .venv-pyodide/bin/activate
           python -m pytest ./tools/pythonpkg/tests

.github/workflows/StagedUpload.yml

@@ -1,5 +1,9 @@
 name: Staged Upload
 on:
+  workflow_call:
+    inputs:
+      override_git_describe:
+        type: string
   workflow_dispatch:
     inputs:
       target_git_describe:

CMakeLists.txt

@@ -978,7 +978,7 @@ function(duckdb_extension_generate_version OUTPUT_VAR WORKING_DIR)
       message(FATAL_ERROR "git is available (at ${GIT_EXECUTABLE}) but has failed to execute 'log -1 --format=%h'.")
     endif()
     execute_process(
-            COMMAND ${GIT_EXECUTABLE} describe --tags
+            COMMAND ${GIT_EXECUTABLE} describe --tags --always
             WORKING_DIRECTORY ${WORKING_DIR}
             RESULT_VARIABLE GIT_RESULT
             OUTPUT_VARIABLE GIT_DESCRIBE

examples/python/duckdb-python.py

@@ -100,7 +100,7 @@
 print(rel.limit(2, offset=1))
 
 # of course these things can be chained
-print(rel.filter('i > 1').project('i + 1').order('j').limit(2))
+print(rel.filter('i > 1').project('i + 1, j').order('j').limit(2))
 
 # aggregate the relation
 print(rel.aggregate("sum(i)"))
@@ -136,7 +136,7 @@
 print(duckdb.distinct(test_df))
 
 # when chaining only the first call needs to include the data frame parameter
-print(duckdb.filter(test_df, 'i > 1').project('i + 1').order('j').limit(2))
+print(duckdb.filter(test_df, 'i > 1').project('i + 1, j').order('j').limit(2))
 
 # turn the relation into something else again
 
@@ -181,5 +181,5 @@
 print(res.df())
 
 # this also works directly on data frames
-res = duckdb.query(test_df, 'my_name_for_test_df', 'SELECT * FROM my_name_for_test_df')
+res = duckdb.query_df(test_df, 'my_name_for_test_df', 'SELECT * FROM my_name_for_test_df')
 print(res.df())

extension/README.md

@@ -170,6 +170,7 @@ duckdb_extension_load(extension_2
     GIT_TAG some_git_hash
 )
 ```
+
 Now to merge the vcpkg.json manifests from these two extension run:
 ```shell   
 make extension_configuration

extension/httpfs/httpfs_config.py

@@ -10,6 +10,13 @@
     os.path.sep.join(x.split('/'))
     for x in [
         'extension/httpfs/' + s
-        for s in ['create_secret_functions.cpp', 'httpfs_extension.cpp', 'httpfs.cpp', 's3fs.cpp', 'crypto.cpp']
+        for s in [
+            'create_secret_functions.cpp',
+            'crypto.cpp',
+            'hffs.cpp',
+            'httpfs.cpp',
+            'httpfs_extension.cpp',
+            's3fs.cpp',
+        ]
     ]
 ]

extension/parquet/parquet_extension.cpp

@@ -654,7 +654,13 @@ class ParquetScanFunction {
 		bool require_extra_columns =
 		    result->multi_file_reader_state && result->multi_file_reader_state->RequiresExtraColumns();
 		if (input.CanRemoveFilterColumns() || require_extra_columns) {
-			result->projection_ids = input.projection_ids;
+			if (!input.projection_ids.empty()) {
+				result->projection_ids = input.projection_ids;
+			} else {
+				result->projection_ids.resize(input.column_ids.size());
+				iota(begin(result->projection_ids), end(result->projection_ids), 0);
+			}
+
 			const auto table_types = bind_data.types;
 			for (const auto &col_idx : input.column_ids) {
 				if (IsRowIdColumnId(col_idx)) {

src/common/error_data.cpp

@@ -1,6 +1,6 @@
 #include "duckdb/common/error_data.hpp"
-#include "duckdb/common/exception.hpp"
 
+#include "duckdb/common/exception.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/to_string.hpp"
 #include "duckdb/common/types.hpp"
@@ -50,7 +50,10 @@ ErrorData::ErrorData(const string &message) : initialized(true), type(ExceptionT
 
 const string &ErrorData::Message() {
 	if (final_message.empty()) {
-		final_message = Exception::ExceptionTypeToString(type) + " Error: " + raw_message;
+		if (type != ExceptionType::UNKNOWN_TYPE) {
+			final_message = Exception::ExceptionTypeToString(type) + " ";
+		}
+		final_message += "Error: " + raw_message;
 		if (type == ExceptionType::INTERNAL) {
 			final_message += "\nThis error signals an assertion failure within DuckDB. This usually occurs due to "
 			                 "unexpected conditions or errors in the program's logic.\nFor more information, see "

src/common/multi_file_reader.cpp

@@ -14,6 +14,9 @@
 
 namespace duckdb {
 
+MultiFileReaderGlobalState::~MultiFileReaderGlobalState() {
+}
+
 MultiFileReader::~MultiFileReader() {
 }
 

src/core_functions/aggregate/holistic/approximate_quantile.cpp

@@ -204,16 +204,25 @@ unique_ptr<FunctionData> BindApproxQuantile(ClientContext &context, AggregateFun
 		throw BinderException("APPROXIMATE QUANTILE can only take constant quantile parameters");
 	}
 	Value quantile_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
+	if (quantile_val.IsNull()) {
+		throw BinderException("APPROXIMATE QUANTILE parameter list cannot be NULL");
+	}
 
 	vector<float> quantiles;
-	if (quantile_val.type().id() != LogicalTypeId::LIST) {
-		quantiles.push_back(CheckApproxQuantile(quantile_val));
-	} else if (quantile_val.IsNull()) {
-		throw BinderException("APPROXIMATE QUANTILE parameter list cannot be NULL");
-	} else {
+	switch (quantile_val.type().id()) {
+	case LogicalTypeId::LIST:
 		for (const auto &element_val : ListValue::GetChildren(quantile_val)) {
 			quantiles.push_back(CheckApproxQuantile(element_val));
 		}
+		break;
+	case LogicalTypeId::ARRAY:
+		for (const auto &element_val : ArrayValue::GetChildren(quantile_val)) {
+			quantiles.push_back(CheckApproxQuantile(element_val));
+		}
+		break;
+	default:
+		quantiles.push_back(CheckApproxQuantile(quantile_val));
+		break;
 	}
 
 	// remove the quantile argument so we can use the unary aggregate

src/core_functions/aggregate/holistic/quantile.cpp

@@ -1509,12 +1509,20 @@ unique_ptr<FunctionData> BindQuantile(ClientContext &context, AggregateFunction
 		throw BinderException("QUANTILE argument must not be NULL");
 	}
 	vector<Value> quantiles;
-	if (quantile_val.type().id() != LogicalTypeId::LIST) {
-		quantiles.push_back(CheckQuantile(quantile_val));
-	} else {
+	switch (quantile_val.type().id()) {
+	case LogicalTypeId::LIST:
 		for (const auto &element_val : ListValue::GetChildren(quantile_val)) {
 			quantiles.push_back(CheckQuantile(element_val));
 		}
+		break;
+	case LogicalTypeId::ARRAY:
+		for (const auto &element_val : ArrayValue::GetChildren(quantile_val)) {
+			quantiles.push_back(CheckQuantile(element_val));
+		}
+		break;
+	default:
+		quantiles.push_back(CheckQuantile(quantile_val));
+		break;
 	}
 
 	Function::EraseArgument(function, arguments, arguments.size() - 1);