Merge pull request #76 from duckdb/update-vendored-0.10.2

Update vendored DuckDB to 0.10.2

binding.gyp

@@ -166,9 +166,9 @@
                 "src/duckdb/src/verification/statement_verifier.cpp", 
                 "src/duckdb/src/verification/unoptimized_statement_verifier.cpp", 
                 "src/duckdb/third_party/fmt/format.cc", 
-                "src/duckdb/third_party/fsst/fsst_avx512.cpp", 
                 "src/duckdb/third_party/fsst/libfsst.cpp", 
                 "src/duckdb/third_party/miniz/miniz.cpp", 
+                "src/duckdb/third_party/re2/re2/bitmap256.cc", 
                 "src/duckdb/third_party/re2/re2/bitstate.cc", 
                 "src/duckdb/third_party/re2/re2/compile.cc", 
                 "src/duckdb/third_party/re2/re2/dfa.cc", 
@@ -271,6 +271,7 @@
                 "src/duckdb/third_party/zstd/compress/zstd_lazy.cpp", 
                 "src/duckdb/third_party/zstd/compress/zstd_ldm.cpp", 
                 "src/duckdb/third_party/zstd/compress/zstd_opt.cpp", 
+                "src/duckdb/third_party/lz4/lz4.cpp", 
                 "src/duckdb/extension/icu/./icu-table-range.cpp", 
                 "src/duckdb/extension/icu/./icu-makedate.cpp", 
                 "src/duckdb/extension/icu/./icu-list-range.cpp", 
@@ -301,30 +302,32 @@
             "include_dirs": [
                 "<!(node -p \"require('node-addon-api').include_dir\")", 
                 "src/duckdb/src/include", 
+                "src/duckdb/third_party/concurrentqueue", 
+                "src/duckdb/third_party/fast_float", 
+                "src/duckdb/third_party/fastpforlib", 
                 "src/duckdb/third_party/fmt/include", 
                 "src/duckdb/third_party/fsst", 
-                "src/duckdb/third_party/re2", 
-                "src/duckdb/third_party/miniz", 
-                "src/duckdb/third_party/utf8proc/include", 
-                "src/duckdb/third_party/utf8proc", 
+                "src/duckdb/third_party/httplib", 
                 "src/duckdb/third_party/hyperloglog", 
-                "src/duckdb/third_party/skiplist", 
-                "src/duckdb/third_party/fastpforlib", 
-                "src/duckdb/third_party/tdigest", 
-                "src/duckdb/third_party/libpg_query/include", 
+                "src/duckdb/third_party/jaro_winkler", 
+                "src/duckdb/third_party/jaro_winkler/details", 
                 "src/duckdb/third_party/libpg_query", 
-                "src/duckdb/third_party/concurrentqueue", 
-                "src/duckdb/third_party/pcg", 
-                "src/duckdb/third_party/httplib", 
-                "src/duckdb/third_party/fast_float", 
+                "src/duckdb/third_party/libpg_query/include", 
+                "src/duckdb/third_party/lz4", 
                 "src/duckdb/third_party/mbedtls", 
                 "src/duckdb/third_party/mbedtls/include", 
                 "src/duckdb/third_party/mbedtls/library", 
-                "src/duckdb/third_party/jaro_winkler", 
-                "src/duckdb/third_party/jaro_winkler/details", 
+                "src/duckdb/third_party/miniz", 
+                "src/duckdb/third_party/pcg", 
+                "src/duckdb/third_party/re2", 
+                "src/duckdb/third_party/skiplist", 
+                "src/duckdb/third_party/tdigest", 
+                "src/duckdb/third_party/utf8proc", 
+                "src/duckdb/third_party/utf8proc/include", 
                 "src/duckdb/extension/parquet/include", 
                 "src/duckdb/third_party/parquet", 
                 "src/duckdb/third_party/thrift", 
+                "src/duckdb/third_party/lz4", 
                 "src/duckdb/third_party/snappy", 
                 "src/duckdb/third_party/zstd/include", 
                 "src/duckdb/third_party/mbedtls", 
@@ -389,9 +392,10 @@
                     {
                         "defines": [
                             "DUCKDB_BUILD_LIBRARY"
-                        ],
+                        ], 
                         "libraries": [
-                            "rstrtmgr.lib", "bcrypt.lib"
+                            "rstrtmgr.lib", 
+                            "bcrypt.lib"
                         ]
                     }
                 ]
@@ -414,4 +418,4 @@
             ]
         }
     ]
-}
+}

binding.gyp.in

@@ -62,6 +62,9 @@
                     {
                         "defines": [
                             "DUCKDB_BUILD_LIBRARY"
+                        ],
+                        "libraries": [
+                            "rstrtmgr.lib", "bcrypt.lib"
                         ]
                     }
                 ]

src/duckdb/extension/icu/icu-timezone.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/exception/conversion_exception.hpp"
 #include "duckdb/function/cast/cast_function_set.hpp"
+#include "duckdb/function/cast_rules.hpp"
 #include "duckdb/main/extension_util.hpp"
 #include "duckdb/parser/parsed_data/create_scalar_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_table_function_info.hpp"
@@ -183,7 +184,8 @@ struct ICUFromNaiveTimestamp : public ICUDateFunc {
 		auto &config = DBConfig::GetConfig(db);
 		auto &casts = config.GetCastFunctions();
 
-		casts.RegisterCastFunction(LogicalType::TIMESTAMP, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
+		const auto implicit_cost = CastRules::ImplicitCast(LogicalType::TIMESTAMP, LogicalType::TIMESTAMP_TZ);
+		casts.RegisterCastFunction(LogicalType::TIMESTAMP, LogicalType::TIMESTAMP_TZ, BindCastFromNaive, implicit_cost);
 		casts.RegisterCastFunction(LogicalType::TIMESTAMP_MS, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
 		casts.RegisterCastFunction(LogicalType::TIMESTAMP_NS, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
 		casts.RegisterCastFunction(LogicalType::TIMESTAMP_S, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);

src/duckdb/extension/icu/icu_extension.cpp

@@ -221,7 +221,7 @@ static void SetICUCalendar(ClientContext &context, SetScope scope, Value &parame
 	}
 }
 
-void IcuExtension::Load(DuckDB &ddb) {
+static void LoadInternal(DuckDB &ddb) {
 	auto &db = *ddb.instance;
 
 	// iterate over all the collations
@@ -275,6 +275,10 @@ void IcuExtension::Load(DuckDB &ddb) {
 	ExtensionUtil::RegisterFunction(db, cal_names);
 }
 
+void IcuExtension::Load(DuckDB &ddb) {
+	LoadInternal(ddb);
+}
+
 std::string IcuExtension::Name() {
 	return "icu";
 }
@@ -285,7 +289,7 @@ extern "C" {
 
 DUCKDB_EXTENSION_API void icu_init(duckdb::DatabaseInstance &db) { // NOLINT
 	duckdb::DuckDB db_wrapper(db);
-	db_wrapper.LoadExtension<duckdb::IcuExtension>();
+	duckdb::LoadInternal(db_wrapper);
 }
 
 DUCKDB_EXTENSION_API const char *icu_version() { // NOLINT

src/duckdb/extension/json/buffered_json_reader.cpp

@@ -44,6 +44,10 @@ bool JSONFileHandle::RequestedReadsComplete() {
 	return requested_reads == actual_reads;
 }
 
+bool JSONFileHandle::LastReadRequested() const {
+	return last_read_requested;
+}
+
 idx_t JSONFileHandle::FileSize() const {
 	return file_size;
 }
@@ -56,6 +60,10 @@ bool JSONFileHandle::CanSeek() const {
 	return can_seek;
 }
 
+bool JSONFileHandle::IsPipe() const {
+	return file_handle->IsPipe();
+}
+
 FileHandle &JSONFileHandle::GetHandle() {
 	return *file_handle;
 }
@@ -193,9 +201,8 @@ BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReade
 void BufferedJSONReader::OpenJSONFile() {
 	lock_guard<mutex> guard(lock);
 	if (!IsOpen()) {
-		auto &file_system = FileSystem::GetFileSystem(context);
-		auto regular_file_handle = file_system.OpenFile(file_name.c_str(), FileFlags::FILE_FLAGS_READ,
-		                                                FileLockType::NO_LOCK, options.compression);
+		auto &fs = FileSystem::GetFileSystem(context);
+		auto regular_file_handle = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ | options.compression);
 		file_handle = make_uniq<JSONFileHandle>(std::move(regular_file_handle), BufferAllocator::Get(context));
 	}
 	Reset();

src/duckdb/extension/json/include/buffered_json_reader.hpp

@@ -60,11 +60,13 @@ struct JSONFileHandle {
 
 	void Reset();
 	bool RequestedReadsComplete();
+	bool LastReadRequested() const;
 
 	idx_t FileSize() const;
 	idx_t Remaining() const;
 
 	bool CanSeek() const;
+	bool IsPipe() const;
 
 	FileHandle &GetHandle();
 

src/duckdb/extension/json/include/json_scan.hpp

@@ -9,13 +9,13 @@
 #pragma once
 
 #include "buffered_json_reader.hpp"
-#include "json_enums.hpp"
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/types/type_map.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
 #include "duckdb/function/table_function.hpp"
+#include "json_enums.hpp"
 #include "json_transform.hpp"
 
 namespace duckdb {
@@ -226,14 +226,20 @@ struct JSONScanLocalState {
 
 private:
 	bool ReadNextBuffer(JSONScanGlobalState &gstate);
-	bool ReadNextBufferInternal(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
-	bool ReadNextBufferSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
-	bool ReadNextBufferNoSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
+	bool ReadNextBufferInternal(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
+	                            bool &file_done);
+	bool ReadNextBufferSeek(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
+	                        bool &file_done);
+	bool ReadNextBufferNoSeek(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
+	                          bool &file_done);
+	AllocatedData AllocateBuffer(JSONScanGlobalState &gstate);
+	data_ptr_t GetReconstructBuffer(JSONScanGlobalState &gstate);
+
 	void SkipOverArrayStart();
 
-	void ReadAndAutoDetect(JSONScanGlobalState &gstate, optional_idx &buffer_index);
-	bool ReconstructFirstObject();
-	void ParseNextChunk();
+	void ReadAndAutoDetect(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index);
+	bool ReconstructFirstObject(JSONScanGlobalState &gstate);
+	void ParseNextChunk(JSONScanGlobalState &gstate);
 
 	void ParseJSON(char *const json_start, const idx_t json_size, const idx_t remaining);
 	void ThrowObjectSizeError(const idx_t object_size);

src/duckdb/extension/json/include/json_serializer.hpp

@@ -25,15 +25,16 @@ struct JsonSerializer : Serializer {
 	void PushValue(yyjson_mut_val *val);
 
 public:
-	explicit JsonSerializer(yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty)
+	explicit JsonSerializer(yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty, bool skip_if_default)
 	    : doc(doc), stack({yyjson_mut_obj(doc)}), skip_if_null(skip_if_null), skip_if_empty(skip_if_empty) {
 		serialize_enum_as_string = true;
-		serialize_default_values = true;
+		serialize_default_values = !skip_if_default;
 	}
 
 	template <class T>
-	static yyjson_mut_val *Serialize(T &value, yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty) {
-		JsonSerializer serializer(doc, skip_if_null, skip_if_empty);
+	static yyjson_mut_val *Serialize(T &value, yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty,
+	                                 bool skip_if_default) {
+		JsonSerializer serializer(doc, skip_if_null, skip_if_empty, skip_if_default);
 		value.Serialize(serializer);
 		return serializer.GetRootObject();
 	}

src/duckdb/extension/json/include/json_structure.hpp

@@ -19,7 +19,7 @@ struct StrpTimeFormat;
 struct JSONStructureNode {
 public:
 	JSONStructureNode();
-	JSONStructureNode(yyjson_val *key_p, yyjson_val *val_p);
+	JSONStructureNode(yyjson_val *key_p, yyjson_val *val_p, const bool ignore_errors);
 
 	//! Disable copy constructors
 	JSONStructureNode(const JSONStructureNode &other) = delete;
@@ -64,7 +64,7 @@ struct JSONStructureDescription {
 	JSONStructureDescription &operator=(JSONStructureDescription &&) noexcept;
 
 	JSONStructureNode &GetOrCreateChild();
-	JSONStructureNode &GetOrCreateChild(yyjson_val *key, yyjson_val *val);
+	JSONStructureNode &GetOrCreateChild(yyjson_val *key, yyjson_val *val, const bool ignore_errors);
 
 public:
 	//! Type of this description
@@ -80,7 +80,7 @@ struct JSONStructureDescription {
 
 struct JSONStructure {
 public:
-	static void ExtractStructure(yyjson_val *val, JSONStructureNode &node);
+	static void ExtractStructure(yyjson_val *val, JSONStructureNode &node, const bool ignore_errors);
 	static LogicalType StructureToType(ClientContext &context, const JSONStructureNode &node, const idx_t max_depth,
 	                                   const double field_appearance_threshold, idx_t depth = 0,
 	                                   idx_t sample_count = DConstants::INVALID_INDEX);

src/duckdb/extension/json/json_functions.cpp

@@ -15,6 +15,9 @@ namespace duckdb {
 using JSONPathType = JSONCommon::JSONPathType;
 
 static JSONPathType CheckPath(const Value &path_val, string &path, size_t &len) {
+	if (path_val.IsNull()) {
+		throw InternalException("JSON path cannot be NULL");
+	}
 	const auto path_str_val = path_val.DefaultCastAs(LogicalType::VARCHAR);
 	auto path_str = path_str_val.GetValueUnsafe<string_t>();
 	len = path_str.GetSize();
@@ -58,9 +61,11 @@ unique_ptr<FunctionData> JSONReadFunctionData::Bind(ClientContext &context, Scal
 	size_t len = 0;
 	JSONPathType path_type = JSONPathType::REGULAR;
 	if (arguments[1]->IsFoldable()) {
-		constant = true;
 		const auto path_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
-		path_type = CheckPath(path_val, path, len);
+		if (!path_val.IsNull()) {
+			constant = true;
+			path_type = CheckPath(path_val, path, len);
+		}
 	}
 	bound_function.arguments[1] = LogicalType::VARCHAR;
 	if (path_type == JSONCommon::JSONPathType::WILDCARD) {

src/duckdb/extension/json/json_functions/json_serialize_plan.cpp

@@ -19,16 +19,19 @@ namespace duckdb {
 struct JsonSerializePlanBindData : public FunctionData {
 	bool skip_if_null = false;
 	bool skip_if_empty = false;
+	bool skip_if_default = false;
 	bool format = false;
 	bool optimize = false;
 
-	JsonSerializePlanBindData(bool skip_if_null_p, bool skip_if_empty_p, bool format_p, bool optimize_p)
-	    : skip_if_null(skip_if_null_p), skip_if_empty(skip_if_empty_p), format(format_p), optimize(optimize_p) {
+	JsonSerializePlanBindData(bool skip_if_null_p, bool skip_if_empty_p, bool skip_if_default_p, bool format_p,
+	                          bool optimize_p)
+	    : skip_if_null(skip_if_null_p), skip_if_empty(skip_if_empty_p), skip_if_default(skip_if_default_p),
+	      format(format_p), optimize(optimize_p) {
 	}
 
 public:
 	unique_ptr<FunctionData> Copy() const override {
-		return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, format, optimize);
+		return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, skip_if_default, format, optimize);
 	}
 	bool Equals(const FunctionData &other_p) const override {
 		return true;
@@ -48,6 +51,7 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
 	// Optional arguments
 	bool skip_if_null = false;
 	bool skip_if_empty = false;
+	bool skip_if_default = false;
 	bool format = false;
 	bool optimize = false;
 
@@ -69,6 +73,11 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
 				throw BinderException("json_serialize_plan: 'skip_empty' argument must be a boolean");
 			}
 			skip_if_empty = BooleanValue::Get(ExpressionExecutor::EvaluateScalar(context, *arg));
+		} else if (arg->alias == "skip_default") {
+			if (arg->return_type.id() != LogicalTypeId::BOOLEAN) {
+				throw BinderException("json_serialize_plan: 'skip_default' argument must be a boolean");
+			}
+			skip_if_default = BooleanValue::Get(ExpressionExecutor::EvaluateScalar(context, *arg));
 		} else if (arg->alias == "format") {
 			if (arg->return_type.id() != LogicalTypeId::BOOLEAN) {
 				throw BinderException("json_serialize_plan: 'format' argument must be a boolean");
@@ -83,7 +92,7 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
 			throw BinderException(StringUtil::Format("json_serialize_plan: Unknown argument '%s'", arg->alias.c_str()));
 		}
 	}
-	return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, format, optimize);
+	return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, skip_if_default, format, optimize);
 }
 
 static bool OperatorSupportsSerialization(LogicalOperator &op, string &operator_name) {
@@ -144,7 +153,8 @@ static void JsonSerializePlanFunction(DataChunk &args, ExpressionState &state, V
 					throw InvalidInputException("Operator '%s' does not support serialization", operator_name);
 				}
 
-				auto plan_json = JsonSerializer::Serialize(*plan, doc, info.skip_if_null, info.skip_if_empty);
+				auto plan_json =
+				    JsonSerializer::Serialize(*plan, doc, info.skip_if_null, info.skip_if_empty, info.skip_if_default);
 				yyjson_mut_arr_append(plans_arr, plan_json);
 			}