Update vendored DuckDB sources to 11dc00c898

CMakeLists.txt

@@ -172,6 +172,7 @@ set(DUCKDB_SRC_FILES
   src/duckdb/src/common/vector_operations/is_distinct_from.cpp
   src/duckdb/src/common/vector_operations/null_operations.cpp
   src/duckdb/src/common/vector_operations/numeric_inplace_operators.cpp
+  src/duckdb/src/common/vector_operations/scalar_executor.cpp
   src/duckdb/src/common/vector_operations/vector_cast.cpp
   src/duckdb/src/common/vector_operations/vector_copy.cpp
   src/duckdb/src/common/vector_operations/vector_hash.cpp

src/duckdb/extension/icu/datetime/calendar.cpp

@@ -832,6 +832,7 @@ void FieldCalendar::AddChecked(CalendarField field, int32_t amount) {
 	switch (field) {
 	case CAL_ERA: {
 		const auto era = GetChecked(CAL_ERA);
+		const auto year = GetChecked(CAL_YEAR);
 		if (failed) {
 			return;
 		}
@@ -842,18 +843,35 @@ void FieldCalendar::AddChecked(CalendarField field, int32_t amount) {
 		}
 		Set(CAL_ERA, sum);
 		PinField(CAL_ERA);
+		// keep the same year within the new era
+		Set(CAL_YEAR, year);
+		PinField(CAL_DATE);
 		return;
 	}
-	case CAL_YEAR:
-	case CAL_YEAR_WOY:
-		// in an era that counts backwards, later years have smaller numbers
-		if (GetChecked(CAL_ERA) == 0 && IsEra0CountingBackward()) {
-			if (!TryMultiply(amount, -1, amount)) {
-				Fail();
-				return;
-			}
+	case CAL_YEAR: {
+		// extended years continue across era boundaries
+		const auto year = GetChecked(CAL_EXTENDED_YEAR);
+		int32_t sum;
+		if (failed || !TryAdd(year, amount, sum)) {
+			Fail();
+			return;
 		}
-		DUCKDB_EXPLICIT_FALLTHROUGH;
+		Set(CAL_EXTENDED_YEAR, sum);
+		PinField(CAL_DATE);
+		return;
+	}
+	case CAL_YEAR_WOY: {
+		// week-based year numbers continue across era boundaries
+		const auto year = GetChecked(CAL_YEAR_WOY);
+		int32_t sum;
+		if (failed || !TryAdd(year, amount, sum)) {
+			Fail();
+			return;
+		}
+		Set(CAL_YEAR_WOY, sum);
+		PinField(CAL_DATE);
+		return;
+	}
 	case CAL_EXTENDED_YEAR:
 	case CAL_MONTH:
 	case CAL_ORDINAL_MONTH: {
@@ -1049,6 +1067,37 @@ int32_t FieldCalendar::FieldDifference(int64_t target, CalendarField field) {
 	failed = false;
 	int32_t min = 0;
 	const auto start = GetTimeChecked();
+	if (field == CAL_ERA) {
+		// eras can start partway through a year, so only count complete eras between the two dates
+		const auto start_era = GetChecked(CAL_ERA);
+		SetTimeChecked(target);
+		const auto target_era = GetChecked(CAL_ERA);
+		const auto difference = int64_t(target_era) - start_era;
+		if (failed) {
+			return 0;
+		}
+		if (difference < NumericLimits<int32_t>::Minimum() || difference > NumericLimits<int32_t>::Maximum()) {
+			Fail();
+			return 0;
+		}
+		// start by applying the difference between the two era numbers
+		auto result = int32_t(difference);
+		SetTimeChecked(start);
+		AddChecked(field, result);
+		const auto reached = GetTimeChecked();
+		if (failed) {
+			return 0;
+		}
+		// check whether applying the era number difference reaches the target without overshooting it
+		if (reached == target || (start < target ? reached < target : reached > target)) {
+			return result;
+		}
+		result += start < target ? -1 : 1;
+		// leave the calendar at the date reached by the returned difference
+		SetTimeChecked(start);
+		AddChecked(field, result);
+		return failed ? 0 : result;
+	}
 
 	// most differences are close to what the average length of the field predicts, which saves
 	// the search below from having to bracket the answer from scratch

src/duckdb/extension/icu/datetime/gregorian.cpp

@@ -177,11 +177,7 @@ int32_t GregorianCalendar::HandleGetExtendedYear() {
 		return InternalGet(CAL_YEAR, EPOCH_YEAR);
 	}
 	case CAL_YEAR_WOY: {
-		auto year_woy = InternalGet(CAL_YEAR_WOY);
-		if (InternalGet(CAL_ERA, AD) == BC) {
-			year_woy = 1 - year_woy;
-		}
-		return HandleGetExtendedYearFromWeekFields(year_woy, InternalGet(CAL_WEEK_OF_YEAR));
+		return HandleGetExtendedYearFromWeekFields(InternalGet(CAL_YEAR_WOY), InternalGet(CAL_WEEK_OF_YEAR));
 	}
 	default:
 		return EPOCH_YEAR;

src/duckdb/extension/icu/datetime/include/calendar.hpp

@@ -125,11 +125,6 @@ class FieldCalendar : public Calendar {
 	//! A table of groups, terminated by a null group
 	using ResolutionTable = const ResolutionGroup *;
 
-	//! Whether the years of the first era count backwards, as in the Gregorian BC era
-	virtual bool IsEra0CountingBackward() const {
-		return false;
-	}
-
 	const TimeZone &GetTimeZone() const {
 		return *zone;
 	}

src/duckdb/extension/icu/datetime/include/coptic.hpp

@@ -48,9 +48,6 @@ class CopticCalendar : public CopticEthiopicCalendar {
 	const char *GetType() const override {
 		return "coptic";
 	}
-	bool IsEra0CountingBackward() const override {
-		return true;
-	}
 	unique_ptr<Calendar> Copy() const override {
 		return unique_ptr<Calendar>(new CopticCalendar(*this));
 	}

src/duckdb/extension/icu/datetime/include/gregorian.hpp

@@ -39,9 +39,6 @@ class GregorianCalendar : public FieldCalendar {
 	const char *GetType() const override {
 		return "gregorian";
 	}
-	bool IsEra0CountingBackward() const override {
-		return true;
-	}
 	unique_ptr<Calendar> Copy() const override {
 		return unique_ptr<Calendar>(new GregorianCalendar(*this));
 	}
@@ -86,9 +83,6 @@ class BuddhistCalendar : public GregorianCalendar {
 	const char *GetType() const override {
 		return "buddhist";
 	}
-	bool IsEra0CountingBackward() const override {
-		return false;
-	}
 	unique_ptr<Calendar> Copy() const override {
 		return unique_ptr<Calendar>(new BuddhistCalendar(*this));
 	}
@@ -141,9 +135,6 @@ class ISO8601Calendar : public GregorianCalendar {
 	const char *GetType() const override {
 		return "iso8601";
 	}
-	bool IsEra0CountingBackward() const override {
-		return false;
-	}
 	unique_ptr<Calendar> Copy() const override {
 		return unique_ptr<Calendar>(new ISO8601Calendar(*this));
 	}

src/duckdb/extension/icu/datetime/include/japanese.hpp

@@ -35,9 +35,6 @@ class JapaneseCalendar : public GregorianCalendar {
 	const char *GetType() const override {
 		return "japanese";
 	}
-	bool IsEra0CountingBackward() const override {
-		return false;
-	}
 	unique_ptr<Calendar> Copy() const override {
 		return unique_ptr<Calendar>(new JapaneseCalendar(*this));
 	}

src/duckdb/extension/icu/icu-datesub.cpp

@@ -8,6 +8,18 @@
 namespace duckdb {
 
 struct ICUCalendarSub : public ICUDateFunc {
+	static unique_ptr<FunctionData> Bind(BindScalarFunctionInput &input) {
+		auto part_value = input.TryGetConstant(0);
+		if (part_value && !part_value->IsNull()) {
+			DatePartSpecifier part;
+			if (TryGetDatePartSpecifier(part_value->GetValue<string>(), part) && part == DatePartSpecifier::ERA) {
+				// date_sub is not monotone for eras because era boundaries can occur partway through a year
+				input.GetBoundFunction().SetArgProperties({});
+			}
+		}
+		return ICUDateFunc::Bind(input);
+	}
+
 	//	ICU only has 32 bit precision for date parts, so it can overflow a high resolution.
 	//	Since there is no difference between ICU and the obvious calculations,
 	//	we make these using the DuckDB internal type.
@@ -189,6 +201,20 @@ ICUDateFunc::part_sub_t ICUDateFunc::SubtractFactory(DatePartSpecifier type) {
 // MS-SQL differences can be computed using ICU by truncating both arguments
 // to the desired part precision and then applying ICU subtraction/difference
 struct ICUCalendarDiff : public ICUDateFunc {
+	static int64_t DifferenceEra(Calendar *calendar, timestamp_tz_t start_date, timestamp_tz_t end_date) {
+		SetTime(calendar, start_date);
+		const auto start_era = ExtractField(calendar, CAL_ERA);
+		SetTime(calendar, end_date);
+		return int64_t(ExtractField(calendar, CAL_ERA)) - start_era;
+	}
+
+	static int64_t DifferenceEra(Calendar *calendar, timestamp_tz_ns_t start_date, timestamp_tz_ns_t end_date) {
+		SetTimeNS(calendar, start_date);
+		const auto start_era = ExtractField(calendar, CAL_ERA);
+		SetTimeNS(calendar, end_date);
+		return int64_t(ExtractField(calendar, CAL_ERA)) - start_era;
+	}
+
 	static timestamp_tz_t TruncateForDiff(Calendar *calendar, timestamp_tz_t date, part_trunc_t trunc_func) {
 		auto micros = SetTime(calendar, date);
 		trunc_func(calendar, micros);
@@ -249,6 +275,9 @@ struct ICUCalendarDiff : public ICUDateFunc {
 				BinaryExecutor::Execute<T, T, int64_t>(
 				    startdate_arg, enddate_arg, result, [&](T start_date, T end_date) -> optional<int64_t> {
 					    if (start_date.IsFinite() && end_date.IsFinite()) {
+						    if (part == DatePartSpecifier::ERA) {
+							    return DifferenceEra(calendar, start_date, end_date);
+						    }
 						    return DifferenceFunc(calendar, start_date, end_date, trunc_func, sub_func);
 					    } else {
 						    return nullopt;
@@ -261,6 +290,9 @@ struct ICUCalendarDiff : public ICUDateFunc {
 			    [&](string_t specifier, T start_date, T end_date) -> optional<int64_t> {
 				    if (start_date.IsFinite() && end_date.IsFinite()) {
 					    const auto part = GetDatePartSpecifier(specifier.GetString());
+					    if (part == DatePartSpecifier::ERA) {
+						    return DifferenceEra(calendar, start_date, end_date);
+					    }
 					    auto trunc_func = DiffTruncationFactory(part);
 					    auto sub_func = SubtractFactory(part);
 					    return DifferenceFunc(calendar, start_date, end_date, trunc_func, sub_func);

src/duckdb/src/common/operator/cast_operators.cpp

@@ -2826,7 +2826,7 @@ bool DoubleToDecimalCast(SRC input, DST &result, CastParameters &parameters, uin
 		return false;
 	}
 	// For some reason PG does not use statistical rounding here (even though it _does_ for integers...)
-	result = Cast::Operation<SRC, DST>(static_cast<SRC>(roundedValue));
+	result = Cast::Operation<double, DST>(roundedValue);
 	return true;
 }
 

src/duckdb/src/common/vector_operations/scalar_executor.cpp

@@ -0,0 +1,26 @@
+#include "duckdb/common/vector_operations/scalar_executor.hpp"
+
+namespace duckdb {
+
+bool ScalarExecutor::PrepareGenericResultValidity(const UnifiedVectorFormat *formats, idx_t format_count,
+                                                  Vector &result, idx_t count, bool preserve_result_validity,
+                                                  bool adds_nulls) {
+	bool inputs_can_have_null = false;
+	for (idx_t input_idx = 0; input_idx < format_count; input_idx++) {
+		if (formats[input_idx].validity.CanHaveNull()) {
+			inputs_can_have_null = true;
+			break;
+		}
+	}
+
+	auto &result_validity = FlatVector::ValidityMutable(result);
+	if (inputs_can_have_null || !preserve_result_validity) {
+		result_validity.Reset(count);
+	} else if (adds_nulls && result_validity.CanHaveNull()) {
+		ValidityMask preserved(result_validity, count);
+		result_validity.Initialize(preserved);
+	}
+	return inputs_can_have_null;
+}
+
+} // namespace duckdb

src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp

@@ -246,11 +246,16 @@ void PhysicalColumnDataScan::BuildPipelines(Pipeline &current, MetaPipeline &met
 			auto &source = cte_source->Cast<PhysicalCTEConsumerSource>();
 			// Prefer direct fanout. Buffered exchange is only used when it can avoid full materialization or
 			// when the consumer can stop early; otherwise this scan reads the materialized working table.
+			auto has_blocking_dependencies = !current.GetDependencies().empty();
+			if (cte.CanRegisterDirectConsumer(current) &&
+			    (!cte.cte_body_has_side_effects || has_blocking_dependencies)) {
+				D_ASSERT(collection);
+				current.AddDependency(cte_dependency);
+				state.AddExternalInputCandidate(current, *this, *cte_source, cte_dependency, cte, source.consumer_idx);
+				state.SetPipelineSource(current, *this);
+				return;
+			}
 			if (cte.TryRegisterDirectConsumer(current, source.consumer_idx)) {
-				auto current_pipeline = current.shared_from_this();
-				current.SetExternalInput();
-				current.AddExternalFinishDependency(cte_dependency);
-				cte_dependency->AddDataflowDependency(current_pipeline);
 				DUCKDB_LOG(current.GetClientContext(), PhysicalOperatorLogType, cte, "PhysicalCTE", "SelectConsumer",
 				           {{"consumer", to_string(source.consumer_idx)}, {"mode", "DIRECT"}});
 				state.SetPipelineSource(current, *cte_source);

src/duckdb/src/execution/operator/set/physical_cte.cpp

@@ -418,12 +418,16 @@ void PhysicalCTE::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline)
 	}
 
 	children[1].get().BuildPipelines(current, meta_pipeline);
-	pipeline_selection_state = CTEPipelineSelectionState::RESOLVED;
+	auto has_unresolved_consumers = exchange && exchange->GetConsumerSummary().unresolved > 0;
+	pipeline_selection_state =
+	    has_unresolved_consumers ? CTEPipelineSelectionState::UNRESOLVED : CTEPipelineSelectionState::RESOLVED;
 
+	bool dependency_added = false;
 	if (exchange && !UseStreamingExchange()) {
-		// All exchange consumers were converted to materialized scans during pipeline construction.
+		// Keep the materialized fallback until graph-dependent consumer selection has completed.
 		auto cte_pipeline = child_meta_pipeline.GetBasePipeline();
 		current.AddDependency(cte_pipeline);
+		dependency_added = true;
 	}
 	if (exchange && last_child_ptr && !current.HasDataflowDependencies()) {
 		for (auto &side_effect_pipeline : side_effect_pipelines) {
@@ -435,19 +439,40 @@ void PhysicalCTE::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline)
 		                                       exchange ? DataflowDependencyMode::SKIP_CONFLICTING
 		                                                : DataflowDependencyMode::INCLUDE);
 	}
+	if (has_unresolved_consumers) {
+		state.AddCTEPipelineSelection(*this, current, child_meta_pipeline.GetBasePipeline(), dependency_added);
+	}
 }
 
 bool PhysicalCTE::TryRegisterDirectConsumer(Pipeline &pipeline, idx_t consumer_idx) {
-	if (!exchange) {
-		return false;
-	}
-	if (!exchange->TryRegisterDirectConsumer(pipeline, consumer_idx)) {
+	if (!CanRegisterDirectConsumer(pipeline)) {
 		return false;
 	}
-	RegisterBatchPreference(pipeline);
+	pipeline.SetExternalInput(GetProducerPipelines());
+	RegisterDirectConsumer(pipeline, consumer_idx);
 	return true;
 }
 
+bool PhysicalCTE::CanRegisterDirectConsumer(Pipeline &pipeline) const {
+	return exchange && exchange->CanRegisterDirectConsumer(pipeline);
+}
+
+void PhysicalCTE::RegisterDirectConsumer(Pipeline &pipeline, idx_t consumer_idx) {
+	D_ASSERT(exchange);
+	exchange->SelectDirectConsumer(pipeline, consumer_idx);
+	RegisterBatchPreference(pipeline);
+}
+
+vector<reference<Pipeline>> PhysicalCTE::GetProducerPipelines() const {
+	D_ASSERT(exchange);
+	return exchange->GetProducerPipelines();
+}
+
+void PhysicalCTE::SetPipelineSelectionResolved() {
+	D_ASSERT(pipeline_selection_state == CTEPipelineSelectionState::UNRESOLVED);
+	pipeline_selection_state = CTEPipelineSelectionState::RESOLVED;
+}
+
 bool PhysicalCTE::ShouldUseBufferedConsumer(Pipeline &pipeline) const {
 	if (!exchange) {
 		return false;

src/duckdb/src/execution/operator/set/physical_recursive_cte_runtime.cpp

@@ -605,7 +605,7 @@ FindInvariantRecursiveMetaPipelines(const vector<shared_ptr<MetaPipeline>> &meta
 			if (!depends_on_variant) {
 				for (auto &entry : meta_pipeline->GetDependencies()) {
 					for (auto &dependency : entry.second) {
-						auto dep_entry = pipeline_to_meta_pipeline.find(dependency.get());
+						auto dep_entry = pipeline_to_meta_pipeline.find(dependency.pipeline.get());
 						if (dep_entry == pipeline_to_meta_pipeline.end()) {
 							continue;
 						}

src/duckdb/src/execution/operator/set/physical_union.cpp

@@ -80,7 +80,9 @@ void PhysicalUnion::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipelin
 		if (order_matters || can_saturate_threads) {
 			// we add dependencies if order matters: union_pipeline comes after all pipelines created by building
 			// current
-			dependencies = meta_pipeline.AddDependenciesFrom(union_pipeline, union_pipeline, false);
+			auto dependency_type =
+			    order_matters ? MetaPipelineDependencyType::REQUIRED : MetaPipelineDependencyType::OPTIONAL_DEPENDENCY;
+			dependencies = meta_pipeline.AddDependenciesFrom(union_pipeline, union_pipeline, false, dependency_type);
 			// we also add dependencies if the LHS child can saturate all available threads
 			// in that case, we recursively make all RHS children depend on the LHS.
 			// This prevents breadth-first plan evaluation

src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "0-alpha38069"
+#define DUCKDB_PATCH_VERSION "0-alpha38143"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 0
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 2
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v2.0.0-alpha38069"
+#define DUCKDB_VERSION "v2.0.0-alpha38143"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "bcd78db0ec"
+#define DUCKDB_SOURCE_ID "11dc00c898"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"