Update vendored DuckDB sources to a14d147c9f

src/duckdb/src/common/enum_util.cpp

@@ -1502,6 +1502,25 @@ DefaultOrderByNullType EnumUtil::FromString<DefaultOrderByNullType>(const char *
 	return static_cast<DefaultOrderByNullType>(StringUtil::StringToEnum(GetDefaultOrderByNullTypeValues(), 5, "DefaultOrderByNullType", value));
 }
 
+const StringUtil::EnumStringLiteral *GetDeleteIdStateValues() {
+	static constexpr StringUtil::EnumStringLiteral values[] {
+		{ static_cast<uint32_t>(DeleteIdState::CONSTANT), "CONSTANT" },
+		{ static_cast<uint32_t>(DeleteIdState::MASKED), "MASKED" },
+		{ static_cast<uint32_t>(DeleteIdState::ARRAY), "ARRAY" }
+	};
+	return values;
+}
+
+template<>
+const char* EnumUtil::ToChars<DeleteIdState>(DeleteIdState value) {
+	return StringUtil::EnumToString(GetDeleteIdStateValues(), 3, "DeleteIdState", static_cast<uint32_t>(value));
+}
+
+template<>
+DeleteIdState EnumUtil::FromString<DeleteIdState>(const char *value) {
+	return static_cast<DeleteIdState>(StringUtil::StringToEnum(GetDeleteIdStateValues(), 3, "DeleteIdState", value));
+}
+
 const StringUtil::EnumStringLiteral *GetDependencyEntryTypeValues() {
 	static constexpr StringUtil::EnumStringLiteral values[] {
 		{ static_cast<uint32_t>(DependencyEntryType::SUBJECT), "SUBJECT" },
@@ -5542,6 +5561,25 @@ VerifyExistenceType EnumUtil::FromString<VerifyExistenceType>(const char *value)
 	return static_cast<VerifyExistenceType>(StringUtil::StringToEnum(GetVerifyExistenceTypeValues(), 3, "VerifyExistenceType", value));
 }
 
+const StringUtil::EnumStringLiteral *GetVersionCompressionResultValues() {
+	static constexpr StringUtil::EnumStringLiteral values[] {
+		{ static_cast<uint32_t>(VersionCompressionResult::FULLY_COMPRESSED), "FULLY_COMPRESSED" },
+		{ static_cast<uint32_t>(VersionCompressionResult::PENDING), "PENDING" },
+		{ static_cast<uint32_t>(VersionCompressionResult::SETTLED), "SETTLED" }
+	};
+	return values;
+}
+
+template<>
+const char* EnumUtil::ToChars<VersionCompressionResult>(VersionCompressionResult value) {
+	return StringUtil::EnumToString(GetVersionCompressionResultValues(), 3, "VersionCompressionResult", static_cast<uint32_t>(value));
+}
+
+template<>
+VersionCompressionResult EnumUtil::FromString<VersionCompressionResult>(const char *value) {
+	return static_cast<VersionCompressionResult>(StringUtil::StringToEnum(GetVersionCompressionResultValues(), 3, "VersionCompressionResult", value));
+}
+
 const StringUtil::EnumStringLiteral *GetVertexTypeValues() {
 	static constexpr StringUtil::EnumStringLiteral values[] {
 		{ static_cast<uint32_t>(VertexType::XY), "XY" },

src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "6-dev72"
+#define DUCKDB_PATCH_VERSION "6-dev80"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 5
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.5.6-dev72"
+#define DUCKDB_VERSION "v1.5.6-dev80"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "52b98657c0"
+#define DUCKDB_SOURCE_ID "a14d147c9f"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

src/duckdb/src/include/duckdb/common/enum_util.hpp

@@ -158,6 +158,8 @@ enum class DecimalBitWidth : uint8_t;
 
 enum class DefaultOrderByNullType : uint8_t;
 
+enum class DeleteIdState : uint8_t;
+
 enum class DependencyEntryType : uint8_t;
 
 enum class DeprecatedIndexType : uint8_t;
@@ -494,6 +496,8 @@ enum class VerificationType : uint8_t;
 
 enum class VerifyExistenceType : uint8_t;
 
+enum class VersionCompressionResult : uint8_t;
+
 enum class VertexType : uint8_t;
 
 enum class WALType : uint8_t;
@@ -696,6 +700,9 @@ const char* EnumUtil::ToChars<DecimalBitWidth>(DecimalBitWidth value);
 template<>
 const char* EnumUtil::ToChars<DefaultOrderByNullType>(DefaultOrderByNullType value);
 
+template<>
+const char* EnumUtil::ToChars<DeleteIdState>(DeleteIdState value);
+
 template<>
 const char* EnumUtil::ToChars<DependencyEntryType>(DependencyEntryType value);
 
@@ -1200,6 +1207,9 @@ const char* EnumUtil::ToChars<VerificationType>(VerificationType value);
 template<>
 const char* EnumUtil::ToChars<VerifyExistenceType>(VerifyExistenceType value);
 
+template<>
+const char* EnumUtil::ToChars<VersionCompressionResult>(VersionCompressionResult value);
+
 template<>
 const char* EnumUtil::ToChars<VertexType>(VertexType value);
 
@@ -1408,6 +1418,9 @@ DecimalBitWidth EnumUtil::FromString<DecimalBitWidth>(const char *value);
 template<>
 DefaultOrderByNullType EnumUtil::FromString<DefaultOrderByNullType>(const char *value);
 
+template<>
+DeleteIdState EnumUtil::FromString<DeleteIdState>(const char *value);
+
 template<>
 DependencyEntryType EnumUtil::FromString<DependencyEntryType>(const char *value);
 
@@ -1912,6 +1925,9 @@ VerificationType EnumUtil::FromString<VerificationType>(const char *value);
 template<>
 VerifyExistenceType EnumUtil::FromString<VerifyExistenceType>(const char *value);
 
+template<>
+VersionCompressionResult EnumUtil::FromString<VersionCompressionResult>(const char *value);
+
 template<>
 VertexType EnumUtil::FromString<VertexType>(const char *value);
 

src/duckdb/src/include/duckdb/common/types.hpp

@@ -514,7 +514,7 @@ struct MapType {
 };
 
 struct UnionType {
-	DUCKDB_API static const idx_t MAX_UNION_MEMBERS = 256;
+	DUCKDB_API static const idx_t MAX_UNION_MEMBERS = 255;
 	DUCKDB_API static idx_t GetMemberCount(const LogicalType &type);
 	DUCKDB_API static const LogicalType &GetMemberType(const LogicalType &type, idx_t index);
 	DUCKDB_API static const string &GetMemberName(const LogicalType &type, idx_t index);

src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/execution/index/index_pointer.hpp"
 #include "duckdb/common/enums/scan_options.hpp"
+#include "duckdb/common/types/validity_mask.hpp"
 
 namespace duckdb {
 class RowGroup;
@@ -21,97 +22,54 @@ class Serializer;
 class Deserializer;
 class FixedSizeAllocator;
 
+//! The type of a serialized chunk info entry
+//! In-memory all chunk infos are represented by ChunkVectorInfo - CONSTANT_INFO (a fully deleted vector) and
+//! EMPTY_INFO (no deletes) only exist as tags in the serialized format
 enum class ChunkInfoType : uint8_t { CONSTANT_INFO, VECTOR_INFO, EMPTY_INFO };
 
-class ChunkInfo {
+//! ChunkVectorInfo holds the version information (the insert and delete ids) of the rows of a single vector within
+//! a row group. The insert and delete ids are stored as constants when all rows share the same id, and as per-row
+//! id arrays otherwise.
+
+//! The result of a CompressVersionIds pass over a vector
+enum class VersionCompressionResult : uint8_t {
+	//! No per-row id arrays remain - there is nothing left to compress
+	FULLY_COMPRESSED,
+	//! Some ids are not yet visible to all transactions - compression can succeed once the
+	//! lowest active start advances past them, without any further modifications
+	PENDING,
+	//! Per-row arrays remain that cannot compress without further modifications
+	//! (some rows are not deleted)
+	SETTLED
+};
+
+//! The storage state of the delete side of a ChunkVectorInfo. Exhaustive and mutually exclusive:
+//! CONSTANT - all rows share constant_delete_id (NOT_DELETED_ID for none-deleted, or a single delete id)
+//! MASKED   - partially deleted, all deleted rows share one committed id (mask_delete_id): which rows are
+//!            deleted is stored in deleted_mask, deleted_data freed
+//! ARRAY    - per-row delete ids materialized in deleted_data
+enum class DeleteIdState : uint8_t { CONSTANT, MASKED, ARRAY };
+
+class ChunkVectorInfo {
 public:
-	ChunkInfo(idx_t start, ChunkInfoType type) : start(start), type(type) {
-	}
-	virtual ~ChunkInfo() {
-	}
+	explicit ChunkVectorInfo(FixedSizeAllocator &allocator, idx_t start, transaction_t insert_id = 0);
+	ChunkVectorInfo(FixedSizeAllocator &allocator, idx_t start, transaction_t insert_id, transaction_t delete_id);
+	~ChunkVectorInfo();
 
 	//! The row index of the first row
 	idx_t start;
-	//! The ChunkInfo type
-	ChunkInfoType type;
 
 public:
 	//! Gets up to max_count entries from the chunk info. If the ret is 0>ret>max_count, the selection vector is filled
 	//! with the tuples
-	virtual idx_t GetSelVector(ScanOptions options, optional_ptr<SelectionVector> sel_vector,
-	                           idx_t max_count) const = 0;
+	idx_t GetSelVector(ScanOptions options, optional_ptr<SelectionVector> sel_vector, idx_t max_count) const;
 	idx_t GetCheckpointRowCount(TransactionData transaction, idx_t max_count);
-	//! Returns whether or not a single row in the ChunkInfo should be used or not for the given transaction
-	virtual bool Fetch(TransactionData transaction, row_t row) = 0;
-	virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
 	idx_t GetCommittedDeletedCount(idx_t max_count) const;
-	virtual bool Cleanup(transaction_t lowest_transaction) const;
-	virtual string ToString(idx_t max_count) const = 0;
-
-	virtual bool HasDeletes(transaction_t transaction_id = MAX_TRANSACTION_ID) const = 0;
-
-	virtual void Write(WriteStream &writer, transaction_t transaction_id) const;
-	static unique_ptr<ChunkInfo> Read(FixedSizeAllocator &allocator, ReadStream &reader);
-
-public:
-	template <class TARGET>
-	TARGET &Cast() {
-		if (type != TARGET::TYPE) {
-			throw InternalException("Failed to cast chunk info to type - query result type mismatch");
-		}
-		return reinterpret_cast<TARGET &>(*this);
-	}
-
-	template <class TARGET>
-	const TARGET &Cast() const {
-		if (type != TARGET::TYPE) {
-			throw InternalException("Failed to cast chunk info to type - query result type mismatch");
-		}
-		return reinterpret_cast<const TARGET &>(*this);
-	}
-};
-
-class ChunkConstantInfo : public ChunkInfo {
-public:
-	static constexpr const ChunkInfoType TYPE = ChunkInfoType::CONSTANT_INFO;
-
-public:
-	explicit ChunkConstantInfo(idx_t start);
-
-	transaction_t insert_id;
-	transaction_t delete_id;
-
-public:
-	idx_t GetSelVector(ScanOptions options, optional_ptr<SelectionVector> sel_vector, idx_t max_count) const override;
-	bool Fetch(TransactionData transaction, row_t row) override;
-	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
-	bool Cleanup(transaction_t lowest_transaction) const override;
-	string ToString(idx_t max_count) const override;
-
-	bool HasDeletes(transaction_t transaction_id = MAX_TRANSACTION_ID) const override;
-
-	void Write(WriteStream &writer, transaction_t transaction_id) const override;
-	static unique_ptr<ChunkInfo> Read(ReadStream &reader);
-
-private:
-	template <class INSERT_OP, class DELETE_OP>
-	idx_t TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const;
-};
-
-class ChunkVectorInfo : public ChunkInfo {
-public:
-	static constexpr const ChunkInfoType TYPE = ChunkInfoType::VECTOR_INFO;
-
-public:
-	explicit ChunkVectorInfo(FixedSizeAllocator &allocator, idx_t start, transaction_t insert_id = 0);
-	~ChunkVectorInfo() override;
-
-public:
-	idx_t GetSelVector(ScanOptions options, optional_ptr<SelectionVector> sel_vector, idx_t max_count) const override;
-	bool Fetch(TransactionData transaction, row_t row) override;
-	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
-	bool Cleanup(transaction_t lowest_transaction) const override;
-	string ToString(idx_t max_count) const override;
+	//! Returns whether or not a single row in the ChunkVectorInfo should be used or not for the given transaction
+	bool Fetch(TransactionData transaction, row_t row);
+	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end);
+	bool Cleanup(transaction_t lowest_transaction) const;
+	string ToString(idx_t max_count) const;
 
 	void Append(idx_t start, idx_t end, transaction_t commit_id);
 
@@ -123,13 +81,27 @@ class ChunkVectorInfo : public ChunkInfo {
 	idx_t Delete(transaction_t transaction_id, row_t rows[], idx_t count);
 	void CommitDelete(transaction_t commit_id, const DeleteInfo &info);
 
-	bool HasDeletes(transaction_t transaction_id = MAX_TRANSACTION_ID) const override;
+	//! Attempts to compress the per-row insert/delete ids into constants
+	//! This is possible when the ids behave identically for all transactions with a start time of at least
+	//! lowest_active_start (i.e. all active and future transactions)
+	VersionCompressionResult CompressVersionIds(transaction_t lowest_active_start);
+	//! Whether a compression pass could achieve anything for this vector (see recheck_compression)
+	bool RecheckCompression() const {
+		return recheck_compression;
+	}
+	//! Verifies (in DEBUG) that a disarmed recheck_compression matches the actual ids: nothing may be
+	//! compressible now or in the future without a modification that re-arms the check
+	void VerifyCachedCompressionState() const;
+
+	bool HasDeletes(transaction_t transaction_id = MAX_TRANSACTION_ID) const;
 	bool AnyDeleted() const;
 	bool HasConstantInsertionId() const;
 	transaction_t ConstantInsertId() const;
+	bool HasConstantDeleteId() const;
+	transaction_t ConstantDeleteId() const;
 
-	void Write(WriteStream &writer, transaction_t transaction_id) const override;
-	static unique_ptr<ChunkInfo> Read(FixedSizeAllocator &allocator, ReadStream &reader);
+	void Write(WriteStream &writer, transaction_t transaction_id) const;
+	static unique_ptr<ChunkVectorInfo> Read(FixedSizeAllocator &allocator, ReadStream &reader);
 
 private:
 	template <class INSERT_OP, class DELETE_OP>
@@ -140,6 +112,13 @@ class ChunkVectorInfo : public ChunkInfo {
 	IndexPointer GetDeletedPointer() const;
 	IndexPointer GetInitializedInsertedPointer();
 	IndexPointer GetInitializedDeletedPointer();
+	//! Frees the per-row delete ids (if any)
+	void FreeDeleteData();
+	//! ARRAY -> MASKED: record alive rows as invalid bits, free the per-row delete array. mask_id is the
+	//! shared committed id of the deleted rows (0 when they are already visible to all transactions)
+	void CompressDeleteToMask(transaction_t mask_id);
+	//! MASKED -> ARRAY: re-materialize the per-row delete array from the bitmask
+	void DecompressDeleteMask();
 
 private:
 	FixedSizeAllocator &allocator;
@@ -150,6 +129,23 @@ class ChunkVectorInfo : public ChunkInfo {
 
 	//! The transaction ids of the transactions that deleted the tuples (if any)
 	IndexPointer deleted_data;
+	//! The constant delete id (if there is only one, e.g. because the entire vector was deleted in one transaction)
+	transaction_t constant_delete_id;
+	//! Bitmask used in the MASKED state: valid bit == the row is deleted, invalid bit == the row is alive.
+	//! Matches the on-disk VECTOR_INFO orientation. Only meaningful when delete_state == DeleteIdState::MASKED.
+	ValidityMask deleted_mask;
+	//! The single committed id shared by every deleted row in the MASKED state. 0 means the deletes are
+	//! visible to all transactions (the value used when read from disk); a non-zero committed id means the
+	//! mask was folded from one committed transaction whose delete is not yet visible to every snapshot.
+	//! Only meaningful when delete_state == DeleteIdState::MASKED.
+	transaction_t mask_delete_id = 0;
+	//! The current delete-side storage state - the single source of truth for the delete side
+	DeleteIdState delete_state = DeleteIdState::CONSTANT;
+	//! Whether a compression pass could achieve anything for this vector: armed by any id modification,
+	//! disarmed when a pass compresses the vector fully or finds it settled (live rows block the collapse
+	//! until a further delete re-arms it). CompressVersionIds returns the cached SETTLED without
+	//! re-scanning the ids while this is false.
+	bool recheck_compression = true;
 };
 
 } // namespace duckdb

src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -229,6 +229,10 @@ class RowGroup : public SegmentBase<RowGroup> {
 	idx_t GetColumnCount() const;
 
 	vector<MetaBlockPointer> CheckpointDeletes(RowGroupWriter &writer);
+	//! Attempts to compress the version information of the row group
+	//! Per-row insert/delete ids that behave identically for all transactions with a start time of at least
+	//! lowest_active_start (i.e. all active and future transactions) are compressed into constants
+	void CompressVersionInfo(transaction_t lowest_active_start);
 
 	//! Direct accessors, fall outside of general use but can be useful to some extensions
 	ColumnData &GetRawColumnData(const StorageIndex &c) const;

src/duckdb/src/include/duckdb/storage/table/row_version_manager.hpp

@@ -36,6 +36,13 @@ class RowVersionManager {
 	idx_t DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count);
 	void CommitDelete(idx_t vector_idx, transaction_t commit_id, const DeleteInfo &info);
 
+	//! Attempts to compress the per-row insert/delete ids of each vector into constants
+	//! This is possible when the ids behave identically for all transactions with a start time of at least
+	//! lowest_active_start (i.e. all active and future transactions)
+	//! Cheap when nothing can have changed: the pass only runs when version ids were modified since the
+	//! last pass, or when a previous pass left ids that can still compress once older transactions finish
+	void CompressVersionIds(transaction_t lowest_active_start);
+
 	vector<MetaBlockPointer> Checkpoint(RowGroupWriter &writer);
 	static shared_ptr<RowVersionManager> Deserialize(MetaBlockPointer delete_pointer, MetadataManager &manager);
 
@@ -45,15 +52,19 @@ class RowVersionManager {
 private:
 	mutex version_lock;
 	FixedSizeAllocator allocator;
-	vector<unique_ptr<ChunkInfo>> vector_info;
+	vector<unique_ptr<ChunkVectorInfo>> vector_info;
 	optional_idx uncheckpointed_delete_commit;
 	vector<MetaBlockPointer> storage_pointers;
+	//! Whether a compression pass may achieve anything: set when version ids are modified, cleared when a
+	//! pass finds no ids that could still compress. For deserialized version info this is derived from the
+	//! deserialized content (with the current storage format checkpointed ids are always settled).
+	bool needs_compression_check = false;
 
 private:
 	FixedSizeAllocator &GetAllocator() {
 		return allocator;
 	}
-	optional_ptr<ChunkInfo> GetChunkInfo(idx_t vector_idx);
+	optional_ptr<ChunkVectorInfo> GetChunkInfo(idx_t vector_idx);
 	ChunkVectorInfo &GetVectorInfo(idx_t vector_idx);
 	void FillVectorInfo(idx_t vector_idx);
 };