From ed026ae046da782dde7e2c859a02a4fa774e5673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Sat, 17 Aug 2024 16:23:11 +0200 Subject: [PATCH 01/10] DuckDBAppender: Add enum support --- .../NativeMethods.LogicalType.cs | 3 + DuckDB.NET.Data/DuckDBAppenderRow.cs | 6 + .../Internal/Writer/EnumVectorDataWriter.cs | 103 ++++++++++++++++++ .../Internal/Writer/VectorDataWriterBase.cs | 4 + .../Writer/VectorDataWriterFactory.cs | 2 +- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 27 +++++ 6 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.LogicalType.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.LogicalType.cs index 1c101577..84afe0c8 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.LogicalType.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.LogicalType.cs @@ -26,6 +26,9 @@ public static class LogicalType [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_enum_internal_type")] public static extern DuckDBType DuckDBEnumInternalType(DuckDBLogicalType type); + [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_enum_dictionary_size")] + public static extern uint DuckDBEnumDictionarySize(DuckDBLogicalType type); + [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_enum_dictionary_value")] public static extern IntPtr DuckDBEnumDictionaryValue(DuckDBLogicalType type, long index); diff --git a/DuckDB.NET.Data/DuckDBAppenderRow.cs b/DuckDB.NET.Data/DuckDBAppenderRow.cs index 66db1994..869bc31b 100644 --- a/DuckDB.NET.Data/DuckDBAppenderRow.cs +++ b/DuckDB.NET.Data/DuckDBAppenderRow.cs @@ -68,6 +68,12 @@ public void EndRow() #endregion + #region Append Enum + + public DuckDBAppenderRow AppendValue(TEnum value) where TEnum : Enum => AppendValueInternal(value); + + #endregion + #region Append Float public DuckDBAppenderRow AppendValue(float? value) => AppendValueInternal(value); diff --git a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs new file mode 100644 index 00000000..aaacd74e --- /dev/null +++ b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using DuckDB.NET.Native; + +namespace DuckDB.NET.Data.Internal.Writer; + +internal sealed unsafe class EnumVectorDataWriter : VectorDataWriterBase +{ + private readonly DuckDBType enumType; + + private readonly Dictionary enumValueIndexDictionary; + + public EnumVectorDataWriter(IntPtr vector, void* vectorData, DuckDBLogicalType logicalType, DuckDBType columnType) : base(vector, vectorData, columnType) + { + enumType = NativeMethods.LogicalType.DuckDBEnumInternalType(logicalType); + uint size = NativeMethods.LogicalType.DuckDBEnumDictionarySize(logicalType); + enumValueIndexDictionary = []; + for (long index = 0; index < size; index++) + { + string enumValue = NativeMethods.LogicalType.DuckDBEnumDictionaryValue(logicalType, index).ToManagedString(); + enumValueIndexDictionary.Add(enumValue, index); + } + } + + internal override bool AppendString(string value, int rowIndex) + { + switch (enumType) + { + case DuckDBType.UnsignedTinyInt: + { + if (enumValueIndexDictionary.TryGetValue(value, out long enumValueIndex) && + enumValueIndex >= byte.MinValue && enumValueIndex <= byte.MaxValue) + { + return AppendValueInternal((byte)enumValueIndex, rowIndex); + } + + return false; + } + case DuckDBType.UnsignedSmallInt: + { + if (enumValueIndexDictionary.TryGetValue(value, out long enumValueIndex) && + enumValueIndex >= ushort.MinValue && enumValueIndex <= ushort.MaxValue) + { + return AppendValueInternal((ushort)enumValueIndex, rowIndex); + } + + return false; + } + case DuckDBType.UnsignedInteger: + { + if (enumValueIndexDictionary.TryGetValue(value, out long enumValueIndex) && + enumValueIndex >= uint.MinValue && enumValueIndex <= uint.MaxValue) + { + return AppendValueInternal((uint)enumValueIndex, rowIndex); + } + + return false; + } + default: + return false; + } + } + + internal override bool AppendEnum(TEnum value, int rowIndex) + { + switch (enumType) + { + case DuckDBType.UnsignedTinyInt: + { + long enumValueIndex = Convert.ToInt64(value); + if (enumValueIndex >= byte.MinValue && enumValueIndex <= byte.MaxValue) + { + return AppendValueInternal((byte)enumValueIndex, rowIndex); + } + + return false; + } + case DuckDBType.UnsignedSmallInt: + { + long enumValueIndex = Convert.ToInt64(value); + if (enumValueIndex >= ushort.MinValue && enumValueIndex <= ushort.MaxValue) + { + return AppendValueInternal((ushort)enumValueIndex, rowIndex); + } + + return false; + } + case DuckDBType.UnsignedInteger: + { + long enumValueIndex = Convert.ToInt64(value); + if (enumValueIndex >= uint.MinValue && enumValueIndex <= uint.MaxValue) + { + return AppendValueInternal((uint)enumValueIndex, rowIndex); + } + + return false; + } + default: + return false; + } + } +} diff --git a/DuckDB.NET.Data/Internal/Writer/VectorDataWriterBase.cs b/DuckDB.NET.Data/Internal/Writer/VectorDataWriterBase.cs index 67c3250c..3ee95bf8 100644 --- a/DuckDB.NET.Data/Internal/Writer/VectorDataWriterBase.cs +++ b/DuckDB.NET.Data/Internal/Writer/VectorDataWriterBase.cs @@ -50,6 +50,8 @@ public void AppendValue(T value, int rowIndex) decimal val => AppendDecimal(val, rowIndex), BigInteger val => AppendBigInteger(val, rowIndex), + Enum val => AppendEnum(val, rowIndex), + string val => AppendString(val, rowIndex), Guid val => AppendGuid(val, rowIndex), DateTime val => AppendDateTime(val, rowIndex), @@ -96,6 +98,8 @@ public void AppendValue(T value, int rowIndex) internal virtual bool AppendBigInteger(BigInteger value, int rowIndex) => ThrowException(); + internal virtual bool AppendEnum(TEnum value, int rowIndex) where TEnum : Enum => ThrowException(); + internal virtual bool AppendCollection(ICollection value, int rowIndex) => ThrowException(); private bool ThrowException() diff --git a/DuckDB.NET.Data/Internal/Writer/VectorDataWriterFactory.cs b/DuckDB.NET.Data/Internal/Writer/VectorDataWriterFactory.cs index 98f828a4..d96d8299 100644 --- a/DuckDB.NET.Data/Internal/Writer/VectorDataWriterFactory.cs +++ b/DuckDB.NET.Data/Internal/Writer/VectorDataWriterFactory.cs @@ -27,7 +27,7 @@ public static unsafe VectorDataWriterBase CreateWriter(IntPtr vector, DuckDBLogi DuckDBType.Blob => new StringVectorDataWriter(vector, dataPointer, columnType), DuckDBType.Varchar => new StringVectorDataWriter(vector, dataPointer, columnType), DuckDBType.Bit => throw new NotImplementedException($"Writing {columnType} to data chunk is not yet supported"), - DuckDBType.Enum => throw new NotImplementedException($"Writing {columnType} to data chunk is not yet supported"), + DuckDBType.Enum => new EnumVectorDataWriter(vector, dataPointer, logicalType, columnType), DuckDBType.Struct => throw new NotImplementedException($"Writing {columnType} to data chunk is not yet supported"), DuckDBType.Decimal => new DecimalVectorDataWriter(vector, dataPointer, logicalType, columnType), DuckDBType.TimestampS => new DateTimeVectorDataWriter(vector, dataPointer, columnType), diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index c7a8828e..69798490 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -275,6 +275,26 @@ public void TemporalValues() result.Select(tuple => tuple.Item8).Should().BeEquivalentTo(dates.Select(TimeOnly.FromDateTime)); } + [Fact] + public void EnumValues() + { + Command.CommandText = "CREATE TYPE test_enum AS ENUM ('test1', 'test2', 'test3')"; + Command.ExecuteNonQuery(); + + Command.CommandText = "CREATE TABLE managedAppenderEnum(a test_enum, b test_enum, c test_enum);"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderEnum")) + { + appender.CreateRow().AppendValue("test1").AppendValue("test2").AppendValue(TestEnum.Test3).EndRow(); + } + + var result = Connection.Query<(string, string, TestEnum)>("SELECT a, b, c FROM managedAppenderEnum").ToList(); + result.Select(tuple => tuple.Item1).Should().BeEquivalentTo("test1"); + result.Select(tuple => tuple.Item2).Should().BeEquivalentTo("test2"); + result.Select(tuple => tuple.Item3).Should().Equal(TestEnum.Test3); + } + [Fact] public void IncompleteRowThrowsException() { @@ -509,4 +529,11 @@ private static string GetQualifiedObjectName(params string[] parts) => Where(p => !string.IsNullOrWhiteSpace(p)). Select(p => '"' + p + '"') ); + + private enum TestEnum : long + { + Test1 = 0, + Test2 = 1, + Test3 = 2, + } } \ No newline at end of file From 8116268bc360e2b8534a7af6c7bc5b3ab85cbcc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Sat, 17 Aug 2024 16:33:29 +0200 Subject: [PATCH 02/10] Refactor code --- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index 69798490..77279059 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -530,7 +530,7 @@ private static string GetQualifiedObjectName(params string[] parts) => Select(p => '"' + p + '"') ); - private enum TestEnum : long + private enum TestEnum { Test1 = 0, Test2 = 1, From 02c583f56cbab38c1badc017d2afeb802a84c28d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Sun, 18 Aug 2024 18:35:30 +0200 Subject: [PATCH 03/10] Improve code --- .../Internal/Writer/EnumVectorDataWriter.cs | 145 +++++++++--------- 1 file changed, 76 insertions(+), 69 deletions(-) diff --git a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs index aaacd74e..9cc5a627 100644 --- a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs +++ b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs @@ -7,97 +7,104 @@ namespace DuckDB.NET.Data.Internal.Writer; internal sealed unsafe class EnumVectorDataWriter : VectorDataWriterBase { + private readonly DuckDBLogicalType logicalType; + private readonly DuckDBType enumType; - private readonly Dictionary enumValueIndexDictionary; + private readonly uint enumDictionarySize; + + private Dictionary? enumValues; public EnumVectorDataWriter(IntPtr vector, void* vectorData, DuckDBLogicalType logicalType, DuckDBType columnType) : base(vector, vectorData, columnType) { + this.logicalType = logicalType; enumType = NativeMethods.LogicalType.DuckDBEnumInternalType(logicalType); - uint size = NativeMethods.LogicalType.DuckDBEnumDictionarySize(logicalType); - enumValueIndexDictionary = []; - for (long index = 0; index < size; index++) + enumDictionarySize = NativeMethods.LogicalType.DuckDBEnumDictionarySize(logicalType); + + uint maxEnumDictionarySize = enumType switch + { + DuckDBType.UnsignedTinyInt => byte.MaxValue, + DuckDBType.UnsignedSmallInt => ushort.MaxValue, + DuckDBType.UnsignedInteger => uint.MaxValue, + _ => throw new NotSupportedException($"The internal enum type must be utinyint, usmallint, or uinteger."), + }; + if (enumDictionarySize > maxEnumDictionarySize) { - string enumValue = NativeMethods.LogicalType.DuckDBEnumDictionaryValue(logicalType, index).ToManagedString(); - enumValueIndexDictionary.Add(enumValue, index); + // This exception should only be thrown if the DuckDB library has a bug. + throw new InvalidOperationException($"The internal enum type is \"{enumType}\" but the enum dictionary size is greater than {maxEnumDictionarySize}."); } + + enumValues = null; } internal override bool AppendString(string value, int rowIndex) { - switch (enumType) + // lazy initialization + if (enumValues == null) { - case DuckDBType.UnsignedTinyInt: - { - if (enumValueIndexDictionary.TryGetValue(value, out long enumValueIndex) && - enumValueIndex >= byte.MinValue && enumValueIndex <= byte.MaxValue) - { - return AppendValueInternal((byte)enumValueIndex, rowIndex); - } - - return false; - } - case DuckDBType.UnsignedSmallInt: - { - if (enumValueIndexDictionary.TryGetValue(value, out long enumValueIndex) && - enumValueIndex >= ushort.MinValue && enumValueIndex <= ushort.MaxValue) - { - return AppendValueInternal((ushort)enumValueIndex, rowIndex); - } - - return false; - } - case DuckDBType.UnsignedInteger: - { - if (enumValueIndexDictionary.TryGetValue(value, out long enumValueIndex) && - enumValueIndex >= uint.MinValue && enumValueIndex <= uint.MaxValue) - { - return AppendValueInternal((uint)enumValueIndex, rowIndex); - } + enumValues = []; + for (uint index = 0; index < enumDictionarySize; index++) + { + string enumValueName = NativeMethods.LogicalType.DuckDBEnumDictionaryValue(logicalType, index).ToManagedString(); + enumValues.Add(enumValueName, index); + } + } - return false; - } - default: - return false; + if (enumValues.TryGetValue(value, out uint enumValue)) + { + // The following casts to byte and ushort are safe because we ensure in the constructor that the value enumDictionarySize is not too high. + return enumType switch + { + DuckDBType.UnsignedTinyInt => AppendValueInternal((byte)enumValue, rowIndex), + DuckDBType.UnsignedSmallInt => AppendValueInternal((ushort)enumValue, rowIndex), + DuckDBType.UnsignedInteger => AppendValueInternal(enumValue, rowIndex), + _ => throw new InvalidOperationException($"Failed to write Enum column because the internal enum type must be utinyint, usmallint, or uinteger."), + }; } + + throw new InvalidOperationException($"Failed to write Enum column because the value \"{value}\" is not valid."); } internal override bool AppendEnum(TEnum value, int rowIndex) { - switch (enumType) + ulong enumValue = ConvertEnumValueToUInt64(value); + if (enumValue <= enumDictionarySize) { - case DuckDBType.UnsignedTinyInt: - { - long enumValueIndex = Convert.ToInt64(value); - if (enumValueIndex >= byte.MinValue && enumValueIndex <= byte.MaxValue) - { - return AppendValueInternal((byte)enumValueIndex, rowIndex); - } + // The following casts to byte, ushort and uint are safe because we ensure in the constructor that the value enumDictionarySize is not too high. + return enumType switch + { + DuckDBType.UnsignedTinyInt => AppendValueInternal((byte)enumValue, rowIndex), + DuckDBType.UnsignedSmallInt => AppendValueInternal((ushort)enumValue, rowIndex), + DuckDBType.UnsignedInteger => AppendValueInternal((uint)enumValue, rowIndex), + _ => throw new InvalidOperationException($"Failed to write Enum column because the internal enum type must be utinyint, usmallint, or uinteger."), + }; + } - return false; - } - case DuckDBType.UnsignedSmallInt: - { - long enumValueIndex = Convert.ToInt64(value); - if (enumValueIndex >= ushort.MinValue && enumValueIndex <= ushort.MaxValue) - { - return AppendValueInternal((ushort)enumValueIndex, rowIndex); - } + throw new InvalidOperationException($"Failed to write Enum column because the value is outside the range (0-{enumDictionarySize})."); + } - return false; - } - case DuckDBType.UnsignedInteger: - { - long enumValueIndex = Convert.ToInt64(value); - if (enumValueIndex >= uint.MinValue && enumValueIndex <= uint.MaxValue) - { - return AppendValueInternal((uint)enumValueIndex, rowIndex); - } + private static ulong ConvertEnumValueToUInt64(TEnum value) where TEnum : Enum + { + switch (Convert.GetTypeCode(value)) + { + case TypeCode.SByte: + return (ulong)Convert.ToSByte(value); + case TypeCode.Byte: + return Convert.ToByte(value); + case TypeCode.Int16: + return (ulong)Convert.ToInt16(value); + case TypeCode.UInt16: + return Convert.ToUInt16(value); + case TypeCode.Int32: + return (ulong)Convert.ToInt32(value); + case TypeCode.UInt32: + return Convert.ToUInt32(value); + case TypeCode.Int64: + return (ulong)Convert.ToInt64(value); + case TypeCode.UInt64: + return Convert.ToUInt64(value); + }; - return false; - } - default: - return false; - } + throw new InvalidOperationException($"Failed to convert the enum value {value} to ulong."); } } From 5c869148de0df66bc5fd9ba4936b84748843f29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Sun, 18 Aug 2024 18:42:16 +0200 Subject: [PATCH 04/10] Improve code --- DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs index 9cc5a627..3b3aabec 100644 --- a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs +++ b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs @@ -68,7 +68,7 @@ internal override bool AppendString(string value, int rowIndex) internal override bool AppendEnum(TEnum value, int rowIndex) { ulong enumValue = ConvertEnumValueToUInt64(value); - if (enumValue <= enumDictionarySize) + if (enumValue < enumDictionarySize) { // The following casts to byte, ushort and uint are safe because we ensure in the constructor that the value enumDictionarySize is not too high. return enumType switch @@ -80,7 +80,7 @@ internal override bool AppendEnum(TEnum value, int rowIndex) }; } - throw new InvalidOperationException($"Failed to write Enum column because the value is outside the range (0-{enumDictionarySize})."); + throw new InvalidOperationException($"Failed to write Enum column because the value is outside the range (0-{enumDictionarySize-1})."); } private static ulong ConvertEnumValueToUInt64(TEnum value) where TEnum : Enum From a3f921e487471539598d807cd3b688e59c834f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Mon, 19 Aug 2024 12:29:49 +0200 Subject: [PATCH 05/10] Improve code --- .../Internal/Writer/EnumVectorDataWriter.cs | 62 +++++++++---------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs index 3b3aabec..f9779c2b 100644 --- a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs +++ b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Threading; using DuckDB.NET.Native; namespace DuckDB.NET.Data.Internal.Writer; @@ -13,7 +14,9 @@ internal sealed unsafe class EnumVectorDataWriter : VectorDataWriterBase private readonly uint enumDictionarySize; - private Dictionary? enumValues; + private readonly Lazy> enumValues; + + private Dictionary EnumValues => enumValues.Value; public EnumVectorDataWriter(IntPtr vector, void* vectorData, DuckDBLogicalType logicalType, DuckDBType columnType) : base(vector, vectorData, columnType) { @@ -34,23 +37,12 @@ public EnumVectorDataWriter(IntPtr vector, void* vectorData, DuckDBLogicalType l throw new InvalidOperationException($"The internal enum type is \"{enumType}\" but the enum dictionary size is greater than {maxEnumDictionarySize}."); } - enumValues = null; + enumValues = new Lazy>(GetEnumValues, LazyThreadSafetyMode.None); } internal override bool AppendString(string value, int rowIndex) { - // lazy initialization - if (enumValues == null) - { - enumValues = []; - for (uint index = 0; index < enumDictionarySize; index++) - { - string enumValueName = NativeMethods.LogicalType.DuckDBEnumDictionaryValue(logicalType, index).ToManagedString(); - enumValues.Add(enumValueName, index); - } - } - - if (enumValues.TryGetValue(value, out uint enumValue)) + if (EnumValues.TryGetValue(value, out uint enumValue)) { // The following casts to byte and ushort are safe because we ensure in the constructor that the value enumDictionarySize is not too high. return enumType switch @@ -83,28 +75,32 @@ internal override bool AppendEnum(TEnum value, int rowIndex) throw new InvalidOperationException($"Failed to write Enum column because the value is outside the range (0-{enumDictionarySize-1})."); } + private Dictionary GetEnumValues() + { + Dictionary enumValues = []; + + for (uint index = 0; index < enumDictionarySize; index++) + { + string enumValueName = NativeMethods.LogicalType.DuckDBEnumDictionaryValue(logicalType, index).ToManagedString(); + enumValues.Add(enumValueName, index); + } + + return enumValues; + } + private static ulong ConvertEnumValueToUInt64(TEnum value) where TEnum : Enum { - switch (Convert.GetTypeCode(value)) + return Convert.GetTypeCode(value) switch { - case TypeCode.SByte: - return (ulong)Convert.ToSByte(value); - case TypeCode.Byte: - return Convert.ToByte(value); - case TypeCode.Int16: - return (ulong)Convert.ToInt16(value); - case TypeCode.UInt16: - return Convert.ToUInt16(value); - case TypeCode.Int32: - return (ulong)Convert.ToInt32(value); - case TypeCode.UInt32: - return Convert.ToUInt32(value); - case TypeCode.Int64: - return (ulong)Convert.ToInt64(value); - case TypeCode.UInt64: - return Convert.ToUInt64(value); + TypeCode.SByte => (ulong)Convert.ToSByte(value), + TypeCode.Byte => Convert.ToByte(value), + TypeCode.Int16 => (ulong)Convert.ToInt16(value), + TypeCode.UInt16 => Convert.ToUInt16(value), + TypeCode.Int32 => (ulong)Convert.ToInt32(value), + TypeCode.UInt32 => Convert.ToUInt32(value), + TypeCode.Int64 => (ulong)Convert.ToInt64(value), + TypeCode.UInt64 => Convert.ToUInt64(value), + _ => throw new InvalidOperationException($"Failed to convert the enum value {value} to ulong."), }; - - throw new InvalidOperationException($"Failed to convert the enum value {value} to ulong."); } } From 64229433606d07a794b85c127f6242a2b3da89fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Mon, 19 Aug 2024 13:21:29 +0200 Subject: [PATCH 06/10] Improve tests --- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 75 +++++++++++++++++-- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index 77279059..eef15c6e 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -11,6 +11,7 @@ using System.Numerics; using Bogus; using Xunit; +using System.Text; namespace DuckDB.NET.Test; @@ -278,21 +279,45 @@ public void TemporalValues() [Fact] public void EnumValues() { - Command.CommandText = "CREATE TYPE test_enum AS ENUM ('test1', 'test2', 'test3')"; + Command.CommandText = GetCreateEnumTypeSql("test_enum1", "test", 10); Command.ExecuteNonQuery(); - Command.CommandText = "CREATE TABLE managedAppenderEnum(a test_enum, b test_enum, c test_enum);"; + Command.CommandText = GetCreateEnumTypeSql("test_enum2", "test", 1000); + Command.ExecuteNonQuery(); + + Command.CommandText = GetCreateEnumTypeSql("test_enum3", "test", 100000); + Command.ExecuteNonQuery(); + + Command.CommandText = "CREATE TABLE managedAppenderEnum(a test_enum1, b test_enum1, c test_enum1, d test_enum1, e test_enum1, f test_enum2, g test_enum2, h test_enum3, i test_enum3);"; Command.ExecuteNonQuery(); using (var appender = Connection.CreateAppender("managedAppenderEnum")) { - appender.CreateRow().AppendValue("test1").AppendValue("test2").AppendValue(TestEnum.Test3).EndRow(); + appender + .CreateRow() + .AppendNullValue() + .AppendNullValue() + .AppendValue("test1") + .AppendValue(TestEnum1.Test2) + .AppendValue(TestEnum1.Test3) + .AppendValue("test327") + .AppendValue(TestEnum2.Test1000) + .AppendValue("test100000") + .AppendValue(TestEnum3.Test6699) + .EndRow(); } - var result = Connection.Query<(string, string, TestEnum)>("SELECT a, b, c FROM managedAppenderEnum").ToList(); - result.Select(tuple => tuple.Item1).Should().BeEquivalentTo("test1"); - result.Select(tuple => tuple.Item2).Should().BeEquivalentTo("test2"); - result.Select(tuple => tuple.Item3).Should().Equal(TestEnum.Test3); + var queryResult = Connection.Query<(string, TestEnum1?, TestEnum1, string, TestEnum1, TestEnum2, string, string, TestEnum3)>("SELECT a, b, c, d, e, f, g, h, i FROM managedAppenderEnum").ToList(); + var result = queryResult[0]; + result.Item1.Should().BeNull(); + result.Item2.Should().BeNull(); + result.Item3.Should().Be(TestEnum1.Test1); + result.Item4.Should().Be("test2"); + result.Item5.Should().Be(TestEnum1.Test3); + result.Item6.Should().Be(TestEnum2.Test327); + result.Item7.Should().Be("test1000"); + result.Item8.Should().Be("test100000"); + result.Item9.Should().Be(TestEnum3.Test6699); } [Fact] @@ -524,16 +549,50 @@ public void ManagedAppenderOnTableAndColumnsWithSpecialCharacters(string schemaN } } + private static string GetCreateEnumTypeSql(string enumName, string enumValueNamePrefix, int count) + { + var stringBuilder = new StringBuilder(); + stringBuilder.AppendFormat(CultureInfo.InvariantCulture, "CREATE TYPE {0} AS ENUM(", enumName); + + for (int i = 1; i <= count; i++) + { + if (i > 1) + { + stringBuilder.Append(','); + } + + stringBuilder.Append('\''); + stringBuilder.Append(enumValueNamePrefix); + stringBuilder.Append(i); + stringBuilder.Append('\''); + } + + stringBuilder.Append(");"); + return stringBuilder.ToString(); + } + private static string GetQualifiedObjectName(params string[] parts) => string.Join('.', parts. Where(p => !string.IsNullOrWhiteSpace(p)). Select(p => '"' + p + '"') ); - private enum TestEnum + private enum TestEnum1 { Test1 = 0, Test2 = 1, Test3 = 2, } + + private enum TestEnum2 : short + { + Test327 = 326, + Test1000 = 999, + } + + private enum TestEnum3 : ulong + { + Test6699 = 6698, + Test100000 = 99999, + } } \ No newline at end of file From 62d976b739083045fdb4ba3702cd100fc33fcf5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Mon, 19 Aug 2024 14:00:55 +0200 Subject: [PATCH 07/10] Improve tests --- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index eef15c6e..da20cea3 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -279,7 +279,7 @@ public void TemporalValues() [Fact] public void EnumValues() { - Command.CommandText = GetCreateEnumTypeSql("test_enum1", "test", 10); + Command.CommandText = GetCreateEnumTypeSql("test_enum1", "test", 3); Command.ExecuteNonQuery(); Command.CommandText = GetCreateEnumTypeSql("test_enum2", "test", 1000); From 7ef6e8d505742a37b37ac6ae06169b358cea58e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Mon, 19 Aug 2024 14:04:55 +0200 Subject: [PATCH 08/10] Remove lazy initialization of enum values because the logical type handle is released too early --- .../Internal/Writer/EnumVectorDataWriter.cs | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs index f9779c2b..5dc6183a 100644 --- a/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs +++ b/DuckDB.NET.Data/Internal/Writer/EnumVectorDataWriter.cs @@ -8,19 +8,14 @@ namespace DuckDB.NET.Data.Internal.Writer; internal sealed unsafe class EnumVectorDataWriter : VectorDataWriterBase { - private readonly DuckDBLogicalType logicalType; - private readonly DuckDBType enumType; private readonly uint enumDictionarySize; - private readonly Lazy> enumValues; - - private Dictionary EnumValues => enumValues.Value; + private readonly Dictionary enumValues; public EnumVectorDataWriter(IntPtr vector, void* vectorData, DuckDBLogicalType logicalType, DuckDBType columnType) : base(vector, vectorData, columnType) { - this.logicalType = logicalType; enumType = NativeMethods.LogicalType.DuckDBEnumInternalType(logicalType); enumDictionarySize = NativeMethods.LogicalType.DuckDBEnumDictionarySize(logicalType); @@ -36,13 +31,18 @@ public EnumVectorDataWriter(IntPtr vector, void* vectorData, DuckDBLogicalType l // This exception should only be thrown if the DuckDB library has a bug. throw new InvalidOperationException($"The internal enum type is \"{enumType}\" but the enum dictionary size is greater than {maxEnumDictionarySize}."); } - - enumValues = new Lazy>(GetEnumValues, LazyThreadSafetyMode.None); + + enumValues = []; + for (uint index = 0; index < enumDictionarySize; index++) + { + string enumValueName = NativeMethods.LogicalType.DuckDBEnumDictionaryValue(logicalType, index).ToManagedString(); + enumValues.Add(enumValueName, index); + } } internal override bool AppendString(string value, int rowIndex) { - if (EnumValues.TryGetValue(value, out uint enumValue)) + if (enumValues.TryGetValue(value, out uint enumValue)) { // The following casts to byte and ushort are safe because we ensure in the constructor that the value enumDictionarySize is not too high. return enumType switch @@ -75,19 +75,6 @@ internal override bool AppendEnum(TEnum value, int rowIndex) throw new InvalidOperationException($"Failed to write Enum column because the value is outside the range (0-{enumDictionarySize-1})."); } - private Dictionary GetEnumValues() - { - Dictionary enumValues = []; - - for (uint index = 0; index < enumDictionarySize; index++) - { - string enumValueName = NativeMethods.LogicalType.DuckDBEnumDictionaryValue(logicalType, index).ToManagedString(); - enumValues.Add(enumValueName, index); - } - - return enumValues; - } - private static ulong ConvertEnumValueToUInt64(TEnum value) where TEnum : Enum { return Convert.GetTypeCode(value) switch From 938cbf21cb7dce9c29521b8cd3745c6e14d47697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Mon, 19 Aug 2024 14:47:55 +0200 Subject: [PATCH 09/10] Improve tests + Fix code --- .../Internal/Writer/ListVectorDataWriter.cs | 19 ++++++++++++++++++- .../DuckDBManagedAppenderListTests.cs | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/DuckDB.NET.Data/Internal/Writer/ListVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/ListVectorDataWriter.cs index ad57c155..f4d6dadb 100644 --- a/DuckDB.NET.Data/Internal/Writer/ListVectorDataWriter.cs +++ b/DuckDB.NET.Data/Internal/Writer/ListVectorDataWriter.cs @@ -82,7 +82,7 @@ internal override bool AppendCollection(ICollection value, int rowIndex) IEnumerable items => WriteItems(items), IEnumerable items => WriteItems(items), - _ => WriteItems((IEnumerable)value) + _ => WriteItemsFallback(value), }; var duckDBListEntry = new DuckDBListEntry(offset, count); @@ -108,6 +108,23 @@ int WriteItems(IEnumerable items) return 0; } + + int WriteItemsFallback(IEnumerable items) + { + if (IsList == false && count != arraySize) + { + throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}"); + } + + var index = 0; + + foreach (var item in items) + { + listItemWriter.AppendValue(item, (int)offset + (index++)); + } + + return 0; + } } private void ResizeVector(int rowIndex, ulong count) diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs index 1adf5eb1..a9a78dff 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs @@ -208,6 +208,16 @@ public void ArrayValuesInt() ListValuesInternal("Integer", faker => faker.Random.Int(), 5); } + [Fact] + public void ListValuesEnum() + { + Command.CommandText = "CREATE TYPE test_enum AS ENUM('test1','test2','test3');"; + Command.ExecuteNonQuery(); + + ListValuesInternal("test_enum", faker => faker.Random.CollectionItem([null, "test1", "test2", "test3"])); + ListValuesInternal("test_enum", faker => faker.Random.CollectionItem([null, TestEnum.Test1, TestEnum.Test2, TestEnum.Test3])); + } + private void ListValuesInternal(string typeName, Func generator, int? length = null) { var rows = 2000; @@ -268,4 +278,11 @@ private void ListValuesInternal(string typeName, Func generator, in .Should().Throw().Where(exception => exception.Message.Contains(length.ToString())); } } + + private enum TestEnum + { + Test1 = 0, + Test2 = 1, + Test3 = 2, + } } \ No newline at end of file From 8ce781f2356329d8f4d0f5e4e618fdd04900f431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Eschgf=C3=A4ller?= Date: Mon, 19 Aug 2024 15:07:20 +0200 Subject: [PATCH 10/10] Improve tests --- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index da20cea3..b6f937a3 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -411,6 +411,35 @@ public void ClosedAdapterThrowException() }).Should().Throw(); } + [Fact] + public void EnumNotValidValueThrowException() + { + Command.CommandText = GetCreateEnumTypeSql("enum_not_valid_value_test_enum", "test", 100); + Command.ExecuteNonQuery(); + + var table = "CREATE TABLE managedAppenderEnumNotValidValueTest(a enum_not_valid_value_test_enum);"; + Command.CommandText = table; + Command.ExecuteNonQuery(); + + Connection.Invoking(dbConnection => + { + using var appender = dbConnection.CreateAppender("managedAppenderEnumNotValidValueTest"); + appender + .CreateRow() + .AppendValue("test12345") + .EndRow(); + }).Should().Throw(); + + Connection.Invoking(dbConnection => + { + using var appender = dbConnection.CreateAppender("managedAppenderEnumNotValidValueTest"); + appender + .CreateRow() + .AppendValue(EnumNotValidValueTestEnum.NotValid) + .EndRow(); + }).Should().Throw(); + } + [Fact] public void TableWithSchema() { @@ -595,4 +624,9 @@ private enum TestEnum3 : ulong Test6699 = 6698, Test100000 = 99999, } + + private enum EnumNotValidValueTestEnum + { + NotValid = 12345, + } } \ No newline at end of file