More precise TimestampNs handling

Giorgi · Oct 21, 2024 · fd5688b · fd5688b
1 parent 7787158
commit fd5688b
Show file tree

Hide file tree

Showing 7 changed files with 56 additions and 29 deletions.
diff --git a/DuckDB.NET.Bindings/DuckDBNativeObjects.cs b/DuckDB.NET.Bindings/DuckDBNativeObjects.cs
@@ -143,12 +143,6 @@ public struct DuckDBTimeTz
 public struct DuckDBTimestampStruct
 {
     public long Micros { get; set; }
-
-    public readonly DateTime ToDateTime()
-    {
-        var ticks = Micros * 10 + Utils.UnixEpochTicks;
-        return new DateTime(ticks);
-    }
 }
 
 [StructLayout(LayoutKind.Sequential)]

diff --git a/DuckDB.NET.Bindings/Utils.cs b/DuckDB.NET.Bindings/Utils.cs
@@ -7,8 +7,6 @@ namespace DuckDB.NET.Native;
 
 public static class Utils
 {
-    internal const long UnixEpochTicks = 621355968000000000;
-
     public static bool IsSuccess(this DuckDBState state)
     {
         return state == DuckDBState.Success;

diff --git a/DuckDB.NET.Data/Extensions/DateTimeExtensions.cs b/DuckDB.NET.Data/Extensions/DateTimeExtensions.cs
@@ -0,0 +1,19 @@
+using System;
+
+namespace DuckDB.NET.Data.Extensions;
+
+//https://stackoverflow.com/a/5359304/239438
+internal static class DateTimeExtensions
+{
+    public const int TicksPerMicrosecond = 10;
+    public const int NanosecondsPerTick = 100;
+
+    public static int Nanoseconds(this DateTime self)
+    {
+#if NET8_0_OR_GREATER
+        return self.Nanosecond;
+#else
+        return (int)(self.Ticks % TimeSpan.TicksPerMillisecond % TicksPerMicrosecond) * NanosecondsPerTick;
+#endif
+    }
+}
diff --git a/DuckDB.NET.Data/Internal/Reader/DateTimeVectorDataReader.cs b/DuckDB.NET.Data/Internal/Reader/DateTimeVectorDataReader.cs
@@ -18,7 +18,6 @@ internal sealed class DateTimeVectorDataReader : VectorDataReaderBase
 
     private static readonly Type TimeOnlyType = typeof(TimeOnly);
     private static readonly Type TimeOnlyNullableType = typeof(TimeOnly?);
-
 #endif
 
     internal unsafe DateTimeVectorDataReader(void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType, string columnName) : base(dataPointer, validityMaskPointer, columnType, columnName)
@@ -86,24 +85,21 @@ protected override T GetValidValue<T>(ulong offset, Type targetType)
             DuckDBType.TimestampTz => ReadTimestamp<T>(offset, targetType),
             DuckDBType.TimestampS => ReadTimestamp<T>(offset, targetType, 1000000),
             DuckDBType.TimestampMs => ReadTimestamp<T>(offset, targetType, 1000),
-            DuckDBType.TimestampNs => ReadTimestamp<T>(offset, targetType, 1, 1000),
+            DuckDBType.TimestampNs => ReadTimestamp<T>(offset, targetType, 1, 1000, true),
             _ => base.GetValidValue<T>(offset, targetType)
         };
     }
 
-    private T ReadTimestamp<T>(ulong offset, Type targetType, int factor = 1, int divisor = 1)
+    private T ReadTimestamp<T>(ulong offset, Type targetType, int factor = 1, int divisor = 1, bool keepNanoseconds = false)
     {
-        var timestampStruct = GetFieldData<DuckDBTimestampStruct>(offset);
-
-        timestampStruct.Micros = timestampStruct.Micros * factor / divisor;
+        var (additionalTicks, timestamp) = ReadTimestamp(offset, factor, divisor, keepNanoseconds);
 
         if (targetType == DateTimeType || targetType == DateTimeNullableType)
         {
-            var dateTime = timestampStruct.ToDateTime();
+            var dateTime = timestamp.ToDateTime().AddTicks(additionalTicks);
             return (T)(object)dateTime;
         }
 
-        var timestamp = NativeMethods.DateTimeHelpers.DuckDBFromTimestamp(timestampStruct);
         return (T)(object)timestamp;
     }
 
@@ -118,7 +114,7 @@ internal override object GetValue(ulong offset, Type targetType)
             DuckDBType.TimestampTz => GetDateTime(offset, targetType),
             DuckDBType.TimestampS => GetDateTime(offset, targetType, 1000000),
             DuckDBType.TimestampMs => GetDateTime(offset, targetType, 1000),
-            DuckDBType.TimestampNs => GetDateTime(offset, targetType, 1, 1000),
+            DuckDBType.TimestampNs => GetDateTime(offset, targetType, 1, 1000, true),
             _ => base.GetValue(offset, targetType)
         };
     }
@@ -176,18 +172,18 @@ private object GetTime(ulong offset, Type targetType)
         return timeOnly;
     }
 
-    private object GetDateTime(ulong offset, Type targetType, int factor = 1, int divisor = 1)
+    private object GetDateTime(ulong offset, Type targetType, int factor = 1, int divisor = 1, bool keepNanoseconds = false)
     {
-        var data = GetFieldData<DuckDBTimestampStruct>(offset);
-
-        data.Micros = (data.Micros * factor / divisor);
+        var (additionalTicks, timestamp) = ReadTimestamp(offset, factor, divisor, keepNanoseconds);
 
         if (targetType == typeof(DateTime))
         {
-            return data.ToDateTime();
+            var dateTime = timestamp.ToDateTime().AddTicks(additionalTicks);
+
+            return dateTime;
         }
 
-        return NativeMethods.DateTimeHelpers.DuckDBFromTimestamp(data);
+        return timestamp;
     }
 
     private object GetDateTimeOffset(ulong offset, Type targetType)
@@ -201,4 +197,20 @@ private object GetDateTimeOffset(ulong offset, Type targetType)
 
         return timeTz;
     }
+
+    private (int additionalTicks, DuckDBTimestamp timestamp) ReadTimestamp(ulong offset, int factor, int divisor, bool keepNanoseconds)
+    {
+        var data = GetFieldData<DuckDBTimestampStruct>(offset);
+        var additionalTicks = 0;
+
+        if (keepNanoseconds)
+        {
+            additionalTicks = (int)(data.Micros % 1000 / 100);
+        }
+
+        data.Micros = (data.Micros * factor / divisor);
+
+        var timestamp = NativeMethods.DateTimeHelpers.DuckDBFromTimestamp(data);
+        return (additionalTicks, timestamp);
+    }
 }
diff --git a/DuckDB.NET.Data/Internal/Writer/DateTimeVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/DateTimeVectorDataWriter.cs
@@ -1,4 +1,5 @@
 using System;
+using DuckDB.NET.Data.Extensions;
 using DuckDB.NET.Native;
 
 namespace DuckDB.NET.Data.Internal.Writer;
@@ -17,6 +18,8 @@ internal override bool AppendDateTime(DateTime value, ulong rowIndex)
         if (ColumnType == DuckDBType.TimestampNs)
         {
             timestamp.Micros *= 1000;
+
+            timestamp.Micros += value.Nanoseconds();
         }
 
         if (ColumnType == DuckDBType.TimestampMs)

diff --git a/DuckDB.NET.Test/DuckDBDataReaderTestAllTypes.cs b/DuckDB.NET.Test/DuckDBDataReaderTestAllTypes.cs
@@ -237,7 +237,7 @@ public void ReadTimeStampNS()
         VerifyDataStruct("timestamp_ns", 17, new List<DateTime>
         {
             new DateTime(1677, 09, 22),
-            new DateTime (2262, 04, 11, 23,47,16).AddTicks(854775 * 10)
+            new DateTime (2262, 04, 11, 23,47,16).AddTicks(8547758)
         }, typeof(DuckDBTimestamp));
     }
 
@@ -633,7 +633,7 @@ public void ReadStructOfFixedArray()
         reader.GetFieldValue<StructOfArrayTest>(columnIndex).Should().BeEquivalentTo(new StructOfArrayTest()
         {
             A = new() { null, 2, 3 },
-            B = new() { "a", null, "c"}
+            B = new() { "a", null, "c" }
         });
 
         reader.Read();

diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs
@@ -247,27 +247,28 @@ public void TemporalValues()
         Command.CommandText = "CREATE TABLE managedAppenderTemporal(a Date, b TimeStamp, c TIMESTAMP_NS, d TIMESTAMP_MS, e TIMESTAMP_S, f TIMESTAMPTZ, g TIMETZ, h Time);";
         Command.ExecuteNonQuery();
 
-        var dates = Enumerable.Range(0, 20).Select(i => new DateTime(1900, 1, 1).AddDays(Random.Shared.Next(1, 50000)).AddSeconds(Random.Shared.Next(3600 * 2, 3600 * 24))).ToList();
+        var dates = Enumerable.Range(0, 20).Select(i => new DateTime(1900, 1, 1).AddDays(Random.Shared.Next(1, 50000))
+                                                        .AddSeconds(Random.Shared.Next(3600 * 2, 3600 * 24))).ToList();
 
         using (var appender = Connection.CreateAppender("managedAppenderTemporal"))
         {
             foreach (var value in dates)
             {
                 appender.CreateRow()
                                 .AppendValue((DateOnly?)DateOnly.FromDateTime(value))
-                                .AppendValue(value).AppendValue(value)
+                                .AppendValue(value).AppendValue(value.AddTicks(1))
                                 .AppendValue(value).AppendValue(value).AppendValue(value)
                                 .AppendValue(value.ToDateTimeOffset(TimeSpan.FromHours(1)))
                                 .AppendValue((TimeOnly?)TimeOnly.FromDateTime(value))
                         .EndRow();
             }
         }
 
-        var result = Connection.Query<(DateOnly, DateTime, DateTime, DateTime, DateTime, DateTime, DateTimeOffset, TimeOnly)>("SELECT a, b, c, d, e, f, g, h FROM managedAppenderTemporal").ToList();
+        var result = Connection.Query<(DateOnly, DateTime, DateTime nanos, DateTime, DateTime, DateTime, DateTimeOffset, TimeOnly)>("SELECT a, b, c, d, e, f, g, h FROM managedAppenderTemporal").ToList();
 
         result.Select(tuple => tuple.Item1).Should().BeEquivalentTo(dates.Select(DateOnly.FromDateTime));
         result.Select(tuple => tuple.Item2).Should().BeEquivalentTo(dates);
-        result.Select(tuple => tuple.Item3).Should().BeEquivalentTo(dates);
+        result.Select(tuple => tuple.nanos).Should().BeEquivalentTo(dates.Select(time => time.AddTicks(1)));
         result.Select(tuple => tuple.Item4).Should().BeEquivalentTo(dates);
         result.Select(tuple => tuple.Item5).Should().BeEquivalentTo(dates);
         result.Select(tuple => tuple.Item6).Should().BeEquivalentTo(dates);