diff --git a/.gitignore b/.gitignore index 906efba..942520a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ tests/data/checkpoints_with_expired_logs/ tests/data/read_null_partitions_from_checkpoint/ tests/data/action_reconciliation/ tests/data/simple_table_with_no_checkpoint/ -tests/data/simple_table_with_no_checkpoint_2/ \ No newline at end of file +tests/data/simple_table_with_no_checkpoint_2/ +.DS_Store \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Settings.cs b/tests/DeltaLake.Tests/Settings.cs index 3bc44cf..dd9aa5f 100644 --- a/tests/DeltaLake.Tests/Settings.cs +++ b/tests/DeltaLake.Tests/Settings.cs @@ -1,5 +1,3 @@ -using System.IO; - namespace DeltaLake.Tests { public static class Settings diff --git a/tests/DeltaLake.Tests/Table/AllTablesEnumerable.cs b/tests/DeltaLake.Tests/Table/AllTablesEnumerable.cs new file mode 100644 index 0000000..49bb28f --- /dev/null +++ b/tests/DeltaLake.Tests/Table/AllTablesEnumerable.cs @@ -0,0 +1,14 @@ +using System.Collections; + +namespace DeltaLake.Tests.Table; +public partial class LoadTests +{ + public class AllTablesEnumerable : IEnumerable + { + public IEnumerator GetEnumerator() => TableHelpers.ValidTables.Select(x => new object[] { + x + }).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/LoadTests.cs b/tests/DeltaLake.Tests/Table/LoadTests.cs new file mode 100644 index 0000000..5ceb81b --- /dev/null +++ b/tests/DeltaLake.Tests/Table/LoadTests.cs @@ -0,0 +1,108 @@ +using DeltaLake.Runtime; +using DeltaLake.Table; + +namespace DeltaLake.Tests.Table; +public partial class LoadTests +{ + public static TableIdentifier[] AllTables = TableHelpers.Tables.Keys.ToArray(); + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + public async Task Load_Table_At_Version_Test(long version) + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = version, + }, + CancellationToken.None); + Assert.Equal(version, table.Version()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + public async Task Table_Load_Version_Test(long version) + { + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), + CancellationToken.None); + Assert.Equal(4, table.Version()); + await table.LoadVersionAsync(version, CancellationToken.None); + Assert.Equal(version, table.Version()); + } + + + [Fact] + public async Task Table_Load_At_Now() + { + var dateTimeOffset = DateTimeOffset.UtcNow; + var location = TableIdentifier.SimpleTable.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = 1, + }, + CancellationToken.None); + Assert.Equal(1, table.Version()); + await table.LoadDateTimeAsync(dateTimeOffset, CancellationToken.None); + Assert.Equal(4, table.Version()); + } + + [Theory] + [InlineData(TableIdentifier.Checkpoints, 1, 12)] + [InlineData(TableIdentifier.CheckpointsVacuumed, 5, 12)] + [InlineData(TableIdentifier.Delta020, 1, 3)] + public async Task Table_Load_Latest(TableIdentifier identifier, int minVersion, int expectedVersion) + { + var location = identifier.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions + { + Version = minVersion, + }, + CancellationToken.None); + Assert.Equal(minVersion, table.Version()); + await table.LoadDateTimeAsync(DateTimeOffset.UtcNow, CancellationToken.None); + Assert.Equal(expectedVersion, table.Version()); + } + + [Theory] + [ClassData(typeof(AllTablesEnumerable))] + public async Task Table_Will_Load_Test(TableIdentifier identifier) + { + var location = identifier.TablePath(); + using var runtime = new DeltaRuntime(RuntimeOptions.Default); + using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), + CancellationToken.None); + switch (identifier) + { + case TableIdentifier.Covid19NYT: + case TableIdentifier.Delta220PartitionedTypes: + case TableIdentifier.SimpleCommit: + case TableIdentifier.Delta08NumericPartition: + case TableIdentifier.Delta08Date: + case TableIdentifier.Golden: + case TableIdentifier.ConcurrentWorkers: + case TableIdentifier.Delta08NullPartition: + case TableIdentifier.Delta08Partition: + case TableIdentifier.Delta08SpecialPartition: + case TableIdentifier.TableWithColumnMapping: + case TableIdentifier.TableWithEdgeTimestamps: + case TableIdentifier.TableWithLiquidClustering: + case TableIdentifier.TableWithoutDvSmall: + Assert.Equal(0, table.Version()); + break; + default: + Assert.NotEqual(0, table.Version()); + break; + } + } +} \ No newline at end of file diff --git a/tests/DeltaLake.Tests/Table/TableHelpers.cs b/tests/DeltaLake.Tests/Table/TableHelpers.cs new file mode 100644 index 0000000..3c2f671 --- /dev/null +++ b/tests/DeltaLake.Tests/Table/TableHelpers.cs @@ -0,0 +1,91 @@ +namespace DeltaLake.Tests.Table; + +public enum TableIdentifier +{ + CheckpointWithPartitions, + Checkpoints, + CheckpointsTombstones, + CheckpointsVacuumed, + ConcurrentWorkers, + Covid19NYT, + Delta020, + Delta08Empty, + Delta08, + Delta08Date, + Delta08NullPartition, + Delta08NumericPartition, + Delta08Partition, + Delta08SpecialPartition, + Delta121OnlyStructStats, + Delta220PartitionedTypes, + DeltaLiveTable, + Golden, + HttpRequests, + Issue1374, + SimpleCommit, + SimpleTable, + SimpleTableFeatures, + SimpleTableWithCdc, + SimpleTableWithCheckPoint, + TableWithColumnMapping, + TableWithDeletionLogs, + TableWithEdgeTimestamps, + TableWithLiquidClustering, + TableWithDvSmall, + TableWithoutDvSmall, + WithCheckpointNoLastcheckpoint, +} +public static class TableHelpers +{ + public static readonly IReadOnlyDictionary Tables = new Dictionary + { + [TableIdentifier.CheckpointWithPartitions] = "checkpoint_with_partitions", + [TableIdentifier.Checkpoints] = "checkpoints", + [TableIdentifier.CheckpointsTombstones] = "checkpoints_tombstones", + [TableIdentifier.CheckpointsVacuumed] = "checkpoints_vacuumed", + [TableIdentifier.ConcurrentWorkers] = "concurrent_workers", + [TableIdentifier.Covid19NYT] = "COVID-19_NYT", + [TableIdentifier.Delta020] = "delta-0.2.0", + [TableIdentifier.Delta08Empty] = "delta-0.8-empty", + [TableIdentifier.Delta08] = "delta-0.8.0", + [TableIdentifier.Delta08Date] = "delta-0.8.0-date", + [TableIdentifier.Delta08NullPartition] = "delta-0.8.0-null-partition", + [TableIdentifier.Delta08NumericPartition] = "delta-0.8.0-numeric-partition", + [TableIdentifier.Delta08Partition] = "delta-0.8.0-partitioned", + [TableIdentifier.Delta08SpecialPartition] = "delta-0.8.0-special-partition", + [TableIdentifier.Delta121OnlyStructStats] = "delta-1.2.1-only-struct-stats", + [TableIdentifier.Delta220PartitionedTypes] = "delta-2.2.0-partitioned-types", + [TableIdentifier.DeltaLiveTable] = "delta-live-table", + [TableIdentifier.Golden] = Path.Join("golden", "data-reader-array-primitives"), + [TableIdentifier.HttpRequests] = "http_requests", + [TableIdentifier.Issue1374] = "issue_1374", + [TableIdentifier.SimpleCommit] = "simple_commit", + [TableIdentifier.SimpleTable] = "simple_table", + [TableIdentifier.SimpleTableFeatures] = "simple_table_features", + [TableIdentifier.SimpleTableWithCdc] = "simple_table_with_cdc", + [TableIdentifier.SimpleTableWithCheckPoint] = "simple_table_with_checkpoint", + [TableIdentifier.TableWithColumnMapping] = "table_with_column_mapping", + [TableIdentifier.TableWithDeletionLogs] = "table_with_deletion_logs", + [TableIdentifier.TableWithEdgeTimestamps] = "table_with_edge_timestamps", + [TableIdentifier.TableWithLiquidClustering] = "table_with_liquid_clustering", + [TableIdentifier.TableWithDvSmall] = "table-with-dv-small", + [TableIdentifier.TableWithoutDvSmall] = "table-without-dv-small", + [TableIdentifier.WithCheckpointNoLastcheckpoint] = "with_checkpoint_no_last_checkpoint", + }; + + public static IEnumerable ValidTables => Tables.Keys.Where(t => t switch + { + TableIdentifier.CheckpointsTombstones => false, + _ => true, + }); + + public static string LogPath(this TableIdentifier tid, string? pathRoot = null) + { + return Path.Join(pathRoot ?? Settings.TestRoot, Tables[tid], "_delta_log"); + } + + public static string TablePath(this TableIdentifier tid, string? pathRoot = null) + { + return Path.Join(pathRoot ?? Settings.TestRoot, Tables[tid]); + } +} diff --git a/tests/DeltaLake.Tests/Table/TableTests.cs b/tests/DeltaLake.Tests/Table/TableTests.cs index 7f94856..ccd9812 100644 --- a/tests/DeltaLake.Tests/Table/TableTests.cs +++ b/tests/DeltaLake.Tests/Table/TableTests.cs @@ -1,8 +1,3 @@ -using System; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using Apache.Arrow; using Apache.Arrow.Memory; using Apache.Arrow.Types; @@ -65,39 +60,6 @@ public async Task Load_Table_Memory_Test() Assert.Equal(4, table.Version()); } - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(3)] - [InlineData(4)] - public async Task Load_Table_At_Version_Test(long version) - { - var location = Path.Join(Settings.TestRoot, "simple_table"); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions - { - Version = version, - }, - CancellationToken.None); - Assert.Equal(version, table.Version()); - } - - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(3)] - [InlineData(4)] - public async Task Table_Load_Version_Test(long version) - { - var location = Path.Join(Settings.TestRoot, "simple_table"); - using var runtime = new DeltaRuntime(RuntimeOptions.Default); - using var table = await DeltaTable.LoadAsync(runtime, location, new TableOptions(), - CancellationToken.None); - Assert.Equal(4, table.Version()); - await table.LoadVersionAsync(version, CancellationToken.None); - Assert.Equal(version, table.Version()); - } - [Fact] public async Task Table_Insert_Test() { diff --git a/tests/DeltaLake.Tests/xunit.runner.json b/tests/DeltaLake.Tests/xunit.runner.json new file mode 100644 index 0000000..13b3983 --- /dev/null +++ b/tests/DeltaLake.Tests/xunit.runner.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "longRunningTestSeconds": 60 +}