From 45c81c47a72d7df761425709d66181a04c1504da Mon Sep 17 00:00:00 2001 From: Tatsuro Shibamura Date: Thu, 28 Sep 2023 16:48:06 +0900 Subject: [PATCH] Support for `GetDynamicPropertyName` configuration (#26) --- CsvHelper.FastDynamic.Tests/CsvReaderTests.cs | 69 +++++++++++++++---- CsvHelper.FastDynamic.Tests/TestData.cs | 2 + CsvHelper.FastDynamic/CsvReaderExtensions.cs | 44 +++++++----- README.md | 28 ++++---- 4 files changed, 99 insertions(+), 44 deletions(-) diff --git a/CsvHelper.FastDynamic.Tests/CsvReaderTests.cs b/CsvHelper.FastDynamic.Tests/CsvReaderTests.cs index 1238a31..3f927e7 100644 --- a/CsvHelper.FastDynamic.Tests/CsvReaderTests.cs +++ b/CsvHelper.FastDynamic.Tests/CsvReaderTests.cs @@ -4,6 +4,8 @@ using System.Linq; using System.Threading.Tasks; +using CsvHelper.Configuration; + using Xunit; namespace CsvHelper.FastDynamic.Tests; @@ -71,19 +73,34 @@ public void GetDynamicRecords_WithMissingHeader() { var csvReader = CreateInMemoryReader_WithMissingHeader(); - var records = csvReader.GetDynamicRecords() - .Cast>() - .ToArray(); + var records = csvReader.GetDynamicRecords(); Assert.NotNull(records); - Assert.Equal(3, records.Length); + Assert.Equal(3, records.Count); for (var i = 0; i < 3; i++) { - Assert.Equal(2, records[i].Count); + Assert.Equal(TestData.CsvRecords[i]["Id"], records[i].Id); + Assert.Equal(TestData.CsvRecords[i]["Name"], records[i].Name); + Assert.Null(records[i].Location); + } + } - Assert.Equal(TestData.CsvRecords[i]["Id"], records[i]["Id"]); - Assert.Equal(TestData.CsvRecords[i]["Name"], records[i]["Name"]); + [Fact] + public void GetDynamicRecords_WithoutHeader() + { + var csvReader = CreateInMemoryReader_WithoutHeader(); + + var records = csvReader.GetDynamicRecords(); + + Assert.NotNull(records); + Assert.Equal(3, records.Count); + + for (var i = 0; i < 3; i++) + { + Assert.Equal(TestData.CsvRecords[i]["Id"], records[i].Column0); + Assert.Equal(TestData.CsvRecords[i]["Name"], records[i].Column1); + Assert.Equal(TestData.CsvRecords[i]["Location"], records[i].Column2); } } @@ -148,19 +165,34 @@ public async Task GetDynamicRecordsAsync_WithMissingHeader() { var csvReader = CreateInMemoryReader_WithMissingHeader(); - var records = (await csvReader.GetDynamicRecordsAsync()) - .Cast>() - .ToArray(); + var records = await csvReader.GetDynamicRecordsAsync(); Assert.NotNull(records); - Assert.Equal(3, records.Length); + Assert.Equal(3, records.Count); for (var i = 0; i < 3; i++) { - Assert.Equal(2, records[i].Count); + Assert.Equal(TestData.CsvRecords[i]["Id"], records[i].Id); + Assert.Equal(TestData.CsvRecords[i]["Name"], records[i].Name); + Assert.Null(records[i].Location); + } + } - Assert.Equal(TestData.CsvRecords[i]["Id"], records[i]["Id"]); - Assert.Equal(TestData.CsvRecords[i]["Name"], records[i]["Name"]); + [Fact] + public async Task GetDynamicRecordsAsync_WithoutHeader() + { + var csvReader = CreateInMemoryReader_WithoutHeader(); + + var records = await csvReader.GetDynamicRecordsAsync(); + + Assert.NotNull(records); + Assert.Equal(3, records.Count); + + for (var i = 0; i < 3; i++) + { + Assert.Equal(TestData.CsvRecords[i]["Id"], records[i].Column0); + Assert.Equal(TestData.CsvRecords[i]["Name"], records[i].Column1); + Assert.Equal(TestData.CsvRecords[i]["Location"], records[i].Column2); } } @@ -285,6 +317,15 @@ private CsvReader CreateInMemoryReader() return new CsvReader(new StringReader(TestData.CsvContent), CultureInfo.InvariantCulture); } + private CsvReader CreateInMemoryReader_WithoutHeader() + { + return new CsvReader(new StringReader(TestData.CsvContentWithoutHeader), new CsvConfiguration(CultureInfo.InvariantCulture) + { + HasHeaderRecord = false, + GetDynamicPropertyName = args => $"Column{args.FieldIndex}" + }); + } + private CsvReader CreateInMemoryReader_WithMissingHeader() { return new CsvReader(new StringReader(TestData.CsvContentWithMissingHeader), CultureInfo.InvariantCulture); diff --git a/CsvHelper.FastDynamic.Tests/TestData.cs b/CsvHelper.FastDynamic.Tests/TestData.cs index 5a085b5..5686068 100644 --- a/CsvHelper.FastDynamic.Tests/TestData.cs +++ b/CsvHelper.FastDynamic.Tests/TestData.cs @@ -6,6 +6,8 @@ public static class TestData { public static readonly string CsvContent = "Id,Name,Location\r\n1,kazuakix,Wakayama\r\n2,daruyanagi,Ehime\r\n3,buchizo,Osaka\r\n"; + public static readonly string CsvContentWithoutHeader = "1,kazuakix,Wakayama\r\n2,daruyanagi,Ehime\r\n3,buchizo,Osaka\r\n"; + public static readonly string CsvContentWithMissingHeader = "Id,Name\r\n1,kazuakix\r\n2,daruyanagi,Ehime\r\n3,buchizo\r\n"; public static readonly IReadOnlyList> CsvRecords = new[] diff --git a/CsvHelper.FastDynamic/CsvReaderExtensions.cs b/CsvHelper.FastDynamic/CsvReaderExtensions.cs index e2e41b9..2aeea5e 100644 --- a/CsvHelper.FastDynamic/CsvReaderExtensions.cs +++ b/CsvHelper.FastDynamic/CsvReaderExtensions.cs @@ -11,7 +11,7 @@ public static class CsvReaderExtensions public static IEnumerable EnumerateDynamicRecords(this CsvReader csvReader) { - if (csvReader.HeaderRecord is null) + if (csvReader.Configuration.HasHeaderRecord && csvReader.HeaderRecord is null) { if (!csvReader.Read()) { @@ -21,19 +21,24 @@ public static IEnumerable EnumerateDynamicRecords(this CsvReader csvRea csvReader.ReadHeader(); } - var csvHeader = new CsvHeader(csvReader.HeaderRecord - .Select((x, i) => csvReader.Configuration.PrepareHeaderForMatch(new PrepareHeaderForMatchArgs(x, i))) - .ToArray()); + if (!csvReader.Read()) + { + yield break; + } + + var csvHeader = new CsvHeader(Enumerable.Range(0, csvReader.HeaderRecord?.Length ?? csvReader.Parser.Count) + .Select((_, i) => csvReader.Configuration.GetDynamicPropertyName(new GetDynamicPropertyNameArgs(i, csvReader.Context))) + .ToArray()); - while (csvReader.Read()) + do { CsvRecord record; try { - var values = new object[csvReader.HeaderRecord.Length]; + var values = new object[csvHeader.FieldNames.Length]; - for (var i = 0; i < csvReader.HeaderRecord.Length; i++) + for (var i = 0; i < csvHeader.FieldNames.Length; i++) { values[i] = csvReader.Parser[i]; } @@ -53,7 +58,8 @@ public static IEnumerable EnumerateDynamicRecords(this CsvReader csvRea } yield return record; - } + + } while (csvReader.Read()); } public static async Task> GetDynamicRecordsAsync(this CsvReader csvReader) @@ -70,7 +76,7 @@ public static async Task> GetDynamicRecordsAsync(this Csv public static async IAsyncEnumerable EnumerateDynamicRecordsAsync(this CsvReader csvReader) { - if (csvReader.HeaderRecord is null) + if (csvReader.Configuration.HasHeaderRecord && csvReader.HeaderRecord is null) { if (!await csvReader.ReadAsync().ConfigureAwait(false)) { @@ -80,19 +86,24 @@ public static async IAsyncEnumerable EnumerateDynamicRecordsAsync(this csvReader.ReadHeader(); } - var csvHeader = new CsvHeader(csvReader.HeaderRecord - .Select((x, i) => csvReader.Configuration.PrepareHeaderForMatch(new PrepareHeaderForMatchArgs(x, i))) - .ToArray()); + if (!await csvReader.ReadAsync().ConfigureAwait(false)) + { + yield break; + } + + var csvHeader = new CsvHeader(Enumerable.Range(0, csvReader.HeaderRecord?.Length ?? csvReader.Parser.Count) + .Select((_, i) => csvReader.Configuration.GetDynamicPropertyName(new GetDynamicPropertyNameArgs(i, csvReader.Context))) + .ToArray()); - while (await csvReader.ReadAsync().ConfigureAwait(false)) + do { CsvRecord record; try { - var values = new object[csvReader.HeaderRecord.Length]; + var values = new object[csvHeader.FieldNames.Length]; - for (var i = 0; i < csvReader.HeaderRecord.Length; i++) + for (var i = 0; i < csvHeader.FieldNames.Length; i++) { values[i] = csvReader.Parser[i]; } @@ -112,6 +123,7 @@ public static async IAsyncEnumerable EnumerateDynamicRecordsAsync(this } yield return record; - } + + } while (await csvReader.ReadAsync().ConfigureAwait(false)); } } diff --git a/README.md b/README.md index ffbd112..d3350fa 100644 --- a/README.md +++ b/README.md @@ -56,33 +56,33 @@ await foreach (var @record in records) ### Dynamic records reader ``` -BenchmarkDotNet v0.13.7, Windows 11 (10.0.22621.2215/22H2/2022Update/SunValley2) +BenchmarkDotNet v0.13.7, Windows 11 (10.0.22621.2361/22H2/2022Update/SunValley2) AMD Ryzen 9 7950X, 1 CPU, 32 logical and 16 physical cores -.NET SDK 7.0.400 - [Host] : .NET 7.0.10 (7.0.1023.36312), X64 RyuJIT AVX2 - DefaultJob : .NET 7.0.10 (7.0.1023.36312), X64 RyuJIT AVX2 +.NET SDK 7.0.401 + [Host] : .NET 7.0.11 (7.0.1123.42427), X64 RyuJIT AVX2 + DefaultJob : .NET 7.0.11 (7.0.1123.42427), X64 RyuJIT AVX2 | Method | Mean | Error | StdDev | Ratio | Gen0 | Gen1 | Allocated | Alloc Ratio | |--------------------- |---------:|--------:|--------:|------:|--------:|--------:|----------:|------------:| -| GetRecords | 684.3 μs | 3.24 μs | 3.03 μs | 1.00 | 31.2500 | 15.6250 | 510.87 KB | 1.00 | -| GetDictionaryRecords | 200.3 μs | 0.71 μs | 0.66 μs | 0.29 | 21.7285 | 21.4844 | 355.05 KB | 0.70 | -| GetDynamicRecords | 163.7 μs | 0.91 μs | 0.85 μs | 0.24 | 14.4043 | 5.1270 | 237.28 KB | 0.46 | -| GetRawRecords | 154.1 μs | 0.45 μs | 0.38 μs | 0.23 | 13.1836 | 5.1270 | 219 KB | 0.43 | +| GetRecords | 679.9 μs | 2.30 μs | 2.04 μs | 1.00 | 31.2500 | 15.6250 | 510.87 KB | 1.00 | +| GetDictionaryRecords | 217.2 μs | 0.26 μs | 0.21 μs | 0.32 | 21.7285 | 21.4844 | 355.05 KB | 0.70 | +| GetDynamicRecords | 159.6 μs | 1.12 μs | 1.04 μs | 0.23 | 14.4043 | 4.6387 | 237.29 KB | 0.46 | +| GetRawRecords | 152.9 μs | 0.84 μs | 0.74 μs | 0.22 | 13.1836 | 5.1270 | 219 KB | 0.43 | ``` ### Dynamic records writer ``` -BenchmarkDotNet v0.13.7, Windows 11 (10.0.22621.2215/22H2/2022Update/SunValley2) +BenchmarkDotNet v0.13.7, Windows 11 (10.0.22621.2361/22H2/2022Update/SunValley2) AMD Ryzen 9 7950X, 1 CPU, 32 logical and 16 physical cores -.NET SDK 7.0.400 - [Host] : .NET 7.0.10 (7.0.1023.36312), X64 RyuJIT AVX2 - DefaultJob : .NET 7.0.10 (7.0.1023.36312), X64 RyuJIT AVX2 +.NET SDK 7.0.401 + [Host] : .NET 7.0.11 (7.0.1123.42427), X64 RyuJIT AVX2 + DefaultJob : .NET 7.0.11 (7.0.1123.42427), X64 RyuJIT AVX2 | Method | Mean | Error | StdDev | Ratio | Gen0 | Gen1 | Allocated | Alloc Ratio | |---------------------------------- |---------:|--------:|--------:|------:|--------:|--------:|----------:|------------:| -| WriteRecords_DynamicObject | 670.2 μs | 4.27 μs | 3.99 μs | 1.00 | 55.6641 | 12.6953 | 914.65 KB | 1.00 | -| WriteDynamicRecords_DynamicObject | 414.1 μs | 1.93 μs | 1.80 μs | 0.62 | 13.6719 | 2.9297 | 225.95 KB | 0.25 | +| WriteRecords_DynamicObject | 659.9 μs | 1.74 μs | 1.54 μs | 1.00 | 55.6641 | 12.6953 | 914.65 KB | 1.00 | +| WriteDynamicRecords_DynamicObject | 391.7 μs | 1.78 μs | 1.66 μs | 0.59 | 13.6719 | 2.9297 | 225.95 KB | 0.25 | ``` ## Thanks