From b5c528f55013aa488ad2f3f426f21e09effb65b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Wagenf=C3=BChr?= Date: Sat, 23 Oct 2021 17:08:28 +0200 Subject: [PATCH] Add the ability to apply rules to a file --- src/CsvProc9000/Csv/CsvColumn.cs | 2 +- src/CsvProc9000/Csv/CsvFile.cs | 18 +--- src/CsvProc9000/Csv/CsvReader.cs | 30 +++--- src/CsvProc9000/Csv/CsvRow.cs | 7 +- src/CsvProc9000/Options/Condition.cs | 13 +++ .../Options/CsvProcessorOptions.cs | 6 +- src/CsvProc9000/Options/Rule.cs | 14 +++ src/CsvProc9000/Processors/CsvProcessor.cs | 95 ++++++++++++++++--- src/CsvProc9000/Workers/CsvWatcherWorker.cs | 6 +- src/CsvProc9000/appsettings.Development.json | 21 +++- 10 files changed, 162 insertions(+), 50 deletions(-) create mode 100644 src/CsvProc9000/Options/Condition.cs create mode 100644 src/CsvProc9000/Options/Rule.cs diff --git a/src/CsvProc9000/Csv/CsvColumn.cs b/src/CsvProc9000/Csv/CsvColumn.cs index 96e2852..92e7e60 100644 --- a/src/CsvProc9000/Csv/CsvColumn.cs +++ b/src/CsvProc9000/Csv/CsvColumn.cs @@ -1,4 +1,4 @@ namespace CsvProc9000.Csv { - public record CsvColumn(int Index, string Name); + public record CsvColumn(string Name); } \ No newline at end of file diff --git a/src/CsvProc9000/Csv/CsvFile.cs b/src/CsvProc9000/Csv/CsvFile.cs index 86d216d..35e3669 100644 --- a/src/CsvProc9000/Csv/CsvFile.cs +++ b/src/CsvProc9000/Csv/CsvFile.cs @@ -1,29 +1,21 @@ using System; using System.Collections.Generic; -using System.Linq; using JetBrains.Annotations; namespace CsvProc9000.Csv { public class CsvFile { - private readonly List _columns = new(); private readonly List _rows = new(); - public CsvFile([NotNull] IEnumerable columns) + public CsvFile([NotNull] string fileName) { - if (columns == null) throw new ArgumentNullException(nameof(columns)); - - var columnsArray = columns.ToArray(); - for (var index = 0; index < columnsArray.Length; index++) - { - _columns.Add(new CsvColumn(index, columnsArray[index])); - } + FileName = fileName ?? throw new ArgumentNullException(nameof(fileName)); } + + public string FileName { get; } - public IReadOnlyCollection Columns => _columns; - - public IReadOnlyCollection Rows => _rows; + public IEnumerable Rows => _rows; public void AddRow([NotNull] CsvRow row) { diff --git a/src/CsvProc9000/Csv/CsvReader.cs b/src/CsvProc9000/Csv/CsvReader.cs index 4e7e0b0..d618fa7 100644 --- a/src/CsvProc9000/Csv/CsvReader.cs +++ b/src/CsvProc9000/Csv/CsvReader.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.IO.Abstractions; @@ -32,44 +33,45 @@ public async Task ReadAsync(string fileName, string delimiter) }; using var csvReader = new CsvHelper.CsvReader(streamReader, configuration); - var csvFile = await CreateCsvFileFromHeader(fileName, csvReader); - await ProcessCsvRows(csvReader, csvFile); + var csvFile = new CsvFile(fileName); + var headers = await GetHeaders(fileName, csvReader); + await ProcessCsvRows(csvReader, csvFile, headers); return csvFile; } - private static async Task CreateCsvFileFromHeader(string fileName, IReader csvReader) + private static async Task> GetHeaders(string fileName, IReader csvReader) { await csvReader.ReadAsync(); if (!csvReader.ReadHeader()) throw new InvalidOperationException($"Could not read header of file {fileName}"); - var header = csvReader.HeaderRecord; - if (!header.Any()) + var headers = csvReader.HeaderRecord; + if (!headers.Any()) throw new InvalidOperationException($"Did not find any headers for file {fileName}"); - - var csvFile = new CsvFile(header); - return csvFile; + + return headers; } - private static async Task ProcessCsvRows(IReader csvReader, CsvFile csvFile) + private static async Task ProcessCsvRows(IReader csvReader, CsvFile csvFile, IEnumerable headers) { + var headersList = headers.ToList(); while (await csvReader.ReadAsync()) { - var row = ProcessCsvRow(csvReader, csvFile); + var row = ProcessCsvRow(csvReader, headersList); csvFile.AddRow(row); } } - private static CsvRow ProcessCsvRow(IReaderRow csvReader, CsvFile csvFile) + private static CsvRow ProcessCsvRow(IReaderRow csvReader, IEnumerable headers) { var csvRow = new CsvRow(); - foreach (var column in csvFile.Columns) + foreach (var header in headers) { - if (!csvReader.TryGetField(column.Index, out var fieldValue)) continue; + if (!csvReader.TryGetField(header, out var fieldValue)) continue; - csvRow.AddField(column, fieldValue); + csvRow.AddOrUpdateField(header, fieldValue); } return csvRow; diff --git a/src/CsvProc9000/Csv/CsvRow.cs b/src/CsvProc9000/Csv/CsvRow.cs index d9391a7..083b221 100644 --- a/src/CsvProc9000/Csv/CsvRow.cs +++ b/src/CsvProc9000/Csv/CsvRow.cs @@ -8,13 +8,14 @@ public class CsvRow { private readonly List _fields = new(); - public IReadOnlyCollection Fields => _fields; + public IEnumerable Fields => _fields; - public void AddField([NotNull] CsvColumn column, [NotNull] string fieldValue) + public void AddOrUpdateField([NotNull] string columnName, [NotNull] string fieldValue) { - if (column == null) throw new ArgumentNullException(nameof(column)); + if (columnName == null) throw new ArgumentNullException(nameof(columnName)); if (fieldValue == null) throw new ArgumentNullException(nameof(fieldValue)); + var column = new CsvColumn(columnName); _fields.Add(new CsvField(column, fieldValue)); } } diff --git a/src/CsvProc9000/Options/Condition.cs b/src/CsvProc9000/Options/Condition.cs new file mode 100644 index 0000000..b761e53 --- /dev/null +++ b/src/CsvProc9000/Options/Condition.cs @@ -0,0 +1,13 @@ +using JetBrains.Annotations; + +namespace CsvProc9000.Options +{ + public class Condition + { + [UsedImplicitly] + public string Field { get; set; } + + [UsedImplicitly] + public string Value { get; set; } + } +} \ No newline at end of file diff --git a/src/CsvProc9000/Options/CsvProcessorOptions.cs b/src/CsvProc9000/Options/CsvProcessorOptions.cs index 84aeffa..af09791 100644 --- a/src/CsvProc9000/Options/CsvProcessorOptions.cs +++ b/src/CsvProc9000/Options/CsvProcessorOptions.cs @@ -1,4 +1,5 @@ -using JetBrains.Annotations; +using System.Collections.Generic; +using JetBrains.Annotations; namespace CsvProc9000.Options { @@ -15,5 +16,8 @@ public class CsvProcessorOptions [UsedImplicitly] public string OutboxDelimiter { get; set; } + + [UsedImplicitly] + public List Rules { get; set; } } } \ No newline at end of file diff --git a/src/CsvProc9000/Options/Rule.cs b/src/CsvProc9000/Options/Rule.cs new file mode 100644 index 0000000..2ec2822 --- /dev/null +++ b/src/CsvProc9000/Options/Rule.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; +using JetBrains.Annotations; + +namespace CsvProc9000.Options +{ + public class Rule + { + [UsedImplicitly] + public List Conditions { get; set; } + + [UsedImplicitly] + public Dictionary Steps { get; set; } + } +} \ No newline at end of file diff --git a/src/CsvProc9000/Processors/CsvProcessor.cs b/src/CsvProc9000/Processors/CsvProcessor.cs index 53bff32..d567088 100644 --- a/src/CsvProc9000/Processors/CsvProcessor.cs +++ b/src/CsvProc9000/Processors/CsvProcessor.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.IO.Abstractions; +using System.Linq; using System.Threading.Tasks; using CsvProc9000.Csv; using CsvProc9000.Options; @@ -24,21 +25,21 @@ public CsvProcessor( [NotNull] ICsvReader csvReader) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _processorOptions = processorOptions?.Value ?? throw new ArgumentNullException(nameof(processorOptions)); + _processorOptions = processorOptions.Value ?? throw new ArgumentNullException(nameof(processorOptions)); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _csvReader = csvReader ?? throw new ArgumentNullException(nameof(csvReader)); } - + public async Task ProcessAsync(IFileInfo file) { _logger.LogDebug("Processor: Waiting until file {File} can be processed", file.FullName); await WaitUntilFileIsUnlockedAsync(file); if (!CanProcess(file)) return; - + await ProcessInternalAsync(file); } - + private bool CanProcess(IFileInfo file) { if (!file.Exists) @@ -46,7 +47,7 @@ private bool CanProcess(IFileInfo file) _logger.LogDebug("Cannot process file {File}, because it does not exist", file.FullName); return false; } - + if (file.IsReadOnly) { _logger.LogDebug("Cannot process file {File}, because it is read-only", file.FullName); @@ -56,7 +57,8 @@ private bool CanProcess(IFileInfo file) // ReSharper disable once InvertIf if (!file.Extension.Equals(".csv")) { - _logger.LogDebug("Cannot process file {File}, because it has file-extensions '{Extension}' instead of '.csv'", + _logger.LogDebug( + "Cannot process file {File}, because it has file-extensions '{Extension}' instead of '.csv'", file.FullName, file.Extension); return false; } @@ -68,19 +70,84 @@ private async Task ProcessInternalAsync(IFileSystemInfo file) { _logger.LogInformation("Processor: Starting to process {File}...", file.FullName); - _logger.LogDebug("Reading in file {File}...", file.FullName); + _logger.LogDebug("Processor: Reading in file {File}...", file.FullName); var csvFile = await _csvReader.ReadAsync(file.FullName, _processorOptions.InboxDelimiter); + + _logger.LogDebug("Processor: Applying rules to file {File}...", file.FullName); + ApplyRulesToFile(csvFile); + } + + // if (_fileSystem.Directory.Exists(_processorOptions.Outbox)) + // _fileSystem.Directory.CreateDirectory(_processorOptions.Outbox); + + private void ApplyRulesToFile(CsvFile csvFile) + { + if (_processorOptions.Rules == null || !_processorOptions.Rules.Any()) + { + _logger.LogWarning("Processor: Cannot process file {File} because there are no rules defined", + csvFile.FileName); + return; + } + + foreach (var rule in _processorOptions.Rules) + ApplyRuleToFile(csvFile, rule); + } + + private void ApplyRuleToFile(CsvFile csvFile, Rule rule) + { + if (rule.Conditions == null || !rule.Conditions.Any()) + { + _logger.LogWarning("Processor: Skipping Rule at index {Index} because it has no conditions", + _processorOptions.Rules.IndexOf(rule)); + return; + } + + foreach (var row in csvFile.Rows) + ApplyRuleToRow(row, rule, csvFile); + } + + private void ApplyRuleToRow(CsvRow row, Rule rule, CsvFile file) + { + if (!MeetsRowConditions(row, rule, file)) return; + + _logger.LogTrace("Processor: File {File} meets rule at index {RuleIndex}. Applying rule...", + file.FileName, _processorOptions.Rules.IndexOf(rule)); + + foreach (var (columnName, fieldValue) in rule.Steps) + { + _logger.LogTrace("Processor: Row at index {RowIndex}: Adding field '{Field}' with value '{FieldValue}'", + _processorOptions.Rules.IndexOf(rule), columnName, fieldValue); + row.AddOrUpdateField(columnName, fieldValue); + } } - private void MoveFileToOutput(IFileInfo file) + private bool MeetsRowConditions(CsvRow row, Rule rule, CsvFile file) { - var fileName = file.Name; - var destinationFileName = _fileSystem.Path.Combine(_processorOptions.Outbox, fileName); + foreach (var condition in rule.Conditions) + { + var field = row.Fields.FirstOrDefault(field => field.Column.Name == condition.Field); + if (field == null) + { + _logger.LogTrace( + "Processor: Row at index {RowIndex} does not meet condition at index {ConditionIndex}, " + + "because the field '{FieldName}' could not be found", + file.Rows.ToList().IndexOf(row), rule.Conditions.IndexOf(condition), condition.Field); + return false; + } - if (_fileSystem.Directory.Exists(_processorOptions.Outbox)) - _fileSystem.Directory.CreateDirectory(_processorOptions.Outbox); + if (field.Value == condition.Value) continue; + + _logger.LogTrace( + "Processor: Row at index {RowIndex} does not meet condition at index {ConditionIndex}, " + + "because the field '{FieldName}' with the value '{FieldValue}' " + + "does not have the desired value '{ConditionValue}'", + file.Rows.ToList().IndexOf(row), rule.Conditions.IndexOf(condition), condition.Field, + field.Value, condition.Value); + return false; + } - file.MoveTo(destinationFileName, true); + // if we get here, all conditions have been met + return true; } private async Task WaitUntilFileIsUnlockedAsync(IFileSystemInfo file) @@ -98,7 +165,7 @@ private bool IsFileLocked(IFileSystemInfo file) { using var stream = _fileSystem.File.Open(file.FullName, FileMode.Open, FileAccess.ReadWrite, FileShare.None); - + // if this succeeds the file is not locked return false; } diff --git a/src/CsvProc9000/Workers/CsvWatcherWorker.cs b/src/CsvProc9000/Workers/CsvWatcherWorker.cs index 60efb56..04b2c8e 100644 --- a/src/CsvProc9000/Workers/CsvWatcherWorker.cs +++ b/src/CsvProc9000/Workers/CsvWatcherWorker.cs @@ -24,13 +24,13 @@ internal sealed class CsvWatcherWorker : BackgroundService public CsvWatcherWorker( [NotNull] ILogger logger, [NotNull] IOptions processorOptions, - [NotNull] IFileSystem fileSystem, + [NotNull] IFileSystem fileSystem, [NotNull] ICsvProcessor csvProcessor) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _csvProcessor = csvProcessor ?? throw new ArgumentNullException(nameof(csvProcessor)); - _processorOptions = processorOptions?.Value ?? throw new ArgumentNullException(nameof(processorOptions)); + _processorOptions = processorOptions.Value ?? throw new ArgumentNullException(nameof(processorOptions)); _fileSystemWatcher = fileSystem.FileSystemWatcher.CreateNew(_processorOptions.Inbox); @@ -39,7 +39,7 @@ public CsvWatcherWorker( protected override Task ExecuteAsync(CancellationToken stoppingToken) { - _logger.LogInformation("Starting to watch for files in {Target}...", _processorOptions.Inbox); + _logger.LogInformation("Watcher: Starting to watch for files in {Target}...", _processorOptions.Inbox); _fileSystemWatcher.EnableRaisingEvents = true; return Task.CompletedTask; diff --git a/src/CsvProc9000/appsettings.Development.json b/src/CsvProc9000/appsettings.Development.json index 28efe9c..fee4aa0 100644 --- a/src/CsvProc9000/appsettings.Development.json +++ b/src/CsvProc9000/appsettings.Development.json @@ -7,6 +7,25 @@ "InboxDelimiter": ";", "Outbox": "X:\\Dev\\__test\\CSV_OUT", - "OutboxDelimiter": ";" + "OutboxDelimiter": ";", + + "Rules": [ + { + "Conditions": [ + { + "Field": "Artikelnummer", + "Value": "16887" + }, + { + "Field": "Versandart", + "Value": "Standardversand" + } + ], + "Steps": { + "SHIPMENTTYPE": "NP", + "Verpackungsart": 0 + } + } + ] } }