Merge pull request #424 from paillave/Scheduler

sql server should works with any adonet driver #423
paillave · Mar 21, 2023 · 15ca0e1 · 15ca0e1
2 parents b580cfb + d9b900d
commit 15ca0e1
Show file tree

Hide file tree

Showing 23 changed files with 72 additions and 27 deletions.
diff --git a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EntityFrameworkCoreExtension</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.Autofac/Paillave.Etl.Autofac.csproj b/src/Paillave.Etl.Autofac/Paillave.Etl.Autofac.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Autofac</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.Bloomberg/Paillave.Etl.Bloomberg.csproj b/src/Paillave.Etl.Bloomberg/Paillave.Etl.Bloomberg.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Bloomberg</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.Dropbox/Paillave.Etl.Dropbox.csproj b/src/Paillave.Etl.Dropbox/Paillave.Etl.Dropbox.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Dropbox</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj b/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.EntityFrameworkCore</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.ExcelFile/Paillave.Etl.ExcelFile.csproj b/src/Paillave.Etl.ExcelFile/Paillave.Etl.ExcelFile.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.ExcelFile</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.ExecutionToolkit/Paillave.Etl.ExecutionToolkit.csproj b/src/Paillave.Etl.ExecutionToolkit/Paillave.Etl.ExecutionToolkit.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.ExecutionToolkit</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.FileSystem/Paillave.Etl.FileSystem.csproj b/src/Paillave.Etl.FileSystem/Paillave.Etl.FileSystem.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.FileSystem</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj b/src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.FromConfigurationConnectors</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.Ftp/Paillave.Etl.Ftp.csproj b/src/Paillave.Etl.Ftp/Paillave.Etl.Ftp.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Ftp</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.Mail/Paillave.Etl.Mail.csproj b/src/Paillave.Etl.Mail/Paillave.Etl.Mail.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Mail</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.Pdf/Paillave.Etl.Pdf.csproj b/src/Paillave.Etl.Pdf/Paillave.Etl.Pdf.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Pdf</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.Sftp/Paillave.Etl.Sftp.csproj b/src/Paillave.Etl.Sftp/Paillave.Etl.Sftp.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Sftp</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.SqlServer/Paillave.Etl.SqlServer.csproj b/src/Paillave.Etl.SqlServer/Paillave.Etl.SqlServer.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.SqlServer</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
@@ -31,6 +31,7 @@
 
   <ItemGroup>
     <PackageReference Include="system.data.sqlclient" Version="4.8.5" allowedVersions="4.8.0"/>
+    <PackageReference Include="system.data.odbc" Version="6.0.11" allowedVersions="6.0.0"/>
   </ItemGroup>
 
 </Project>
diff --git a/src/Paillave.Etl.SqlServer/SqlCommandValueProvider.cs b/src/Paillave.Etl.SqlServer/SqlCommandValueProvider.cs
@@ -2,6 +2,7 @@
 using Paillave.Etl.SqlServer.Core;
 using System;
 using System.Collections.Generic;
+using System.Data;
 using System.Data.SqlClient;
 using System.Linq;
 using System.Linq.Expressions;
@@ -61,7 +62,7 @@ public class SqlCommandValueProvider<TIn, TOut> : ValuesProviderBase<TIn, TOut>
         public SqlCommandValueProvider(SqlCommandValueProviderArgs<TIn, TOut> args) => (_args) = (args);
         public override ProcessImpact PerformanceImpact => ProcessImpact.Average;
         public override ProcessImpact MemoryFootPrint => ProcessImpact.Light;
-        private TOut CreateRecord(SqlDataReader record, IList<SqlResultFieldDefinition> fieldDefinitions)
+        private TOut CreateRecord(IDataReader record, IList<SqlResultFieldDefinition> fieldDefinitions)
         {
             IDictionary<string, object> values = new Dictionary<string, object>();
             for (int i = 0; i < record.FieldCount; i++)
@@ -73,8 +74,11 @@ private TOut CreateRecord(SqlDataReader record, IList<SqlResultFieldDefinition>
 
         public override void PushValues(TIn input, Action<TOut> push, CancellationToken cancellationToken, IExecutionContext context)
         {
-            var sqlConnection = _args.ConnectionName == null ? context.DependencyResolver.Resolve<SqlConnection>() : context.DependencyResolver.Resolve<SqlConnection>(_args.ConnectionName);
-            var command = new SqlCommand(_args.SqlQuery, sqlConnection);
+            var sqlConnection = _args.ConnectionName == null ? context.DependencyResolver.Resolve<IDbConnection>() : context.DependencyResolver.Resolve<IDbConnection>(_args.ConnectionName);
+            var command = sqlConnection.CreateCommand();
+            command.CommandText = _args.SqlQuery;
+            command.CommandType = CommandType.Text;
+            // var command = new SqlCommand(_args.SqlQuery, sqlConnection);
             Regex getParamRegex = new Regex(@"@(?<param>\w*)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
             var allMatches = getParamRegex.Matches(_args.SqlQuery).ToList().Select(match => match.Groups["param"].Value).Distinct().ToList();
             foreach (var parameterName in allMatches)

diff --git a/src/Paillave.Etl.SqlServer/SqlServerSaveStreamNode.cs b/src/Paillave.Etl.SqlServer/SqlServerSaveStreamNode.cs
@@ -7,6 +7,7 @@
 using System.Data.SqlClient;
 using System.Reflection;
 using System.Linq.Expressions;
+using System.Data;
 
 namespace Paillave.Etl.SqlServer
 {
@@ -92,12 +93,15 @@ private string GetSqlStatement()
         private void ProcessItem(TValue item, string connectionName)
         {
             var resolver = this.ExecutionContext.DependencyResolver;
-            var sqlConnection = connectionName == null ? resolver.Resolve<SqlConnection>() : resolver.Resolve<SqlConnection>(connectionName);
+            var sqlConnection = connectionName == null ? resolver.Resolve<IDbConnection>() : resolver.Resolve<IDbConnection>(connectionName);
             // List<PropertyInfo> pivot = base.Args.Pivot == null ? new List<PropertyInfo>() : base.Args.Pivot.GetPropertyInfos();
             // List<PropertyInfo> computed = base.Args.Computed == null ? new List<PropertyInfo>() : base.Args.Computed.GetPropertyInfos();
             // var sqlQuery = CreateSqlQuery(base.Args.Table, typeof(TIn).GetProperties().ToList(), pivot, computed);
             var sqlStatement = GetSqlStatement();
-            var command = new SqlCommand(sqlStatement, sqlConnection);
+            var command=sqlConnection.CreateCommand();
+            command.CommandText = sqlStatement;
+            command.CommandType = CommandType.Text;
+            // var command = new SqlCommand(sqlStatement, sqlConnection);
             // Regex getParamRegex = new Regex(@"@(?<param>\w*)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
             // var allMatches = getParamRegex.Matches(base.Args.SqlQuery).ToList().Select(match => match.Groups["param"].Value).Distinct().ToList();
             foreach (var parameterName in _inPropertyInfos.Keys.Except(_computed.Select(i => i.Name)))
@@ -109,7 +113,7 @@ private void ProcessItem(TValue item, string connectionName)
                     UpdateRecord(reader, item);
         }
 
-        private void UpdateRecord(SqlDataReader record, TValue item)
+        private void UpdateRecord(IDataReader record, TValue item)
         {
             IDictionary<string, object> values = new Dictionary<string, object>();
             for (int i = 0; i < record.FieldCount; i++)

diff --git a/src/Paillave.Etl.SqlServer/ToSqlCommandStreamNode.cs b/src/Paillave.Etl.SqlServer/ToSqlCommandStreamNode.cs
@@ -6,6 +6,7 @@
 using System.Data.SqlClient;
 using System.Reflection;
 using System.Text.RegularExpressions;
+using System.Data;
 
 namespace Paillave.Etl.SqlServer
 {
@@ -34,8 +35,12 @@ protected override TStream CreateOutputStream(ToSqlCommandArgs<TIn, TStream, TVa
         public void ProcessItem(TValue item, string connectionName)
         {
             var resolver = this.ExecutionContext.DependencyResolver;
-            var sqlConnection = connectionName == null ? resolver.Resolve<SqlConnection>() : resolver.Resolve<SqlConnection>(connectionName);
-            var command = new SqlCommand(base.Args.SqlQuery, sqlConnection);
+            var sqlConnection = connectionName == null ? resolver.Resolve<IDbConnection>() : resolver.Resolve<IDbConnection>(connectionName);
+            var command = sqlConnection.CreateCommand();
+            command.CommandText = base.Args.SqlQuery;
+            command.CommandType = CommandType.Text;
+
+            // var command = new SqlCommand(base.Args.SqlQuery, sqlConnection);
             Regex getParamRegex = new Regex(@"@(?<param>\w*)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
             var allMatches = getParamRegex.Matches(base.Args.SqlQuery).ToList().Select(match => match.Groups["param"].Value).Distinct().ToList();
             foreach (var parameterName in allMatches)

diff --git a/src/Paillave.Etl.TextFile/Paillave.Etl.TextFile.csproj b/src/Paillave.Etl.TextFile/Paillave.Etl.TextFile.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.TextFile</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.XmlFile/Paillave.Etl.XmlFile.csproj b/src/Paillave.Etl.XmlFile/Paillave.Etl.XmlFile.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.XmlFile</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl.Zip/Paillave.Etl.Zip.csproj b/src/Paillave.Etl.Zip/Paillave.Etl.Zip.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Zip</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Etl/Paillave.Etl.csproj b/src/Paillave.Etl/Paillave.Etl.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Core</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Paillave.Pdf/Paillave.Pdf.csproj b/src/Paillave.Pdf/Paillave.Pdf.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.Pdf</PackageId>
-    <Version>2.1.2-beta</Version>
+    <Version>2.1.3-beta</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

diff --git a/src/Tutorials/SimpleTutorial/Program copy 2.cs b/src/Tutorials/SimpleTutorial/Program copy 2.cs
@@ -6,6 +6,37 @@
 
 
 var testXmlContent = @"<?xml version=""1.0"" encoding=""UTF-8"" ?>
+<CompactData
+	xmlns=""http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message""
+	xmlns:xsi=""http://www.w3.org/2001/XMLSchema-instance"" xsi:schemaLocation=""http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message https://stats.ecb.europa.eu/stats/vocabulary/sdmx/2.0/SDMXMessage.xsd"">
+	<Header>
+		<ID>EXR-HIST_2023-02-17</ID>
+		<Test>false</Test>
+		<Name xml:lang=""en"">Euro foreign exchange reference rates</Name>
+		<Prepared>2023-02-17T15:55:10</Prepared>
+		<Sender id=""4F0"">
+			<Name xml:lang=""en"">European Central Bank</Name>
+			<Contact>
+				<Department xml:lang=""en"">DG Statistics</Department>
+				<URI>mailto:[email protected]</URI>
+			</Contact>
+		</Sender>
+		<DataSetAgency>ECB</DataSetAgency>
+		<DataSetID>ECB_EXR_WEB</DataSetID>
+		<Extracted>2023-02-17T15:55:10</Extracted>
+	</Header>
+	<DataSet
+		xmlns=""http://www.ecb.europa.eu/vocabulary/stats/exr/1"" xsi:schemaLocation=""http://www.ecb.europa.eu/vocabulary/stats/exr/1 https://stats.ecb.europa.eu/stats/vocabulary/exr/1/2006-09-04/sdmx-compact.xsd"">
+		<Group CURRENCY=""USD"" CURRENCY_DENOM=""EUR"" EXR_TYPE=""SP00"" EXR_SUFFIX=""A"" DECIMALS=""4"" UNIT=""USD"" UNIT_MULT=""0"" TITLE_COMPL=""ECB reference exchange rate, US dollar/Euro, 2:15 pm (C.E.T.)"" />
+		<Series FREQ=""D"" CURRENCY=""USD"" CURRENCY_DENOM=""EUR"" EXR_TYPE=""SP00"" EXR_SUFFIX=""A"" TIME_FORMAT=""P1D"" COLLECTION=""A"">
+			<Obs TIME_PERIOD=""1999-01-04"" OBS_VALUE=""1.1789"" OBS_STATUS=""A"" OBS_CONF=""F"" />
+			<Obs TIME_PERIOD=""1999-01-05"" OBS_VALUE=""1.1790"" OBS_STATUS=""A"" OBS_CONF=""F"" />
+			<Obs TIME_PERIOD=""1999-01-06"" OBS_VALUE=""1.1743"" OBS_STATUS=""A"" OBS_CONF=""F"" />
+			<Obs TIME_PERIOD=""1999-01-07"" OBS_VALUE=""1.1632"" OBS_STATUS=""A"" OBS_CONF=""F"" />
+			<Obs TIME_PERIOD=""2023-02-16"" OBS_VALUE=""1.0700"" OBS_STATUS=""A"" OBS_CONF=""F"" />
+			<Obs TIME_PERIOD=""2023-02-17"" OBS_VALUE=""1.0625"" OBS_STATUS=""A"" OBS_CONF=""F"" />
+		</Series>
+	</DataSet>
 </CompactData>";
 var processRunner = StreamProcessRunner.Create<string>(DefineProcess);
 processRunner.DebugNodeStream += (sender, e) =>