diff --git a/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj b/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj index 8800081f8c..345ab3cbf3 100644 --- a/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj +++ b/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj @@ -7,7 +7,6 @@ - diff --git a/csharp/src/Drivers/Apache/Apache.Arrow.Adbc.Drivers.Apache.csproj b/csharp/src/Drivers/Apache/Apache.Arrow.Adbc.Drivers.Apache.csproj index 13efc4e0ba..28d6929b69 100644 --- a/csharp/src/Drivers/Apache/Apache.Arrow.Adbc.Drivers.Apache.csproj +++ b/csharp/src/Drivers/Apache/Apache.Arrow.Adbc.Drivers.Apache.csproj @@ -6,7 +6,6 @@ - diff --git a/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj b/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj index 6a311fd930..ec718b6376 100644 --- a/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj +++ b/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj @@ -4,7 +4,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs index 0862f86d4d..3deac62831 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs @@ -44,7 +44,7 @@ public static void CanClientExecuteUpdate( Adbc.Client.AdbcConnection adbcConnection, TestConfiguration testConfiguration, string[] queries, - List expectedResults, + IReadOnlyList expectedResults, string? environmentName = null) { if (adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection)); diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/TestBase.cs b/csharp/test/Apache.Arrow.Adbc.Tests/TestBase.cs index 46f52584a8..8bbd2df116 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/TestBase.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/TestBase.cs @@ -376,6 +376,16 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object await SelectAndValidateValuesAsync(selectStatement, [value], expectedLength); } + private static T? ArrowArrayAs(IArrowArray arrowArray) + where T : IArrowArray + { + if (arrowArray is T t) + { + return t; + } + return default; + } + /// /// Selects a single value and validates it equality with expected value and number of results. /// @@ -391,8 +401,23 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object int actualLength = 0; using (IArrowArrayStream stream = queryResult.Stream ?? throw new InvalidOperationException("stream is null")) { + Dictionary> valueGetters = new() + { + { ArrowTypeId.Decimal128, (a, i) => ArrowArrayAs(a)?.GetSqlDecimal(i) }, + { ArrowTypeId.Double, (a, i) => ArrowArrayAs(a)?.GetValue(i) }, + { ArrowTypeId.Float, (a, i) => ArrowArrayAs(a)?.GetValue(i) }, + { ArrowTypeId.Int64, (a, i) => ArrowArrayAs(a)?.GetValue(i) }, + { ArrowTypeId.Int32, (a, i) => ArrowArrayAs(a)?.GetValue(i) }, + { ArrowTypeId.Int16, (a, i) => ArrowArrayAs(a)?.GetValue(i) }, + { ArrowTypeId.Int8, (a, i) => ArrowArrayAs(a)?.GetValue(i) }, + { ArrowTypeId.String, (a, i) => ArrowArrayAs(a)?.GetString(i) }, + { ArrowTypeId.Timestamp, (a, i) => ArrowArrayAs(a)?.GetTimestamp(i) }, + { ArrowTypeId.Date32, (a, i) => ArrowArrayAs(a)?.GetDateTimeOffset(i) }, + { ArrowTypeId.Boolean, (a, i) => ArrowArrayAs(a)?.GetValue(i) }, + }; // Assume first column Field field = stream.Schema.GetFieldByIndex(0); + Int32Array? indexArray = null; while (true) { using (RecordBatch nextBatch = await stream.ReadNextRecordBatchAsync()) @@ -400,62 +425,6 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object if (nextBatch == null) { break; } switch (field.DataType) { - case Decimal128Type: - Decimal128Array decimalArray = (Decimal128Array)nextBatch.Column(0); - actualLength += decimalArray.Length; - ValidateValue((i) => values?[i], decimalArray.Length, (i) => decimalArray.GetSqlDecimal(i)); - break; - case DoubleType: - DoubleArray doubleArray = (DoubleArray)nextBatch.Column(0); - actualLength += doubleArray.Length; - ValidateValue((i) => values?[i], doubleArray.Length, (i) => doubleArray.GetValue(i)); - break; - case FloatType: - FloatArray floatArray = (FloatArray)nextBatch.Column(0); - actualLength += floatArray.Length; - ValidateValue((i) => values?[i], floatArray.Length, (i) => floatArray.GetValue(i)); - break; - case Int64Type: - Int64Array int64Array = (Int64Array)nextBatch.Column(0); - actualLength += int64Array.Length; - ValidateValue((i) => values?[i], int64Array.Length, (i) => int64Array.GetValue(i)); - break; - case Int32Type: - Int32Array intArray = (Int32Array)nextBatch.Column(0); - actualLength += intArray.Length; - ValidateValue((i) => values?[i], intArray.Length, (i) => intArray.GetValue(i)); - break; - case Int16Type: - Int16Array shortArray = (Int16Array)nextBatch.Column(0); - actualLength += shortArray.Length; - ValidateValue((i) => values?[i], shortArray.Length, (i) => shortArray.GetValue(i)); - break; - case Int8Type: - Int8Array tinyIntArray = (Int8Array)nextBatch.Column(0); - actualLength += tinyIntArray.Length; - ValidateValue((i) => values?[i], tinyIntArray.Length, (i) => tinyIntArray.GetValue(i)); - break; - case StringType: - StringArray stringArray = (StringArray)nextBatch.Column(0); - actualLength += stringArray.Length; - ValidateValue((i) => values?[i], stringArray.Length, (i) => stringArray.GetString(i)); - break; - case TimestampType: - TimestampArray timestampArray = (TimestampArray)nextBatch.Column(0); - actualLength += timestampArray.Length; - ValidateValue((i) => values?[i], timestampArray.Length, (i) => timestampArray.GetTimestamp(i)); - break; - case Date32Type: - Date32Array date32Array = (Date32Array)nextBatch.Column(0); - actualLength += date32Array.Length; - ValidateValue((i) => values?[i], date32Array.Length, (i) => date32Array.GetDateTimeOffset(i)); - break; - case BooleanType: - BooleanArray booleanArray = (BooleanArray)nextBatch.Column(0); - Int32Array? indexArray = hasIndexColumn ? (Int32Array)nextBatch.Column(1) : null; - actualLength += booleanArray.Length; - ValidateValue((i) => values?[i], booleanArray.Length, (i) => booleanArray.GetValue(i), indexArray); - break; case BinaryType: BinaryArray binaryArray = (BinaryArray)nextBatch.Column(0); actualLength += binaryArray.Length; @@ -464,10 +433,20 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object case NullType: NullArray nullArray = (NullArray)nextBatch.Column(0); actualLength += nullArray.Length; - ValidateValue((i) => values?[i] == null, nullArray.Length, (i) => nullArray.IsNull(i)); + ValidateValue(nullArray.Length, (i) => values?[i] == null, (i) => nullArray.IsNull(i)); break; default: - Assert.Fail($"Unhandled datatype {field.DataType}"); + if (valueGetters.TryGetValue(field.DataType.TypeId, out Func? valueGetter)) + { + IArrowArray array = nextBatch.Column(0); + actualLength += array.Length; + indexArray = hasIndexColumn ? (Int32Array)nextBatch.Column(1) : null; + ValidateValue(array.Length, (i) => values?[i], (i) => valueGetter(array, i), indexArray, array.IsNull); + } + else + { + Assert.Fail($"Unhandled datatype {field.DataType}"); + } break; } @@ -497,15 +476,20 @@ private static void ValidateBinaryArrayValue(Func expectedValues, /// /// Validates a single values for all results (in the batch). /// - /// The value to validate. /// The length of the current batch/array. + /// The value to validate. /// The getter function to retrieve the actual value. - private static void ValidateValue(Func value, int length, Func getter, Int32Array? indexColumn = null) + /// + private static void ValidateValue(int length, Func value, Func getter, Int32Array? indexColumn = null, Func? isNullEvaluator = default) { for (int i = 0; i < length; i++) { int valueIndex = indexColumn?.GetValue(i) ?? i; object? expected = value(valueIndex); + if (isNullEvaluator != null) + { + Assert.Equal(expected == null, isNullEvaluator(i)); + } object? actual = getter(i); Assert.Equal(expected, actual); } diff --git a/csharp/test/Drivers/Apache/Apache.Arrow.Adbc.Tests.Drivers.Apache.csproj b/csharp/test/Drivers/Apache/Apache.Arrow.Adbc.Tests.Drivers.Apache.csproj index 6198d2bb34..8dacbaeadd 100644 --- a/csharp/test/Drivers/Apache/Apache.Arrow.Adbc.Tests.Drivers.Apache.csproj +++ b/csharp/test/Drivers/Apache/Apache.Arrow.Adbc.Tests.Drivers.Apache.csproj @@ -12,7 +12,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + + diff --git a/csharp/test/Drivers/Apache/ApacheTestConfiguration.cs b/csharp/test/Drivers/Apache/ApacheTestConfiguration.cs index 7d12292030..fb62ccd9a7 100644 --- a/csharp/test/Drivers/Apache/ApacheTestConfiguration.cs +++ b/csharp/test/Drivers/Apache/ApacheTestConfiguration.cs @@ -27,9 +27,6 @@ public class ApacheTestConfiguration : TestConfiguration [JsonPropertyName("port")] public string Port { get; set; } = string.Empty; - [JsonPropertyName("token"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string Token { get; set; } = string.Empty; - [JsonPropertyName("path"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string Path { get; set; } = string.Empty; @@ -54,5 +51,13 @@ public class ApacheTestConfiguration : TestConfiguration [JsonPropertyName("http_request_timeout_ms"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string HttpRequestTimeoutMilliseconds { get; set; } = string.Empty; + [JsonPropertyName("type"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string Type { get; set; } = string.Empty; + + [JsonPropertyName("data_type_conv"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string DataTypeConversion { get; set; } = string.Empty; + + [JsonPropertyName("tls_options"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string TlsOptions { get; set; } = string.Empty; } } diff --git a/csharp/test/Drivers/Apache/Common/BinaryBooleanValueTests.cs b/csharp/test/Drivers/Apache/Common/BinaryBooleanValueTests.cs new file mode 100644 index 0000000000..fb6d355443 --- /dev/null +++ b/csharp/test/Drivers/Apache/Common/BinaryBooleanValueTests.cs @@ -0,0 +1,154 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading.Tasks; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; +using Xunit; +using Xunit.Abstractions; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common +{ + // TODO: When supported, use prepared statements instead of SQL string literals + // Which will better test how the driver handles values sent/received + + /// + /// Validates that specific binary and boolean values can be inserted, retrieved and targeted correctly + /// + public abstract class BinaryBooleanValueTests : TestBase + where TConfig : TestConfiguration + where TEnv : HiveServer2TestEnvironment + { + public BinaryBooleanValueTests(ITestOutputHelper output, TestEnvironment.Factory testEnvFactory) + : base(output, testEnvFactory) { } + + public static IEnumerable ByteArrayData(int size) + { + var rnd = new Random(); + byte[] bytes = new byte[size]; + rnd.NextBytes(bytes); + yield return new object[] { bytes }; + } + + /// + /// Validates if driver can send and receive specific Binary values correctly. + /// + [SkippableTheory] + [InlineData(null)] + [MemberData(nameof(ByteArrayData), 0)] + [MemberData(nameof(ByteArrayData), 2)] + [MemberData(nameof(ByteArrayData), 1024)] + public async Task TestBinaryData(byte[]? value) + { + string columnName = "BINARYTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "BINARY")); + string? formattedValue = value != null ? $"X'{BitConverter.ToString(value).Replace("-", "")}'" : null; + await ValidateInsertSelectDeleteSingleValueAsync( + table.TableName, + columnName, + value, + formattedValue); + } + + /// + /// Validates if driver can send and receive specific Boolean values correctly. + /// + [SkippableTheory] + [InlineData(null)] + [InlineData(true)] + [InlineData(false)] + public async Task TestBooleanData(bool? value) + { + string columnName = "BOOLEANTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "BOOLEAN")); + string? formattedValue = value == null ? null : $"{value?.ToString(CultureInfo.InvariantCulture)}"; + await ValidateInsertSelectDeleteTwoValuesAsync( + table.TableName, + columnName, + value, + formattedValue); + } + + /// + /// Validates if driver can receive specific NULL values correctly. + /// + [SkippableTheory] + [InlineData("NULL")] + [InlineData("CAST(NULL AS INT)")] + [InlineData("CAST(NULL AS BIGINT)")] + [InlineData("CAST(NULL AS SMALLINT)")] + [InlineData("CAST(NULL AS TINYINT)")] + [InlineData("CAST(NULL AS FLOAT)")] + [InlineData("CAST(NULL AS DOUBLE)")] + [InlineData("CAST(NULL AS DECIMAL(38,0))")] + [InlineData("CAST(NULL AS STRING)")] + [InlineData("CAST(NULL AS VARCHAR(10))")] + [InlineData("CAST(NULL AS CHAR(10))")] + [InlineData("CAST(NULL AS BOOLEAN)")] + [InlineData("CAST(NULL AS BINARY)")] + [InlineData("CAST(NULL AS MAP)")] + [InlineData("CAST(NULL AS STRUCT)")] + [InlineData("CAST(NULL AS ARRAY)")] + public async Task TestNullData(string projectionClause) + { + string selectStatement = $"SELECT {projectionClause};"; + // Note: by default, this returns as String type, not NULL type. + await SelectAndValidateValuesAsync(selectStatement, (object?)null, 1); + } + + [SkippableTheory] + [InlineData(1)] + [InlineData(7)] + [InlineData(8)] + [InlineData(9)] + [InlineData(15)] + [InlineData(16)] + [InlineData(17)] + [InlineData(23)] + [InlineData(24)] + [InlineData(25)] + [InlineData(31)] + [InlineData(32)] // Full integer + [InlineData(33)] + [InlineData(39)] + [InlineData(40)] + [InlineData(41)] + [InlineData(47)] + [InlineData(48)] + [InlineData(49)] + [InlineData(63)] + [InlineData(64)] // Full 2 integers + [InlineData(65)] + public async Task TestMultilineNullData(int numberOfValues) + { + Random rnd = new(); + int percentIsNull = 50; + + object?[] values = new object?[numberOfValues]; + for (int i = 0; i < numberOfValues; i++) + { + values[i] = rnd.Next(0, 100) < percentIsNull ? null : rnd.Next(0, 2) != 0; + } + string columnName = "BOOLEANTYPE"; + string indexColumnName = "INDEXCOL"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}, {2} {3}", indexColumnName, "INT", columnName, "BOOLEAN")); + await ValidateInsertSelectDeleteMultipleValuesAsync(table.TableName, columnName, indexColumnName, values); + } + } +} diff --git a/csharp/test/Drivers/Apache/Common/ClientTests.cs b/csharp/test/Drivers/Apache/Common/ClientTests.cs new file mode 100644 index 0000000000..e3b0309d08 --- /dev/null +++ b/csharp/test/Drivers/Apache/Common/ClientTests.cs @@ -0,0 +1,214 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; +using Apache.Arrow.Adbc.Tests.Xunit; +using Xunit; +using Xunit.Abstractions; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common +{ + /// + /// Class for testing the ADBC Client using the Spark ADBC driver. + /// + /// + /// Tests are ordered to ensure data is created for the other + /// queries to run. + /// Note: This test creates/replaces the table identified in the configuration (metadata/table). + /// It uses the test collection "TableCreateTestCollection" to ensure it does not run + /// as the same time as any other tests that may create/update the same table. + /// + [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] + [Collection("TableCreateTestCollection")] + public abstract class ClientTests : TestBase + where TConfig : TestConfiguration + where TEnv : HiveServer2TestEnvironment + { + public ClientTests(ITestOutputHelper? outputHelper, TestEnvironment.Factory testEnvFactory) + : base(outputHelper, testEnvFactory) + { + Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable)); + } + + /// + /// Validates if the client execute updates. + /// + [SkippableFact, Order(1)] + public void CanClientExecuteUpdate() + { + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) + { + adbcConnection.Open(); + + string[] queries = GetQueries(); + + var expectedResults = GetUpdateExpectedResults(); + Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, TestConfiguration, queries, expectedResults); + } + } + + protected abstract IReadOnlyList GetUpdateExpectedResults(); + + /// + /// Validates if the client can get the schema. + /// + [SkippableFact, Order(2)] + public void CanClientGetSchema() + { + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) + { + Tests.ClientTests.CanClientGetSchema(adbcConnection, TestConfiguration, $"SELECT * FROM {TestConfiguration.Metadata.Table}"); + } + } + + /// + /// Validates if the client can connect to a live server and + /// parse the results. + /// + [SkippableFact, Order(3)] + public void CanClientExecuteQuery() + { + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) + { + Tests.ClientTests.CanClientExecuteQuery(adbcConnection, TestConfiguration); + } + } + + /// + /// Validates if the client can connect to a live server and + /// parse the results. + /// + [SkippableFact, Order(5)] + public void CanClientExecuteEmptyQuery() + { + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) + { + Tests.ClientTests.CanClientExecuteQuery( + adbcConnection, + TestConfiguration, + customQuery: $"SELECT * FROM {TestConfiguration.Metadata.Table} WHERE FALSE", + expectedResultsCount: 0); + } + } + + /// + /// Validates if the client is retrieving and converting values + /// to the expected types. + /// + [SkippableFact, Order(4)] + public void VerifyTypesAndValues() + { + using (Adbc.Client.AdbcConnection dbConnection = GetAdbcConnection()) + { + SampleDataBuilder sampleDataBuilder = GetSampleDataBuilder(); + + Tests.ClientTests.VerifyTypesAndValues(dbConnection, sampleDataBuilder); + } + } + + [SkippableFact] + public void VerifySchemaTablesWithNoConstraints() + { + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(includeTableConstraints: false)) + { + adbcConnection.Open(); + + string schema = "Tables"; + + var tables = adbcConnection.GetSchema(schema); + + Assert.True(tables.Rows.Count > 0, $"No tables were found in the schema '{schema}'"); + } + } + + [SkippableFact] + public void VerifySchemaTables() + { + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) + { + adbcConnection.Open(); + + var collections = adbcConnection.GetSchema("MetaDataCollections"); + Assert.Equal(7, collections.Rows.Count); + Assert.Equal(2, collections.Columns.Count); + + var restrictions = adbcConnection.GetSchema("Restrictions"); + Assert.Equal(11, restrictions.Rows.Count); + Assert.Equal(3, restrictions.Columns.Count); + + var catalogs = adbcConnection.GetSchema("Catalogs"); + Assert.Single(catalogs.Columns); + var catalog = (string?)catalogs.Rows[0].ItemArray[0]; + + catalogs = adbcConnection.GetSchema("Catalogs", new[] { catalog }); + Assert.Equal(1, catalogs.Rows.Count); + + string random = "X" + Guid.NewGuid().ToString("N"); + + catalogs = adbcConnection.GetSchema("Catalogs", new[] { random }); + Assert.Equal(0, catalogs.Rows.Count); + + var schemas = adbcConnection.GetSchema("Schemas", new[] { catalog }); + Assert.Equal(2, schemas.Columns.Count); + var schema = (string?)schemas.Rows[0].ItemArray[1]; + + schemas = adbcConnection.GetSchema("Schemas", new[] { catalog, schema }); + Assert.Equal(1, schemas.Rows.Count); + + schemas = adbcConnection.GetSchema("Schemas", new[] { random }); + Assert.Equal(0, schemas.Rows.Count); + + schemas = adbcConnection.GetSchema("Schemas", new[] { catalog, random }); + Assert.Equal(0, schemas.Rows.Count); + + schemas = adbcConnection.GetSchema("Schemas", new[] { random, random }); + Assert.Equal(0, schemas.Rows.Count); + + var tableTypes = adbcConnection.GetSchema("TableTypes"); + Assert.Single(tableTypes.Columns); + + var tables = adbcConnection.GetSchema("Tables", new[] { catalog, schema }); + Assert.Equal(4, tables.Columns.Count); + + tables = adbcConnection.GetSchema("Tables", new[] { catalog, random }); + Assert.Equal(0, tables.Rows.Count); + + tables = adbcConnection.GetSchema("Tables", new[] { random, schema }); + Assert.Equal(0, tables.Rows.Count); + + tables = adbcConnection.GetSchema("Tables", new[] { random, random }); + Assert.Equal(0, tables.Rows.Count); + + tables = adbcConnection.GetSchema("Tables", new[] { catalog, schema, random }); + Assert.Equal(0, tables.Rows.Count); + + var columns = adbcConnection.GetSchema("Columns", new[] { catalog, schema }); + Assert.Equal(16, columns.Columns.Count); + } + } + + private Adbc.Client.AdbcConnection GetAdbcConnection(bool includeTableConstraints = true) + { + return new Adbc.Client.AdbcConnection( + NewDriver, GetDriverParameters(TestConfiguration), + [] + ); + } + } +} diff --git a/csharp/test/Drivers/Apache/Common/ComplexTypesValueTests.cs b/csharp/test/Drivers/Apache/Common/ComplexTypesValueTests.cs new file mode 100644 index 0000000000..9a785c0604 --- /dev/null +++ b/csharp/test/Drivers/Apache/Common/ComplexTypesValueTests.cs @@ -0,0 +1,81 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Threading.Tasks; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; +using Xunit; +using Xunit.Abstractions; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common +{ + // TODO: When supported, use prepared statements instead of SQL string literals + // Which will better test how the driver handles values sent/received + + /// + /// Validates that specific complex structured types can be inserted, retrieved and targeted correctly + /// + public class ComplexTypesValueTests : TestBase + where TConfig : TestConfiguration + where TEnv : HiveServer2TestEnvironment + { + public ComplexTypesValueTests(ITestOutputHelper output, TestEnvironment.Factory testEnvFactory) + : base(output, testEnvFactory) { } + + /// + /// Validates if driver can send and receive specific array of integer values correctly. + /// + [SkippableTheory] + [InlineData("ARRAY(CAST(1 AS INT), 2, 3)", "[1,2,3]")] + [InlineData("ARRAY(CAST(1 AS LONG), 2, 3)", "[1,2,3]")] + [InlineData("ARRAY(CAST(1 AS DOUBLE), 2, 3)", "[1.0,2.0,3.0]")] + [InlineData("ARRAY(CAST(1 AS NUMERIC(38,0)), 2, 3)", "[1,2,3]")] + [InlineData("ARRAY(CAST('John Doe' AS STRING), 2, 3)", """["John Doe","2","3"]""")] + // Note: Timestamp returned adjusted to UTC. + [InlineData("ARRAY(CAST('2024-01-01T00:00:00-07:00' AS TIMESTAMP), CAST('2024-02-02T02:02:02+01:30' AS TIMESTAMP), CAST('2024-03-03T03:03:03Z' AS TIMESTAMP))", """[2024-01-01 07:00:00,2024-02-02 00:32:02,2024-03-03 03:03:03]""")] + [InlineData("ARRAY(CAST('2024-01-01T00:00:00Z' AS DATE), CAST('2024-02-02T02:02:02Z' AS DATE), CAST('2024-03-03T03:03:03Z' AS DATE))", """[2024-01-01,2024-02-02,2024-03-03]""")] + [InlineData("ARRAY(INTERVAL 123 YEARS 11 MONTHS, INTERVAL 5 YEARS, INTERVAL 6 MONTHS)", """[123-11,5-0,0-6]""")] + public async Task TestArrayData(string projection, string value) + { + string selectStatement = $"SELECT {projection};"; + await SelectAndValidateValuesAsync(selectStatement, value, 1); + } + + /// + /// Validates if driver can send and receive specific map values correctly. + /// + [SkippableTheory] + [InlineData("MAP(1, 'John Doe', 2, 'Jane Doe', 3, 'Jack Doe')", """{1:"John Doe",2:"Jane Doe",3:"Jack Doe"}""")] + [InlineData("MAP('John Doe', 1, 'Jane Doe', 2, 'Jack Doe', 3)", """{"Jack Doe":3,"Jane Doe":2,"John Doe":1}""")] + public async Task TestMapData(string projection, string value) + { + string selectStatement = $"SELECT {projection};"; + await SelectAndValidateValuesAsync(selectStatement, value, 1); + } + + /// + /// Validates if driver can send and receive specific map values correctly. + /// + [SkippableTheory] + [InlineData("STRUCT(CAST(1 AS INT), CAST('John Doe' AS STRING))", """{"col1":1,"col2":"John Doe"}""")] + [InlineData("STRUCT(CAST('John Doe' AS STRING), CAST(1 AS INT))", """{"col1":"John Doe","col2":1}""")] + public async Task TestStructData(string projection, string value) + { + string selectStatement = $"SELECT {projection};"; + await SelectAndValidateValuesAsync(selectStatement, value, 1); + } + } +} diff --git a/csharp/test/Drivers/Apache/Common/DateTimeValueTests.cs b/csharp/test/Drivers/Apache/Common/DateTimeValueTests.cs new file mode 100644 index 0000000000..089de26a6c --- /dev/null +++ b/csharp/test/Drivers/Apache/Common/DateTimeValueTests.cs @@ -0,0 +1,145 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading.Tasks; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; +using Xunit; +using Xunit.Abstractions; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common +{ + // TODO: When supported, use prepared statements instead of SQL string literals + // Which will better test how the driver handles values sent/received + + /// + /// Validates that specific date, timestamp and interval values can be inserted, retrieved and targeted correctly + /// + public abstract class DateTimeValueTests : TestBase + where TConfig : TestConfiguration + where TEnv : HiveServer2TestEnvironment + { + // Spark handles microseconds but not nanoseconds. Truncated to 6 decimal places. + const string DateTimeZoneFormat = "yyyy-MM-dd'T'HH:mm:ss'.'ffffffK"; + const string DateTimeFormat = "yyyy-MM-dd' 'HH:mm:ss"; + protected const string DateFormat = "yyyy-MM-dd"; + + private static readonly DateTimeOffset[] s_timestampValues = + [ +#if NET5_0_OR_GREATER + DateTimeOffset.UnixEpoch, +#endif + DateTimeOffset.MinValue, + DateTimeOffset.MaxValue, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow.ToOffset(TimeSpan.FromHours(4)) + ]; + + public DateTimeValueTests(ITestOutputHelper output, TestEnvironment.Factory testEnvFactory) + : base(output, testEnvFactory) { } + + /// + /// Validates if driver can send and receive specific Timstamp values correctly + /// + [SkippableTheory] + [MemberData(nameof(TimestampData), "TIMESTAMP")] + public async Task TestTimestampData(DateTimeOffset value, string columnType) + { + string columnName = "TIMESTAMPTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, columnType)); + + string format = TestEnvironment.GetValueForProtocolVersion(DateTimeFormat, DateTimeZoneFormat)!; + string formattedValue = $"{value.ToString(format, CultureInfo.InvariantCulture)}"; + DateTimeOffset truncatedValue = DateTimeOffset.ParseExact(formattedValue, format, CultureInfo.InvariantCulture); + + object expectedValue = TestEnvironment.GetValueForProtocolVersion(formattedValue, truncatedValue)!; + await ValidateInsertSelectDeleteSingleValueAsync( + table.TableName, + columnName, + expectedValue, + "TO_TIMESTAMP(" + QuoteValue(formattedValue) + ")"); + } + + /// + /// Validates if driver can send and receive specific no timezone Timstamp values correctly + /// + [SkippableTheory] + [MemberData(nameof(TimestampData), "DATE")] + public async Task TestDateData(DateTimeOffset value, string columnType) + { + string columnName = "DATETYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, columnType)); + + string formattedValue = $"{value.ToString(DateFormat, CultureInfo.InvariantCulture)}"; + DateTimeOffset truncatedValue = DateTimeOffset.ParseExact(formattedValue, DateFormat, CultureInfo.InvariantCulture); + + // Remove timezone offset + object expectedValue = TestEnvironment.GetValueForProtocolVersion(formattedValue, new DateTimeOffset(truncatedValue.DateTime, TimeSpan.Zero))!; + await ValidateInsertSelectDeleteSingleValueAsync( + table.TableName, + columnName, + expectedValue, + "TO_DATE(" + QuoteValue(formattedValue) + ")"); + } + + /// + /// Tests INTERVAL data types (YEAR-MONTH and DAY-SECOND). + /// + /// The INTERVAL to test. + /// The expected return value. + /// + [SkippableTheory] + [InlineData("INTERVAL 1 YEAR", "1-0")] + [InlineData("INTERVAL 1 YEAR 2 MONTH", "1-2")] + [InlineData("INTERVAL 2 MONTHS", "0-2")] + [InlineData("INTERVAL -1 YEAR", "-1-0")] + [InlineData("INTERVAL -1 YEAR 2 MONTH", "-0-10")] + [InlineData("INTERVAL -2 YEAR 2 MONTH", "-1-10")] + [InlineData("INTERVAL 1 YEAR -2 MONTH", "0-10")] + [InlineData("INTERVAL 178956970 YEAR", "178956970-0")] + [InlineData("INTERVAL 178956969 YEAR 11 MONTH", "178956969-11")] + [InlineData("INTERVAL -178956970 YEAR", "-178956970-0")] + [InlineData("INTERVAL 0 DAYS 0 HOURS 0 MINUTES 0 SECONDS", "0 00:00:00.000000000")] + [InlineData("INTERVAL 1 DAYS", "1 00:00:00.000000000")] + [InlineData("INTERVAL 2 HOURS", "0 02:00:00.000000000")] + [InlineData("INTERVAL 3 MINUTES", "0 00:03:00.000000000")] + [InlineData("INTERVAL 4 SECONDS", "0 00:00:04.000000000")] + [InlineData("INTERVAL 1 DAYS 2 HOURS", "1 02:00:00.000000000")] + [InlineData("INTERVAL 1 DAYS 2 HOURS 3 MINUTES", "1 02:03:00.000000000")] + [InlineData("INTERVAL 1 DAYS 2 HOURS 3 MINUTES 4 SECONDS", "1 02:03:04.000000000")] + [InlineData("INTERVAL 1 DAYS 2 HOURS 3 MINUTES 4.123123123 SECONDS", "1 02:03:04.123123000")] // Only to microseconds + [InlineData("INTERVAL 106751990 DAYS 23 HOURS 59 MINUTES 59.999999 SECONDS", "106751990 23:59:59.999999000")] + [InlineData("INTERVAL 106751991 DAYS 0 HOURS 0 MINUTES 0 SECONDS", "106751991 00:00:00.000000000")] + [InlineData("INTERVAL -106751991 DAYS 0 HOURS 0 MINUTES 0 SECONDS", "-106751991 00:00:00.000000000")] + [InlineData("INTERVAL -106751991 DAYS 23 HOURS 59 MINUTES 59.999999 SECONDS", "-106751990 00:00:00.000001000")] + public async Task TestIntervalData(string intervalClause, string value) + { + string selectStatement = $"SELECT {intervalClause} AS INTERVAL_VALUE;"; + await SelectAndValidateValuesAsync(selectStatement, value, 1); + } + + public static IEnumerable TimestampData(string columnType) + { + foreach (DateTimeOffset timestamp in s_timestampValues) + { + yield return new object[] { timestamp, columnType }; + } + } + } +} diff --git a/csharp/test/Drivers/Apache/Common/DriverTests.cs b/csharp/test/Drivers/Apache/Common/DriverTests.cs new file mode 100644 index 0000000000..33c466cfc4 --- /dev/null +++ b/csharp/test/Drivers/Apache/Common/DriverTests.cs @@ -0,0 +1,672 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Apache.Arrow.Adbc.Drivers.Apache.Spark; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; +using Apache.Arrow.Adbc.Tests.Metadata; +using Apache.Arrow.Adbc.Tests.Xunit; +using Apache.Arrow.Ipc; +using Xunit; +using Xunit.Abstractions; +using ColumnTypeId = Apache.Arrow.Adbc.Drivers.Apache.Spark.SparkConnection.ColumnTypeId; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common +{ + /// + /// Class for testing the Spark ADBC driver connection tests. + /// + /// + /// Tests are ordered to ensure data is created for the other + /// queries to run. + /// Note: This test creates/replaces the table identified in the configuration (metadata/table). + /// It uses the test collection "TableCreateTestCollection" to ensure it does not run + /// as the same time as any other tests that may create/update the same table. + /// + [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] + [Collection("TableCreateTestCollection")] + public abstract class DriverTests : TestBase + where TConfig : ApacheTestConfiguration + where TEnv : HiveServer2TestEnvironment + { + /// + /// Supported Spark data types as a subset of + /// + private enum SupportedSparkDataType : short + { + ARRAY = ColumnTypeId.ARRAY, + BIGINT = ColumnTypeId.BIGINT, + BINARY = ColumnTypeId.BINARY, + BOOLEAN = ColumnTypeId.BOOLEAN, + CHAR = ColumnTypeId.CHAR, + DATE = ColumnTypeId.DATE, + DECIMAL = ColumnTypeId.DECIMAL, + DOUBLE = ColumnTypeId.DOUBLE, + FLOAT = ColumnTypeId.FLOAT, + INTEGER = ColumnTypeId.INTEGER, + JAVA_OBJECT = ColumnTypeId.JAVA_OBJECT, + LONGNVARCHAR = ColumnTypeId.LONGNVARCHAR, + LONGVARBINARY = ColumnTypeId.LONGVARBINARY, + LONGVARCHAR = ColumnTypeId.LONGVARCHAR, + NCHAR = ColumnTypeId.NCHAR, + NULL = ColumnTypeId.NULL, + NUMERIC = ColumnTypeId.NUMERIC, + NVARCHAR = ColumnTypeId.NVARCHAR, + REAL = ColumnTypeId.REAL, + SMALLINT = ColumnTypeId.SMALLINT, + STRUCT = ColumnTypeId.STRUCT, + TIMESTAMP = ColumnTypeId.TIMESTAMP, + TINYINT = ColumnTypeId.TINYINT, + VARBINARY = ColumnTypeId.VARBINARY, + VARCHAR = ColumnTypeId.VARCHAR, + } + + private static List DefaultTableTypes => new() { "TABLE", "VIEW" }; + + public DriverTests(ITestOutputHelper? outputHelper, TestEnvironment.Factory testEnvFactory) + : base(outputHelper, testEnvFactory) + { + Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable)); + } + + /// + /// Validates if the driver can execute update statements. + /// + [SkippableFact, Order(1)] + public void CanExecuteUpdate() + { + AdbcConnection adbcConnection = NewConnection(); + + string[] queries = GetQueries(); + + //List expectedResults = TestEnvironment.ServerType != SparkServerType.Databricks + // ? + // [ + // -1, // CREATE TABLE + // 1, // INSERT + // 1, // INSERT + // 1, // INSERT + // //1, // UPDATE + // //1, // DELETE + // ] + // : + // [ + // -1, // CREATE TABLE + // 1, // INSERT + // 1, // INSERT + // 1, // INSERT + // 1, // UPDATE + // 1, // DELETE + // ]; + + var expectedResults = GetUpdateExpectedResults(); + for (int i = 0; i < queries.Length; i++) + { + string query = queries[i]; + using AdbcStatement statement = adbcConnection.CreateStatement(); + statement.SqlQuery = query; + + UpdateResult updateResult = statement.ExecuteUpdate(); + + if (ValidateAffectedRows) Assert.Equal(expectedResults[i], updateResult.AffectedRows); + } + } + + protected abstract IReadOnlyList GetUpdateExpectedResults(); + + /// + /// Validates if the driver can call GetInfo. + /// + [SkippableFact, Order(2)] + public async Task CanGetInfo() + { + AdbcConnection adbcConnection = NewConnection(); + + // Test the supported info codes + List handledCodes = new List() + { + AdbcInfoCode.DriverName, + AdbcInfoCode.DriverVersion, + AdbcInfoCode.VendorName, + AdbcInfoCode.DriverArrowVersion, + AdbcInfoCode.VendorVersion, + AdbcInfoCode.VendorSql + }; + using IArrowArrayStream stream = adbcConnection.GetInfo(handledCodes); + + RecordBatch recordBatch = await stream.ReadNextRecordBatchAsync(); + UInt32Array infoNameArray = (UInt32Array)recordBatch.Column("info_name"); + + List expectedValues = new List() + { + "DriverName", + "DriverVersion", + "VendorName", + "DriverArrowVersion", + "VendorVersion", + "VendorSql" + }; + + for (int i = 0; i < infoNameArray.Length; i++) + { + AdbcInfoCode? value = (AdbcInfoCode?)infoNameArray.GetValue(i); + DenseUnionArray valueArray = (DenseUnionArray)recordBatch.Column("info_value"); + + Assert.Contains(value.ToString(), expectedValues); + + switch (value) + { + case AdbcInfoCode.VendorSql: + // TODO: How does external developer know the second field is the boolean field? + BooleanArray booleanArray = (BooleanArray)valueArray.Fields[1]; + bool? boolValue = booleanArray.GetValue(i); + OutputHelper?.WriteLine($"{value}={boolValue}"); + Assert.True(boolValue); + break; + default: + StringArray stringArray = (StringArray)valueArray.Fields[0]; + string stringValue = stringArray.GetString(i); + OutputHelper?.WriteLine($"{value}={stringValue}"); + Assert.NotNull(stringValue); + break; + } + } + + // Test the unhandled info codes. + List unhandledCodes = new List() + { + AdbcInfoCode.VendorArrowVersion, + AdbcInfoCode.VendorSubstrait, + AdbcInfoCode.VendorSubstraitMaxVersion + }; + using IArrowArrayStream stream2 = adbcConnection.GetInfo(unhandledCodes); + + recordBatch = await stream2.ReadNextRecordBatchAsync(); + infoNameArray = (UInt32Array)recordBatch.Column("info_name"); + + List unexpectedValues = new List() + { + "VendorArrowVersion", + "VendorSubstrait", + "VendorSubstraitMaxVersion" + }; + for (int i = 0; i < infoNameArray.Length; i++) + { + AdbcInfoCode? value = (AdbcInfoCode?)infoNameArray.GetValue(i); + DenseUnionArray valueArray = (DenseUnionArray)recordBatch.Column("info_value"); + + Assert.Contains(value.ToString(), unexpectedValues); + switch (value) + { + case AdbcInfoCode.VendorSql: + BooleanArray booleanArray = (BooleanArray)valueArray.Fields[1]; + Assert.Null(booleanArray.GetValue(i)); + break; + default: + StringArray stringArray = (StringArray)valueArray.Fields[0]; + Assert.Null(stringArray.GetString(i)); + break; + } + } + } + + /// + /// Validates if the driver can call GetObjects with GetObjectsDepth as Catalogs with CatalogPattern as a pattern. + /// + /// + [SkippableFact, Order(3)] + public abstract void CanGetObjectsCatalogs(string? pattern); + + /// + /// Validates if the driver can call GetObjects with GetObjectsDepth as Catalogs with CatalogPattern as a pattern. + /// + /// + protected void GetObjectsCatalogsTest(string? pattern) + { + string? catalogName = TestConfiguration.Metadata.Catalog; + string? schemaName = TestConfiguration.Metadata.Schema; + + using IArrowArrayStream stream = Connection.GetObjects( + depth: AdbcConnection.GetObjectsDepth.Catalogs, + catalogPattern: pattern, + dbSchemaPattern: null, + tableNamePattern: null, + tableTypes: DefaultTableTypes, + columnNamePattern: null); + + using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; + + List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); + AdbcCatalog? catalog = catalogs.Where((catalog) => string.Equals(catalog.Name, catalogName)).FirstOrDefault(); + + Assert.True(pattern == string.Empty && catalog == null || catalog != null, "catalog should not be null"); + } + + + /// + /// Validates if the driver can call GetObjects with GetObjectsDepth as DbSchemas with DbSchemaName as a pattern. + /// + [SkippableFact, Order(4)] + public abstract void CanGetObjectsDbSchemas(string dbSchemaPattern); + + /// + /// Validates if the driver can call GetObjects with GetObjectsDepth as DbSchemas with DbSchemaName as a pattern. + /// + protected void GetObjectsDbSchemasTest(string dbSchemaPattern) + { + // need to add the database + string? databaseName = TestConfiguration.Metadata.Catalog; + string? schemaName = TestConfiguration.Metadata.Schema; + + using IArrowArrayStream stream = Connection.GetObjects( + depth: AdbcConnection.GetObjectsDepth.DbSchemas, + catalogPattern: databaseName, + dbSchemaPattern: dbSchemaPattern, + tableNamePattern: null, + tableTypes: DefaultTableTypes, + columnNamePattern: null); + + using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; + + List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); + + List? dbSchemas = catalogs + .Where(c => string.Equals(c.Name, databaseName)) + .Select(c => c.DbSchemas) + .FirstOrDefault(); + AdbcDbSchema? dbSchema = dbSchemas?.Where((dbSchema) => string.Equals(dbSchema.Name, schemaName)).FirstOrDefault(); + + Assert.True(dbSchema != null, "dbSchema should not be null"); + } + + /// + /// Validates if the driver can call GetObjects with GetObjectsDepth as Tables with TableName as a pattern. + /// + [SkippableFact, Order(5)] + public abstract void CanGetObjectsTables(string tableNamePattern); + + /// + /// Validates if the driver can call GetObjects with GetObjectsDepth as Tables with TableName as a pattern. + /// + protected void GetObjectsTablesTest(string tableNamePattern) + { + // need to add the database + string? databaseName = TestConfiguration.Metadata.Catalog; + string? schemaName = TestConfiguration.Metadata.Schema; + string? tableName = TestConfiguration.Metadata.Table; + + using IArrowArrayStream stream = Connection.GetObjects( + depth: AdbcConnection.GetObjectsDepth.Tables, + catalogPattern: databaseName, + dbSchemaPattern: schemaName, + tableNamePattern: tableNamePattern, + tableTypes: DefaultTableTypes, + columnNamePattern: null); + + using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; + + List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); + + List? tables = catalogs + .Where(c => string.Equals(c.Name, databaseName)) + .Select(c => c.DbSchemas) + .FirstOrDefault() + ?.Where(s => string.Equals(s.Name, schemaName)) + .Select(s => s.Tables) + .FirstOrDefault(); + + AdbcTable? table = tables?.Where((table) => string.Equals(table.Name, tableName)).FirstOrDefault(); + Assert.True(table != null, "table should not be null"); + // TODO: Determine why this is returned blank. + //Assert.Equal("TABLE", table.Type); + } + + /// + /// Validates if the driver can call GetObjects for GetObjectsDepth as All. + /// + [SkippableFact, Order(6)] + public void CanGetObjectsAll() + { + // need to add the database + string? databaseName = TestConfiguration.Metadata.Catalog; + string? schemaName = TestConfiguration.Metadata.Schema; + string? tableName = TestConfiguration.Metadata.Table; + string? columnName = null; + + using IArrowArrayStream stream = Connection.GetObjects( + depth: AdbcConnection.GetObjectsDepth.All, + catalogPattern: databaseName, + dbSchemaPattern: schemaName, + tableNamePattern: tableName, + tableTypes: DefaultTableTypes, + columnNamePattern: columnName); + + using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; + + List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); + AdbcTable? table = catalogs + .Where(c => string.Equals(c.Name, databaseName)) + .Select(c => c.DbSchemas) + .FirstOrDefault() + ?.Where(s => string.Equals(s.Name, schemaName)) + .Select(s => s.Tables) + .FirstOrDefault() + ?.Where(t => string.Equals(t.Name, tableName)) + .FirstOrDefault(); + + Assert.True(table != null, "table should not be null"); + // TODO: Determine why this is returned blank. + //Assert.Equal("TABLE", table.Type); + List? columns = table.Columns; + + Assert.True(columns != null, "Columns cannot be null"); + Assert.Equal(TestConfiguration.Metadata.ExpectedColumnCount, columns.Count); + + for (int i = 0; i < columns.Count; i++) + { + // Verify column metadata is returned/consistent. + AdbcColumn column = columns[i]; + Assert.Equal(i + 1, column.OrdinalPosition); + Assert.False(string.IsNullOrEmpty(column.Name)); + Assert.False(string.IsNullOrEmpty(column.XdbcTypeName)); + Assert.False(Regex.IsMatch(column.XdbcTypeName, @"[_,\d\<\>\(\)]", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), + "Unexpected character found in field XdbcTypeName"); + + var supportedTypes = Enum.GetValues(typeof(SupportedSparkDataType)).Cast(); + Assert.Contains((SupportedSparkDataType)column.XdbcSqlDataType!, supportedTypes); + Assert.Equal(column.XdbcDataType, column.XdbcSqlDataType); + + Assert.NotNull(column.XdbcDataType); + Assert.Contains((SupportedSparkDataType)column.XdbcDataType!, supportedTypes); + + HashSet typesHaveColumnSize = new() + { + (short)SupportedSparkDataType.DECIMAL, + (short)SupportedSparkDataType.NUMERIC, + (short)SupportedSparkDataType.CHAR, + (short)SupportedSparkDataType.VARCHAR, + }; + HashSet typesHaveDecimalDigits = new() + { + (short)SupportedSparkDataType.DECIMAL, + (short)SupportedSparkDataType.NUMERIC, + }; + + bool typeHasColumnSize = typesHaveColumnSize.Contains(column.XdbcDataType.Value); + Assert.Equal(column.XdbcColumnSize.HasValue, typeHasColumnSize); + + bool typeHasDecimalDigits = typesHaveDecimalDigits.Contains(column.XdbcDataType.Value); + Assert.Equal(column.XdbcDecimalDigits.HasValue, typeHasDecimalDigits); + + Assert.False(string.IsNullOrEmpty(column.Remarks)); + + Assert.NotNull(column.XdbcColumnDef); + + Assert.NotNull(column.XdbcNullable); + Assert.Contains(new short[] { 1, 0 }, i => i == column.XdbcNullable); + + Assert.NotNull(column.XdbcIsNullable); + Assert.Contains(new string[] { "YES", "NO" }, i => i.Equals(column.XdbcIsNullable)); + + Assert.NotNull(column.XdbcIsAutoIncrement); + + Assert.Null(column.XdbcCharOctetLength); + Assert.Null(column.XdbcDatetimeSub); + Assert.Null(column.XdbcNumPrecRadix); + Assert.Null(column.XdbcScopeCatalog); + Assert.Null(column.XdbcScopeSchema); + Assert.Null(column.XdbcScopeTable); + } + } + + /// + /// Validates if the driver can call GetObjects with GetObjectsDepth as Tables with TableName as a Special Character. + /// + [SkippableTheory, Order(7)] + [InlineData("MyIdentifier")] + [InlineData("ONE")] + [InlineData("mYiDentifier")] + [InlineData("3rd_identifier")] + // Note: Tables in 'hive_metastore' only support ASCII alphabetic, numeric and underscore. + public void CanGetObjectsTablesWithSpecialCharacter(string tableName) + { + string catalogName = TestConfiguration.Metadata.Catalog; + string schemaPrefix = Guid.NewGuid().ToString().Replace("-", ""); + using TemporarySchema schema = TemporarySchema.NewTemporarySchemaAsync(catalogName, Statement).Result; + string schemaName = schema.SchemaName; + string catalogFormatted = string.IsNullOrEmpty(catalogName) ? string.Empty : DelimitIdentifier(catalogName) + "."; + string fullTableName = $"{catalogFormatted}{DelimitIdentifier(schemaName)}.{DelimitIdentifier(tableName)}"; + using TemporaryTable temporaryTable = TemporaryTable.NewTemporaryTableAsync(Statement, fullTableName, $"CREATE TABLE IF NOT EXISTS {fullTableName} (INDEX INT)").Result; + + using IArrowArrayStream stream = Connection.GetObjects( + depth: AdbcConnection.GetObjectsDepth.Tables, + catalogPattern: catalogName, + dbSchemaPattern: schemaName, + tableNamePattern: tableName, + tableTypes: DefaultTableTypes, + columnNamePattern: null); + + using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; + + List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); + + List? tables = catalogs + .Where(c => string.Equals(c.Name, catalogName)) + .Select(c => c.DbSchemas) + .FirstOrDefault() + ?.Where(s => string.Equals(s.Name, schemaName)) + .Select(s => s.Tables) + .FirstOrDefault(); + + AdbcTable? table = tables?.FirstOrDefault(); + + Assert.True(table != null, "table should not be null"); + Assert.Equal(tableName, table.Name, true); + } + + /// + /// Validates if the driver can call GetTableSchema. + /// + [SkippableFact, Order(8)] + public void CanGetTableSchema() + { + AdbcConnection adbcConnection = NewConnection(); + + string? catalogName = TestConfiguration.Metadata.Catalog; + string? schemaName = TestConfiguration.Metadata.Schema; + string tableName = TestConfiguration.Metadata.Table!; + + Schema schema = adbcConnection.GetTableSchema(catalogName, schemaName, tableName); + + int numberOfFields = schema.FieldsList.Count; + + Assert.Equal(TestConfiguration.Metadata.ExpectedColumnCount, numberOfFields); + } + + /// + /// Validates if the driver can call GetTableTypes. + /// + [SkippableFact, Order(9)] + public async Task CanGetTableTypes() + { + AdbcConnection adbcConnection = NewConnection(); + + using IArrowArrayStream arrowArrayStream = adbcConnection.GetTableTypes(); + + RecordBatch recordBatch = await arrowArrayStream.ReadNextRecordBatchAsync(); + + StringArray stringArray = (StringArray)recordBatch.Column("table_type"); + + List known_types = new List + { + "TABLE", "VIEW" + }; + + int results = 0; + + for (int i = 0; i < stringArray.Length; i++) + { + string value = stringArray.GetString(i); + + if (known_types.Contains(value)) + { + results++; + } + } + + Assert.Equal(known_types.Count, results); + } + + /// + /// Validates if the driver can connect to a live server and + /// parse the results. + /// + [SkippableTheory, Order(10)] + [InlineData(0.1)] + [InlineData(0.25)] + [InlineData(1.0)] + [InlineData(2.0)] + [InlineData(null)] + public void CanExecuteQuery(double? batchSizeFactor) + { + // Ensure all records can be retrieved, independent of the batch size. + TConfig testConfiguration = (TConfig)TestConfiguration.Clone(); + long expectedResultCount = testConfiguration.ExpectedResultsCount; + long nonZeroExpectedResultCount = expectedResultCount == 0 ? 1 : expectedResultCount; + testConfiguration.BatchSize = batchSizeFactor != null ? ((long)(nonZeroExpectedResultCount * batchSizeFactor)).ToString() : string.Empty; + OutputHelper?.WriteLine($"BatchSize: {testConfiguration.BatchSize}. ExpectedResultCount: {expectedResultCount}"); + + using AdbcConnection adbcConnection = NewConnection(testConfiguration); + + using AdbcStatement statement = adbcConnection.CreateStatement(); + statement.SqlQuery = TestConfiguration.Query; + OutputHelper?.WriteLine(statement.SqlQuery); + + QueryResult queryResult = statement.ExecuteQuery(); + + Tests.DriverTests.CanExecuteQuery(queryResult, TestConfiguration.ExpectedResultsCount); + } + + /// + /// Validates if the driver can connect to a live server and + /// parse the results using the asynchronous methods. + /// + [SkippableFact, Order(11)] + public async Task CanExecuteQueryAsync() + { + using AdbcConnection adbcConnection = NewConnection(); + using AdbcStatement statement = adbcConnection.CreateStatement(); + + statement.SqlQuery = TestConfiguration.Query; + QueryResult queryResult = await statement.ExecuteQueryAsync(); + + await Tests.DriverTests.CanExecuteQueryAsync(queryResult, TestConfiguration.ExpectedResultsCount); + } + + /// + /// Validates if the driver can connect to a live server and + /// perform and update asynchronously. + /// + [SkippableFact, Order(12)] + public async Task CanExecuteUpdateAsync() + { + using AdbcConnection adbcConnection = NewConnection(); + using AdbcStatement statement = adbcConnection.CreateStatement(); + using TemporaryTable temporaryTable = await NewTemporaryTableAsync(statement, "INDEX INT"); + + statement.SqlQuery = GetInsertStatement(temporaryTable.TableName, "INDEX", "1"); + UpdateResult updateResult = await statement.ExecuteUpdateAsync(); + + if (ValidateAffectedRows) Assert.Equal(1, updateResult.AffectedRows); + } + + [SkippableFact, Order(13)] + public void CanDetectInvalidAuthentication() + { + AdbcDriver driver = NewDriver; + Assert.NotNull(driver); + Dictionary parameters = GetDriverParameters(TestConfiguration); + + bool hasToken = parameters.TryGetValue(SparkParameters.Token, out var token) && !string.IsNullOrEmpty(token); + bool hasUsername = parameters.TryGetValue(AdbcOptions.Username, out var username) && !string.IsNullOrEmpty(username); + bool hasPassword = parameters.TryGetValue(AdbcOptions.Password, out var password) && !string.IsNullOrEmpty(password); + if (hasToken) + { + parameters[SparkParameters.Token] = "invalid-token"; + } + else if (hasUsername && hasPassword) + { + parameters[AdbcOptions.Password] = "invalid-password"; + } + else + { + Assert.Fail($"Unexpected configuration. Must provide '{SparkParameters.Token}' or '{AdbcOptions.Username}' and '{AdbcOptions.Password}'."); + } + + AdbcDatabase database = driver.Open(parameters); + AggregateException exception = Assert.ThrowsAny(() => database.Connect(parameters)); + OutputHelper?.WriteLine(exception.Message); + } + + [SkippableFact, Order(14)] + public void CanDetectInvalidServer() + { + AdbcDriver driver = NewDriver; + Assert.NotNull(driver); + Dictionary parameters = GetDriverParameters(TestConfiguration); + + bool hasUri = parameters.TryGetValue(AdbcOptions.Uri, out var uri) && !string.IsNullOrEmpty(uri); + bool hasHostName = parameters.TryGetValue(SparkParameters.HostName, out var hostName) && !string.IsNullOrEmpty(hostName); + if (hasUri) + { + parameters[AdbcOptions.Uri] = "http://unknownhost.azure.com/cliservice"; + } + else if (hasHostName) + { + parameters[SparkParameters.HostName] = "unknownhost.azure.com"; + } + else + { + Assert.Fail($"Unexpected configuration. Must provide '{AdbcOptions.Uri}' or '{SparkParameters.HostName}'."); + } + + AdbcDatabase database = driver.Open(parameters); + AggregateException exception = Assert.ThrowsAny(() => database.Connect(parameters)); + OutputHelper?.WriteLine(exception.Message); + } + + /// + /// Validates if the driver can connect to a live server and + /// parse the results using the asynchronous methods. + /// + [SkippableFact, Order(15)] + public async Task CanExecuteQueryAsyncEmptyResult() + { + using AdbcConnection adbcConnection = NewConnection(); + using AdbcStatement statement = adbcConnection.CreateStatement(); + + statement.SqlQuery = $"SELECT * from {TestConfiguration.Metadata.Table} WHERE FALSE"; + QueryResult queryResult = await statement.ExecuteQueryAsync(); + + await Tests.DriverTests.CanExecuteQueryAsync(queryResult, 0); + } + } +} diff --git a/csharp/test/Drivers/Apache/Common/NumericValueTests.cs b/csharp/test/Drivers/Apache/Common/NumericValueTests.cs new file mode 100644 index 0000000000..99ad341e3d --- /dev/null +++ b/csharp/test/Drivers/Apache/Common/NumericValueTests.cs @@ -0,0 +1,274 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Data.SqlTypes; +using System.Threading.Tasks; +using Apache.Arrow.Adbc.Drivers.Apache.Hive2; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; +using Xunit; +using Xunit.Abstractions; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common +{ + // TODO: When supported, use prepared statements instead of SQL string literals + // Which will better test how the driver handles values sent/received + + public abstract class NumericValueTests : TestBase + where TConfig : TestConfiguration + where TEnv : HiveServer2TestEnvironment + { + /// + /// Validates that specific numeric values can be inserted, retrieved and targeted correctly + /// + public NumericValueTests(ITestOutputHelper output, TestEnvironment.Factory testEnvFactory) + : base(output, testEnvFactory) { } + + /// + /// Validates if driver can send and receive specific Integer values correctly + /// + [SkippableTheory] + [InlineData(-1)] + [InlineData(0)] + [InlineData(1)] + public async Task TestIntegerSanity(int value) + { + string columnName = "INTTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} INT", columnName)); + await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); + } + + /// + /// Validates if driver can handle the largest / smallest numbers + /// + [SkippableTheory] + [InlineData(int.MaxValue)] + [InlineData(int.MinValue)] + public async Task TestIntegerMinMax(int value) + { + string columnName = "INTTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} INT", columnName)); + await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); + } + + /// + /// Validates if driver can handle the largest / smallest numbers + /// + [SkippableTheory] + [InlineData(long.MaxValue)] + [InlineData(long.MinValue)] + public async Task TestLongMinMax(long value) + { + string columnName = "BIGINTTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} BIGINT", columnName)); + await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); + } + + /// + /// Validates if driver can handle the largest / smallest numbers + /// + [SkippableTheory] + [InlineData(short.MaxValue)] + [InlineData(short.MinValue)] + public async Task TestSmallIntMinMax(short value) + { + string columnName = "SMALLINTTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} SMALLINT", columnName)); + await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); + } + + /// + /// Validates if driver can handle the largest / smallest numbers + /// + [SkippableTheory] + [InlineData(sbyte.MaxValue)] + [InlineData(sbyte.MinValue)] + public async Task TestTinyIntMinMax(sbyte value) + { + string columnName = "TINYINTTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} TINYINT", columnName)); + await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); + } + + /// + /// Validates if driver can handle smaller Number type correctly + /// + [SkippableTheory] + [InlineData("-1")] + [InlineData("0")] + [InlineData("1")] + [InlineData("99")] + [InlineData("-99")] + public async Task TestSmallNumberRange(string value) + { + string columnName = "SMALLNUMBER"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(2,0)", columnName)); + object? expectedValue = TestEnvironment.GetValueForProtocolVersion(value, new SqlDecimal(double.Parse(value))); + await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, expectedValue); + } + + /// + /// Validates if driver correctly errors out when the values exceed the column's limit + /// + [SkippableTheory] + [InlineData(-100)] + [InlineData(100)] + [InlineData(int.MaxValue)] + [InlineData(int.MinValue)] + public async Task TestSmallNumberRangeOverlimit(int value) + { + string columnName = "SMALLNUMBER"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(2,0)", columnName)); + await Assert.ThrowsAsync( + async () => await ValidateInsertSelectDeleteSingleValueAsync( + table.TableName, + columnName, TestEnvironment.GetValueForProtocolVersion(value.ToString(), new SqlDecimal(value)))); + } + + /// + /// Validates if driver can handle a large scale Number type correctly + /// + [SkippableTheory] + [InlineData("0E-37")] + [InlineData("-2.0030000000000000000000000000000000000")] + [InlineData("4.8500000000000000000000000000000000000")] + [InlineData("1E-37")] + [InlineData("9.5545204502636499875576383003668916798")] + public async Task TestLargeScaleNumberRange(string value) + { + string columnName = "LARGESCALENUMBER"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,37)", columnName)); + await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, TestEnvironment.GetValueForProtocolVersion(value, new SqlDecimal(double.Parse(value)))); + } + + /// + /// Validates if driver can error handle when input goes beyond a large scale Number type + /// + [SkippableTheory] + [InlineData("-10")] + [InlineData("10")] + [InlineData("99999999999999999999999999999999999999")] + [InlineData("-99999999999999999999999999999999999999")] + public async Task TestLargeScaleNumberOverlimit(string value) + { + string columnName = "LARGESCALENUMBER"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,37)", columnName)); + await Assert.ThrowsAsync(async () => await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, SqlDecimal.Parse(value))); + } + + /// + /// Validates if driver can handle a small scale Number type correctly + /// + [SkippableTheory] + [InlineData("0.00")] + [InlineData("4.85")] + [InlineData("-999999999999999999999999999999999999.99")] + [InlineData("999999999999999999999999999999999999.99")] + public async Task TestSmallScaleNumberRange(string value) + { + string columnName = "SMALLSCALENUMBER"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,2)", columnName)); + await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, TestEnvironment.GetValueForProtocolVersion(value, SqlDecimal.Parse(value))); + } + + /// + /// Validates if driver can error handle when an insert goes beyond a small scale Number type correctly + /// + [SkippableTheory] + [InlineData("-99999999999999999999999999999999999999")] + [InlineData("99999999999999999999999999999999999999")] + public async Task TestSmallScaleNumberOverlimit(string value) + { + string columnName = "SMALLSCALENUMBER"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,2)", columnName)); + await Assert.ThrowsAsync(async () => await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, SqlDecimal.Parse(value))); + } + + + /// + /// Tests that decimals are rounded as expected. + /// Snowflake allows inserts of scales beyond the data type size, but storage of value will round it up or down + /// + [SkippableTheory] + [InlineData(2.467, 2.47)] + [InlineData(-672.613, -672.61)] + public async Task TestRoundingNumbers(decimal input, decimal output) + { + string columnName = "SMALLSCALENUMBER"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,2)", columnName)); + SqlDecimal value = new SqlDecimal(input); + SqlDecimal returned = new SqlDecimal(output); + await InsertSingleValueAsync(table.TableName, columnName, value.ToString()); + await SelectAndValidateValuesAsync(table.TableName, columnName, TestEnvironment.GetValueForProtocolVersion(output.ToString(), returned), 1); + string whereClause = GetWhereClause(columnName, returned); + if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1); + } + + /// + /// Validates if driver can handle floating point number type correctly + /// + [SkippableTheory] + [InlineData(0)] + [InlineData(0.2)] + [InlineData(15e-03)] + [InlineData(1.234E+2)] + [InlineData(double.NegativeInfinity)] + [InlineData(double.PositiveInfinity)] + [InlineData(double.NaN)] + [InlineData(double.MinValue)] + [InlineData(double.MaxValue)] + public async Task TestDoubleValuesInsertSelectDelete(double value) + { + string columnName = "DOUBLETYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DOUBLE", columnName)); + string valueString = ConvertDoubleToString(value); + await InsertSingleValueAsync(table.TableName, columnName, valueString); + await SelectAndValidateValuesAsync(table.TableName, columnName, value, 1); + string whereClause = GetWhereClause(columnName, value); + if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1); + } + + /// + /// Validates if driver can handle floating point number type correctly + /// + [SkippableTheory] + [InlineData(0)] + [InlineData(25)] + [InlineData(float.NegativeInfinity)] + [InlineData(float.PositiveInfinity)] + [InlineData(float.NaN)] + // TODO: Solve server issue when non-integer float value is used in where clause. + //[InlineData(25.1)] + //[InlineData(0.2)] + //[InlineData(15e-03)] + //[InlineData(1.234E+2)] + //[InlineData(float.MinValue)] + //[InlineData(float.MaxValue)] + public async Task TestFloatValuesInsertSelectDelete(float value) + { + string columnName = "FLOATTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} FLOAT", columnName)); + string valueString = ConvertFloatToString(value); + await InsertSingleValueAsync(table.TableName, columnName, valueString); + object doubleValue = (double)value; + // Spark over HTTP returns float as double whereas Spark on Databricks returns float. + object floatValue = TestEnvironment.DataTypeConversion.HasFlag(DataTypeConversion.Scalar) ? value : doubleValue; + await SelectAndValidateValuesAsync(table.TableName, columnName, floatValue, 1); + string whereClause = GetWhereClause(columnName, value); + if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1); + } + } +} diff --git a/csharp/test/Drivers/Apache/Common/StatementTests.cs b/csharp/test/Drivers/Apache/Common/StatementTests.cs new file mode 100644 index 0000000000..69eec0dd26 --- /dev/null +++ b/csharp/test/Drivers/Apache/Common/StatementTests.cs @@ -0,0 +1,125 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; +using Apache.Arrow.Adbc.Tests.Xunit; +using Xunit; +using Xunit.Abstractions; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common +{ + /// + /// Class for testing the Snowflake ADBC driver connection tests. + /// + /// + /// Tests are ordered to ensure data is created for the other + /// queries to run. + /// + [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] + public abstract class StatementTests : TestBase + where TConfig : ApacheTestConfiguration + where TEnv : HiveServer2TestEnvironment + { + private static List DefaultTableTypes => ["TABLE", "VIEW"]; + + public StatementTests(ITestOutputHelper? outputHelper, TestEnvironment.Factory testEnvFactory) + : base(outputHelper, testEnvFactory) + { + Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable)); + } + + /// + /// Validates if the SetOption handle valid/invalid data correctly for the PollTime option. + /// + [SkippableTheory] + [InlineData("-1", true)] + [InlineData("zero", true)] + [InlineData("-2147483648", true)] + [InlineData("2147483648", true)] + [InlineData("0")] + [InlineData("1")] + [InlineData("2147483647")] + public void CanSetOptionPollTime(string value, bool throws = false) + { + var testConfiguration = TestConfiguration.Clone() as TConfig; + testConfiguration!.PollTimeMilliseconds = value; + if (throws) + { + Assert.Throws(() => NewConnection(testConfiguration).CreateStatement()); + } + + AdbcStatement statement = NewConnection().CreateStatement(); + if (throws) + { + Assert.Throws(() => statement.SetOption(Adbc.Drivers.Apache.Hive2.HiveServer2Statement.Options.PollTimeMilliseconds, value)); + } + else + { + statement.SetOption(Adbc.Drivers.Apache.Hive2.HiveServer2Statement.Options.PollTimeMilliseconds, value); + } + } + + /// + /// Validates if the SetOption handle valid/invalid data correctly for the BatchSize option. + /// + [SkippableTheory] + [InlineData("-1", true)] + [InlineData("one", true)] + [InlineData("-2147483648", true)] + [InlineData("2147483648", false)] + [InlineData("9223372036854775807", false)] + [InlineData("9223372036854775808", true)] + [InlineData("0", true)] + [InlineData("1")] + [InlineData("2147483647")] + public void CanSetOptionBatchSize(string value, bool throws = false) + { + var testConfiguration = TestConfiguration.Clone() as TConfig; + testConfiguration!.BatchSize = value; + if (throws) + { + Assert.Throws(() => NewConnection(testConfiguration).CreateStatement()); + } + + AdbcStatement statement = NewConnection().CreateStatement(); + if (throws) + { + Assert.Throws(() => statement!.SetOption(Adbc.Drivers.Apache.Hive2.HiveServer2Statement.Options.BatchSize, value)); + } + else + { + statement.SetOption(Adbc.Drivers.Apache.Hive2.HiveServer2Statement.Options.BatchSize, value); + } + } + + /// + /// Validates if the driver can execute update statements. + /// + [SkippableFact, Order(1)] + public async Task CanInteractUsingSetOptions() + { + const string columnName = "INDEX"; + Statement.SetOption(Adbc.Drivers.Apache.Hive2.HiveServer2Statement.Options.PollTimeMilliseconds, "100"); + Statement.SetOption(Adbc.Drivers.Apache.Hive2.HiveServer2Statement.Options.BatchSize, "10"); + using TemporaryTable temporaryTable = await NewTemporaryTableAsync(Statement, $"{columnName} INT"); + await ValidateInsertSelectDeleteSingleValueAsync(temporaryTable.TableName, columnName, 1); + } + } +} diff --git a/csharp/test/Drivers/Apache/Common/StringValueTests.cs b/csharp/test/Drivers/Apache/Common/StringValueTests.cs new file mode 100644 index 0000000000..e861f7ebfa --- /dev/null +++ b/csharp/test/Drivers/Apache/Common/StringValueTests.cs @@ -0,0 +1,129 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Apache.Arrow.Adbc.Drivers.Apache.Hive2; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; +using Xunit; +using Xunit.Abstractions; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common +{ + // TODO: When supported, use prepared statements instead of SQL string literals + // Which will better test how the driver handles values sent/received + + /// + /// Validates that specific string and character values can be inserted, retrieved and targeted correctly + /// + public abstract class StringValueTests : TestBase + where TConfig : TestConfiguration + where TEnv : HiveServer2TestEnvironment + { + public StringValueTests(ITestOutputHelper output, TestEnvironment.Factory testEnvFactory) + : base(output, testEnvFactory) { } + + public static IEnumerable ByteArrayData(int size) + { + var rnd = new Random(); + byte[] bytes = new byte[size]; + rnd.NextBytes(bytes); + yield return new object[] { bytes }; + } + + /// + /// Validates if driver can send and receive specific String values correctly. + /// + [SkippableTheory] + [InlineData(null)] + [InlineData("")] + [InlineData("你好")] + [InlineData(" Leading and trailing spaces ")] + protected virtual async Task TestStringData(string? value) + { + string columnName = "STRINGTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "STRING")); + await ValidateInsertSelectDeleteSingleValueAsync( + table.TableName, + columnName, + value, + value != null ? QuoteValue(value) : value); + } + + /// + /// Validates if driver can send and receive specific VARCHAR values correctly. + /// + [SkippableTheory] + [InlineData(null)] + [InlineData("")] + [InlineData("你好")] + [InlineData(" Leading and trailing spaces ")] + protected virtual async Task TestVarcharData(string? value) + { + string columnName = "VARCHARTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "VARCHAR(100)")); + await ValidateInsertSelectDeleteSingleValueAsync( + table.TableName, + columnName, + value, + value != null ? QuoteValue(value) : value); + } + + /// + /// Validates if driver can send and receive specific VARCHAR values correctly. + /// + [SkippableTheory] + [InlineData(null)] + [InlineData("")] + [InlineData("你好")] + [InlineData(" Leading and trailing spaces ")] + protected virtual async Task TestCharData(string? value) + { + string columnName = "CHARTYPE"; + int fieldLength = 100; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, $"CHAR({fieldLength})")); + + string? formattedValue = value != null ? QuoteValue(value.PadRight(fieldLength)) : value; + string? paddedValue = value != null ? value.PadRight(fieldLength) : value; + + await InsertSingleValueAsync(table.TableName, columnName, formattedValue); + await SelectAndValidateValuesAsync(table.TableName, columnName, paddedValue, 1, formattedValue); + string whereClause = GetWhereClause(columnName, formattedValue ?? paddedValue); + if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1); + } + + /// + /// Validates if driver fails to insert invalid length of VARCHAR value. + /// + [SkippableTheory] + [InlineData("String whose length is too long for VARCHAR(10).", new string[] { "Exceeds", "length limitation: 10" }, null)] + protected virtual async Task TestVarcharExceptionData(string value, string[] expectedTexts, string? expectedSqlState) + { + string columnName = "VARCHARTYPE"; + using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "VARCHAR(10)")); + AdbcException exception = await Assert.ThrowsAsync(async () => await ValidateInsertSelectDeleteSingleValueAsync( + table.TableName, + columnName, + value, + value != null ? QuoteValue(value) : value)); + + AssertContainsAll(expectedTexts, exception.Message); + Assert.Equal(expectedSqlState, exception.SqlState); + } + } +} diff --git a/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs b/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs new file mode 100644 index 0000000000..f141b293c5 --- /dev/null +++ b/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs @@ -0,0 +1,40 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using Apache.Arrow.Adbc.Drivers.Apache.Hive2; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2 +{ + public abstract class HiveServer2TestEnvironment : TestEnvironment + where TConfig : TestConfiguration + { + public HiveServer2TestEnvironment(Func getConnection) + : base(getConnection) + { + } + + internal DataTypeConversion DataTypeConversion => ((HiveServer2Connection)Connection).DataTypeConversion; + + public string? GetValueForProtocolVersion(string? unconvertedValue, string? convertedValue) => + ((HiveServer2Connection)Connection).DataTypeConversion.HasFlag(DataTypeConversion.None) ? unconvertedValue : convertedValue; + + public object? GetValueForProtocolVersion(object? unconvertedValue, object? convertedValue) => + ((HiveServer2Connection)Connection).DataTypeConversion.HasFlag(DataTypeConversion.None) ? unconvertedValue : convertedValue; + + } +} diff --git a/csharp/test/Drivers/Apache/Spark/BinaryBooleanValueTests.cs b/csharp/test/Drivers/Apache/Spark/BinaryBooleanValueTests.cs index 403c4ac017..192db59e37 100644 --- a/csharp/test/Drivers/Apache/Spark/BinaryBooleanValueTests.cs +++ b/csharp/test/Drivers/Apache/Spark/BinaryBooleanValueTests.cs @@ -15,136 +15,15 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Threading.Tasks; -using Xunit; using Xunit.Abstractions; namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - // TODO: When supported, use prepared statements instead of SQL string literals - // Which will better test how the driver handles values sent/received - - /// - /// Validates that specific binary and boolean values can be inserted, retrieved and targeted correctly - /// - public class BinaryBooleanValueTests : TestBase + public class BinaryBooleanValueTests : Common.BinaryBooleanValueTests { - public BinaryBooleanValueTests(ITestOutputHelper output) : base(output, new SparkTestEnvironment.Factory()) { } - - public static IEnumerable ByteArrayData(int size) - { - var rnd = new Random(); - byte[] bytes = new byte[size]; - rnd.NextBytes(bytes); - yield return new object[] { bytes }; - } - - /// - /// Validates if driver can send and receive specific Binary values correctly. - /// - [SkippableTheory] - [InlineData(null)] - [MemberData(nameof(ByteArrayData), 0)] - [MemberData(nameof(ByteArrayData), 2)] - [MemberData(nameof(ByteArrayData), 1024)] - public async Task TestBinaryData(byte[]? value) - { - string columnName = "BINARYTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "BINARY")); - string? formattedValue = value != null ? $"X'{BitConverter.ToString(value).Replace("-", "")}'" : null; - await ValidateInsertSelectDeleteSingleValueAsync( - table.TableName, - columnName, - value, - formattedValue); - } - - /// - /// Validates if driver can send and receive specific Boolean values correctly. - /// - [SkippableTheory] - [InlineData(null)] - [InlineData(true)] - [InlineData(false)] - public async Task TestBooleanData(bool? value) + public BinaryBooleanValueTests(ITestOutputHelper output) + : base(output, new SparkTestEnvironment.Factory()) { - string columnName = "BOOLEANTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "BOOLEAN")); - string? formattedValue = value == null ? null : $"{value?.ToString(CultureInfo.InvariantCulture)}"; - await ValidateInsertSelectDeleteTwoValuesAsync( - table.TableName, - columnName, - value, - formattedValue); - } - - /// - /// Validates if driver can receive specific NULL values correctly. - /// - [SkippableTheory] - [InlineData("NULL")] - [InlineData("CAST(NULL AS INT)")] - [InlineData("CAST(NULL AS BIGINT)")] - [InlineData("CAST(NULL AS SMALLINT)")] - [InlineData("CAST(NULL AS TINYINT)")] - [InlineData("CAST(NULL AS FLOAT)")] - [InlineData("CAST(NULL AS DOUBLE)")] - [InlineData("CAST(NULL AS DECIMAL(38,0))")] - [InlineData("CAST(NULL AS STRING)")] - [InlineData("CAST(NULL AS VARCHAR(10))")] - [InlineData("CAST(NULL AS CHAR(10))")] - [InlineData("CAST(NULL AS BOOLEAN)")] - [InlineData("CAST(NULL AS BINARY)")] - [InlineData("CAST(NULL AS MAP)")] - [InlineData("CAST(NULL AS STRUCT)")] - [InlineData("CAST(NULL AS ARRAY)")] - public async Task TestNullData(string projectionClause) - { - string selectStatement = $"SELECT {projectionClause};"; - // Note: by default, this returns as String type, not NULL type. - await SelectAndValidateValuesAsync(selectStatement, (object?)null, 1); - } - - [SkippableTheory] - [InlineData(1)] - [InlineData(7)] - [InlineData(8)] - [InlineData(9)] - [InlineData(15)] - [InlineData(16)] - [InlineData(17)] - [InlineData(23)] - [InlineData(24)] - [InlineData(25)] - [InlineData(31)] - [InlineData(32)] // Full integer - [InlineData(33)] - [InlineData(39)] - [InlineData(40)] - [InlineData(41)] - [InlineData(47)] - [InlineData(48)] - [InlineData(49)] - [InlineData(63)] - [InlineData(64)] // Full 2 integers - [InlineData(65)] - public async Task TestMultilineNullData(int numberOfValues) - { - Random rnd = new(); - int percentIsNull = 50; - - object?[] values = new object?[numberOfValues]; - for (int i = 0; i < numberOfValues; i++) - { - values[i] = rnd.Next(0, 100) < percentIsNull ? null : rnd.Next(0, 2) != 0; - } - string columnName = "BOOLEANTYPE"; - string indexColumnName = "INDEXCOL"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}, {2} {3}", indexColumnName, "INT", columnName, "BOOLEAN")); - await ValidateInsertSelectDeleteMultipleValuesAsync(table.TableName, columnName, indexColumnName, values); } } } diff --git a/csharp/test/Drivers/Apache/Spark/ClientTests.cs b/csharp/test/Drivers/Apache/Spark/ClientTests.cs index f2288f4200..d5230749ae 100644 --- a/csharp/test/Drivers/Apache/Spark/ClientTests.cs +++ b/csharp/test/Drivers/Apache/Spark/ClientTests.cs @@ -1,4 +1,4 @@ -/* +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -15,213 +15,42 @@ * limitations under the License. */ -using System; using System.Collections.Generic; using Apache.Arrow.Adbc.Drivers.Apache.Spark; -using Apache.Arrow.Adbc.Tests.Xunit; -using Xunit; using Xunit.Abstractions; namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - /// - /// Class for testing the ADBC Client using the Spark ADBC driver. - /// - /// - /// Tests are ordered to ensure data is created for the other - /// queries to run. - /// Note: This test creates/replaces the table identified in the configuration (metadata/table). - /// It uses the test collection "TableCreateTestCollection" to ensure it does not run - /// as the same time as any other tests that may create/update the same table. - /// - [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] - [Collection("TableCreateTestCollection")] - public class ClientTests : TestBase + public class ClientTests : Common.ClientTests { - public ClientTests(ITestOutputHelper? outputHelper) : base(outputHelper, new SparkTestEnvironment.Factory()) + public ClientTests(ITestOutputHelper? outputHelper) + : base(outputHelper, new SparkTestEnvironment.Factory()) { - Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable)); } - /// - /// Validates if the client execute updates. - /// - [SkippableFact, Order(1)] - public void CanClientExecuteUpdate() + protected override IReadOnlyList GetUpdateExpectedResults() { - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) - { - adbcConnection.Open(); - - string[] queries = GetQueries(); - int affectedRows = ValidateAffectedRows ? 1 : -1; - - List expectedResults = TestEnvironment.ServerType != SparkServerType.Databricks - ? [ - -1, // CREATE TABLE - affectedRows, // INSERT - affectedRows, // INSERT - affectedRows, // INSERT - //1, // UPDATE - //1, // DELETE - ] - : [ - -1, // CREATE TABLE - affectedRows, // INSERT - affectedRows, // INSERT - affectedRows, // INSERT - affectedRows, // UPDATE - affectedRows, // DELETE - ]; - - Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, TestConfiguration, queries, expectedResults); - } - } - - /// - /// Validates if the client can get the schema. - /// - [SkippableFact, Order(2)] - public void CanClientGetSchema() - { - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) - { - Tests.ClientTests.CanClientGetSchema(adbcConnection, TestConfiguration, $"SELECT * FROM {TestConfiguration.Metadata.Table}"); - } - } - - /// - /// Validates if the client can connect to a live server and - /// parse the results. - /// - [SkippableFact, Order(3)] - public void CanClientExecuteQuery() - { - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) - { - Tests.ClientTests.CanClientExecuteQuery(adbcConnection, TestConfiguration); - } - } - - /// - /// Validates if the client can connect to a live server and - /// parse the results. - /// - [SkippableFact, Order(5)] - public void CanClientExecuteEmptyQuery() - { - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) - { - Tests.ClientTests.CanClientExecuteQuery( - adbcConnection, - TestConfiguration, - customQuery: $"SELECT * FROM {TestConfiguration.Metadata.Table} WHERE FALSE", - expectedResultsCount: 0); - } - } - - /// - /// Validates if the client is retrieving and converting values - /// to the expected types. - /// - [SkippableFact, Order(4)] - public void VerifyTypesAndValues() - { - using (Adbc.Client.AdbcConnection dbConnection = GetAdbcConnection()) - { - SampleDataBuilder sampleDataBuilder = GetSampleDataBuilder(); - - Tests.ClientTests.VerifyTypesAndValues(dbConnection, sampleDataBuilder); - } - } - - [SkippableFact] - public void VerifySchemaTablesWithNoConstraints() - { - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(includeTableConstraints: false)) - { - adbcConnection.Open(); - - string schema = "Tables"; - - var tables = adbcConnection.GetSchema(schema); - - Assert.True(tables.Rows.Count > 0, $"No tables were found in the schema '{schema}'"); - } - } - - [SkippableFact] - public void VerifySchemaTables() - { - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) - { - adbcConnection.Open(); - - var collections = adbcConnection.GetSchema("MetaDataCollections"); - Assert.Equal(7, collections.Rows.Count); - Assert.Equal(2, collections.Columns.Count); - - var restrictions = adbcConnection.GetSchema("Restrictions"); - Assert.Equal(11, restrictions.Rows.Count); - Assert.Equal(3, restrictions.Columns.Count); - - var catalogs = adbcConnection.GetSchema("Catalogs"); - Assert.Single(catalogs.Columns); - var catalog = (string?)catalogs.Rows[0].ItemArray[0]; - - catalogs = adbcConnection.GetSchema("Catalogs", new[] { catalog }); - Assert.Equal(1, catalogs.Rows.Count); - - string random = "X" + Guid.NewGuid().ToString("N"); - - catalogs = adbcConnection.GetSchema("Catalogs", new[] { random }); - Assert.Equal(0, catalogs.Rows.Count); - - var schemas = adbcConnection.GetSchema("Schemas", new[] { catalog }); - Assert.Equal(2, schemas.Columns.Count); - var schema = (string?)schemas.Rows[0].ItemArray[1]; - - schemas = adbcConnection.GetSchema("Schemas", new[] { catalog, schema }); - Assert.Equal(1, schemas.Rows.Count); - - schemas = adbcConnection.GetSchema("Schemas", new[] { random }); - Assert.Equal(0, schemas.Rows.Count); - - schemas = adbcConnection.GetSchema("Schemas", new[] { catalog, random }); - Assert.Equal(0, schemas.Rows.Count); - - schemas = adbcConnection.GetSchema("Schemas", new[] { random, random }); - Assert.Equal(0, schemas.Rows.Count); - - var tableTypes = adbcConnection.GetSchema("TableTypes"); - Assert.Single(tableTypes.Columns); - - var tables = adbcConnection.GetSchema("Tables", new[] { catalog, schema }); - Assert.Equal(4, tables.Columns.Count); - - tables = adbcConnection.GetSchema("Tables", new[] { catalog, random }); - Assert.Equal(0, tables.Rows.Count); - - tables = adbcConnection.GetSchema("Tables", new[] { random, schema }); - Assert.Equal(0, tables.Rows.Count); - - tables = adbcConnection.GetSchema("Tables", new[] { random, random }); - Assert.Equal(0, tables.Rows.Count); - - tables = adbcConnection.GetSchema("Tables", new[] { catalog, schema, random }); - Assert.Equal(0, tables.Rows.Count); - - var columns = adbcConnection.GetSchema("Columns", new[] { catalog, schema }); - Assert.Equal(16, columns.Columns.Count); - } + int affectedRows = ValidateAffectedRows ? 1 : -1; + return GetUpdateExpecteResults(affectedRows, TestEnvironment.ServerType == SparkServerType.Databricks); } - private Adbc.Client.AdbcConnection GetAdbcConnection(bool includeTableConstraints = true) + internal static IReadOnlyList GetUpdateExpecteResults(int affectedRows, bool isDatabricks) { - return new Adbc.Client.AdbcConnection( - NewDriver, GetDriverParameters(TestConfiguration), - [] - ); + return !isDatabricks + ? [ + -1, // CREATE TABLE + affectedRows, // INSERT + affectedRows, // INSERT + affectedRows, // INSERT + ] + : [ + -1, // CREATE TABLE + affectedRows, // INSERT + affectedRows, // INSERT + affectedRows, // INSERT + affectedRows, // UPDATE + affectedRows, // DELETE + ]; } } } diff --git a/csharp/test/Drivers/Apache/Spark/ComplexTypesValueTests.cs b/csharp/test/Drivers/Apache/Spark/ComplexTypesValueTests.cs index e65194b6f2..c2ca23b60a 100644 --- a/csharp/test/Drivers/Apache/Spark/ComplexTypesValueTests.cs +++ b/csharp/test/Drivers/Apache/Spark/ComplexTypesValueTests.cs @@ -15,65 +15,15 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Xunit; using Xunit.Abstractions; namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - // TODO: When supported, use prepared statements instead of SQL string literals - // Which will better test how the driver handles values sent/received - - /// - /// Validates that specific complex structured types can be inserted, retrieved and targeted correctly - /// - public class ComplexTypesValueTests : TestBase + public class ComplexTypesValueTests : Common.ComplexTypesValueTests { - public ComplexTypesValueTests(ITestOutputHelper output) : base(output, new SparkTestEnvironment.Factory()) { } - - /// - /// Validates if driver can send and receive specific array of integer values correctly. - /// - [SkippableTheory] - [InlineData("ARRAY(CAST(1 AS INT), 2, 3)", "[1,2,3]")] - [InlineData("ARRAY(CAST(1 AS LONG), 2, 3)", "[1,2,3]")] - [InlineData("ARRAY(CAST(1 AS DOUBLE), 2, 3)", "[1.0,2.0,3.0]")] - [InlineData("ARRAY(CAST(1 AS NUMERIC(38,0)), 2, 3)", "[1,2,3]")] - [InlineData("ARRAY(CAST('John Doe' AS STRING), 2, 3)", """["John Doe","2","3"]""")] - // Note: Timestamp returned adjusted to UTC. - [InlineData("ARRAY(CAST('2024-01-01T00:00:00-07:00' AS TIMESTAMP), CAST('2024-02-02T02:02:02+01:30' AS TIMESTAMP), CAST('2024-03-03T03:03:03Z' AS TIMESTAMP))", """[2024-01-01 07:00:00,2024-02-02 00:32:02,2024-03-03 03:03:03]""")] - [InlineData("ARRAY(CAST('2024-01-01T00:00:00Z' AS DATE), CAST('2024-02-02T02:02:02Z' AS DATE), CAST('2024-03-03T03:03:03Z' AS DATE))", """[2024-01-01,2024-02-02,2024-03-03]""")] - [InlineData("ARRAY(INTERVAL 123 YEARS 11 MONTHS, INTERVAL 5 YEARS, INTERVAL 6 MONTHS)", """[123-11,5-0,0-6]""")] - public async Task TestArrayData(string projection, string value) - { - string selectStatement = $"SELECT {projection};"; - await SelectAndValidateValuesAsync(selectStatement, value, 1); - } - - /// - /// Validates if driver can send and receive specific map values correctly. - /// - [SkippableTheory] - [InlineData("MAP(1, 'John Doe', 2, 'Jane Doe', 3, 'Jack Doe')", """{1:"John Doe",2:"Jane Doe",3:"Jack Doe"}""")] - [InlineData("MAP('John Doe', 1, 'Jane Doe', 2, 'Jack Doe', 3)", """{"Jack Doe":3,"Jane Doe":2,"John Doe":1}""")] - public async Task TestMapData(string projection, string value) - { - string selectStatement = $"SELECT {projection};"; - await SelectAndValidateValuesAsync(selectStatement, value, 1); - } - - /// - /// Validates if driver can send and receive specific map values correctly. - /// - [SkippableTheory] - [InlineData("STRUCT(CAST(1 AS INT), CAST('John Doe' AS STRING))", """{"col1":1,"col2":"John Doe"}""")] - [InlineData("STRUCT(CAST('John Doe' AS STRING), CAST(1 AS INT))", """{"col1":"John Doe","col2":1}""")] - public async Task TestStructData(string projection, string value) + public ComplexTypesValueTests(ITestOutputHelper output) + : base(output, new SparkTestEnvironment.Factory()) { - string selectStatement = $"SELECT {projection};"; - await SelectAndValidateValuesAsync(selectStatement, value, 1); } } } diff --git a/csharp/test/Drivers/Apache/Spark/DateTimeValueTests.cs b/csharp/test/Drivers/Apache/Spark/DateTimeValueTests.cs index 81c872f5a2..e9d832ab9e 100644 --- a/csharp/test/Drivers/Apache/Spark/DateTimeValueTests.cs +++ b/csharp/test/Drivers/Apache/Spark/DateTimeValueTests.cs @@ -16,7 +16,6 @@ */ using System; -using System.Collections.Generic; using System.Globalization; using System.Threading.Tasks; using Apache.Arrow.Adbc.Drivers.Apache.Spark; @@ -25,55 +24,18 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - // TODO: When supported, use prepared statements instead of SQL string literals - // Which will better test how the driver handles values sent/received - - /// - /// Validates that specific date, timestamp and interval values can be inserted, retrieved and targeted correctly - /// - public class DateTimeValueTests : TestBase + public class DateTimeValueTests : Common.DateTimeValueTests { - // Spark handles microseconds but not nanoseconds. Truncated to 6 decimal places. - const string DateTimeZoneFormat = "yyyy-MM-dd'T'HH:mm:ss'.'ffffffK"; - const string DateTimeFormat = "yyyy-MM-dd' 'HH:mm:ss"; - const string DateFormat = "yyyy-MM-dd"; - - private static readonly DateTimeOffset[] s_timestampValues = - [ -#if NET5_0_OR_GREATER - DateTimeOffset.UnixEpoch, -#endif - DateTimeOffset.MinValue, - DateTimeOffset.MaxValue, - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow.ToOffset(TimeSpan.FromHours(4)) - ]; - - public DateTimeValueTests(ITestOutputHelper output) : base(output, new SparkTestEnvironment.Factory()) { } + public DateTimeValueTests(ITestOutputHelper output) + : base(output, new SparkTestEnvironment.Factory()) + { } - /// - /// Validates if driver can send and receive specific Timstamp values correctly - /// [SkippableTheory] - [MemberData(nameof(TimestampData), "TIMESTAMP")] [MemberData(nameof(TimestampData), "TIMESTAMP_LTZ")] - public async Task TestTimestampData(DateTimeOffset value, string columnType) + public async Task TestTimestampDataDatabricks(DateTimeOffset value, string columnType) { - Skip.If((TestEnvironment.ServerType != SparkServerType.Databricks) && (columnType.Equals("TIMESTAMP_LTZ") || value == DateTimeOffset.MaxValue)); - - string columnName = "TIMESTAMPTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, columnType)); - - string format = TestEnvironment.GetValueForProtocolVersion(DateTimeFormat, DateTimeZoneFormat)!; - string formattedValue = $"{value.ToString(format, CultureInfo.InvariantCulture)}"; - DateTimeOffset truncatedValue = DateTimeOffset.ParseExact(formattedValue, format, CultureInfo.InvariantCulture); - - object expectedValue = TestEnvironment.GetValueForProtocolVersion(formattedValue, truncatedValue)!; - await ValidateInsertSelectDeleteSingleValueAsync( - table.TableName, - columnName, - expectedValue , - "TO_TIMESTAMP(" + QuoteValue(formattedValue) + ")"); + Skip.If(TestEnvironment.ServerType != SparkServerType.Databricks); + await base.TestTimestampData(value, columnType); } /// @@ -81,11 +43,9 @@ await ValidateInsertSelectDeleteSingleValueAsync( /// [SkippableTheory] [MemberData(nameof(TimestampData), "TIMESTAMP_NTZ")] - public async Task TestTimestampNoTimezoneData(DateTimeOffset value, string columnType) + public async Task TestTimestampNoTimezoneDataDatabricks(DateTimeOffset value, string columnType) { - // Note: Minimum value falls outside range of valid values on server when no time zone is included. Cannot be selected - Skip.If(value == DateTimeOffset.MinValue || TestEnvironment.ServerType != SparkServerType.Databricks); - + Skip.If(TestEnvironment.ServerType != SparkServerType.Databricks); string columnName = "TIMESTAMPTYPE"; using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, columnType)); @@ -99,71 +59,5 @@ await ValidateInsertSelectDeleteSingleValueAsync( new DateTimeOffset(truncatedValue.DateTime, TimeSpan.Zero), QuoteValue(formattedValue)); } - - /// - /// Validates if driver can send and receive specific no timezone Timstamp values correctly - /// - [SkippableTheory] - [MemberData(nameof(TimestampData), "DATE")] - public async Task TestDateData(DateTimeOffset value, string columnType) - { - string columnName = "DATETYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, columnType)); - - string formattedValue = $"{value.ToString(DateFormat, CultureInfo.InvariantCulture)}"; - DateTimeOffset truncatedValue = DateTimeOffset.ParseExact(formattedValue, DateFormat, CultureInfo.InvariantCulture); - - // Remove timezone offset - object expectedValue = TestEnvironment.GetValueForProtocolVersion(formattedValue, new DateTimeOffset(truncatedValue.DateTime, TimeSpan.Zero))!; - await ValidateInsertSelectDeleteSingleValueAsync( - table.TableName, - columnName, - expectedValue, - "TO_DATE(" + QuoteValue(formattedValue) + ")"); - } - - /// - /// Tests INTERVAL data types (YEAR-MONTH and DAY-SECOND). - /// - /// The INTERVAL to test. - /// The expected return value. - /// - [SkippableTheory] - [InlineData("INTERVAL 1 YEAR", "1-0")] - [InlineData("INTERVAL 1 YEAR 2 MONTH", "1-2")] - [InlineData("INTERVAL 2 MONTHS", "0-2")] - [InlineData("INTERVAL -1 YEAR", "-1-0")] - [InlineData("INTERVAL -1 YEAR 2 MONTH", "-0-10")] - [InlineData("INTERVAL -2 YEAR 2 MONTH", "-1-10")] - [InlineData("INTERVAL 1 YEAR -2 MONTH", "0-10")] - [InlineData("INTERVAL 178956970 YEAR", "178956970-0")] - [InlineData("INTERVAL 178956969 YEAR 11 MONTH", "178956969-11")] - [InlineData("INTERVAL -178956970 YEAR", "-178956970-0")] - [InlineData("INTERVAL 0 DAYS 0 HOURS 0 MINUTES 0 SECONDS", "0 00:00:00.000000000")] - [InlineData("INTERVAL 1 DAYS", "1 00:00:00.000000000")] - [InlineData("INTERVAL 2 HOURS", "0 02:00:00.000000000")] - [InlineData("INTERVAL 3 MINUTES", "0 00:03:00.000000000")] - [InlineData("INTERVAL 4 SECONDS", "0 00:00:04.000000000")] - [InlineData("INTERVAL 1 DAYS 2 HOURS", "1 02:00:00.000000000")] - [InlineData("INTERVAL 1 DAYS 2 HOURS 3 MINUTES", "1 02:03:00.000000000")] - [InlineData("INTERVAL 1 DAYS 2 HOURS 3 MINUTES 4 SECONDS", "1 02:03:04.000000000")] - [InlineData("INTERVAL 1 DAYS 2 HOURS 3 MINUTES 4.123123123 SECONDS", "1 02:03:04.123123000")] // Only to microseconds - [InlineData("INTERVAL 106751990 DAYS 23 HOURS 59 MINUTES 59.999999 SECONDS", "106751990 23:59:59.999999000")] - [InlineData("INTERVAL 106751991 DAYS 0 HOURS 0 MINUTES 0 SECONDS", "106751991 00:00:00.000000000")] - [InlineData("INTERVAL -106751991 DAYS 0 HOURS 0 MINUTES 0 SECONDS", "-106751991 00:00:00.000000000")] - [InlineData("INTERVAL -106751991 DAYS 23 HOURS 59 MINUTES 59.999999 SECONDS", "-106751990 00:00:00.000001000")] - public async Task TestIntervalData(string intervalClause, string value) - { - string selectStatement = $"SELECT {intervalClause} AS INTERVAL_VALUE;"; - await SelectAndValidateValuesAsync(selectStatement, value, 1); - } - - public static IEnumerable TimestampData(string columnType) - { - foreach (DateTimeOffset timestamp in s_timestampValues) - { - yield return new object[] { timestamp, columnType }; - } - } } } diff --git a/csharp/test/Drivers/Apache/Spark/DriverTests.cs b/csharp/test/Drivers/Apache/Spark/DriverTests.cs index cd6b4c22af..6970777c22 100644 --- a/csharp/test/Drivers/Apache/Spark/DriverTests.cs +++ b/csharp/test/Drivers/Apache/Spark/DriverTests.cs @@ -15,638 +15,47 @@ * limitations under the License. */ -using System; using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading.Tasks; using Apache.Arrow.Adbc.Drivers.Apache.Spark; -using Apache.Arrow.Adbc.Tests.Metadata; -using Apache.Arrow.Adbc.Tests.Xunit; -using Apache.Arrow.Ipc; using Xunit; using Xunit.Abstractions; -using ColumnTypeId = Apache.Arrow.Adbc.Drivers.Apache.Spark.SparkConnection.ColumnTypeId; namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - /// - /// Class for testing the Spark ADBC driver connection tests. - /// - /// - /// Tests are ordered to ensure data is created for the other - /// queries to run. - /// Note: This test creates/replaces the table identified in the configuration (metadata/table). - /// It uses the test collection "TableCreateTestCollection" to ensure it does not run - /// as the same time as any other tests that may create/update the same table. - /// - [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] - [Collection("TableCreateTestCollection")] - public class DriverTests : TestBase + public class DriverTests : Common.DriverTests { - /// - /// Supported Spark data types as a subset of - /// - private enum SupportedSparkDataType : short + public DriverTests(ITestOutputHelper? outputHelper) + : base(outputHelper, new SparkTestEnvironment.Factory()) { - ARRAY = ColumnTypeId.ARRAY, - BIGINT = ColumnTypeId.BIGINT, - BINARY = ColumnTypeId.BINARY, - BOOLEAN = ColumnTypeId.BOOLEAN, - CHAR = ColumnTypeId.CHAR, - DATE = ColumnTypeId.DATE, - DECIMAL = ColumnTypeId.DECIMAL, - DOUBLE = ColumnTypeId.DOUBLE, - FLOAT = ColumnTypeId.FLOAT, - INTEGER = ColumnTypeId.INTEGER, - JAVA_OBJECT = ColumnTypeId.JAVA_OBJECT, - LONGNVARCHAR = ColumnTypeId.LONGNVARCHAR, - LONGVARBINARY = ColumnTypeId.LONGVARBINARY, - LONGVARCHAR = ColumnTypeId.LONGVARCHAR, - NCHAR = ColumnTypeId.NCHAR, - NULL = ColumnTypeId.NULL, - NUMERIC = ColumnTypeId.NUMERIC, - NVARCHAR = ColumnTypeId.NVARCHAR, - REAL = ColumnTypeId.REAL, - SMALLINT = ColumnTypeId.SMALLINT, - STRUCT = ColumnTypeId.STRUCT, - TIMESTAMP = ColumnTypeId.TIMESTAMP, - TINYINT = ColumnTypeId.TINYINT, - VARBINARY = ColumnTypeId.VARBINARY, - VARCHAR = ColumnTypeId.VARCHAR, } - private static List DefaultTableTypes => new() { "TABLE", "VIEW" }; - - public DriverTests(ITestOutputHelper? outputHelper) : base(outputHelper, new SparkTestEnvironment.Factory()) - { - Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable)); - } - - /// - /// Validates if the driver can execute update statements. - /// - [SkippableFact, Order(1)] - public void CanExecuteUpdate() - { - AdbcConnection adbcConnection = NewConnection(); - - string[] queries = GetQueries(); - - List expectedResults = TestEnvironment.ServerType != SparkServerType.Databricks - ? - [ - -1, // CREATE TABLE - 1, // INSERT - 1, // INSERT - 1, // INSERT - //1, // UPDATE - //1, // DELETE - ] - : - [ - -1, // CREATE TABLE - 1, // INSERT - 1, // INSERT - 1, // INSERT - 1, // UPDATE - 1, // DELETE - ]; - - for (int i = 0; i < queries.Length; i++) - { - string query = queries[i]; - using AdbcStatement statement = adbcConnection.CreateStatement(); - statement.SqlQuery = query; - - UpdateResult updateResult = statement.ExecuteUpdate(); - - if (ValidateAffectedRows) Assert.Equal(expectedResults[i], updateResult.AffectedRows); - } - } - - /// - /// Validates if the driver can call GetInfo. - /// - [SkippableFact, Order(2)] - public async Task CanGetInfo() - { - AdbcConnection adbcConnection = NewConnection(); - - // Test the supported info codes - List handledCodes = new List() - { - AdbcInfoCode.DriverName, - AdbcInfoCode.DriverVersion, - AdbcInfoCode.VendorName, - AdbcInfoCode.DriverArrowVersion, - AdbcInfoCode.VendorVersion, - AdbcInfoCode.VendorSql - }; - using IArrowArrayStream stream = adbcConnection.GetInfo(handledCodes); - - RecordBatch recordBatch = await stream.ReadNextRecordBatchAsync(); - UInt32Array infoNameArray = (UInt32Array)recordBatch.Column("info_name"); - - List expectedValues = new List() - { - "DriverName", - "DriverVersion", - "VendorName", - "DriverArrowVersion", - "VendorVersion", - "VendorSql" - }; - - for (int i = 0; i < infoNameArray.Length; i++) - { - AdbcInfoCode? value = (AdbcInfoCode?)infoNameArray.GetValue(i); - DenseUnionArray valueArray = (DenseUnionArray)recordBatch.Column("info_value"); - - Assert.Contains(value.ToString(), expectedValues); - - switch (value) - { - case AdbcInfoCode.VendorSql: - // TODO: How does external developer know the second field is the boolean field? - BooleanArray booleanArray = (BooleanArray)valueArray.Fields[1]; - bool? boolValue = booleanArray.GetValue(i); - OutputHelper?.WriteLine($"{value}={boolValue}"); - Assert.True(boolValue); - break; - default: - StringArray stringArray = (StringArray)valueArray.Fields[0]; - string stringValue = stringArray.GetString(i); - OutputHelper?.WriteLine($"{value}={stringValue}"); - Assert.NotNull(stringValue); - break; - } - } - - // Test the unhandled info codes. - List unhandledCodes = new List() - { - AdbcInfoCode.VendorArrowVersion, - AdbcInfoCode.VendorSubstrait, - AdbcInfoCode.VendorSubstraitMaxVersion - }; - using IArrowArrayStream stream2 = adbcConnection.GetInfo(unhandledCodes); - - recordBatch = await stream2.ReadNextRecordBatchAsync(); - infoNameArray = (UInt32Array)recordBatch.Column("info_name"); - - List unexpectedValues = new List() - { - "VendorArrowVersion", - "VendorSubstrait", - "VendorSubstraitMaxVersion" - }; - for (int i = 0; i < infoNameArray.Length; i++) - { - AdbcInfoCode? value = (AdbcInfoCode?)infoNameArray.GetValue(i); - DenseUnionArray valueArray = (DenseUnionArray)recordBatch.Column("info_value"); - - Assert.Contains(value.ToString(), unexpectedValues); - switch (value) - { - case AdbcInfoCode.VendorSql: - BooleanArray booleanArray = (BooleanArray)valueArray.Fields[1]; - Assert.Null(booleanArray.GetValue(i)); - break; - default: - StringArray stringArray = (StringArray)valueArray.Fields[0]; - Assert.Null(stringArray.GetString(i)); - break; - } - } - } - - /// - /// Validates if the driver can call GetObjects with GetObjectsDepth as Catalogs with CatalogPattern as a pattern. - /// - /// - [SkippableTheory, Order(3)] + [SkippableTheory] [MemberData(nameof(CatalogNamePatternData))] - public void GetGetObjectsCatalogs(string? pattern) + public override void CanGetObjectsCatalogs(string? pattern) { - string? catalogName = TestConfiguration.Metadata.Catalog; - string? schemaName = TestConfiguration.Metadata.Schema; - - using IArrowArrayStream stream = Connection.GetObjects( - depth: AdbcConnection.GetObjectsDepth.Catalogs, - catalogPattern: pattern, - dbSchemaPattern: null, - tableNamePattern: null, - tableTypes: DefaultTableTypes, - columnNamePattern: null); - - using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; - - List catalogs = GetObjectsParser.ParseCatalog(recordBatch, null); - AdbcCatalog? catalog = catalogs.Where((catalog) => string.Equals(catalog.Name, catalogName)).FirstOrDefault(); - - Assert.True((pattern == string.Empty && catalog == null) || catalog != null, "catalog should not be null"); + GetObjectsCatalogsTest(pattern); } - /// - /// Validates if the driver can call GetObjects with GetObjectsDepth as DbSchemas with DbSchemaName as a pattern. - /// - [SkippableTheory, Order(4)] + [SkippableTheory] [MemberData(nameof(DbSchemasNamePatternData))] - public void CanGetObjectsDbSchemas(string dbSchemaPattern) + public override void CanGetObjectsDbSchemas(string dbSchemaPattern) { - // need to add the database - string? databaseName = TestConfiguration.Metadata.Catalog; - string? schemaName = TestConfiguration.Metadata.Schema; - - using IArrowArrayStream stream = Connection.GetObjects( - depth: AdbcConnection.GetObjectsDepth.DbSchemas, - catalogPattern: databaseName, - dbSchemaPattern: dbSchemaPattern, - tableNamePattern: null, - tableTypes: DefaultTableTypes, - columnNamePattern: null); - - using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; - - List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); - - List? dbSchemas = catalogs - .Where(c => string.Equals(c.Name, databaseName)) - .Select(c => c.DbSchemas) - .FirstOrDefault(); - AdbcDbSchema? dbSchema = dbSchemas?.Where((dbSchema) => string.Equals(dbSchema.Name, schemaName)).FirstOrDefault(); - - Assert.True(dbSchema != null, "dbSchema should not be null"); + GetObjectsDbSchemasTest(dbSchemaPattern); } - /// - /// Validates if the driver can call GetObjects with GetObjectsDepth as Tables with TableName as a pattern. - /// - [SkippableTheory, Order(5)] + [SkippableTheory] [MemberData(nameof(TableNamePatternData))] - public void CanGetObjectsTables(string tableNamePattern) - { - // need to add the database - string? databaseName = TestConfiguration.Metadata.Catalog; - string? schemaName = TestConfiguration.Metadata.Schema; - string? tableName = TestConfiguration.Metadata.Table; - - using IArrowArrayStream stream = Connection.GetObjects( - depth: AdbcConnection.GetObjectsDepth.Tables, - catalogPattern: databaseName, - dbSchemaPattern: schemaName, - tableNamePattern: tableNamePattern, - tableTypes: DefaultTableTypes, - columnNamePattern: null); - - using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; - - List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); - - List? tables = catalogs - .Where(c => string.Equals(c.Name, databaseName)) - .Select(c => c.DbSchemas) - .FirstOrDefault() - ?.Where(s => string.Equals(s.Name, schemaName)) - .Select(s => s.Tables) - .FirstOrDefault(); - - AdbcTable? table = tables?.Where((table) => string.Equals(table.Name, tableName)).FirstOrDefault(); - Assert.True(table != null, "table should not be null"); - // TODO: Determine why this is returned blank. - //Assert.Equal("TABLE", table.Type); - } - - /// - /// Validates if the driver can call GetObjects for GetObjectsDepth as All. - /// - [SkippableFact, Order(6)] - public void CanGetObjectsAll() - { - // need to add the database - string? databaseName = TestConfiguration.Metadata.Catalog; - string? schemaName = TestConfiguration.Metadata.Schema; - string? tableName = TestConfiguration.Metadata.Table; - string? columnName = null; - - using IArrowArrayStream stream = Connection.GetObjects( - depth: AdbcConnection.GetObjectsDepth.All, - catalogPattern: databaseName, - dbSchemaPattern: schemaName, - tableNamePattern: tableName, - tableTypes: DefaultTableTypes, - columnNamePattern: columnName); - - using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; - - List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); - AdbcTable? table = catalogs - .Where(c => string.Equals(c.Name, databaseName)) - .Select(c => c.DbSchemas) - .FirstOrDefault() - ?.Where(s => string.Equals(s.Name, schemaName)) - .Select(s => s.Tables) - .FirstOrDefault() - ?.Where(t => string.Equals(t.Name, tableName)) - .FirstOrDefault(); - - Assert.True(table != null, "table should not be null"); - // TODO: Determine why this is returned blank. - //Assert.Equal("TABLE", table.Type); - List? columns = table.Columns; - - Assert.True(columns != null, "Columns cannot be null"); - Assert.Equal(TestConfiguration.Metadata.ExpectedColumnCount, columns.Count); - - for (int i = 0; i < columns.Count; i++) - { - // Verify column metadata is returned/consistent. - AdbcColumn column = columns[i]; - Assert.Equal(i + 1, column.OrdinalPosition); - Assert.False(string.IsNullOrEmpty(column.Name)); - Assert.False(string.IsNullOrEmpty(column.XdbcTypeName)); - Assert.False(Regex.IsMatch(column.XdbcTypeName, @"[_,\d\<\>\(\)]", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), - "Unexpected character found in field XdbcTypeName"); - - var supportedTypes = Enum.GetValues(typeof(SupportedSparkDataType)).Cast(); - Assert.Contains((SupportedSparkDataType)column.XdbcSqlDataType!, supportedTypes); - Assert.Equal(column.XdbcDataType, column.XdbcSqlDataType); - - Assert.NotNull(column.XdbcDataType); - Assert.Contains((SupportedSparkDataType)column.XdbcDataType!, supportedTypes); - - HashSet typesHaveColumnSize = new() - { - (short)SupportedSparkDataType.DECIMAL, - (short)SupportedSparkDataType.NUMERIC, - (short)SupportedSparkDataType.CHAR, - (short)SupportedSparkDataType.VARCHAR, - }; - HashSet typesHaveDecimalDigits = new() - { - (short)SupportedSparkDataType.DECIMAL, - (short)SupportedSparkDataType.NUMERIC, - }; - - bool typeHasColumnSize = typesHaveColumnSize.Contains(column.XdbcDataType.Value); - Assert.Equal(column.XdbcColumnSize.HasValue, typeHasColumnSize); - - bool typeHasDecimalDigits = typesHaveDecimalDigits.Contains(column.XdbcDataType.Value); - Assert.Equal(column.XdbcDecimalDigits.HasValue, typeHasDecimalDigits); - - Assert.False(string.IsNullOrEmpty(column.Remarks)); - - Assert.NotNull(column.XdbcColumnDef); - - Assert.NotNull(column.XdbcNullable); - Assert.Contains(new short[] { 1, 0 }, i => i == column.XdbcNullable); - - Assert.NotNull(column.XdbcIsNullable); - Assert.Contains(new string[] { "YES", "NO" }, i => i.Equals(column.XdbcIsNullable)); - - Assert.NotNull(column.XdbcIsAutoIncrement); - - Assert.Null(column.XdbcCharOctetLength); - Assert.Null(column.XdbcDatetimeSub); - Assert.Null(column.XdbcNumPrecRadix); - Assert.Null(column.XdbcScopeCatalog); - Assert.Null(column.XdbcScopeSchema); - Assert.Null(column.XdbcScopeTable); - } - } - - /// - /// Validates if the driver can call GetObjects with GetObjectsDepth as Tables with TableName as a Special Character. - /// - [SkippableTheory, Order(7)] - [InlineData("MyIdentifier")] - [InlineData("ONE")] - [InlineData("mYiDentifier")] - [InlineData("3rd_identifier")] - // Note: Tables in 'hive_metastore' only support ASCII alphabetic, numeric and underscore. - public void CanGetObjectsTablesWithSpecialCharacter(string tableName) + public override void CanGetObjectsTables(string tableNamePattern) { - string catalogName = TestConfiguration.Metadata.Catalog; - string schemaPrefix = Guid.NewGuid().ToString().Replace("-", ""); - using TemporarySchema schema = TemporarySchema.NewTemporarySchemaAsync(catalogName, Statement).Result; - string schemaName = schema.SchemaName; - string catalogFormatted = string.IsNullOrEmpty(catalogName) ? string.Empty : DelimitIdentifier(catalogName) + "."; - string fullTableName = $"{catalogFormatted}{DelimitIdentifier(schemaName)}.{DelimitIdentifier(tableName)}"; - using TemporaryTable temporaryTable = TemporaryTable.NewTemporaryTableAsync(Statement, fullTableName, $"CREATE TABLE IF NOT EXISTS {fullTableName} (INDEX INT)").Result; - - using IArrowArrayStream stream = Connection.GetObjects( - depth: AdbcConnection.GetObjectsDepth.Tables, - catalogPattern: catalogName, - dbSchemaPattern: schemaName, - tableNamePattern: tableName, - tableTypes: DefaultTableTypes, - columnNamePattern: null); - - using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; - - List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); - - List? tables = catalogs - .Where(c => string.Equals(c.Name, catalogName)) - .Select(c => c.DbSchemas) - .FirstOrDefault() - ?.Where(s => string.Equals(s.Name, schemaName)) - .Select(s => s.Tables) - .FirstOrDefault(); - - AdbcTable? table = tables?.FirstOrDefault(); - - Assert.True(table != null, "table should not be null"); - Assert.Equal(tableName, table.Name, true); + GetObjectsTablesTest(tableNamePattern); } - /// - /// Validates if the driver can call GetTableSchema. - /// - [SkippableFact, Order(8)] - public void CanGetTableSchema() + protected override IReadOnlyList GetUpdateExpectedResults() { - AdbcConnection adbcConnection = NewConnection(); - - string? catalogName = TestConfiguration.Metadata.Catalog; - string? schemaName = TestConfiguration.Metadata.Schema; - string tableName = TestConfiguration.Metadata.Table!; - - Schema schema = adbcConnection.GetTableSchema(catalogName, schemaName, tableName); - - int numberOfFields = schema.FieldsList.Count; - - Assert.Equal(TestConfiguration.Metadata.ExpectedColumnCount, numberOfFields); + int affectedRows = ValidateAffectedRows ? 1 : -1; + return ClientTests.GetUpdateExpecteResults(affectedRows, TestEnvironment.ServerType == SparkServerType.Databricks); } - /// - /// Validates if the driver can call GetTableTypes. - /// - [SkippableFact, Order(9)] - public async Task CanGetTableTypes() - { - AdbcConnection adbcConnection = NewConnection(); - - using IArrowArrayStream arrowArrayStream = adbcConnection.GetTableTypes(); - - RecordBatch recordBatch = await arrowArrayStream.ReadNextRecordBatchAsync(); - - StringArray stringArray = (StringArray)recordBatch.Column("table_type"); - - List known_types = new List - { - "TABLE", "VIEW" - }; - - int results = 0; - - for (int i = 0; i < stringArray.Length; i++) - { - string value = stringArray.GetString(i); - - if (known_types.Contains(value)) - { - results++; - } - } - - Assert.Equal(known_types.Count, results); - } - - /// - /// Validates if the driver can connect to a live server and - /// parse the results. - /// - [SkippableTheory, Order(10)] - [InlineData(0.1)] - [InlineData(0.25)] - [InlineData(1.0)] - [InlineData(2.0)] - [InlineData(null)] - public void CanExecuteQuery(double? batchSizeFactor) - { - // Ensure all records can be retrieved, independent of the batch size. - SparkTestConfiguration testConfiguration = (SparkTestConfiguration)TestConfiguration.Clone(); - long expectedResultCount = testConfiguration.ExpectedResultsCount; - long nonZeroExpectedResultCount = (expectedResultCount == 0 ? 1 : expectedResultCount); - testConfiguration.BatchSize = batchSizeFactor != null ? ((long)(nonZeroExpectedResultCount * batchSizeFactor)).ToString() : string.Empty; - OutputHelper?.WriteLine($"BatchSize: {testConfiguration.BatchSize}. ExpectedResultCount: {expectedResultCount}"); - - using AdbcConnection adbcConnection = NewConnection(testConfiguration); - - using AdbcStatement statement = adbcConnection.CreateStatement(); - statement.SqlQuery = TestConfiguration.Query; - OutputHelper?.WriteLine(statement.SqlQuery); - - QueryResult queryResult = statement.ExecuteQuery(); - - Tests.DriverTests.CanExecuteQuery(queryResult, TestConfiguration.ExpectedResultsCount); - } - - /// - /// Validates if the driver can connect to a live server and - /// parse the results using the asynchronous methods. - /// - [SkippableFact, Order(11)] - public async Task CanExecuteQueryAsync() - { - using AdbcConnection adbcConnection = NewConnection(); - using AdbcStatement statement = adbcConnection.CreateStatement(); - - statement.SqlQuery = TestConfiguration.Query; - QueryResult queryResult = await statement.ExecuteQueryAsync(); - - await Tests.DriverTests.CanExecuteQueryAsync(queryResult, TestConfiguration.ExpectedResultsCount); - } - - /// - /// Validates if the driver can connect to a live server and - /// perform and update asynchronously. - /// - [SkippableFact, Order(12)] - public async Task CanExecuteUpdateAsync() - { - using AdbcConnection adbcConnection = NewConnection(); - using AdbcStatement statement = adbcConnection.CreateStatement(); - using TemporaryTable temporaryTable = await NewTemporaryTableAsync(statement, "INDEX INT"); - - statement.SqlQuery = GetInsertStatement(temporaryTable.TableName, "INDEX", "1"); - UpdateResult updateResult = await statement.ExecuteUpdateAsync(); - - if (ValidateAffectedRows) Assert.Equal(1, updateResult.AffectedRows); - } - - [SkippableFact, Order(13)] - public void CanDetectInvalidAuthentication() - { - AdbcDriver driver = NewDriver; - Assert.NotNull(driver); - Dictionary parameters = GetDriverParameters(TestConfiguration); - - bool hasToken = parameters.TryGetValue(SparkParameters.Token, out var token) && !string.IsNullOrEmpty(token); - bool hasUsername = parameters.TryGetValue(AdbcOptions.Username, out var username) && !string.IsNullOrEmpty(username); - bool hasPassword = parameters.TryGetValue(AdbcOptions.Password, out var password) && !string.IsNullOrEmpty(password); - if (hasToken) - { - parameters[SparkParameters.Token] = "invalid-token"; - } - else if (hasUsername && hasPassword) - { - parameters[AdbcOptions.Password] = "invalid-password"; - } - else - { - Assert.Fail($"Unexpected configuration. Must provide '{SparkParameters.Token}' or '{AdbcOptions.Username}' and '{AdbcOptions.Password}'."); - } - - AdbcDatabase database = driver.Open(parameters); - AggregateException exception = Assert.ThrowsAny(() => database.Connect(parameters)); - OutputHelper?.WriteLine(exception.Message); - } - - [SkippableFact, Order(14)] - public void CanDetectInvalidServer() - { - AdbcDriver driver = NewDriver; - Assert.NotNull(driver); - Dictionary parameters = GetDriverParameters(TestConfiguration); - - bool hasUri = parameters.TryGetValue(AdbcOptions.Uri, out var uri) && !string.IsNullOrEmpty(uri); - bool hasHostName = parameters.TryGetValue(SparkParameters.HostName, out var hostName) && !string.IsNullOrEmpty(hostName); - if (hasUri) - { - parameters[AdbcOptions.Uri] = "http://unknownhost.azure.com/cliservice"; - } - else if (hasHostName) - { - parameters[SparkParameters.HostName] = "unknownhost.azure.com"; - } - else - { - Assert.Fail($"Unexpected configuration. Must provide '{AdbcOptions.Uri}' or '{SparkParameters.HostName}'."); - } - - AdbcDatabase database = driver.Open(parameters); - AggregateException exception = Assert.ThrowsAny(() => database.Connect(parameters)); - OutputHelper?.WriteLine(exception.Message); - } - - /// - /// Validates if the driver can connect to a live server and - /// parse the results using the asynchronous methods. - /// - [SkippableFact, Order(15)] - public async Task CanExecuteQueryAsyncEmptyResult() - { - using AdbcConnection adbcConnection = NewConnection(); - using AdbcStatement statement = adbcConnection.CreateStatement(); - - statement.SqlQuery = $"SELECT * from {TestConfiguration.Metadata.Table} WHERE FALSE"; - QueryResult queryResult = await statement.ExecuteQueryAsync(); - - await Tests.DriverTests.CanExecuteQueryAsync(queryResult, 0); - } public static IEnumerable CatalogNamePatternData() { diff --git a/csharp/test/Drivers/Apache/Spark/NumericValueTests.cs b/csharp/test/Drivers/Apache/Spark/NumericValueTests.cs index 326caff13d..417cfc3c64 100644 --- a/csharp/test/Drivers/Apache/Spark/NumericValueTests.cs +++ b/csharp/test/Drivers/Apache/Spark/NumericValueTests.cs @@ -15,258 +15,15 @@ * limitations under the License. */ -using System; -using System.Data.SqlTypes; -using System.Threading.Tasks; -using Apache.Arrow.Adbc.Drivers.Apache.Hive2; -using Apache.Arrow.Adbc.Drivers.Apache.Spark; -using Xunit; using Xunit.Abstractions; namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - // TODO: When supported, use prepared statements instead of SQL string literals - // Which will better test how the driver handles values sent/received - - public class NumericValueTests : TestBase + public class NumericValueTests : Common.NumericValueTests { - /// - /// Validates that specific numeric values can be inserted, retrieved and targeted correctly - /// - public NumericValueTests(ITestOutputHelper output) : base(output, new SparkTestEnvironment.Factory()) { } - - /// - /// Validates if driver can send and receive specific Integer values correctly - /// - [SkippableTheory] - [InlineData(-1)] - [InlineData(0)] - [InlineData(1)] - public async Task TestIntegerSanity(int value) - { - string columnName = "INTTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} INT", columnName)); - await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); - } - - /// - /// Validates if driver can handle the largest / smallest numbers - /// - [SkippableTheory] - [InlineData(int.MaxValue)] - [InlineData(int.MinValue)] - public async Task TestIntegerMinMax(int value) - { - string columnName = "INTTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} INT", columnName)); - await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); - } - - /// - /// Validates if driver can handle the largest / smallest numbers - /// - [SkippableTheory] - [InlineData(long.MaxValue)] - [InlineData(long.MinValue)] - public async Task TestLongMinMax(long value) - { - string columnName = "BIGINTTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} BIGINT", columnName)); - await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); - } - - /// - /// Validates if driver can handle the largest / smallest numbers - /// - [SkippableTheory] - [InlineData(short.MaxValue)] - [InlineData(short.MinValue)] - public async Task TestSmallIntMinMax(short value) - { - string columnName = "SMALLINTTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} SMALLINT", columnName)); - await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); - } - - /// - /// Validates if driver can handle the largest / smallest numbers - /// - [SkippableTheory] - [InlineData(sbyte.MaxValue)] - [InlineData(sbyte.MinValue)] - public async Task TestTinyIntMinMax(sbyte value) - { - string columnName = "TINYINTTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} TINYINT", columnName)); - await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, value); - } - - /// - /// Validates if driver can handle smaller Number type correctly - /// - [SkippableTheory] - [InlineData("-1")] - [InlineData("0")] - [InlineData("1")] - [InlineData("99")] - [InlineData("-99")] - public async Task TestSmallNumberRange(string value) - { - string columnName = "SMALLNUMBER"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(2,0)", columnName)); - object? expectedValue = TestEnvironment.GetValueForProtocolVersion(value, new SqlDecimal(double.Parse(value))); - await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, expectedValue); - } - - /// - /// Validates if driver correctly errors out when the values exceed the column's limit - /// - [SkippableTheory] - [InlineData(-100)] - [InlineData(100)] - [InlineData(int.MaxValue)] - [InlineData(int.MinValue)] - public async Task TestSmallNumberRangeOverlimit(int value) - { - string columnName = "SMALLNUMBER"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(2,0)", columnName)); - await Assert.ThrowsAsync( - async () => await ValidateInsertSelectDeleteSingleValueAsync( - table.TableName, - columnName, TestEnvironment.GetValueForProtocolVersion(value.ToString(), new SqlDecimal(value)))); - } - - /// - /// Validates if driver can handle a large scale Number type correctly - /// - [SkippableTheory] - [InlineData("0E-37")] - [InlineData("-2.0030000000000000000000000000000000000")] - [InlineData("4.8500000000000000000000000000000000000")] - [InlineData("1E-37")] - [InlineData("9.5545204502636499875576383003668916798")] - public async Task TestLargeScaleNumberRange(string value) - { - string columnName = "LARGESCALENUMBER"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,37)", columnName)); - await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, TestEnvironment.GetValueForProtocolVersion(value, new SqlDecimal(double.Parse(value)))); - } - - /// - /// Validates if driver can error handle when input goes beyond a large scale Number type - /// - [SkippableTheory] - [InlineData("-10")] - [InlineData("10")] - [InlineData("99999999999999999999999999999999999999")] - [InlineData("-99999999999999999999999999999999999999")] - public async Task TestLargeScaleNumberOverlimit(string value) - { - string columnName = "LARGESCALENUMBER"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,37)", columnName)); - await Assert.ThrowsAsync(async () => await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, SqlDecimal.Parse(value))); - } - - /// - /// Validates if driver can handle a small scale Number type correctly - /// - [SkippableTheory] - [InlineData("0.00")] - [InlineData("4.85")] - [InlineData("-999999999999999999999999999999999999.99")] - [InlineData("999999999999999999999999999999999999.99")] - public async Task TestSmallScaleNumberRange(string value) - { - string columnName = "SMALLSCALENUMBER"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,2)", columnName)); - await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, TestEnvironment.GetValueForProtocolVersion(value, SqlDecimal.Parse(value))); - } - - /// - /// Validates if driver can error handle when an insert goes beyond a small scale Number type correctly - /// - [SkippableTheory] - [InlineData("-99999999999999999999999999999999999999")] - [InlineData("99999999999999999999999999999999999999")] - public async Task TestSmallScaleNumberOverlimit(string value) - { - string columnName = "SMALLSCALENUMBER"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,2)", columnName)); - await Assert.ThrowsAsync(async () => await ValidateInsertSelectDeleteSingleValueAsync(table.TableName, columnName, SqlDecimal.Parse(value))); - } - - - /// - /// Tests that decimals are rounded as expected. - /// Snowflake allows inserts of scales beyond the data type size, but storage of value will round it up or down - /// - [SkippableTheory] - [InlineData(2.467, 2.47)] - [InlineData(-672.613, -672.61)] - public async Task TestRoundingNumbers(decimal input, decimal output) - { - string columnName = "SMALLSCALENUMBER"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DECIMAL(38,2)", columnName)); - SqlDecimal value = new SqlDecimal(input); - SqlDecimal returned = new SqlDecimal(output); - await InsertSingleValueAsync(table.TableName, columnName, value.ToString()); - await SelectAndValidateValuesAsync(table.TableName, columnName, TestEnvironment.GetValueForProtocolVersion(output.ToString(), returned), 1); - string whereClause = GetWhereClause(columnName, returned); - if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1); - } - - /// - /// Validates if driver can handle floating point number type correctly - /// - [SkippableTheory] - [InlineData(0)] - [InlineData(0.2)] - [InlineData(15e-03)] - [InlineData(1.234E+2)] - [InlineData(double.NegativeInfinity)] - [InlineData(double.PositiveInfinity)] - [InlineData(double.NaN)] - [InlineData(double.MinValue)] - [InlineData(double.MaxValue)] - public async Task TestDoubleValuesInsertSelectDelete(double value) - { - string columnName = "DOUBLETYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} DOUBLE", columnName)); - string valueString = ConvertDoubleToString(value); - await InsertSingleValueAsync(table.TableName, columnName, valueString); - await SelectAndValidateValuesAsync(table.TableName, columnName, value, 1); - string whereClause = GetWhereClause(columnName, value); - if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1); - } - - /// - /// Validates if driver can handle floating point number type correctly - /// - [SkippableTheory] - [InlineData(0)] - [InlineData(25)] - [InlineData(float.NegativeInfinity)] - [InlineData(float.PositiveInfinity)] - [InlineData(float.NaN)] - // TODO: Solve server issue when non-integer float value is used in where clause. - //[InlineData(25.1)] - //[InlineData(0.2)] - //[InlineData(15e-03)] - //[InlineData(1.234E+2)] - //[InlineData(float.MinValue)] - //[InlineData(float.MaxValue)] - public async Task TestFloatValuesInsertSelectDelete(float value) + public NumericValueTests(ITestOutputHelper output) + : base(output, new SparkTestEnvironment.Factory()) { - string columnName = "FLOATTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} FLOAT", columnName)); - string valueString = ConvertFloatToString(value); - await InsertSingleValueAsync(table.TableName, columnName, valueString); - object doubleValue = (double)value; - // Spark over HTTP returns float as double whereas Spark on Databricks returns float. - object floatValue = TestEnvironment.ServerType == SparkServerType.Databricks || TestEnvironment.DataTypeConversion.HasFlag(DataTypeConversion.Scalar) ? value : doubleValue; - await base.SelectAndValidateValuesAsync(table.TableName, columnName, floatValue, 1); - string whereClause = GetWhereClause(columnName, value); - if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1); } } } diff --git a/csharp/test/Drivers/Apache/Spark/SparkTestConfiguration.cs b/csharp/test/Drivers/Apache/Spark/SparkTestConfiguration.cs index 7eb513d314..5ada5abeb3 100644 --- a/csharp/test/Drivers/Apache/Spark/SparkTestConfiguration.cs +++ b/csharp/test/Drivers/Apache/Spark/SparkTestConfiguration.cs @@ -22,13 +22,8 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark public class SparkTestConfiguration : ApacheTestConfiguration { - [JsonPropertyName("type"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string Type { get; set; } = string.Empty; + [JsonPropertyName("token"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string Token { get; set; } = string.Empty; - [JsonPropertyName("data_type_conv"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string DataTypeConversion { get; set; } = string.Empty; - - [JsonPropertyName("tls_options"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string TlsOptions { get; set; } = string.Empty; } } diff --git a/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs b/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs index 06b6fe2463..54c5368536 100644 --- a/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs +++ b/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs @@ -21,11 +21,12 @@ using System.Text; using Apache.Arrow.Adbc.Drivers.Apache.Hive2; using Apache.Arrow.Adbc.Drivers.Apache.Spark; +using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2; using Apache.Arrow.Types; namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - public class SparkTestEnvironment : TestEnvironment + public class SparkTestEnvironment : HiveServer2TestEnvironment { public class Factory : Factory { @@ -49,12 +50,6 @@ public override string GetCreateTemporaryTableStatement(string tableName, string return string.Format("CREATE TABLE {0} ({1})", tableName, columns); } - public string? GetValueForProtocolVersion(string? hiveValue, string? databrickValue) => - ServerType != SparkServerType.Databricks && ((HiveServer2Connection)Connection).DataTypeConversion.HasFlag(DataTypeConversion.None) ? hiveValue : databrickValue; - - public object? GetValueForProtocolVersion(object? hiveValue, object? databrickValue) => - ServerType != SparkServerType.Databricks && ((HiveServer2Connection)Connection).DataTypeConversion.HasFlag(DataTypeConversion.None) ? hiveValue : databrickValue; - public override string Delimiter => "`"; public override Dictionary GetDriverParameters(SparkTestConfiguration testConfiguration) @@ -123,8 +118,6 @@ public override Dictionary GetDriverParameters(SparkTestConfigur internal SparkServerType ServerType => ((SparkConnection)Connection).ServerType; - internal DataTypeConversion DataTypeConversion => ((SparkConnection)Connection).DataTypeConversion; - public override string VendorVersion => ((HiveServer2Connection)Connection).VendorVersion; public override bool SupportsDelete => ServerType == SparkServerType.Databricks; diff --git a/csharp/test/Drivers/Apache/Spark/SqlTypeNameParserTests.cs b/csharp/test/Drivers/Apache/Spark/SqlTypeNameParserTests.cs index 388a8f82ae..8ec4f5390f 100644 --- a/csharp/test/Drivers/Apache/Spark/SqlTypeNameParserTests.cs +++ b/csharp/test/Drivers/Apache/Spark/SqlTypeNameParserTests.cs @@ -17,10 +17,6 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading.Tasks; using Apache.Arrow.Adbc.Drivers.Apache.Spark; using Xunit; using Xunit.Abstractions; diff --git a/csharp/test/Drivers/Apache/Spark/StatementTests.cs b/csharp/test/Drivers/Apache/Spark/StatementTests.cs index 08a738c768..25d27179ac 100644 --- a/csharp/test/Drivers/Apache/Spark/StatementTests.cs +++ b/csharp/test/Drivers/Apache/Spark/StatementTests.cs @@ -15,108 +15,15 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Apache.Arrow.Adbc.Drivers.Apache.Spark; -using Apache.Arrow.Adbc.Tests.Xunit; -using Xunit; using Xunit.Abstractions; namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - /// - /// Class for testing the Snowflake ADBC driver connection tests. - /// - /// - /// Tests are ordered to ensure data is created for the other - /// queries to run. - /// - [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] - public class StatementTests : TestBase + public class StatementTests : Common.StatementTests { - private static List DefaultTableTypes => new() { "TABLE", "VIEW" }; - - public StatementTests(ITestOutputHelper? outputHelper) : base(outputHelper, new SparkTestEnvironment.Factory()) - { - Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable)); - } - - /// - /// Validates if the SetOption handle valid/invalid data correctly for the PollTime option. - /// - [SkippableTheory] - [InlineData("-1", true)] - [InlineData("zero", true)] - [InlineData("-2147483648", true)] - [InlineData("2147483648", true)] - [InlineData("0")] - [InlineData("1")] - [InlineData("2147483647")] - public void CanSetOptionPollTime(string value, bool throws = false) - { - var testConfiguration = TestConfiguration.Clone() as SparkTestConfiguration; - testConfiguration!.PollTimeMilliseconds = value; - if (throws) - { - Assert.Throws(() => NewConnection(testConfiguration).CreateStatement()); - } - - AdbcStatement statement = NewConnection().CreateStatement(); - if (throws) - { - Assert.Throws(() => statement.SetOption(SparkStatement.Options.PollTimeMilliseconds, value)); - } - else - { - statement.SetOption(SparkStatement.Options.PollTimeMilliseconds, value); - } - } - - /// - /// Validates if the SetOption handle valid/invalid data correctly for the BatchSize option. - /// - [SkippableTheory] - [InlineData("-1", true)] - [InlineData("one", true)] - [InlineData("-2147483648", true)] - [InlineData("2147483648", false)] - [InlineData("9223372036854775807", false)] - [InlineData("9223372036854775808", true)] - [InlineData("0", true)] - [InlineData("1")] - [InlineData("2147483647")] - public void CanSetOptionBatchSize(string value, bool throws = false) - { - var testConfiguration = TestConfiguration.Clone() as SparkTestConfiguration; - testConfiguration!.BatchSize = value; - if (throws) - { - Assert.Throws(() => NewConnection(testConfiguration).CreateStatement()); - } - - AdbcStatement statement = NewConnection().CreateStatement(); - if (throws) - { - Assert.Throws(() => statement!.SetOption(SparkStatement.Options.BatchSize, value)); - } - else - { - statement.SetOption(SparkStatement.Options.BatchSize, value); - } - } - - /// - /// Validates if the driver can execute update statements. - /// - [SkippableFact, Order(1)] - public async Task CanInteractUsingSetOptions() + public StatementTests(ITestOutputHelper? outputHelper) + : base(outputHelper, new SparkTestEnvironment.Factory()) { - const string columnName = "INDEX"; - Statement.SetOption(SparkStatement.Options.PollTimeMilliseconds, "100"); - Statement.SetOption(SparkStatement.Options.BatchSize, "10"); - using TemporaryTable temporaryTable = await NewTemporaryTableAsync(Statement, $"{columnName} INT"); - await ValidateInsertSelectDeleteSingleValueAsync(temporaryTable.TableName, columnName, 1); } } } diff --git a/csharp/test/Drivers/Apache/Spark/StringValueTests.cs b/csharp/test/Drivers/Apache/Spark/StringValueTests.cs index 875eb2ea1e..ef4e6f1fb4 100644 --- a/csharp/test/Drivers/Apache/Spark/StringValueTests.cs +++ b/csharp/test/Drivers/Apache/Spark/StringValueTests.cs @@ -15,127 +15,52 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; using System.Threading.Tasks; -using Apache.Arrow.Adbc.Drivers.Apache.Hive2; using Apache.Arrow.Adbc.Drivers.Apache.Spark; using Xunit; using Xunit.Abstractions; namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark { - // TODO: When supported, use prepared statements instead of SQL string literals - // Which will better test how the driver handles values sent/received - - /// - /// Validates that specific string and character values can be inserted, retrieved and targeted correctly - /// - public class StringValueTests : TestBase + public class StringValueTests(ITestOutputHelper output) + : Common.StringValueTests(output, new SparkTestEnvironment.Factory()) { - public StringValueTests(ITestOutputHelper output) : base(output, new SparkTestEnvironment.Factory()) { } - - public static IEnumerable ByteArrayData(int size) - { - var rnd = new Random(); - byte[] bytes = new byte[size]; - rnd.NextBytes(bytes); - yield return new object[] { bytes }; - } - - /// - /// Validates if driver can send and receive specific String values correctly. - /// [SkippableTheory] - [InlineData(null)] - [InlineData("")] - [InlineData("你好")] [InlineData("String contains formatting characters tab\t, newline\n, carriage return\r.", SparkServerType.Databricks)] - [InlineData(" Leading and trailing spaces ")] - internal async Task TestStringData(string? value, SparkServerType? serverType = default) + internal async Task TestStringDataDatabricks(string? value, SparkServerType serverType) { - Skip.If(serverType != null && TestEnvironment.ServerType != serverType); - string columnName = "STRINGTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "STRING")); - await ValidateInsertSelectDeleteSingleValueAsync( - table.TableName, - columnName, - value, - value != null ? QuoteValue(value) : value); + Skip.If(TestEnvironment.ServerType != serverType); + await TestStringData(value); } - /// - /// Validates if driver can send and receive specific VARCHAR values correctly. - /// [SkippableTheory] - [InlineData(null)] - [InlineData("")] - [InlineData("你好")] [InlineData("String contains formatting characters tab\t, newline\n, carriage return\r.", SparkServerType.Databricks)] - [InlineData(" Leading and trailing spaces ")] - internal async Task TestVarcharData(string? value, SparkServerType? serverType = default) + internal async Task TestVarcharDataDatabricks(string? value, SparkServerType serverType) { - Skip.If(serverType != null && TestEnvironment.ServerType != serverType); - string columnName = "VARCHARTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "VARCHAR(100)")); - await ValidateInsertSelectDeleteSingleValueAsync( - table.TableName, - columnName, - value, - value != null ? QuoteValue(value) : value); + Skip.If(TestEnvironment.ServerType != serverType); + await TestVarcharData(value); } - private bool IsBelowMinimumVersion(string? minVersion) => minVersion != null && VendorVersionAsVersion < Version.Parse(minVersion); - - /// - /// Validates if driver can send and receive specific VARCHAR values correctly. - /// [SkippableTheory] - [InlineData(null)] - [InlineData("")] - [InlineData("你好")] [InlineData("String contains formatting characters tab\t, newline\n, carriage return\r.", SparkServerType.Databricks)] - [InlineData(" Leading and trailing spaces ")] - internal async Task TestCharData(string? value, SparkServerType? serverType = default) + internal async Task TestCharDataDatabricks(string? value, SparkServerType serverType) { - Skip.If(serverType != null && TestEnvironment.ServerType != serverType); - string columnName = "CHARTYPE"; - int fieldLength = 100; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, $"CHAR({fieldLength})")); - - string? formattedValue = value != null ? QuoteValue(value.PadRight(fieldLength)) : value; - string? paddedValue = value != null ? value.PadRight(fieldLength) : value; + Skip.If(TestEnvironment.ServerType != serverType); + await TestCharData(value); + } - await InsertSingleValueAsync(table.TableName, columnName, formattedValue); - await SelectAndValidateValuesAsync(table.TableName, columnName, paddedValue, 1, formattedValue); - string whereClause = GetWhereClause(columnName, formattedValue ?? paddedValue); - if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1); + protected override async Task TestVarcharExceptionData(string value, string[] expectedTexts, string? expectedSqlState) + { + Skip.If(TestEnvironment.ServerType == SparkServerType.Databricks); + await base.TestVarcharExceptionData(value, expectedTexts, expectedSqlState); } - /// - /// Validates if driver fails to insert invalid length of VARCHAR value. - /// [SkippableTheory] - [InlineData("String whose length is too long for VARCHAR(10).")] - public async Task TestVarcharExceptionData(string value) + [InlineData("String whose length is too long for VARCHAR(10).", new string[] { "DELTA_EXCEED_CHAR_VARCHAR_LIMIT", "DeltaInvariantViolationException" }, "22001")] + public async Task TestVarcharExceptionDataDatabricks(string value, string[] expectedTexts, string? expectedSqlState) { - string columnName = "VARCHARTYPE"; - using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "VARCHAR(10)")); - AdbcException exception = await Assert.ThrowsAsync(async () => await ValidateInsertSelectDeleteSingleValueAsync( - table.TableName, - columnName, - value, - value != null ? QuoteValue(value) : value)); - - bool serverTypeDatabricks = TestEnvironment.ServerType == SparkServerType.Databricks; - string[] expectedTexts = serverTypeDatabricks - ? ["DELTA_EXCEED_CHAR_VARCHAR_LIMIT", "DeltaInvariantViolationException"] - : ["Exceeds", "length limitation: 10"]; - AssertContainsAll(expectedTexts, exception.Message); - - string? expectedSqlState = serverTypeDatabricks ? "22001" : null; - Assert.Equal(expectedSqlState, exception.SqlState); + Skip.IfNot(TestEnvironment.ServerType == SparkServerType.Databricks, $"Server type: {TestEnvironment.ServerType}"); + await base.TestVarcharExceptionData(value, expectedTexts, expectedSqlState); } - } } diff --git a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj index 01f21523ce..04f2c92bc0 100644 --- a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj +++ b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj @@ -11,7 +11,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj index 1b20a2157f..265b67827a 100644 --- a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj +++ b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj @@ -11,7 +11,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj index f657ed09d1..f660e7a645 100644 --- a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj @@ -22,7 +22,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/java/pom.xml b/java/pom.xml index 7c2a314feb..66cc7b43fe 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -97,7 +97,7 @@ 18.0.0 - 3.48.2 + 3.48.3 11 11