Skip to content

Commit

Permalink
resolve merge issues
Browse files Browse the repository at this point in the history
  • Loading branch information
David Coe committed Dec 4, 2024
2 parents 4fe3bc9 + 73bbc8a commit ab46983
Show file tree
Hide file tree
Showing 31 changed files with 1,977 additions and 1,621 deletions.
1 change: 0 additions & 1 deletion csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Apache.Arrow" Version="18.1.0" />
<PackageReference Include="System.Text.Json" Version="8.0.5" />
</ItemGroup>
<ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible($(TargetFramework), 'net6.0'))">
<Compile Remove="C\NativeLibrary.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

<ItemGroup>
<PackageReference Include="ApacheThrift" Version="0.21.0" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />
<PackageReference Include="System.Text.Json" Version="8.0.5" />
</ItemGroup>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.Net.Http.WinHttpHandler" Version="8.0.2" Condition="'$(TargetFrameworkIdentifier)' == '.NETStandard'" />
<PackageReference Include="Apache.Arrow.Flight" Version="18.0.0" />
<PackageReference Include="Apache.Arrow.Flight" Version="18.1.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Apache.Arrow.Adbc\Apache.Arrow.Adbc.csproj" />
Expand Down
2 changes: 1 addition & 1 deletion csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public static void CanClientExecuteUpdate(
Adbc.Client.AdbcConnection adbcConnection,
TestConfiguration testConfiguration,
string[] queries,
List<int> expectedResults,
IReadOnlyList<int> expectedResults,
string? environmentName = null)
{
if (adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection));
Expand Down
104 changes: 44 additions & 60 deletions csharp/test/Apache.Arrow.Adbc.Tests/TestBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,16 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object
await SelectAndValidateValuesAsync(selectStatement, [value], expectedLength);
}

private static T? ArrowArrayAs<T>(IArrowArray arrowArray)
where T : IArrowArray
{
if (arrowArray is T t)
{
return t;
}
return default;
}

/// <summary>
/// Selects a single value and validates it equality with expected value and number of results.
/// </summary>
Expand All @@ -391,71 +401,30 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object
int actualLength = 0;
using (IArrowArrayStream stream = queryResult.Stream ?? throw new InvalidOperationException("stream is null"))
{
Dictionary<ArrowTypeId, Func<IArrowArray, int, object?>> valueGetters = new()
{
{ ArrowTypeId.Decimal128, (a, i) => ArrowArrayAs<Decimal128Array>(a)?.GetSqlDecimal(i) },
{ ArrowTypeId.Double, (a, i) => ArrowArrayAs<DoubleArray>(a)?.GetValue(i) },
{ ArrowTypeId.Float, (a, i) => ArrowArrayAs<FloatArray>(a)?.GetValue(i) },
{ ArrowTypeId.Int64, (a, i) => ArrowArrayAs<Int64Array>(a)?.GetValue(i) },
{ ArrowTypeId.Int32, (a, i) => ArrowArrayAs<Int32Array>(a)?.GetValue(i) },
{ ArrowTypeId.Int16, (a, i) => ArrowArrayAs<Int16Array>(a)?.GetValue(i) },
{ ArrowTypeId.Int8, (a, i) => ArrowArrayAs<Int8Array>(a)?.GetValue(i) },
{ ArrowTypeId.String, (a, i) => ArrowArrayAs<StringArray>(a)?.GetString(i) },
{ ArrowTypeId.Timestamp, (a, i) => ArrowArrayAs<TimestampArray>(a)?.GetTimestamp(i) },
{ ArrowTypeId.Date32, (a, i) => ArrowArrayAs<Date32Array>(a)?.GetDateTimeOffset(i) },
{ ArrowTypeId.Boolean, (a, i) => ArrowArrayAs<BooleanArray>(a)?.GetValue(i) },
};
// Assume first column
Field field = stream.Schema.GetFieldByIndex(0);
Int32Array? indexArray = null;
while (true)
{
using (RecordBatch nextBatch = await stream.ReadNextRecordBatchAsync())
{
if (nextBatch == null) { break; }
switch (field.DataType)
{
case Decimal128Type:
Decimal128Array decimalArray = (Decimal128Array)nextBatch.Column(0);
actualLength += decimalArray.Length;
ValidateValue((i) => values?[i], decimalArray.Length, (i) => decimalArray.GetSqlDecimal(i));
break;
case DoubleType:
DoubleArray doubleArray = (DoubleArray)nextBatch.Column(0);
actualLength += doubleArray.Length;
ValidateValue((i) => values?[i], doubleArray.Length, (i) => doubleArray.GetValue(i));
break;
case FloatType:
FloatArray floatArray = (FloatArray)nextBatch.Column(0);
actualLength += floatArray.Length;
ValidateValue((i) => values?[i], floatArray.Length, (i) => floatArray.GetValue(i));
break;
case Int64Type:
Int64Array int64Array = (Int64Array)nextBatch.Column(0);
actualLength += int64Array.Length;
ValidateValue((i) => values?[i], int64Array.Length, (i) => int64Array.GetValue(i));
break;
case Int32Type:
Int32Array intArray = (Int32Array)nextBatch.Column(0);
actualLength += intArray.Length;
ValidateValue((i) => values?[i], intArray.Length, (i) => intArray.GetValue(i));
break;
case Int16Type:
Int16Array shortArray = (Int16Array)nextBatch.Column(0);
actualLength += shortArray.Length;
ValidateValue((i) => values?[i], shortArray.Length, (i) => shortArray.GetValue(i));
break;
case Int8Type:
Int8Array tinyIntArray = (Int8Array)nextBatch.Column(0);
actualLength += tinyIntArray.Length;
ValidateValue((i) => values?[i], tinyIntArray.Length, (i) => tinyIntArray.GetValue(i));
break;
case StringType:
StringArray stringArray = (StringArray)nextBatch.Column(0);
actualLength += stringArray.Length;
ValidateValue((i) => values?[i], stringArray.Length, (i) => stringArray.GetString(i));
break;
case TimestampType:
TimestampArray timestampArray = (TimestampArray)nextBatch.Column(0);
actualLength += timestampArray.Length;
ValidateValue((i) => values?[i], timestampArray.Length, (i) => timestampArray.GetTimestamp(i));
break;
case Date32Type:
Date32Array date32Array = (Date32Array)nextBatch.Column(0);
actualLength += date32Array.Length;
ValidateValue((i) => values?[i], date32Array.Length, (i) => date32Array.GetDateTimeOffset(i));
break;
case BooleanType:
BooleanArray booleanArray = (BooleanArray)nextBatch.Column(0);
Int32Array? indexArray = hasIndexColumn ? (Int32Array)nextBatch.Column(1) : null;
actualLength += booleanArray.Length;
ValidateValue((i) => values?[i], booleanArray.Length, (i) => booleanArray.GetValue(i), indexArray);
break;
case BinaryType:
BinaryArray binaryArray = (BinaryArray)nextBatch.Column(0);
actualLength += binaryArray.Length;
Expand All @@ -464,10 +433,20 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object
case NullType:
NullArray nullArray = (NullArray)nextBatch.Column(0);
actualLength += nullArray.Length;
ValidateValue((i) => values?[i] == null, nullArray.Length, (i) => nullArray.IsNull(i));
ValidateValue(nullArray.Length, (i) => values?[i] == null, (i) => nullArray.IsNull(i));
break;
default:
Assert.Fail($"Unhandled datatype {field.DataType}");
if (valueGetters.TryGetValue(field.DataType.TypeId, out Func<IArrowArray, int, object?>? valueGetter))
{
IArrowArray array = nextBatch.Column(0);
actualLength += array.Length;
indexArray = hasIndexColumn ? (Int32Array)nextBatch.Column(1) : null;
ValidateValue(array.Length, (i) => values?[i], (i) => valueGetter(array, i), indexArray, array.IsNull);
}
else
{
Assert.Fail($"Unhandled datatype {field.DataType}");
}
break;

}
Expand Down Expand Up @@ -497,15 +476,20 @@ private static void ValidateBinaryArrayValue(Func<int, object?> expectedValues,
/// <summary>
/// Validates a single values for all results (in the batch).
/// </summary>
/// <param name="value">The value to validate.</param>
/// <param name="length">The length of the current batch/array.</param>
/// <param name="value">The value to validate.</param>
/// <param name="getter">The getter function to retrieve the actual value.</param>
private static void ValidateValue(Func<int, object?> value, int length, Func<int, object?> getter, Int32Array? indexColumn = null)
/// <param name="indexColumn"></param>
private static void ValidateValue(int length, Func<int, object?> value, Func<int, object?> getter, Int32Array? indexColumn = null, Func<int, bool>? isNullEvaluator = default)
{
for (int i = 0; i < length; i++)
{
int valueIndex = indexColumn?.GetValue(i) ?? i;
object? expected = value(valueIndex);
if (isNullEvaluator != null)
{
Assert.Equal(expected == null, isNullEvaluator(i));
}
object? actual = getter(i);
Assert.Equal<object>(expected, actual);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Xunit.SkippableFact" Version="1.4.13" />
<PackageReference Include="Xunit.SkippableFact" Version="1.5.23" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />
</ItemGroup>

<ItemGroup>
Expand Down
11 changes: 8 additions & 3 deletions csharp/test/Drivers/Apache/ApacheTestConfiguration.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ public class ApacheTestConfiguration : TestConfiguration
[JsonPropertyName("port")]
public string Port { get; set; } = string.Empty;

[JsonPropertyName("token"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string Token { get; set; } = string.Empty;

[JsonPropertyName("path"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string Path { get; set; } = string.Empty;

Expand All @@ -54,5 +51,13 @@ public class ApacheTestConfiguration : TestConfiguration
[JsonPropertyName("http_request_timeout_ms"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string HttpRequestTimeoutMilliseconds { get; set; } = string.Empty;

[JsonPropertyName("type"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string Type { get; set; } = string.Empty;

[JsonPropertyName("data_type_conv"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string DataTypeConversion { get; set; } = string.Empty;

[JsonPropertyName("tls_options"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string TlsOptions { get; set; } = string.Empty;
}
}
154 changes: 154 additions & 0 deletions csharp/test/Drivers/Apache/Common/BinaryBooleanValueTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System;
using System.Collections.Generic;
using System.Globalization;
using System.Threading.Tasks;
using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2;
using Xunit;
using Xunit.Abstractions;

namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common
{
// TODO: When supported, use prepared statements instead of SQL string literals
// Which will better test how the driver handles values sent/received

/// <summary>
/// Validates that specific binary and boolean values can be inserted, retrieved and targeted correctly
/// </summary>
public abstract class BinaryBooleanValueTests<TConfig, TEnv> : TestBase<TConfig, TEnv>
where TConfig : TestConfiguration
where TEnv : HiveServer2TestEnvironment<TConfig>
{
public BinaryBooleanValueTests(ITestOutputHelper output, TestEnvironment<TConfig>.Factory<TEnv> testEnvFactory)
: base(output, testEnvFactory) { }

public static IEnumerable<object[]> ByteArrayData(int size)
{
var rnd = new Random();
byte[] bytes = new byte[size];
rnd.NextBytes(bytes);
yield return new object[] { bytes };
}

/// <summary>
/// Validates if driver can send and receive specific Binary values correctly.
/// </summary>
[SkippableTheory]
[InlineData(null)]
[MemberData(nameof(ByteArrayData), 0)]
[MemberData(nameof(ByteArrayData), 2)]
[MemberData(nameof(ByteArrayData), 1024)]
public async Task TestBinaryData(byte[]? value)
{
string columnName = "BINARYTYPE";
using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "BINARY"));
string? formattedValue = value != null ? $"X'{BitConverter.ToString(value).Replace("-", "")}'" : null;
await ValidateInsertSelectDeleteSingleValueAsync(
table.TableName,
columnName,
value,
formattedValue);
}

/// <summary>
/// Validates if driver can send and receive specific Boolean values correctly.
/// </summary>
[SkippableTheory]
[InlineData(null)]
[InlineData(true)]
[InlineData(false)]
public async Task TestBooleanData(bool? value)
{
string columnName = "BOOLEANTYPE";
using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "BOOLEAN"));
string? formattedValue = value == null ? null : $"{value?.ToString(CultureInfo.InvariantCulture)}";
await ValidateInsertSelectDeleteTwoValuesAsync(
table.TableName,
columnName,
value,
formattedValue);
}

/// <summary>
/// Validates if driver can receive specific NULL values correctly.
/// </summary>
[SkippableTheory]
[InlineData("NULL")]
[InlineData("CAST(NULL AS INT)")]
[InlineData("CAST(NULL AS BIGINT)")]
[InlineData("CAST(NULL AS SMALLINT)")]
[InlineData("CAST(NULL AS TINYINT)")]
[InlineData("CAST(NULL AS FLOAT)")]
[InlineData("CAST(NULL AS DOUBLE)")]
[InlineData("CAST(NULL AS DECIMAL(38,0))")]
[InlineData("CAST(NULL AS STRING)")]
[InlineData("CAST(NULL AS VARCHAR(10))")]
[InlineData("CAST(NULL AS CHAR(10))")]
[InlineData("CAST(NULL AS BOOLEAN)")]
[InlineData("CAST(NULL AS BINARY)")]
[InlineData("CAST(NULL AS MAP<STRING, INT>)")]
[InlineData("CAST(NULL AS STRUCT<NAME: STRING>)")]
[InlineData("CAST(NULL AS ARRAY<INT>)")]
public async Task TestNullData(string projectionClause)
{
string selectStatement = $"SELECT {projectionClause};";
// Note: by default, this returns as String type, not NULL type.
await SelectAndValidateValuesAsync(selectStatement, (object?)null, 1);
}

[SkippableTheory]
[InlineData(1)]
[InlineData(7)]
[InlineData(8)]
[InlineData(9)]
[InlineData(15)]
[InlineData(16)]
[InlineData(17)]
[InlineData(23)]
[InlineData(24)]
[InlineData(25)]
[InlineData(31)]
[InlineData(32)] // Full integer
[InlineData(33)]
[InlineData(39)]
[InlineData(40)]
[InlineData(41)]
[InlineData(47)]
[InlineData(48)]
[InlineData(49)]
[InlineData(63)]
[InlineData(64)] // Full 2 integers
[InlineData(65)]
public async Task TestMultilineNullData(int numberOfValues)
{
Random rnd = new();
int percentIsNull = 50;

object?[] values = new object?[numberOfValues];
for (int i = 0; i < numberOfValues; i++)
{
values[i] = rnd.Next(0, 100) < percentIsNull ? null : rnd.Next(0, 2) != 0;
}
string columnName = "BOOLEANTYPE";
string indexColumnName = "INDEXCOL";
using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}, {2} {3}", indexColumnName, "INT", columnName, "BOOLEAN"));
await ValidateInsertSelectDeleteMultipleValuesAsync(table.TableName, columnName, indexColumnName, values);
}
}
}
Loading

0 comments on commit ab46983

Please sign in to comment.