Skip to content

Commit

Permalink
feat: respect iox::column_type::field metadata when mapping query (#132)
Browse files Browse the repository at this point in the history
  • Loading branch information
NguyenHoangSon96 authored Dec 18, 2024
1 parent 115c12d commit 41045ca
Show file tree
Hide file tree
Showing 12 changed files with 460 additions and 64 deletions.
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
## 1.0.0 [unreleased]

### Features

1. [#132](https://github.com/InfluxCommunity/influxdb3-csharp/pull/132): Respect iox::column_type::field metadata when
mapping query results into values.
- iox::column_type::field::integer: => Long
- iox::column_type::field::uinteger: => Long
- iox::column_type::field::float: => Double
- iox::column_type::field::string: => String
- iox::column_type::field::boolean: => Boolean

## 0.9.0 [unreleased]

## 0.8.0 [2024-09-13]
Expand Down
2 changes: 1 addition & 1 deletion Client.Test/Internal/AssemblyHelperTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public void GetAssemblyVersion()
var version = AssemblyHelper.GetVersion();
Assert.Multiple(() =>
{
Assert.That(Version.Parse(version).Major, Is.EqualTo(0));
Assert.That(Version.Parse(version).Major, Is.EqualTo(1));
Assert.That(Version.Parse(version).Minor, Is.GreaterThanOrEqualTo(0));
Assert.That(Version.Parse(version).Build, Is.EqualTo(0));
Assert.That(Version.Parse(version).Revision, Is.EqualTo(0));
Expand Down
44 changes: 44 additions & 0 deletions Client.Test/Internal/RecordBatchConverterTest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
using System;
using Apache.Arrow;
using Apache.Arrow.Types;
using InfluxDB3.Client.Internal;
using InfluxDB3.Client.Write;

namespace InfluxDB3.Client.Test.Internal;

public class RecordBatchConverterTest
{
[Test]
public void ConvertToPointDataValue()
{
Schema schema;
RecordBatch recordBatch;
PointDataValues point;

var stringField = new Field("measurement", StringType.Default, true);
var stringArray = new StringArray.Builder().Append("host").Build();
schema = new Schema(new[] { stringField, }, null);
recordBatch = new RecordBatch(schema, new[] { stringArray }, 1);
point = RecordBatchConverter.ConvertToPointDataValue(recordBatch, 0);
Assert.That(point.GetMeasurement(), Is.EqualTo("host"));

Assert.Multiple(() =>
{
var now = new DateTimeOffset();

var timeField = new Field("time", TimestampType.Default, true);
var timeArray = new TimestampArray.Builder().Append(now).Build();
schema = new Schema(new[] { timeField, }, null);
recordBatch = new RecordBatch(schema, new[] { timeArray }, 1);

point = RecordBatchConverter.ConvertToPointDataValue(recordBatch, 0);
Assert.That(point.GetTimestamp(), Is.EqualTo(TimestampConverter.GetNanoTime(now.UtcDateTime)));

timeField = new Field("test", TimestampType.Default, true);
schema = new Schema(new[] { timeField, }, null);
recordBatch = new RecordBatch(schema, new[] { timeArray }, 1);
point = RecordBatchConverter.ConvertToPointDataValue(recordBatch, 0);
Assert.That(point.GetField("test"), Is.EqualTo(now));
});
}
}
2 changes: 1 addition & 1 deletion Client.Test/Internal/RestClientTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public async Task UserAgent()

var requests = MockServer.LogEntries.ToList();

Assert.That(requests[0].RequestMessage.Headers?["User-Agent"][0], Does.StartWith("influxdb3-csharp/0."));
Assert.That(requests[0].RequestMessage.Headers?["User-Agent"][0], Does.StartWith("influxdb3-csharp/1."));
Assert.That(requests[0].RequestMessage.Headers?["User-Agent"][0], Does.EndWith(".0.0"));
}

Expand Down
26 changes: 26 additions & 0 deletions Client.Test/Internal/TimestampConverterTest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
using System;
using System.Numerics;
using InfluxDB3.Client.Internal;

namespace InfluxDB3.Client.Test.Internal;

public class TimestampConverterTest
{
private static readonly DateTime EpochStart = new(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);

[Test]
public void GetNanoTime()
{
var now = DateTime.Now;

var localTime = DateTime.SpecifyKind(now, DateTimeKind.Local);
var timestamp = TimestampConverter.GetNanoTime(localTime);
BigInteger nanoTime = now.ToUniversalTime().Subtract(EpochStart).Ticks * 100;
Assert.That(nanoTime, Is.EqualTo(timestamp));

var unspecifiedTime = DateTime.SpecifyKind(now, DateTimeKind.Unspecified);
timestamp = TimestampConverter.GetNanoTime(unspecifiedTime);
nanoTime = DateTime.SpecifyKind(now, DateTimeKind.Utc).Subtract(EpochStart).Ticks * 100;
Assert.That(nanoTime, Is.EqualTo(timestamp));
}
}
142 changes: 142 additions & 0 deletions Client.Test/Internal/TypeCastTest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
using System.Collections.Generic;
using Apache.Arrow;
using Apache.Arrow.Types;
using InfluxDB3.Client.Internal;

namespace InfluxDB3.Client.Test.Internal;

public class TypeCastTest
{
[Test]
public void IsNumber()
{
Assert.Multiple(() =>
{
Assert.That(TypeCasting.IsNumber(1), Is.True);
Assert.That(TypeCasting.IsNumber(1.2), Is.True);
Assert.That(TypeCasting.IsNumber(-1.2), Is.True);
});

Assert.Multiple(() =>
{
Assert.That(TypeCasting.IsNumber('1'), Is.False);
Assert.That(TypeCasting.IsNumber('a'), Is.False);
Assert.That(TypeCasting.IsNumber(true), Is.False);
Assert.That(TypeCasting.IsNumber(null), Is.False);
});
}

[Test]
public void GetMappedValue()
{
// If pass the correct value type to GetMappedValue() it will return the value with a correct type
// If pass the incorrect value type to GetMappedValue() it will NOT throws any error but return the passed value
const string fieldName = "test";

var field = GenerateIntField(fieldName);
Assert.Multiple(() =>
{
Assert.That(TypeCasting.GetMappedValue(field, 1)!, Is.EqualTo(1));
Assert.That(TypeCasting.GetMappedValue(field, "a")!, Is.EqualTo("a"));
});

field = GenerateUIntField(fieldName);
Assert.Multiple(() =>
{
Assert.That(TypeCasting.GetMappedValue(field, 1)!, Is.EqualTo(1));
Assert.That(TypeCasting.GetMappedValue(field, -1)!, Is.EqualTo(-1));
Assert.That(TypeCasting.GetMappedValue(field, "a")!, Is.EqualTo("a"));
});

field = GenerateDoubleField(fieldName);
Assert.Multiple(() =>
{
Assert.That(TypeCasting.GetMappedValue(field, 1.2)!, Is.EqualTo(1.2));
Assert.That(TypeCasting.GetMappedValue(field, "a")!, Is.EqualTo("a"));
});

field = GenerateBooleanField(fieldName);
Assert.Multiple(() =>
{
Assert.That(TypeCasting.GetMappedValue(field, true)!, Is.EqualTo(true));
Assert.That(TypeCasting.GetMappedValue(field, 10)!, Is.EqualTo(10));
});

field = GenerateStringField(fieldName);
Assert.Multiple(() =>
{
Assert.That(TypeCasting.GetMappedValue(field, "a")!, Is.EqualTo("a"));
Assert.That(TypeCasting.GetMappedValue(field, 10)!, Is.EqualTo(10));
});


field = GenerateIntFieldTestTypeMeta(fieldName);
Assert.That(TypeCasting.GetMappedValue(field, 1)!, Is.EqualTo(1));
}

private static Field GenerateIntFieldTestTypeMeta(string fieldName)
{
var meta = new Dictionary<string, string>
{
{
"iox::column::type", "iox::column_type::field::test"
}
};
return new Field(fieldName, Int64Type.Default, true, meta);
}

private static Field GenerateIntField(string fieldName)
{
var meta = new Dictionary<string, string>
{
{
"iox::column::type", "iox::column_type::field::integer"
}
};
return new Field(fieldName, Int64Type.Default, true, meta);
}

private static Field GenerateUIntField(string fieldName)
{
var meta = new Dictionary<string, string>
{
{
"iox::column::type", "iox::column_type::field::uinteger"
}
};
return new Field(fieldName, UInt64Type.Default, true, meta);
}

private static Field GenerateDoubleField(string fieldName)
{
var meta = new Dictionary<string, string>
{
{
"iox::column::type", "iox::column_type::field::float"
}
};
return new Field(fieldName, DoubleType.Default, true, meta);
}

private static Field GenerateBooleanField(string fieldName)
{
var meta = new Dictionary<string, string>
{
{
"iox::column::type", "iox::column_type::field::boolean"
}
};
return new Field(fieldName, BooleanType.Default, true, meta);
}

private static Field GenerateStringField(string fieldName)
{
var meta = new Dictionary<string, string>
{
{
"iox::column::type", "iox::column_type::field::string"
}
};
return new Field(fieldName, StringType.Default, true, meta);
}
}
2 changes: 1 addition & 1 deletion Client/Client.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
</Description>
<Authors>InfluxDB3.Client Contributors</Authors>
<AssemblyName>InfluxDB3.Client</AssemblyName>
<VersionPrefix>0.9.0</VersionPrefix>
<VersionPrefix>1.0.0</VersionPrefix>
<VersionSuffix>dev</VersionSuffix>

<PackageId>InfluxDB3.Client</PackageId>
Expand Down
76 changes: 15 additions & 61 deletions Client/InfluxDBClient.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
Expand Down Expand Up @@ -457,19 +458,23 @@ public InfluxDBClient() : this(
await foreach (var batch in QueryBatches(query, queryType, database, namedParameters, headers)
.ConfigureAwait(false))
{
var rowCount = batch.Column(0).Length;
for (var i = 0; i < rowCount; i++)
for (var i = 0; i < batch.Column(0).Length; i++)
{
var row = new List<object?>();
for (var j = 0; j < batch.ColumnCount; j++)
var columnCount = batch.ColumnCount;
var row = new object?[columnCount];
for (var j = 0; j < columnCount; j++)
{
if (batch.Column(j) is ArrowArray array)
if (batch.Column(j) is not ArrowArray array)
{
row.Add(array.GetObjectValue(i));
continue;
}
row[j] = TypeCasting.GetMappedValue(
batch.Schema.FieldsList[j],
array.GetObjectValue(i)
);
}

yield return row.ToArray();
yield return row;
}
}
}
Expand Down Expand Up @@ -523,60 +528,9 @@ public async IAsyncEnumerable<PointDataValues> QueryPoints(string query, QueryTy
await foreach (var batch in QueryBatches(query, queryType, database, namedParameters, headers)
.ConfigureAwait(false))
{
var rowCount = batch.Column(0).Length;
for (var i = 0; i < rowCount; i++)
for (var i = 0; i < batch.Column(0).Length; i++)
{
var point = new PointDataValues();
for (var j = 0; j < batch.ColumnCount; j++)
{
var schema = batch.Schema.FieldsList[j];
var fullName = schema.Name;

if (batch.Column(j) is not ArrowArray array)
continue;

var objectValue = array.GetObjectValue(i);
if (objectValue is null)
continue;

if ((fullName == "measurement" || fullName == "iox::measurement") && objectValue is string)
{
point = point.SetMeasurement((string)objectValue);
continue;
}

if (!schema.HasMetadata)
{
if (fullName == "time" && objectValue is DateTimeOffset timestamp)
{
point = point.SetTimestamp(timestamp);
}
else
// just push as field If you don't know what type is it
point = point.SetField(fullName, objectValue);

continue;
}

var type = schema.Metadata["iox::column::type"];
var parts = type.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
var valueType = parts[2];
// string fieldType = parts.Length > 3 ? parts[3] : "";

if (valueType == "field")
{
point = point.SetField(fullName, objectValue);
}
else if (valueType == "tag")
{
point = point.SetTag(fullName, (string)objectValue);
}
else if (valueType == "timestamp" && objectValue is DateTimeOffset timestamp)
{
point = point.SetTimestamp(timestamp);
}
}

var point = RecordBatchConverter.ConvertToPointDataValue(batch, i);
yield return point;
}
}
Expand Down Expand Up @@ -853,7 +807,7 @@ internal static HttpClient CreateAndConfigureHttpClient(ClientConfig config)
if (handler.SupportsAutomaticDecompression)
{
handler.AutomaticDecompression =
System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate;
DecompressionMethods.GZip | DecompressionMethods.Deflate;
}

if (handler.SupportsProxy && config.Proxy != null)
Expand Down
Loading

0 comments on commit 41045ca

Please sign in to comment.