Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(csharp/src/Drivers/Apache): make Apache driver tests inheritable #2341

Merged
merged 6 commits into from
Dec 3, 2024
Merged

feat(csharp/src/Drivers/Apache): make Apache driver tests inheritable #2341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
16e8a71
feat(csharp/src/Drivers/Apache): make Apace driver tests inheritable
birschick-bq Nov 28, 2024
13493cb
remove trailing space
birschick-bq Nov 28, 2024
d078e3c
refactor test configuration.
birschick-bq Dec 2, 2024
f3395e6
Merge remote-tracking branch 'upstream/main' into dev/birschick-bq/in…
birschick-bq Dec 3, 2024
9f8407a
remove duplicate package reference
birschick-bq Dec 3, 2024
db6da84
code review suggested improvements
birschick-bq Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public class ClientTests
public static void CanClientExecuteUpdate(Adbc.Client.AdbcConnection adbcConnection,
TestConfiguration testConfiguration,
string[] queries,
List<int> expectedResults)
IReadOnlyList<int> expectedResults)
{
if (adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection));
if (testConfiguration == null) throw new ArgumentNullException(nameof(testConfiguration));
Expand Down
104 changes: 44 additions & 60 deletions csharp/test/Apache.Arrow.Adbc.Tests/TestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,16 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object
await SelectAndValidateValuesAsync(selectStatement, [value], expectedLength);
}

private static T? ArrowArrayAs<T>(IArrowArray arrowArray)
where T : IArrowArray
{
if (arrowArray is T t)
{
return t;
}
return default;
}

/// <summary>
/// Selects a single value and validates it equality with expected value and number of results.
/// </summary>
Expand All @@ -391,71 +401,30 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object
int actualLength = 0;
using (IArrowArrayStream stream = queryResult.Stream ?? throw new InvalidOperationException("stream is null"))
{
Dictionary<ArrowTypeId, Func<IArrowArray, int, object?>> valueGetters = new()
{
{ ArrowTypeId.Decimal128, (a, i) => ArrowArrayAs<Decimal128Array>(a)?.GetSqlDecimal(i) },
{ ArrowTypeId.Double, (a, i) => ArrowArrayAs<DoubleArray>(a)?.GetValue(i) },
{ ArrowTypeId.Float, (a, i) => ArrowArrayAs<FloatArray>(a)?.GetValue(i) },
{ ArrowTypeId.Int64, (a, i) => ArrowArrayAs<Int64Array>(a)?.GetValue(i) },
{ ArrowTypeId.Int32, (a, i) => ArrowArrayAs<Int32Array>(a)?.GetValue(i) },
{ ArrowTypeId.Int16, (a, i) => ArrowArrayAs<Int16Array>(a)?.GetValue(i) },
{ ArrowTypeId.Int8, (a, i) => ArrowArrayAs<Int8Array>(a)?.GetValue(i) },
{ ArrowTypeId.String, (a, i) => ArrowArrayAs<StringArray>(a)?.GetString(i) },
{ ArrowTypeId.Timestamp, (a, i) => ArrowArrayAs<TimestampArray>(a)?.GetTimestamp(i) },
{ ArrowTypeId.Date32, (a, i) => ArrowArrayAs<Date32Array>(a)?.GetDateTimeOffset(i) },
{ ArrowTypeId.Boolean, (a, i) => ArrowArrayAs<BooleanArray>(a)?.GetValue(i) },
};
// Assume first column
Field field = stream.Schema.GetFieldByIndex(0);
Int32Array? indexArray = null;
while (true)
{
using (RecordBatch nextBatch = await stream.ReadNextRecordBatchAsync())
{
if (nextBatch == null) { break; }
switch (field.DataType)
{
case Decimal128Type:
Decimal128Array decimalArray = (Decimal128Array)nextBatch.Column(0);
actualLength += decimalArray.Length;
ValidateValue((i) => values?[i], decimalArray.Length, (i) => decimalArray.GetSqlDecimal(i));
break;
case DoubleType:
DoubleArray doubleArray = (DoubleArray)nextBatch.Column(0);
actualLength += doubleArray.Length;
ValidateValue((i) => values?[i], doubleArray.Length, (i) => doubleArray.GetValue(i));
break;
case FloatType:
FloatArray floatArray = (FloatArray)nextBatch.Column(0);
actualLength += floatArray.Length;
ValidateValue((i) => values?[i], floatArray.Length, (i) => floatArray.GetValue(i));
break;
case Int64Type:
Int64Array int64Array = (Int64Array)nextBatch.Column(0);
actualLength += int64Array.Length;
ValidateValue((i) => values?[i], int64Array.Length, (i) => int64Array.GetValue(i));
break;
case Int32Type:
Int32Array intArray = (Int32Array)nextBatch.Column(0);
actualLength += intArray.Length;
ValidateValue((i) => values?[i], intArray.Length, (i) => intArray.GetValue(i));
break;
case Int16Type:
Int16Array shortArray = (Int16Array)nextBatch.Column(0);
actualLength += shortArray.Length;
ValidateValue((i) => values?[i], shortArray.Length, (i) => shortArray.GetValue(i));
break;
case Int8Type:
Int8Array tinyIntArray = (Int8Array)nextBatch.Column(0);
actualLength += tinyIntArray.Length;
ValidateValue((i) => values?[i], tinyIntArray.Length, (i) => tinyIntArray.GetValue(i));
break;
case StringType:
StringArray stringArray = (StringArray)nextBatch.Column(0);
actualLength += stringArray.Length;
ValidateValue((i) => values?[i], stringArray.Length, (i) => stringArray.GetString(i));
break;
case TimestampType:
TimestampArray timestampArray = (TimestampArray)nextBatch.Column(0);
actualLength += timestampArray.Length;
ValidateValue((i) => values?[i], timestampArray.Length, (i) => timestampArray.GetTimestamp(i));
break;
case Date32Type:
Date32Array date32Array = (Date32Array)nextBatch.Column(0);
actualLength += date32Array.Length;
ValidateValue((i) => values?[i], date32Array.Length, (i) => date32Array.GetDateTimeOffset(i));
break;
case BooleanType:
BooleanArray booleanArray = (BooleanArray)nextBatch.Column(0);
Int32Array? indexArray = hasIndexColumn ? (Int32Array)nextBatch.Column(1) : null;
actualLength += booleanArray.Length;
ValidateValue((i) => values?[i], booleanArray.Length, (i) => booleanArray.GetValue(i), indexArray);
break;
case BinaryType:
BinaryArray binaryArray = (BinaryArray)nextBatch.Column(0);
actualLength += binaryArray.Length;
Expand All @@ -464,10 +433,20 @@ protected async Task SelectAndValidateValuesAsync(string selectStatement, object
case NullType:
NullArray nullArray = (NullArray)nextBatch.Column(0);
actualLength += nullArray.Length;
ValidateValue((i) => values?[i] == null, nullArray.Length, (i) => nullArray.IsNull(i));
ValidateValue(nullArray.Length, (i) => values?[i] == null, (i) => nullArray.IsNull(i));
break;
default:
Assert.Fail($"Unhandled datatype {field.DataType}");
if (valueGetters.TryGetValue(field.DataType.TypeId, out Func<IArrowArray, int, object?>? valueGetter))
{
IArrowArray array = nextBatch.Column(0);
actualLength += array.Length;
indexArray = hasIndexColumn ? (Int32Array)nextBatch.Column(1) : null;
ValidateValue(array.Length, (i) => values?[i], (i) => valueGetter(array, i), indexArray, array.IsNull);
}
else
{
Assert.Fail($"Unhandled datatype {field.DataType}");
}
break;

}
Expand Down Expand Up @@ -497,15 +476,20 @@ private static void ValidateBinaryArrayValue(Func<int, object?> expectedValues,
/// <summary>
/// Validates a single values for all results (in the batch).
/// </summary>
/// <param name="value">The value to validate.</param>
/// <param name="length">The length of the current batch/array.</param>
/// <param name="value">The value to validate.</param>
/// <param name="getter">The getter function to retrieve the actual value.</param>
private static void ValidateValue(Func<int, object?> value, int length, Func<int, object?> getter, Int32Array? indexColumn = null)
/// <param name="indexColumn"></param>
private static void ValidateValue(int length, Func<int, object?> value, Func<int, object?> getter, Int32Array? indexColumn = null, Func<int, bool>? isNullEvaluator = default)
{
for (int i = 0; i < length; i++)
{
int valueIndex = indexColumn?.GetValue(i) ?? i;
object? expected = value(valueIndex);
if (isNullEvaluator != null)
{
Assert.Equal(expected == null, isNullEvaluator(i));
}
object? actual = getter(i);
Assert.Equal<object>(expected, actual);
}
Expand Down
11 changes: 8 additions & 3 deletions csharp/test/Drivers/Apache/ApacheTestConfiguration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ public class ApacheTestConfiguration : TestConfiguration
[JsonPropertyName("port")]
public string Port { get; set; } = string.Empty;

[JsonPropertyName("token"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string Token { get; set; } = string.Empty;

[JsonPropertyName("path"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string Path { get; set; } = string.Empty;

Expand All @@ -54,5 +51,13 @@ public class ApacheTestConfiguration : TestConfiguration
[JsonPropertyName("http_request_timeout_ms"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string HttpRequestTimeoutMilliseconds { get; set; } = string.Empty;

[JsonPropertyName("type"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string Type { get; set; } = string.Empty;

[JsonPropertyName("data_type_conv"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string DataTypeConversion { get; set; } = string.Empty;

[JsonPropertyName("tls_options"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string TlsOptions { get; set; } = string.Empty;
}
}
154 changes: 154 additions & 0 deletions csharp/test/Drivers/Apache/Common/BinaryBooleanValueTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System;
using System.Collections.Generic;
using System.Globalization;
using System.Threading.Tasks;
using Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2;
using Xunit;
using Xunit.Abstractions;

namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Common
{
// TODO: When supported, use prepared statements instead of SQL string literals
// Which will better test how the driver handles values sent/received

/// <summary>
/// Validates that specific binary and boolean values can be inserted, retrieved and targeted correctly
/// </summary>
public abstract class BinaryBooleanValueTests<TConfig, TEnv> : TestBase<TConfig, TEnv>
where TConfig : TestConfiguration
where TEnv : HiveServer2TestEnvironment<TConfig>
{
public BinaryBooleanValueTests(ITestOutputHelper output, TestEnvironment<TConfig>.Factory<TEnv> testEnvFactory)
: base(output, testEnvFactory) { }

public static IEnumerable<object[]> ByteArrayData(int size)
{
var rnd = new Random();
byte[] bytes = new byte[size];
rnd.NextBytes(bytes);
yield return new object[] { bytes };
}

/// <summary>
/// Validates if driver can send and receive specific Binary values correctly.
/// </summary>
[SkippableTheory]
[InlineData(null)]
[MemberData(nameof(ByteArrayData), 0)]
[MemberData(nameof(ByteArrayData), 2)]
[MemberData(nameof(ByteArrayData), 1024)]
public async Task TestBinaryData(byte[]? value)
{
string columnName = "BINARYTYPE";
using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "BINARY"));
string? formattedValue = value != null ? $"X'{BitConverter.ToString(value).Replace("-", "")}'" : null;
await ValidateInsertSelectDeleteSingleValueAsync(
table.TableName,
columnName,
value,
formattedValue);
}

/// <summary>
/// Validates if driver can send and receive specific Boolean values correctly.
/// </summary>
[SkippableTheory]
[InlineData(null)]
[InlineData(true)]
[InlineData(false)]
public async Task TestBooleanData(bool? value)
{
string columnName = "BOOLEANTYPE";
using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}", columnName, "BOOLEAN"));
string? formattedValue = value == null ? null : $"{value?.ToString(CultureInfo.InvariantCulture)}";
await ValidateInsertSelectDeleteTwoValuesAsync(
table.TableName,
columnName,
value,
formattedValue);
}

/// <summary>
/// Validates if driver can receive specific NULL values correctly.
/// </summary>
[SkippableTheory]
[InlineData("NULL")]
[InlineData("CAST(NULL AS INT)")]
[InlineData("CAST(NULL AS BIGINT)")]
[InlineData("CAST(NULL AS SMALLINT)")]
[InlineData("CAST(NULL AS TINYINT)")]
[InlineData("CAST(NULL AS FLOAT)")]
[InlineData("CAST(NULL AS DOUBLE)")]
[InlineData("CAST(NULL AS DECIMAL(38,0))")]
[InlineData("CAST(NULL AS STRING)")]
[InlineData("CAST(NULL AS VARCHAR(10))")]
[InlineData("CAST(NULL AS CHAR(10))")]
[InlineData("CAST(NULL AS BOOLEAN)")]
[InlineData("CAST(NULL AS BINARY)")]
[InlineData("CAST(NULL AS MAP<STRING, INT>)")]
[InlineData("CAST(NULL AS STRUCT<NAME: STRING>)")]
[InlineData("CAST(NULL AS ARRAY<INT>)")]
public async Task TestNullData(string projectionClause)
{
string selectStatement = $"SELECT {projectionClause};";
// Note: by default, this returns as String type, not NULL type.
await SelectAndValidateValuesAsync(selectStatement, (object?)null, 1);
}

[SkippableTheory]
[InlineData(1)]
[InlineData(7)]
[InlineData(8)]
[InlineData(9)]
[InlineData(15)]
[InlineData(16)]
[InlineData(17)]
[InlineData(23)]
[InlineData(24)]
[InlineData(25)]
[InlineData(31)]
[InlineData(32)] // Full integer
[InlineData(33)]
[InlineData(39)]
[InlineData(40)]
[InlineData(41)]
[InlineData(47)]
[InlineData(48)]
[InlineData(49)]
[InlineData(63)]
[InlineData(64)] // Full 2 integers
[InlineData(65)]
public async Task TestMultilineNullData(int numberOfValues)
{
Random rnd = new();
int percentIsNull = 50;

object?[] values = new object?[numberOfValues];
for (int i = 0; i < numberOfValues; i++)
{
values[i] = rnd.Next(0, 100) < percentIsNull ? null : rnd.Next(0, 2) != 0;
}
string columnName = "BOOLEANTYPE";
string indexColumnName = "INDEXCOL";
using TemporaryTable table = await NewTemporaryTableAsync(Statement, string.Format("{0} {1}, {2} {3}", indexColumnName, "INT", columnName, "BOOLEAN"));
await ValidateInsertSelectDeleteMultipleValuesAsync(table.TableName, columnName, indexColumnName, values);
}
}
}
Loading
Loading