Skip to content

Commit

Permalink
fix data type parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
David Coe committed Jan 10, 2025
1 parent 94143dd commit f4e3126
Show file tree
Hide file tree
Showing 5 changed files with 114 additions and 12 deletions.
20 changes: 20 additions & 0 deletions csharp/src/Drivers/BigQuery/AssemblyInfo.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System.Runtime.CompilerServices;

[assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Tests.Drivers.BigQuery, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")]
28 changes: 21 additions & 7 deletions csharp/src/Drivers/BigQuery/BigQueryConnection.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
Expand Down Expand Up @@ -47,7 +48,7 @@ public class BigQueryConnection : AdbcConnection
const string infoVendorName = "BigQuery";
const string infoDriverArrowVersion = "1.0.0";

readonly AdbcInfoCode[] infoSupportedCodes = new [] {
readonly AdbcInfoCode[] infoSupportedCodes = new[] {
AdbcInfoCode.DriverName,
AdbcInfoCode.DriverVersion,
AdbcInfoCode.DriverArrowVersion,
Expand Down Expand Up @@ -530,11 +531,15 @@ private StructArray GetColumnSchema(
ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]);
remarksBuilder.Append("");

string dataType = ToTypeName(GetValue(row["data_type"]));
string dataType = ToTypeName(GetValue(row["data_type"]), out string suffix);

if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL"))
if ((dataType.StartsWith("NUMERIC") ||
dataType.StartsWith("DECIMAL") ||
dataType.StartsWith("BIGNUMERIC") ||
dataType.StartsWith("BIGDECIMAL"))
&& !string.IsNullOrEmpty(suffix))
{
ParsedDecimalValues values = ParsePrecisionAndScale(dataType);
ParsedDecimalValues values = ParsePrecisionAndScale(suffix);
xdbcColumnSizeBuilder.Append(values.Precision);
xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale));
}
Expand Down Expand Up @@ -752,10 +757,19 @@ private string PatternToRegEx(string? pattern)
return builder.ToString();
}

private string ToTypeName(string type)
private string ToTypeName(string type, out string suffix)
{
int index = Math.Min(type.IndexOf("("), type.IndexOf("<"));
suffix = string.Empty;

int index = type.IndexOf("(");
if (index == -1)
index = type.IndexOf("<");

string dataType = index == -1 ? type : type.Substring(0, index);

if (index > -1)
suffix = type.Substring(dataType.Length);

return dataType;
}

Expand Down Expand Up @@ -965,7 +979,7 @@ private ParsedDecimalValues ParsePrecisionAndScale(string type)
public override IArrowArrayStream GetTableTypes()
{
StringArray.Builder tableTypesBuilder = new StringArray.Builder();
tableTypesBuilder.AppendRange(new string[] { "BASE TABLE", "VIEW" });
tableTypesBuilder.AppendRange(BigQueryTableTypes.TableTypes);

IArrowArray[] dataArrays = new IArrowArray[]
{
Expand Down
25 changes: 25 additions & 0 deletions csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System.Collections.Generic;

namespace Apache.Arrow.Adbc.Drivers.BigQuery
{
internal static class BigQueryTableTypes
{
public static List<string> TableTypes = new List<string> { "BASE TABLE", "VIEW", "CLONE", "SNAPSHOT" };
}
}
11 changes: 11 additions & 0 deletions csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,17 @@ public static List<AdbcCatalog> ParseCatalog(RecordBatch recordBatch, string? sc
{
if (constraintsArray == null) return null;

// constraint details may not be loaded correctly if the depth wasn't Columns
try
{
if (constraintsArray.Fields.Count == 0)
return null;
}
catch (NullReferenceException)
{
return null;
}

List<AdbcConstraint> constraints = new List<AdbcConstraint>();

StringArray name = (StringArray)constraintsArray.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_name")]; // constraint_name | utf8
Expand Down
42 changes: 37 additions & 5 deletions csharp/test/Drivers/BigQuery/DriverTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Apache.Arrow.Adbc.Drivers.BigQuery;
using Apache.Arrow.Adbc.Tests.Metadata;
using Apache.Arrow.Adbc.Tests.Xunit;
using Apache.Arrow.Ipc;
Expand Down Expand Up @@ -116,7 +117,7 @@ public void CanGetObjects()
catalogPattern: catalogName,
dbSchemaPattern: schemaName,
tableNamePattern: tableName,
tableTypes: new List<string> { "BASE TABLE", "VIEW" },
tableTypes: BigQueryTableTypes.TableTypes,
columnNamePattern: columnName);

RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;
Expand All @@ -134,6 +135,40 @@ public void CanGetObjects()
Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns?.Count);
}

[SkippableFact, Order(3)]
public void CanGetObjectsTables()
{
string? catalogName = _testConfiguration.Metadata.Catalog;
string? schemaName = _testConfiguration.Metadata.Schema;
string? tableName = _testConfiguration.Metadata.Table;

AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration);

IArrowArrayStream stream = adbcConnection.GetObjects(
depth: AdbcConnection.GetObjectsDepth.Tables,
catalogPattern: catalogName,
dbSchemaPattern: schemaName,
tableNamePattern: null,
tableTypes: BigQueryTableTypes.TableTypes,
columnNamePattern: null);

RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;

List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName);

List<AdbcTable>? tables = catalogs
.Where(c => string.Equals(c.Name, catalogName))
.Select(c => c.DbSchemas)
.FirstOrDefault()
?.Where(s => string.Equals(s.Name, schemaName))
.Select(s => s.Tables)
.FirstOrDefault();

AdbcTable? table = tables?.Where((table) => string.Equals(table.Name, tableName)).FirstOrDefault();
Assert.True(table != null, "table should not be null");
Assert.Equal("BASE TABLE", table.Type);
}

/// <summary>
/// Validates if the driver can call GetTableSchema.
/// </summary>
Expand Down Expand Up @@ -167,10 +202,7 @@ public void CanGetTableTypes()

StringArray stringArray = (StringArray)recordBatch.Column("table_type");

List<string> known_types = new List<string>
{
"BASE TABLE", "VIEW"
};
List<string> known_types = BigQueryTableTypes.TableTypes;

int results = 0;

Expand Down

0 comments on commit f4e3126

Please sign in to comment.