feat(csharp/src/Drivers/Apache): convert Double to Float for Apache Spark on scalar conversion

csharp/src/Drivers/Apache/Hive2/HiveServer2Reader.cs

@@ -63,6 +63,11 @@ internal class HiveServer2Reader : IArrowArrayStream
                 { ArrowTypeId.Decimal128, ConvertToDecimal128 },
                 { ArrowTypeId.Timestamp, ConvertToTimestamp },
             };
+        private static readonly IReadOnlyDictionary<ArrowTypeId, Func<DoubleArray, IArrowType, IArrowArray>> s_arrowDoubleConverters =
+            new Dictionary<ArrowTypeId, Func<DoubleArray, IArrowType, IArrowArray>>()
+            {
+                { ArrowTypeId.Float, ConvertToFloat },
+            };
 
         public HiveServer2Reader(
             HiveServer2Statement statement,
@@ -147,14 +152,22 @@ private static IArrowArray GetArray(TColumn column, IArrowType? expectedArrowTyp
                 Func<StringArray, IArrowType, IArrowArray> converter = s_arrowStringConverters[expectedArrowType.TypeId];
                 return converter(stringArray, expectedArrowType);
             }
+            else if (expectedArrowType != null && arrowArray is DoubleArray doubleArray && s_arrowDoubleConverters.ContainsKey(expectedArrowType.TypeId))
+            {
+                // Perform a conversion from double to another (float) type.
+                Func<DoubleArray, IArrowType, IArrowArray> converter = s_arrowDoubleConverters[expectedArrowType.TypeId];
+                return converter(doubleArray, expectedArrowType);
+            }
             return arrowArray;
         }
 
         internal static Date32Array ConvertToDate32(StringArray array, IArrowType _)
         {
             const DateTimeStyles DateTimeStyles = DateTimeStyles.AllowWhiteSpaces;
-            var resultArray = new Date32Array.Builder();
             int length = array.Length;
+            var resultArray = new Date32Array
+                .Builder()
+                .Reserve(length);
             for (int i = 0; i < length; i++)
             {
                 // Work with UTF8 string.
@@ -178,6 +191,20 @@ internal static Date32Array ConvertToDate32(StringArray array, IArrowType _)
             return resultArray.Build();
         }
 
+        internal static FloatArray ConvertToFloat(DoubleArray array, IArrowType _)
+        {
+            int length = array.Length;
+            var resultArray = new FloatArray
+                .Builder()
+                .Reserve(length);
+            for (int i = 0; i < length; i++)
+            {
+                resultArray.Append((float?)array.GetValue(i));
+            }
+
+            return resultArray.Build();
+        }
+
         internal static bool TryParse(ReadOnlySpan<byte> date, out DateTime dateTime)
         {
             if (date.Length == KnownFormatDateLength
@@ -204,12 +231,14 @@ internal static bool TryParse(ReadOnlySpan<byte> date, out DateTime dateTime)
 
         private static Decimal128Array ConvertToDecimal128(StringArray array, IArrowType schemaType)
         {
+            int length = array.Length;
             // Using the schema type to get the precision and scale.
             Decimal128Type decimalType = (Decimal128Type)schemaType;
-            var resultArray = new Decimal128Array.Builder(decimalType);
+            var resultArray = new Decimal128Array
+                .Builder(decimalType)
+                .Reserve(length);
             Span<byte> buffer = stackalloc byte[decimalType.ByteWidth];
 
-            int length = array.Length;
             for (int i = 0; i < length; i++)
             {
                 // Work with UTF8 string.
@@ -235,9 +264,11 @@ private static Decimal128Array ConvertToDecimal128(StringArray array, IArrowType
         internal static TimestampArray ConvertToTimestamp(StringArray array, IArrowType _)
         {
             const DateTimeStyles DateTimeStyles = DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces;
-            // Match the precision of the server
-            var resultArrayBuilder = new TimestampArray.Builder(TimeUnit.Microsecond);
             int length = array.Length;
+            // Match the precision of the server
+            var resultArrayBuilder = new TimestampArray
+                .Builder(TimeUnit.Microsecond)
+                .Reserve(length);
             for (int i = 0; i < length; i++)
             {
                 // Work with UTF8 string.

csharp/src/Drivers/Apache/Hive2/HiveServer2SchemaParser.cs

@@ -31,8 +31,8 @@ public override IArrowType GetArrowType(TPrimitiveTypeEntry thriftType, DataType
                 TTypeId.BIGINT_TYPE => Int64Type.Default,
                 TTypeId.BINARY_TYPE => BinaryType.Default,
                 TTypeId.BOOLEAN_TYPE => BooleanType.Default,
-                TTypeId.DOUBLE_TYPE
-                or TTypeId.FLOAT_TYPE => DoubleType.Default,
+                TTypeId.DOUBLE_TYPE => DoubleType.Default,
+                TTypeId.FLOAT_TYPE => convertScalar ? FloatType.Default : DoubleType.Default,
                 TTypeId.INT_TYPE => Int32Type.Default,
                 TTypeId.SMALLINT_TYPE => Int16Type.Default,
                 TTypeId.TINYINT_TYPE => Int8Type.Default,

csharp/src/Drivers/Apache/Spark/README.md

@@ -84,7 +84,7 @@ The following table depicts how the Spark ADBC driver converts a Spark type to a
 | DATE*                | *String*   | *string* | Date32 | DateTime |
 | DECIMAL*             | *String*   | *string* | Decimal128 | SqlDecimal |
 | DOUBLE               | Double     | double | | |
-| FLOAT                | *Double*   | *double* | | |
+| FLOAT                | *Double*   | *double* | Float | float |
 | INT                  | Int32      | int | | |
 | INTERVAL_DAY_TIME+   | String     | string | | |
 | INTERVAL_YEAR_MONTH+ | String     | string | | |

csharp/test/Drivers/Apache/Spark/NumericValueTests.cs

@@ -263,7 +263,7 @@ public async Task TestFloatValuesInsertSelectDelete(float value)
             await InsertSingleValueAsync(table.TableName, columnName, valueString);
             object doubleValue = (double)value;
             // Spark over HTTP returns float as double whereas Spark on Databricks returns float.
-            object floatValue = TestEnvironment.ServerType != SparkServerType.Databricks ? doubleValue : value;
+            object floatValue = TestEnvironment.ServerType == SparkServerType.Databricks || TestEnvironment.DataTypeConversion.HasFlag(DataTypeConversion.Scalar) ? value : doubleValue;
             await base.SelectAndValidateValuesAsync(table.TableName, columnName, floatValue, 1);
             string whereClause = GetWhereClause(columnName, value);
             if (SupportsDelete) await DeleteFromTableAsync(table.TableName, whereClause, 1);

csharp/test/Drivers/Apache/Spark/Resources/SparkData.sql

@@ -14,7 +14,7 @@
  -- See the License for the specific language governing permissions and
  -- limitations under the License.
 
-CREATE OR REPLACE TABLE IF NOT EXISTS {ADBC_CATALOG}.{ADBC_DATASET}.{ADBC_TABLE} (
+CREATE TABLE IF NOT EXISTS {ADBC_CATALOG}.{ADBC_DATASET}.{ADBC_TABLE} (
   id LONG,
   byte BYTE,
   short SHORT,

csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs

@@ -123,6 +123,8 @@ public override Dictionary<string, string> GetDriverParameters(SparkTestConfigur
 
         internal SparkServerType ServerType => ((SparkConnection)Connection).ServerType;
 
+        internal DataTypeConversion DataTypeConversion => ((SparkConnection)Connection).DataTypeConversion;
+
         public override string VendorVersion => ((HiveServer2Connection)Connection).VendorVersion;
 
         public override bool SupportsDelete => ServerType == SparkServerType.Databricks;
@@ -139,10 +141,11 @@ public override string GetInsertStatement(string tableName, string columnName, s
         public override SampleDataBuilder GetSampleDataBuilder()
         {
             SampleDataBuilder sampleDataBuilder = new();
-            Type floatNetType = ServerType == SparkServerType.Databricks ? typeof(float) : typeof(double);
-            Type floatArrowType = ServerType == SparkServerType.Databricks ? typeof(FloatType) : typeof(DoubleType);
+            bool dataTypeIsFloat = ServerType == SparkServerType.Databricks || DataTypeConversion.HasFlag(DataTypeConversion.Scalar);
+            Type floatNetType = dataTypeIsFloat ? typeof(float) : typeof(double);
+            Type floatArrowType = dataTypeIsFloat ? typeof(FloatType) : typeof(DoubleType);
             object floatValue;
-            if (ServerType == SparkServerType.Databricks)
+            if (dataTypeIsFloat)
                 floatValue = 1f;
             else
                 floatValue = 1d;