Optionally disable validate_decimal_precision check in DecimalBuilder.append_value for interop test

arrow/src/array/array_binary.rs

@@ -1486,7 +1486,7 @@ mod tests {
             192, 219, 180, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 36, 75, 238, 253,
             255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
         ];
-        let array_data = ArrayData::builder(DataType::Decimal(23, 6))
+        let array_data = ArrayData::builder(DataType::Decimal(38, 6))
             .len(2)
             .add_buffer(Buffer::from(&values[..]))
             .build()
@@ -1498,6 +1498,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn test_decimal_append_error_value() {
         let mut decimal_builder = DecimalBuilder::new(10, 5, 3);
         let mut result = decimal_builder.append_value(123456);
@@ -1506,9 +1507,15 @@ mod tests {
             "Invalid argument error: 123456 is too large to store in a Decimal of precision 5. Max is 99999",
             error.to_string()
         );
+
+        unsafe {
+            decimal_builder.disable_value_validation();
+        }
+        result = decimal_builder.append_value(123456);
+        assert!(result.is_ok());
         decimal_builder.append_value(12345).unwrap();
         let arr = decimal_builder.finish();
-        assert_eq!("12.345", arr.value_as_string(0));
+        assert_eq!("12.345", arr.value_as_string(1));
 
         decimal_builder = DecimalBuilder::new(10, 2, 1);
         result = decimal_builder.append_value(100);
@@ -1517,18 +1524,21 @@ mod tests {
             "Invalid argument error: 100 is too large to store in a Decimal of precision 2. Max is 99",
             error.to_string()
         );
+
+        unsafe {
+            decimal_builder.disable_value_validation();
+        }
+        result = decimal_builder.append_value(100);
+        assert!(result.is_ok());
         decimal_builder.append_value(99).unwrap();
         result = decimal_builder.append_value(-100);
-        error = result.unwrap_err();
-        assert_eq!(
-            "Invalid argument error: -100 is too small to store in a Decimal of precision 2. Min is -99",
-            error.to_string()
-        );
+        assert!(result.is_ok());
         decimal_builder.append_value(-99).unwrap();
         let arr = decimal_builder.finish();
-        assert_eq!("9.9", arr.value_as_string(0));
-        assert_eq!("-9.9", arr.value_as_string(1));
+        assert_eq!("9.9", arr.value_as_string(1));
+        assert_eq!("-9.9", arr.value_as_string(3));
     }
+
     #[test]
     fn test_decimal_from_iter_values() {
         let array = DecimalArray::from_iter_values(vec![-100, 0, 101].into_iter());

arrow/src/array/builder.rs

@@ -1163,6 +1163,10 @@ pub struct DecimalBuilder {
     builder: FixedSizeListBuilder<UInt8Builder>,
     precision: usize,
     scale: usize,
+
+    /// Should i128 values be validated for compatibility with scale and precision?
+    /// defaults to true
+    value_validation: bool,
 }
 
 impl<OffsetSize: OffsetSizeTrait> ArrayBuilder for GenericBinaryBuilder<OffsetSize> {
@@ -1453,16 +1457,32 @@ impl DecimalBuilder {
             builder: FixedSizeListBuilder::new(values_builder, byte_width),
             precision,
             scale,
+            value_validation: true,
         }
     }
 
+    /// Disable validation
+    ///
+    /// # Safety
+    ///
+    /// After disabling validation, caller must ensure that appended values are compatible
+    /// for the specified precision and scale.
+    pub unsafe fn disable_value_validation(&mut self) {
+        self.value_validation = false;
+    }
+
     /// Appends a byte slice into the builder.
     ///
     /// Automatically calls the `append` method to delimit the slice appended in as a
     /// distinct array element.
     #[inline]
     pub fn append_value(&mut self, value: i128) -> Result<()> {
-        let value = validate_decimal_precision(value, self.precision)?;
+        let value = if self.value_validation {
+            validate_decimal_precision(value, self.precision)?
+        } else {
+            value
+        };
+
         let value_as_bytes = Self::from_i128_to_fixed_size_bytes(
             value,
             self.builder.value_length() as usize,
@@ -1478,7 +1498,7 @@ impl DecimalBuilder {
         self.builder.append(true)
     }
 
-    fn from_i128_to_fixed_size_bytes(v: i128, size: usize) -> Result<Vec<u8>> {
+    pub(crate) fn from_i128_to_fixed_size_bytes(v: i128, size: usize) -> Result<Vec<u8>> {
         if size > 16 {
             return Err(ArrowError::InvalidArgumentError(
                 "DecimalBuilder only supports values up to 16 bytes.".to_string(),
@@ -3418,14 +3438,14 @@ mod tests {
 
     #[test]
     fn test_decimal_builder() {
-        let mut builder = DecimalBuilder::new(30, 23, 6);
+        let mut builder = DecimalBuilder::new(30, 38, 6);
 
         builder.append_value(8_887_000_000).unwrap();
         builder.append_null().unwrap();
         builder.append_value(-8_887_000_000).unwrap();
         let decimal_array: DecimalArray = builder.finish();
 
-        assert_eq!(&DataType::Decimal(23, 6), decimal_array.data_type());
+        assert_eq!(&DataType::Decimal(38, 6), decimal_array.data_type());
         assert_eq!(3, decimal_array.len());
         assert_eq!(1, decimal_array.null_count());
         assert_eq!(32, decimal_array.value_offset(2));

arrow/src/array/data.rs

@@ -18,7 +18,7 @@
 //! Contains `ArrayData`, a generic representation of Arrow array data which encapsulates
 //! common attributes and operations for Arrow array.
 
-use crate::datatypes::{DataType, IntervalUnit, UnionMode};
+use crate::datatypes::{validate_decimal_precision, DataType, IntervalUnit, UnionMode};
 use crate::error::{ArrowError, Result};
 use crate::{bitmap::Bitmap, datatypes::ArrowNativeType};
 use crate::{
@@ -999,6 +999,21 @@ impl ArrayData {
 
     pub fn validate_dictionary_offset(&self) -> Result<()> {
         match &self.data_type {
+            DataType::Decimal(p, _) => {
+                let values_buffer = &self.buffers[0];
+
+                for pos in 0..values_buffer.len() {
+                    let raw_val = unsafe {
+                        std::slice::from_raw_parts(
+                            values_buffer.as_ptr().add(pos),
+                            16_usize,
+                        )
+                    };
+                    let value = i128::from_le_bytes(raw_val.try_into().unwrap());
+                    validate_decimal_precision(value, *p)?;
+                }
+                Ok(())
+            }
             DataType::Utf8 => self.validate_utf8::<i32>(),
             DataType::LargeUtf8 => self.validate_utf8::<i64>(),
             DataType::Binary => self.validate_offsets_full::<i32>(self.buffers[1].len()),
@@ -1492,8 +1507,9 @@ mod tests {
     use std::ptr::NonNull;
 
     use crate::array::{
-        make_array, Array, BooleanBuilder, Int32Array, Int32Builder, Int64Array,
-        StringArray, StructBuilder, UInt64Array,
+        make_array, Array, BooleanBuilder, DecimalBuilder, FixedSizeListBuilder,
+        Int32Array, Int32Builder, Int64Array, StringArray, StructBuilder, UInt64Array,
+        UInt8Builder,
     };
     use crate::buffer::Buffer;
     use crate::datatypes::Field;
@@ -2707,4 +2723,36 @@ mod tests {
 
         assert_eq!(array, &expected);
     }
+
+    #[test]
+    #[cfg(not(feature = "force_validate"))]
+    fn test_decimal_full_validation() {
+        let values_builder = UInt8Builder::new(10);
+        let byte_width = 16;
+        let mut fixed_size_builder =
+            FixedSizeListBuilder::new(values_builder, byte_width);
+        let value_as_bytes = DecimalBuilder::from_i128_to_fixed_size_bytes(
+            123456,
+            fixed_size_builder.value_length() as usize,
+        )
+        .unwrap();
+        fixed_size_builder
+            .values()
+            .append_slice(value_as_bytes.as_slice())
+            .unwrap();
+        fixed_size_builder.append(true).unwrap();
+        let fixed_size_array = fixed_size_builder.finish();
+
+        // Build ArrayData for Decimal
+        let builder = ArrayData::builder(DataType::Decimal(5, 3))
+            .len(fixed_size_array.len())
+            .add_buffer(fixed_size_array.data_ref().child_data()[0].buffers()[0].clone());
+        let array_data = unsafe { builder.build_unchecked() };
+        let validation_result = array_data.validate_full();
+        let error = validation_result.unwrap_err();
+        assert_eq!(
+            "Invalid argument error: 123456 is too large to store in a Decimal of precision 5. Max is 99999",
+            error.to_string()
+        );
+    }
 }

arrow/src/array/transform/mod.rs

@@ -721,6 +721,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn test_decimal() {
         let decimal_array =
             create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3);
@@ -734,6 +735,7 @@ mod tests {
         assert_eq!(array, expected);
     }
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn test_decimal_offset() {
         let decimal_array =
             create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3);
@@ -748,6 +750,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn test_decimal_null_offset_nulls() {
         let decimal_array =
             create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3);

arrow/src/csv/reader.rs

@@ -1203,8 +1203,8 @@ mod tests {
     fn test_csv_reader_with_decimal() {
         let schema = Schema::new(vec![
             Field::new("city", DataType::Utf8, false),
-            Field::new("lat", DataType::Decimal(26, 6), false),
-            Field::new("lng", DataType::Decimal(26, 6), false),
+            Field::new("lat", DataType::Decimal(38, 6), false),
+            Field::new("lng", DataType::Decimal(38, 6), false),
         ]);
 
         let file = File::open("test/data/decimal_test.csv").unwrap();

arrow/src/ffi.rs

@@ -907,6 +907,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn test_decimal_round_trip() -> Result<()> {
         // create an array natively
         let original_array = [Some(12345_i128), Some(-12345_i128), None]

arrow/src/ipc/reader.rs

@@ -1036,6 +1036,7 @@ mod tests {
     use crate::{datatypes, util::integration_util::*};
 
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn read_generated_files_014() {
         let testdata = crate::util::test_util::arrow_test_data();
         let version = "0.14.1";
@@ -1156,6 +1157,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn read_generated_streams_014() {
         let testdata = crate::util::test_util::arrow_test_data();
         let version = "0.14.1";

arrow/src/ipc/writer.rs

@@ -1078,6 +1078,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn read_and_rewrite_generated_files_014() {
         let testdata = crate::util::test_util::arrow_test_data();
         let version = "0.14.1";
@@ -1130,6 +1131,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(not(feature = "force_validate"))]
     fn read_and_rewrite_generated_streams_014() {
         let testdata = crate::util::test_util::arrow_test_data();
         let version = "0.14.1";

integration-testing/src/lib.rs

@@ -593,6 +593,10 @@ fn array_from_json(
         }
         DataType::Decimal(precision, scale) => {
             let mut b = DecimalBuilder::new(json_col.count, *precision, *scale);
+            // C++ interop tests involve incompatible decimal values
+            unsafe {
+                b.disable_value_validation();
+            }
             for (is_valid, value) in json_col
                 .validity
                 .as_ref()