Add support for arrays of arbitrary size (serde big array)

CHANGELOG.md

@@ -5,6 +5,51 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+* Add support for arrays of arbitrary size.
+    This feature requires Rust 1.51+.
+
+    ```rust
+    // Rust
+    #[serde_as(as = "[[_; 64]; 33]")]
+    value: [[u8; 64]; 33],
+
+    // JSON
+    "value": [[0,0,0,0,0,...], [0,0,0,...], ...],
+    ```
+
+    Mapping of arrays was available before, but limited to arrays of length 32.
+    All conversion methods are available for the array elements.
+
+    This is similar to the existing [`serde-big-array`] crate with three important improvements:
+
+    1. Support for the `serde_as` annotation.
+    2. Supports non-copy elements (see [serde-big-array#6][serde-big-array-copy]).
+    3. Supports arbitrary nestings of arrays (see [serde-big-array#7][serde-big-array-nested]).
+
+[`serde-big-array`]: https://crates.io/crates/serde-big-array
+[serde-big-array-copy]: https://github.com/est31/serde-big-array/issues/6
+[serde-big-array-nested]: https://github.com/est31/serde-big-array/issues/7
+
+* Arrays with tuple elements can now be deserialized from  a map.
+    This feature requires Rust 1.51+.
+
+    ```rust
+    // Rust
+    #[serde_as(as = "BTreeMap<_, _>")]
+    value: [(String, u16); 3],
+
+    // JSON
+    "value": {
+        "a": 1,
+        "b": 2,
+        "c": 3
+    },
+    ```
+
 ## [1.6.4]
 
 ### Fixed

Cargo.toml

@@ -36,6 +36,7 @@ macros = ["serde_with_macros"]
 chrono_crate = {package = "chrono", version = "0.4.1", features = ["serde"], optional = true}
 doc-comment = {version = "0.3.3", optional = true}
 hex = {version = "0.4.2", optional = true}
+rustversion = "1.0.0"
 serde = {version = "1.0.75", features = ["derive"]}
 serde_json = {version = "1.0.1", optional = true}
 serde_with_macros = {path = "./serde_with_macros", version = "1.4.1", optional = true}
@@ -48,7 +49,6 @@ mime = "0.3.16"
 pretty_assertions = "0.6.1"
 regex = {version = "1.3.9", default-features = false, features = ["std"]}
 ron = "0.6"
-rustversion = "1.0.0"
 serde-xml-rs = "0.4.1"
 serde_derive = "1.0.75"
 serde_json = {version = "1.0.25", features = ["preserve_order"]}

src/de/const_arrays.rs

@@ -0,0 +1,148 @@
+use super::*;
+use crate::utils::{MapIter, SeqIter};
+use serde::de::*;
+use std::collections::{BTreeMap, HashMap};
+use std::fmt;
+use std::mem::MaybeUninit;
+
+// TODO this should probably be moved into the utils module when const generics are available for MSRV
+
+/// # Safety
+/// The code follow exactly the pattern of initializing an array element-by-element from the standard library.
+/// https://doc.rust-lang.org/nightly/std/mem/union.MaybeUninit.html#initializing-an-array-element-by-element
+fn array_from_iterator<I, T, E, const N: usize>(
+    mut iter: I,
+    expected: &dyn Expected,
+) -> Result<[T; N], E>
+where
+    I: Iterator<Item = Result<T, E>>,
+    E: Error,
+{
+    fn drop_array_elems<T, const N: usize>(num: usize, mut arr: [MaybeUninit<T>; N]) {
+        (&mut arr[0..num]).iter_mut().for_each(|elem| {
+            // TODO This would be better with assume_init_drop nightly function
+            // https://github.com/rust-lang/rust/issues/63567
+            unsafe { std::ptr::drop_in_place(elem.as_mut_ptr()) };
+        });
+    }
+
+    // Create an uninitialized array of `MaybeUninit`. The `assume_init` is
+    // safe because the type we are claiming to have initialized here is a
+    // bunch of `MaybeUninit`s, which do not require initialization.
+    //
+    // TODO could be simplified with nightly maybe_uninit_uninit_array feature
+    // https://doc.rust-lang.org/nightly/std/mem/union.MaybeUninit.html#method.uninit_array
+    let mut arr: [MaybeUninit<T>; N] = unsafe { MaybeUninit::uninit().assume_init() };
+
+    // Dropping a `MaybeUninit` does nothing. Thus using raw pointer
+    // assignment instead of `ptr::write` does not cause the old
+    // uninitialized value to be dropped. Also if there is a panic during
+    // this loop, we have a memory leak, but there is no memory safety
+    // issue.
+    for (idx, elem) in (&mut arr[..]).iter_mut().enumerate() {
+        *elem = match iter.next() {
+            Some(Ok(value)) => MaybeUninit::new(value),
+            Some(Err(err)) => {
+                drop_array_elems(idx, arr);
+                return Err(err);
+            }
+            None => {
+                drop_array_elems(idx, arr);
+                return Err(Error::invalid_length(idx, expected));
+            }
+        };
+    }
+
+    // Everything is initialized. Transmute the array to the
+    // initialized type.
+    // A normal transmute is not possible because of:
+    // https://github.com/rust-lang/rust/issues/61956
+    Ok(unsafe { std::mem::transmute_copy::<_, [T; N]>(&arr) })
+}
+
+impl<'de, T, As, const N: usize> DeserializeAs<'de, [T; N]> for [As; N]
+where
+    As: DeserializeAs<'de, T>,
+{
+    fn deserialize_as<D>(deserializer: D) -> Result<[T; N], D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        struct ArrayVisitor<T, const M: usize>(PhantomData<T>);
+
+        impl<'de, T, As, const M: usize> Visitor<'de> for ArrayVisitor<DeserializeAsWrap<T, As>, M>
+        where
+            As: DeserializeAs<'de, T>,
+        {
+            type Value = [T; M];
+
+            fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+                formatter.write_fmt(format_args!("an array of size {}", M))
+            }
+
+            fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+            where
+                A: SeqAccess<'de>,
+            {
+                array_from_iterator(
+                    SeqIter::new(seq).map(|res: Result<DeserializeAsWrap<T, As>, A::Error>| {
+                        res.map(|t| t.into_inner())
+                    }),
+                    &self,
+                )
+            }
+        }
+
+        deserializer.deserialize_tuple(N, ArrayVisitor::<DeserializeAsWrap<T, As>, N>(PhantomData))
+    }
+}
+
+macro_rules! tuple_seq_as_map_impl_intern {
+    ($tyorig:ty, $ty:ident <KAs, VAs>) => {
+        #[allow(clippy::implicit_hasher)]
+        impl<'de, K, KAs, V, VAs, const N: usize> DeserializeAs<'de, $tyorig> for $ty<KAs, VAs>
+        where
+            KAs: DeserializeAs<'de, K>,
+            VAs: DeserializeAs<'de, V>,
+        {
+            fn deserialize_as<D>(deserializer: D) -> Result<$tyorig, D::Error>
+            where
+                D: Deserializer<'de>,
+            {
+                struct MapVisitor<K, KAs, V, VAs, const M: usize> {
+                    marker: PhantomData<(K, KAs, V, VAs)>,
+                }
+
+                impl<'de, K, KAs, V, VAs, const M: usize> Visitor<'de> for MapVisitor<K, KAs, V, VAs, M>
+                where
+                    KAs: DeserializeAs<'de, K>,
+                    VAs: DeserializeAs<'de, V>,
+                {
+                    type Value = [(K, V); M];
+
+                    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+                        formatter.write_fmt(format_args!("a map of length {}", M))
+                    }
+
+                    fn visit_map<A>(self, access: A) -> Result<Self::Value, A::Error>
+                    where
+                        A: MapAccess<'de>,
+                    {
+                        array_from_iterator(MapIter::new(access).map(
+                            |res: Result<(DeserializeAsWrap<K, KAs>, DeserializeAsWrap<V, VAs>), A::Error>| {
+                                res.map(|(k, v)| (k.into_inner(), v.into_inner()))
+                            }
+                        ), &self)
+                    }
+                }
+
+                let visitor = MapVisitor::<K, KAs, V, VAs, N> {
+                    marker: PhantomData,
+                };
+                deserializer.deserialize_map(visitor)
+            }
+        }
+    }
+}
+tuple_seq_as_map_impl_intern!([(K, V); N], BTreeMap<KAs, VAs>);
+tuple_seq_as_map_impl_intern!([(K, V); N], HashMap<KAs, VAs>);

src/de/impls.rs

@@ -314,88 +314,6 @@ tuple_impl!(14 0 T0 As0 1 T1 As1 2 T2 As2 3 T3 As3 4 T4 As4 5 T5 As5 6 T6 As6 7
 tuple_impl!(15 0 T0 As0 1 T1 As1 2 T2 As2 3 T3 As3 4 T4 As4 5 T5 As5 6 T6 As6 7 T7 As7 8 T8 As8 9 T9 As9 10 T10 As10 11 T11 As11 12 T12 As12 13 T13 As13 14 T14 As14);
 tuple_impl!(16 0 T0 As0 1 T1 As1 2 T2 As2 3 T3 As3 4 T4 As4 5 T5 As5 6 T6 As6 7 T7 As7 8 T8 As8 9 T9 As9 10 T10 As10 11 T11 As11 12 T12 As12 13 T13 As13 14 T14 As14 15 T15 As15);
 
-macro_rules! array_impl {
-    ($len:literal $($idx:tt)*) => {
-        impl<'de, T, As> DeserializeAs<'de, [T; $len]> for [As; $len]
-        where
-            As: DeserializeAs<'de, T>,
-        {
-            fn deserialize_as<D>(deserializer: D) -> Result<[T; $len], D::Error>
-            where
-                D: Deserializer<'de>,
-            {
-                struct ArrayVisitor<T>(PhantomData<T>);
-
-                impl<'de, T, As> Visitor<'de>
-                    for ArrayVisitor<DeserializeAsWrap<T, As>>
-                where
-                    As: DeserializeAs<'de, T>,
-                {
-                    type Value = [T; $len];
-
-                    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
-                        formatter.write_str(concat!("an array of size ", $len))
-                    }
-
-                    #[allow(non_snake_case)]
-                    // Because of 0-size arrays
-                    #[allow(unused_variables, unused_mut)]
-                    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
-                    where
-                        A: SeqAccess<'de>,
-                    {
-                        Ok([$(
-                            match seq.next_element::<DeserializeAsWrap<T, As>>()? {
-                                Some(value) => value.into_inner(),
-                                None => return Err(Error::invalid_length($idx, &self)),
-                            },
-                        )*])
-                    }
-                }
-
-                deserializer.deserialize_tuple(
-                    $len,
-                    ArrayVisitor::<DeserializeAsWrap<T, As>>(PhantomData),
-                )
-            }
-        }
-    };
-}
-
-array_impl!(0);
-array_impl!(1 0);
-array_impl!(2 0 1);
-array_impl!(3 0 1 2);
-array_impl!(4 0 1 2 3);
-array_impl!(5 0 1 2 3 4);
-array_impl!(6 0 1 2 3 4 5);
-array_impl!(7 0 1 2 3 4 5 6);
-array_impl!(8 0 1 2 3 4 5 6 7);
-array_impl!(9 0 1 2 3 4 5 6 7 8);
-array_impl!(10 0 1 2 3 4 5 6 7 8 9);
-array_impl!(11 0 1 2 3 4 5 6 7 8 9 10);
-array_impl!(12 0 1 2 3 4 5 6 7 8 9 10 11);
-array_impl!(13 0 1 2 3 4 5 6 7 8 9 10 11 12);
-array_impl!(14 0 1 2 3 4 5 6 7 8 9 10 11 12 13);
-array_impl!(15 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14);
-array_impl!(16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15);
-array_impl!(17 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16);
-array_impl!(18 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17);
-array_impl!(19 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18);
-array_impl!(20 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19);
-array_impl!(21 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20);
-array_impl!(22 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21);
-array_impl!(23 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22);
-array_impl!(24 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23);
-array_impl!(25 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24);
-array_impl!(26 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25);
-array_impl!(27 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26);
-array_impl!(28 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27);
-array_impl!(29 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28);
-array_impl!(30 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29);
-array_impl!(31 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30);
-array_impl!(32 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31);
-
 impl<'de, T: Deserialize<'de>> DeserializeAs<'de, T> for Same {
     fn deserialize_as<D>(deserializer: D) -> Result<T, D::Error>
     where

src/de/legacy_arrays.rs

@@ -0,0 +1,85 @@
+use super::*;
+use std::fmt;
+use serde::de::*;
+
+macro_rules! array_impl {
+    ($len:literal $($idx:tt)*) => {
+        impl<'de, T, As> DeserializeAs<'de, [T; $len]> for [As; $len]
+        where
+            As: DeserializeAs<'de, T>,
+        {
+            fn deserialize_as<D>(deserializer: D) -> Result<[T; $len], D::Error>
+            where
+                D: Deserializer<'de>,
+            {
+                struct ArrayVisitor<T>(PhantomData<T>);
+
+                impl<'de, T, As> Visitor<'de>
+                    for ArrayVisitor<DeserializeAsWrap<T, As>>
+                where
+                    As: DeserializeAs<'de, T>,
+                {
+                    type Value = [T; $len];
+
+                    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+                        formatter.write_str(concat!("an array of size ", $len))
+                    }
+
+                    #[allow(non_snake_case)]
+                    // Because of 0-size arrays
+                    #[allow(unused_variables, unused_mut)]
+                    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+                    where
+                        A: SeqAccess<'de>,
+                    {
+                        Ok([$(
+                            match seq.next_element::<DeserializeAsWrap<T, As>>()? {
+                                Some(value) => value.into_inner(),
+                                None => return Err(Error::invalid_length($idx, &self)),
+                            },
+                        )*])
+                    }
+                }
+
+                deserializer.deserialize_tuple(
+                    $len,
+                    ArrayVisitor::<DeserializeAsWrap<T, As>>(PhantomData),
+                )
+            }
+        }
+    };
+}
+
+array_impl!(0);
+array_impl!(1 0);
+array_impl!(2 0 1);
+array_impl!(3 0 1 2);
+array_impl!(4 0 1 2 3);
+array_impl!(5 0 1 2 3 4);
+array_impl!(6 0 1 2 3 4 5);
+array_impl!(7 0 1 2 3 4 5 6);
+array_impl!(8 0 1 2 3 4 5 6 7);
+array_impl!(9 0 1 2 3 4 5 6 7 8);
+array_impl!(10 0 1 2 3 4 5 6 7 8 9);
+array_impl!(11 0 1 2 3 4 5 6 7 8 9 10);
+array_impl!(12 0 1 2 3 4 5 6 7 8 9 10 11);
+array_impl!(13 0 1 2 3 4 5 6 7 8 9 10 11 12);
+array_impl!(14 0 1 2 3 4 5 6 7 8 9 10 11 12 13);
+array_impl!(15 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14);
+array_impl!(16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15);
+array_impl!(17 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16);
+array_impl!(18 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17);
+array_impl!(19 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18);
+array_impl!(20 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19);
+array_impl!(21 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20);
+array_impl!(22 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21);
+array_impl!(23 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22);
+array_impl!(24 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23);
+array_impl!(25 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24);
+array_impl!(26 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25);
+array_impl!(27 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26);
+array_impl!(28 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27);
+array_impl!(29 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28);
+array_impl!(30 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29);
+array_impl!(31 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30);
+array_impl!(32 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31);