From 0cd02620030884bca19a5e8e283eb2758af8a11c Mon Sep 17 00:00:00 2001
From: bluss <bluss@users.noreply.github.com>
Date: Sat, 17 Apr 2021 18:20:39 +0200
Subject: [PATCH] shape: Factor out common layout check

---
 src/impl_methods.rs | 48 +++++++++++++++++++++------------------------
 1 file changed, 22 insertions(+), 26 deletions(-)

diff --git a/src/impl_methods.rs b/src/impl_methods.rs
index 8493cf497..e56beef9b 100644
--- a/src/impl_methods.rs
+++ b/src/impl_methods.rs
@@ -1642,14 +1642,8 @@ where
         A: Clone,
         S: Data,
     {
-        if size_of_shape_checked(&shape) != Ok(self.dim.size()) {
-            return Err(error::incompatible_shapes(&self.dim, &shape));
-        }
-        let layout = self.layout_impl();
-        if order == Order::Automatic {
-            order = preferred_order_for_layout(layout);
-        }
-
+        let layout = self.layout_and_order(&shape, &mut order)?;
+        // safe because: the number of elements is preserved and it's contiguous
         unsafe {
             if layout.is(Layout::CORDER) && order == Order::RowMajor {
                 let strides = shape.default_strides();
@@ -1672,6 +1666,24 @@ where
         }
     }
 
+    /// Check if `shape` is valid for reshaping the array (in terms of number of elements),
+    /// and also compute the Layout of self and resolve Order if it is Automatic.
+    ///
+    /// After returning successfully, `order` is either RowMajor or ColumnMajor.
+    fn layout_and_order<E>(&self, shape: &E, order: &mut Order) -> Result<Layout, ShapeError>
+    where
+        E: Dimension,
+    {
+        if size_of_shape_checked(shape) != Ok(self.dim.size()) {
+            return Err(error::incompatible_shapes(&self.dim, shape));
+        }
+        let layout = self.layout_impl();
+        if *order == Order::Automatic {
+            *order = preferred_order_for_layout(layout);
+        }
+        Ok(layout)
+    }
+
     /// Transform the array into `shape`; any shape with the same number of elements is accepted,
     /// but the source array or view must be in contiguous and stored in standard row-major (C) or
     /// column-major (Fortran) memory order.
@@ -1707,15 +1719,7 @@ where
     where
         E: Dimension,
     {
-        if size_of_shape_checked(&shape) != Ok(self.dim.size()) {
-            return Err(error::incompatible_shapes(&self.dim, &shape));
-        }
-
-        let layout = self.layout_impl();
-        if order == Order::Automatic {
-            order = preferred_order_for_layout(layout);
-        }
-
+        let layout = self.layout_and_order(&shape, &mut order)?;
         // safe because: the number of elements is preserved and it's contiguous
         unsafe {
             if layout.is(Layout::CORDER) && order == Order::RowMajor {
@@ -1769,15 +1773,7 @@ where
     where
         E: Dimension,
     {
-        if size_of_shape_checked(&shape) != Ok(self.dim.size()) {
-            return Err(error::incompatible_shapes(&self.dim, &shape));
-        }
-
-        let layout = self.layout_impl();
-        if order == Order::Automatic {
-            order = preferred_order_for_layout(layout);
-        }
-
+        let layout = self.layout_and_order(&shape, &mut order)?;
         // safe because: the number of elements is preserved and it's contiguous
         unsafe {
             if layout.is(Layout::CORDER) && order == Order::RowMajor {