onnx/ops/resize add Interpolator::Nearest

[openmynet]

add Interpolator::Nearest to support onnxruntime upsample

[kali]

Thanks for your interest. Is getting the left one always the right move ? I would have expected something like if x_ratio < 0.5 { y_left } else { y_right } ... plus the handling of the various rounding ties cases.

Is this unlocking new tests from the onnx test suite ? There is a handful of them covering the nearest case in https://github.com/onnx/onnx/blob/main/docs/Operators.md#examples-108 ... You can find out by running: cargo test --ignored resize in harness/onnx-test-suite. If it is the case, the include-passing-ignored.sh script can update the test list text files for you.

[openmynet]

I didn't consider some places, I tried the method compatible with CoordTransformer in onnx. CoordTransformer::HalfPixel seems to be incompatible with upsample when outputting structures, AlignCorners and Asymmetric do not have this problem .
The following part of the code comes from onnx/src/ops/resize:

use ndarray::{ArrayBase, Axis, Dim, IxDynImpl, OwnedRepr};
use tract_onnx;
#[test]
fn run_upsamle_nearest() {
    // Nearest - Asymmetric 
    // https://github.com/onnx/onnx/blob/main/docs/Operators.md#examples-108  // resize_upsample_sizes_nearest
    let expected = vec![
        vec![1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0],
        vec![1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0],
        vec![1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0],
        vec![1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0],
        vec![3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0],
        vec![3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0],
        vec![3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0],
    ]
    .into_iter()
    .flatten()
    .collect::<Vec<_>>();
    let expected =
        tract_onnx::prelude::tract_ndarray::Array4::from_shape_vec((1, 1, 7, 8), expected)
            .unwrap()
            .into_dyn();
    let input: Vec<f32> = vec![1.0, 2.0, 3.0, 4.0]; // [[[ [1, 2], [3, 4], ]]]
    let input = tract_onnx::prelude::tract_ndarray::Array::from_shape_vec((1, 1, 2, 2), input)
        .unwrap()
        .into_dyn();
    let output_shape = vec![1, 1, 7, 8];
    let output = upsamle_nearest(
        input,
        &output_shape,
        CoordTransformer::Asymmetric,
        Interpolator::Nearest,
    );
    assert_eq!(output, expected);

    // Nearest - AlignCorners
    // https://github.com/onnx/onnx/blob/main/docs/Operators.md#examples-108  // resize_upsample_sizes_nearest_floor_align_corners
    let output_shape = vec![1, 1, 8, 8];
    let expected = vec![
        vec![1.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0],
        vec![1.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0],
        vec![1.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0],
        vec![5.0, 5.0, 5.0, 6.0, 6.0, 7.0, 7.0, 8.0],
        vec![5.0, 5.0, 5.0, 6.0, 6.0, 7.0, 7.0, 8.0],
        vec![9.0, 9.0, 9.0, 10.0, 10.0, 11.0, 11.0, 12.0],
        vec![9.0, 9.0, 9.0, 10.0, 10.0, 11.0, 11.0, 12.0],
        vec![13.0, 13.0, 13.0, 14.0, 14.0, 15.0, 15.0, 16.0],
    ]
    .into_iter()
    .flatten()
    .collect::<Vec<_>>();
    let expected =
        tract_onnx::prelude::tract_ndarray::Array4::from_shape_vec((1, 1, 8, 8), expected)
            .unwrap()
            .into_dyn();

    let input: Vec<f32> = vec![
        vec![1.0, 2.0, 3.0, 4.0],
        vec![5.0, 6.0, 7.0, 8.0],
        vec![9.0, 10.0, 11.0, 12.0],
        vec![13.0, 14.0, 15.0, 16.0],
    ]
    .into_iter()
    .flatten()
    .collect::<Vec<_>>();
    let input = tract_onnx::prelude::tract_ndarray::Array::from_shape_vec((1, 1, 4, 4), input)
        .unwrap()
        .into_dyn();

    let output = upsamle_nearest(
        input,
        &output_shape,
        CoordTransformer::AlignCorners,
        Interpolator::NearestFloorAlignCorners,
    );
    assert_eq!(output, expected);
}

#[derive(Clone, Debug, Hash)]
enum CoordTransformer {
    HalfPixel,
    AlignCorners,
    Asymmetric,
}
impl CoordTransformer {
    fn transform(&self, x_out: usize, scale: f32, len_in: usize, len_out: usize) -> f32 {
        match self {
            CoordTransformer::HalfPixel => {
                (x_out as f32 + 0.5) * scale - 0.5 
            }
            CoordTransformer::AlignCorners => {
                (x_out as f32 * (len_in as f32 - 1.0)) / (len_out as f32 - 1.0)
            }
            CoordTransformer::Asymmetric => (x_out as f32) / scale,
        }
    }
}

#[derive(Clone, Debug, Hash)]
enum Interpolator {
    Linear,
    Nearest,
    NearestFloorAlignCorners,
    // NearestCeilHalfPixel,
}

impl Interpolator {
    fn interpolate(&self, y_left: f32, y_right: f32, x_ratio: f32) -> f32 {
        match self {
            Interpolator::Linear => y_left * (1.0 - x_ratio) + y_right * x_ratio,
            Interpolator::Nearest => {
                y_left
            }
            // Interpolator::NearestCeilHalfPixel => {
            //    CoordTransformer::HalfPixel outputs structure incompatible with upsample 
            // }
            Interpolator::NearestFloorAlignCorners => {
                y_left * (1.0 - x_ratio.floor()) + y_right * x_ratio.floor()
            }
        }
    }
}
// from https://github.com/sonos/tract/blob/main/onnx/src/ops/resize.rs //  line:132
fn upsamle_nearest(
    input: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
    output_shape: &Vec<usize>,
    coord_transformer: CoordTransformer,
    interpolator: Interpolator,
) -> ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>> {
    let mut data = input;
    for axis in 0..data.ndim() {
        if output_shape[axis] == data.shape()[axis] {
            continue;
        } else if output_shape[axis] > data.shape()[axis] {
            let scale = output_shape[axis] as f32 / data.shape()[axis] as f32;
            let mut new_shape: tract_onnx::prelude::TVec<usize> = data.shape().into();
            new_shape[axis] = output_shape[axis];
            data = tract_onnx::prelude::tract_ndarray::ArrayD::from_shape_fn(
                &*new_shape,
                |co_o| -> f32 {
                    let x_out = co_o[axis];
                    //self.coord_transformer.transform
                    let x_in = coord_transformer.transform(
                        x_out,
                        scale,
                        data.shape()[axis],
                        new_shape[axis],
                    );
                    let mut co_i = co_o.clone();
                    let x_left = (x_in as usize).min(data.shape()[axis] - 1).max(0);
                    co_i[axis] = x_left;
                    let y_left = data[&co_i];
                    let x_right = (x_left + 1).min(data.shape()[axis] - 1);
                    co_i[axis] = x_right;
                    let y_right = data[&co_i];
                    let x_frac = x_in - x_left as f32;

                    // self.interpolator.interpolate
                    interpolator.interpolate(y_left, y_right, x_frac)
                },
            )
        }
    }
    data
}

[kali]

I don't I understand the point you're making here. I am not convinced the implementation you propose does the right thing. Can we find a unit test somewhere that can convince me otherwise ?