From ecb5681e20134cd8d283445e6aea8f11008cf810 Mon Sep 17 00:00:00 2001 From: Animesh Jain Date: Sat, 1 Feb 2020 01:38:08 +0000 Subject: [PATCH 1/3] [QNN] Optimize lowering for requantize and FixedPointMultiply. --- src/relay/qnn/op/requantize.cc | 6 +++++- src/relay/qnn/util.cc | 10 +++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/relay/qnn/op/requantize.cc b/src/relay/qnn/op/requantize.cc index 29c8e3e915cf..7c1231e97346 100644 --- a/src/relay/qnn/op/requantize.cc +++ b/src/relay/qnn/op/requantize.cc @@ -103,7 +103,11 @@ Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale, shifted_int64_t = Add(Cast(output_zero_point, hp_dtype), scaled_int64_t); } - // 4) Clip to the out_dtype min/max. + // 4) Clip to the out_dtype min/max. Skip clipping if out_dtype is Int32. The fixed point + // multiplication keeps the value in int32 range. + if (out_dtype == DataType::Int(32)) { + return Cast(shifted_int64_t, out_dtype); + } auto q_min = GetQmin(out_dtype); auto q_max = GetQmax(out_dtype); auto clipped_t = Clip(shifted_int64_t, q_min, q_max); diff --git a/src/relay/qnn/util.cc b/src/relay/qnn/util.cc index cd0e68824129..d80fc812b3f2 100644 --- a/src/relay/qnn/util.cc +++ b/src/relay/qnn/util.cc @@ -149,6 +149,7 @@ Expr FixedPointMultiplyPerChannel(Expr tensor, std::vector multipliers, // 1) Calculating the integer multiplier and integer shift. These are calculated per axis/per // channel. std::vector fixed_pt_multipliers, lshifts, rshifts; + bool is_lshift_required = false; for (auto multiplier : multipliers) { int32_t fixed_pt_multiplier, shift; std::tie(fixed_pt_multiplier, shift) = GetFixedPointMultiplierShift(multiplier); @@ -157,12 +158,15 @@ Expr FixedPointMultiplyPerChannel(Expr tensor, std::vector multipliers, fixed_pt_multipliers.push_back(fixed_pt_multiplier); lshifts.push_back(lshift); rshifts.push_back(rshift); + is_lshift_required |= (lshift != 0); } // 2) Multiply the integer multiplier. Convert lefts shifts into expr and multiply. - auto lshift_expr = MakeConstantTensor(hp_dtype, {n_channels}, lshifts); - auto exp_lshift_expr = ExpandBiasToMatchAxis(lshift_expr, n_dim, {channel_axis}); - tensor = LeftShift(tensor, exp_lshift_expr); + if (is_lshift_required) { + auto lshift_expr = MakeConstantTensor(hp_dtype, {n_channels}, lshifts); + auto exp_lshift_expr = ExpandBiasToMatchAxis(lshift_expr, n_dim, {channel_axis}); + tensor = LeftShift(tensor, exp_lshift_expr); + } // 3) Perform the multiplication in higher precision. // The scalar is a fixed point value of int32 where the decimal point is From a6e2d3414b5836811fe783e397af7288de502144 Mon Sep 17 00:00:00 2001 From: Animesh Jain Date: Tue, 4 Feb 2020 01:07:16 +0000 Subject: [PATCH 2/3] Add check for requantize scale gt 1. --- src/relay/qnn/op/requantize.cc | 18 ++++++++++++++---- src/relay/qnn/util.cc | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/relay/qnn/op/requantize.cc b/src/relay/qnn/op/requantize.cc index 7c1231e97346..7e99ad7938d9 100644 --- a/src/relay/qnn/op/requantize.cc +++ b/src/relay/qnn/op/requantize.cc @@ -67,6 +67,9 @@ Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale, tensor = Subtract(tensor, Cast(input_zero_point, hp_dtype)); } + // Check if multiplier is greater than 1. + bool is_multiplier_gt_one = false; + // 2) If the input and output scales are same, we can skip the fixed point multiplication. Check // if the input scale is per-tensor or per-channel. If it is per-tensor, there is single scale for // the whole tensor. For per-channel (aka per-axis), there is a vector of scales for the input @@ -78,6 +81,9 @@ Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale, float input_scale_float = GetScalarFromConstant(input_scale); double double_multiplier = static_cast(input_scale_float) / static_cast(output_scale_float); + if (double_multiplier > 1) { + is_multiplier_gt_one = true; + } // Skip if input and output scales are same. if (!IsEqualScalar(input_scale, output_scale)) { scaled_int64_t = @@ -88,8 +94,12 @@ Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale, std::vector double_multipliers; auto input_axis_scales = GetFloatVectorFromConstant(input_scale); for (auto input_axis_scale : input_axis_scales) { - double_multipliers.push_back(static_cast(input_axis_scale) / - static_cast(output_scale_float)); + double multiplier = + static_cast(input_axis_scale) / static_cast(output_scale_float); + double_multipliers.push_back(multiplier); + if (multiplier > 1) { + is_multiplier_gt_one = true; + } } int axis = param->axis; axis = (axis == -1) ? input_shape.size() - 1 : axis; @@ -104,8 +114,8 @@ Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale, } // 4) Clip to the out_dtype min/max. Skip clipping if out_dtype is Int32. The fixed point - // multiplication keeps the value in int32 range. - if (out_dtype == DataType::Int(32)) { + // multiplication keeps the value in int32 range if the requantize scale is less than 1. + if (out_dtype == DataType::Int(32) && !is_multiplier_gt_one) { return Cast(shifted_int64_t, out_dtype); } auto q_min = GetQmin(out_dtype); diff --git a/src/relay/qnn/util.cc b/src/relay/qnn/util.cc index d80fc812b3f2..fad37bcab98d 100644 --- a/src/relay/qnn/util.cc +++ b/src/relay/qnn/util.cc @@ -158,7 +158,7 @@ Expr FixedPointMultiplyPerChannel(Expr tensor, std::vector multipliers, fixed_pt_multipliers.push_back(fixed_pt_multiplier); lshifts.push_back(lshift); rshifts.push_back(rshift); - is_lshift_required |= (lshift != 0); + is_lshift_required = is_lshift_required | (lshift != 0); } // 2) Multiply the integer multiplier. Convert lefts shifts into expr and multiply. From 385a3d05ce076c9ca0bdba7f7448909006821993 Mon Sep 17 00:00:00 2001 From: Animesh Jain Date: Tue, 4 Feb 2020 17:20:46 +0000 Subject: [PATCH 3/3] Added test case. --- tests/python/relay/test_op_qnn_requantize.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/python/relay/test_op_qnn_requantize.py b/tests/python/relay/test_op_qnn_requantize.py index bd37cb989f46..b682498cb10b 100644 --- a/tests/python/relay/test_op_qnn_requantize.py +++ b/tests/python/relay/test_op_qnn_requantize.py @@ -311,6 +311,21 @@ def test_per_channel_different_scale(): rounding=rounding) verify(mod, (golden_data, golden_output)) + # Have input scale > output scale + golden_data = np.arange(-5, 5, 1).astype('int32').reshape((5,2)) + golden_output = np.array([-10, -2, -6, -1, -2, 0, 2, 1, 6, 2]).reshape((5, 2)) + + for rounding in roundings: + mod = get_mod(data_shape=(5, 2), + data_dtype='int32', + out_dtype="int8", + input_scale=[1.0, 0.25], + output_scale=0.5, + axis=1, + rounding=rounding) + verify(mod, (golden_data, golden_output)) + + if __name__ == "__main__": test_same_scale() test_downscale()