diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 56d7fba985ce1..f13d8cac85e52 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -114,6 +114,25 @@ struct SimpleOpTypeSetTeller : public Teller { "sign", "silu", "logical_not", "reciprocal", "tanh_shrink", "logsigmoid", "erf", "bitwise_not", "equal", "not_equal", "rsqrt"}; + + // Static shape does not support 0 or 1 dim's input. + if (!with_dynamic_shape) { + auto inputs = desc.Inputs(); + for (auto iter : inputs) { + for (auto var_name : iter.second) { + auto* block = desc.Block(); + if (block) { + auto* var_desc = block->FindVar(var_name); + // Can't get feed op's TensorDesc + if (op_type != "feed" && var_desc && !var_desc->Persistable()) { + const auto shape = var_desc->GetShape(); + if (shape.size() == 1 || shape.size() == 0) return false; + } + } + } + } + } + if (act_op_list.find(op_type) != act_op_list.end()) { auto* block = desc.Block(); if (block == nullptr) { @@ -122,15 +141,6 @@ struct SimpleOpTypeSetTeller : public Teller { "the pass."; return false; } - auto x_var_name = desc.Input("X")[0]; - auto* x_var_desc = block->FindVar(x_var_name); - const auto x_shape = x_var_desc->GetShape(); - if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) { - VLOG(3) << op_type - << " op does not support input's dim is 1 or 0 in tensorrt " - "static shape mode."; - return false; - } #if !IS_TRT_VERSION_GE(7000) if (op_type == "erf") { VLOG(3) << op_type << " op does not support tensorrt."; @@ -138,6 +148,9 @@ struct SimpleOpTypeSetTeller : public Teller { } #endif #if !IS_TRT_VERSION_GE(8600) + auto x_var_name = desc.Input("X")[0]; + auto* x_var_desc = block->FindVar(x_var_name); + const auto x_shape = x_var_desc->GetShape(); if (x_shape.size() == 0 && unary_list.find(op_type) != unary_list.end()) { VLOG(3) << op_type << " op does not support 0 dim input when TensorRT < 8.6."; @@ -145,24 +158,6 @@ struct SimpleOpTypeSetTeller : public Teller { } #endif } - // In static shape in Paddle-TRT, we can't allow that one op has a - // 1D intermediate tensor as input. - if (!with_dynamic_shape) { - auto inputs = desc.Inputs(); - for (auto iter : inputs) { - for (auto var_name : iter.second) { - auto* block = desc.Block(); - if (block) { - auto* var_desc = block->FindVar(var_name); - // Can't get feed op's TensorDesc - if (op_type != "feed" && var_desc && !var_desc->Persistable()) { - const auto shape = var_desc->GetShape(); - if (shape.size() == 1) return false; - } - } - } - } - } if (op_type == "dropout") { /* @@ -1505,6 +1500,7 @@ struct SimpleOpTypeSetTeller : public Teller { "elementwise op."; return false; } + if (x_var_desc->Persistable() && !with_dynamic_shape) { VLOG(3) << "Input X is a parameter which is not supported for " diff --git a/test/ir/inference/test_trt_convert_elementwise.py b/test/ir/inference/test_trt_convert_elementwise.py index a6faff0787be5..0ac4a2ba46209 100644 --- a/test/ir/inference/test_trt_convert_elementwise.py +++ b/test/ir/inference/test_trt_convert_elementwise.py @@ -1214,5 +1214,161 @@ def test(self): self.run_test() +class TrtConvertElementwise0D(TrtLayerAutoScanTest): + def is_program_valid(self, program_config: ProgramConfig) -> bool: + return True + + def sample_program_configs(self): + def generate_input(dims, op_type): + shape = [] + if dims == 0: + shape = [] + elif dims == 1: + shape = [8] + elif dims == 2: + shape = [1, 8] + elif dims == 3: + shape = [1, 8, 8] + else: + shape = [1, 8, 8, 8] + + # elementwise_floordiv is integer only + if op_type == "elementwise_floordiv": + return np.random.randint( + low=1, high=10000, size=shape, dtype=np.int32 + ) + elif op_type == "elementwise_mod": + return np.random.uniform(low=0.1, high=1.0, size=shape).astype( + np.float32 + ) + else: + return np.random.random(shape).astype(np.float32) + + for dims in [[0, 0], [0, 1], [0, 2], [1, 0], [2, 0]]: + for op_type in [ + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", + "elementwise_floordiv", + "elementwise_mod", + ]: + for axis in [-1 if dims[0] == 1 or dims[0] == 0 else 1]: + self.dims = dims[0] + dics = [{"axis": axis}] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data"], + "Y": ["weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + "outputs_dtype": { + "output_data": np.float32 + if op_type != "elementwise_floordiv" + else np.int32 + }, + } + ] + ops = self.generate_op_config(ops_config) + + program_config = ProgramConfig( + ops=ops, + weights={ + "weight": TensorConfig( + data_gen=partial( + generate_input, dims[1], op_type + ) + ) + }, + inputs={ + "input_data": TensorConfig( + data_gen=partial( + generate_input, dims[0], op_type + ) + ), + }, + outputs=["output_data"], + ) + + yield program_config + + def sample_predictor_configs( + self, program_config + ) -> (paddle_infer.Config, List[int], float): + def generate_dynamic_shape(attrs): + # The input.dims[1] must be equal to the weight's length. + if self.dims == 0: + self.dynamic_shape.min_input_shape = {"input_data": []} + self.dynamic_shape.max_input_shape = {"input_data": []} + self.dynamic_shape.opt_input_shape = {"input_data": []} + if self.dims == 1: + self.dynamic_shape.min_input_shape = {"input_data": [1]} + self.dynamic_shape.max_input_shape = {"input_data": [16]} + self.dynamic_shape.opt_input_shape = {"input_data": [8]} + elif self.dims == 2: + self.dynamic_shape.min_input_shape = {"input_data": [1, 8]} + self.dynamic_shape.max_input_shape = {"input_data": [4, 8]} + self.dynamic_shape.opt_input_shape = {"input_data": [2, 8]} + elif self.dims == 3: + self.dynamic_shape.min_input_shape = {"input_data": [1, 1, 4]} + self.dynamic_shape.max_input_shape = {"input_data": [4, 16, 16]} + self.dynamic_shape.opt_input_shape = {"input_data": [2, 8, 8]} + elif self.dims == 4: + self.dynamic_shape.min_input_shape = { + "input_data": [1, 8, 8, 8] + } + self.dynamic_shape.max_input_shape = { + "input_data": [4, 8, 8, 8] + } + self.dynamic_shape.opt_input_shape = { + "input_data": [4, 8, 8, 8] + } + + def clear_dynamic_shape(): + self.dynamic_shape.max_input_shape = {} + self.dynamic_shape.min_input_shape = {} + self.dynamic_shape.opt_input_shape = {} + + def generate_trt_nodes_num(attrs, dynamic_shape): + if not dynamic_shape and (self.dims == 1 or self.dims == 0): + return 0, 3 + return 1, 2 + + attrs = [ + program_config.ops[i].attrs for i in range(len(program_config.ops)) + ] + + # for static_shape + clear_dynamic_shape() + self.trt_param.precision = paddle_infer.PrecisionType.Float32 + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, False + ), (1e-5, 1e-5) + self.trt_param.precision = paddle_infer.PrecisionType.Half + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, False + ), (1e-3, 1e-3) + + # # for dynamic_shape + generate_dynamic_shape(attrs) + self.trt_param.precision = paddle_infer.PrecisionType.Float32 + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, True + ), (1e-5, 1e-5) + self.trt_param.precision = paddle_infer.PrecisionType.Half + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, True + ), (1e-3, 1e-3) + + def test(self): + self.run_test() + + if __name__ == "__main__": unittest.main() diff --git a/test/ir/inference/test_trt_convert_equal.py b/test/ir/inference/test_trt_convert_equal.py index 4993e830f190b..5879a003d9546 100644 --- a/test/ir/inference/test_trt_convert_equal.py +++ b/test/ir/inference/test_trt_convert_equal.py @@ -40,54 +40,64 @@ def generate_input(shape): return np.random.random(shape).astype(np.float32) for op_type in ["equal", "not_equal"]: - for batch in [1, 2, 4]: - for shape in [[batch, 1], [batch, 1, 32], [batch, 1, 16, 32]]: - for axis in [-1 if len(shape) == 1 else 1]: - self.dims = len(shape) - dics = [{"axis": axis}, {"in_dtype": 0, "out_dtype": 5}] - ops_config = [ - { - "op_type": op_type, - "op_inputs": { - "X": ["input_data1"], - "Y": ["input_data2"], - }, - "op_outputs": {"Out": ["compare_output_data"]}, - "op_attrs": dics[0], - "outputs_dtype": { - "compare_output_data": np.bool_ - }, + for shape in [[], [1, 1], [1, 1, 32], [1, 1, 16, 32]]: + for axis in [-1 if len(shape) == 1 or len(shape) == 0 else 1]: + self.dims = len(shape) + dics = [{"axis": axis}, {"in_dtype": 0, "out_dtype": 5}] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data1"], + "Y": ["input_data2"], }, - { - "op_type": "cast", - "op_inputs": {"X": ["compare_output_data"]}, - "op_outputs": {"Out": ["output_data"]}, - "op_attrs": dics[1], - "outputs_dtype": {"output_data": np.float32}, - }, - ] - ops = self.generate_op_config(ops_config) - - program_config = ProgramConfig( - ops=ops, - weights={}, - inputs={ - "input_data1": TensorConfig( - data_gen=partial(generate_input, shape) - ), - "input_data2": TensorConfig( - data_gen=partial(generate_input, shape) - ), - }, - outputs=["output_data"], - ) - yield program_config + "op_outputs": {"Out": ["compare_output_data"]}, + "op_attrs": dics[0], + "outputs_dtype": {"compare_output_data": np.bool_}, + }, + { + "op_type": "cast", + "op_inputs": {"X": ["compare_output_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[1], + "outputs_dtype": {"output_data": np.float32}, + }, + ] + ops = self.generate_op_config(ops_config) + + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data1": TensorConfig( + data_gen=partial(generate_input, shape) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, shape) + ), + }, + outputs=["output_data"], + ) + yield program_config def sample_predictor_configs( self, program_config ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. + if self.dims == 0: + self.dynamic_shape.min_input_shape = { + "input_data1": [], + "input_data2": [], + } + self.dynamic_shape.max_input_shape = { + "input_data1": [], + "input_data2": [], + } + self.dynamic_shape.opt_input_shape = { + "input_data1": [], + "input_data2": [], + } if self.dims == 2: self.dynamic_shape.min_input_shape = { "input_data1": [1, 1],