[TorchFX] Model transformer tests (#2920)

### Changes

Model transformation tests are presented

### Reason for changes

To cover TorchFX model transformations by tests

### Related tickets

#2775 

### Tests

* test_model_insertion_transformation
* test_constant_update_transformation
* test_constant_update_transformation_no_constant
* TestQDQInsertion
* test_node_removal_transformation

nncf/experimental/torch/fx/transformations.py

@@ -116,12 +116,13 @@ def leaf_module_insertion_transformation(model: torch.fx.GraphModule):
     return leaf_module_insertion_transformation
 
 
-def bias_update_transformation_builder(node: NNCFNode, value: torch.Tensor) -> TransformationFNType:
+def bias_update_transformation_builder(node: NNCFNode, value: torch.Tensor, input_port_id: int) -> TransformationFNType:
     """
     Return transformation which updates constant of the given node with bias to the given value.
 
     :param node: Node with bias which requires bias constant update.
     :param value: New value to use as the bias constant.
+    :param input_port_id: Input port id to get constant node from.
     :return: Transformation which updates constant of the given node with bias to the given value.
     """
 
@@ -137,7 +138,7 @@ def bias_update_transformation(model: torch.fx.GraphModule):
             raise nncf.InternalError(f"Node {graph_node.name} has {len(add_nodes)} outputs with adds, 1 expected")
 
         bias_node = add_nodes[0]
-        constant_update_fn(model, bias_node, value, input_port_id=1)
+        constant_update_fn(model, bias_node, value, input_port_id=input_port_id)
 
     return bias_update_transformation
 
@@ -228,7 +229,7 @@ def qdq_insertion_transformation_builder(
 
     def qdq_insertion_transformation(model: torch.fx.GraphModule):
         if any(tp.target_type != TargetType.OPERATION_WITH_WEIGHTS for tp in target_points) and len(target_points) > 1:
-            raise RuntimeError(
+            raise nncf.InternalError(
                 "Insertion of shared qdq pair for the weights is not supported."
                 " Please use non shared qdq pairs for the weights quantization."
             )

nncf/quantization/algorithms/bias_correction/torch_fx_backend.py

@@ -45,7 +45,7 @@ def target_point(target_type: TargetType, target_node_name: str, port_id: int) -
     def create_bias_correction_command(
         node: NNCFNode, bias_value: Tensor, nncf_graph: NNCFGraph
     ) -> FXApplyTransformationCommand:
-        return FXApplyTransformationCommand(bias_update_transformation_builder(node, bias_value.data))
+        return FXApplyTransformationCommand(bias_update_transformation_builder(node, bias_value.data, input_port_id=1))
 
     @staticmethod
     def model_extraction_command(

nncf/quantization/algorithms/fast_bias_correction/torch_fx_backend.py

@@ -41,7 +41,7 @@ def target_point(target_type: TargetType, target_node_name: str, port_id: int) -
     def create_bias_correction_command(
         node: NNCFNode, bias_value: Tensor, nncf_graph: NNCFGraph
     ) -> FXApplyTransformationCommand:
-        return FXApplyTransformationCommand(bias_update_transformation_builder(node, bias_value.data))
+        return FXApplyTransformationCommand(bias_update_transformation_builder(node, bias_value.data, input_port_id=1))
 
     @staticmethod
     def model_extraction_command(

nncf/quantization/algorithms/min_max/torch_fx_backend.py

@@ -12,6 +12,7 @@
 from typing import Dict, List, Optional, Set, Tuple
 
 import torch
+from torch.quantization.fake_quantize import FakeQuantize
 
 import nncf
 import nncf.torch.graph.operator_metatypes as om
@@ -240,7 +241,7 @@ def _create_quantizer(
         scale_shape: Tuple,
         parameters: FakeQuantizeParameters,
         target_type: TargetType,
-    ) -> BaseQuantizer:
+    ) -> FakeQuantize:
         mode = quantizer_config.mode
         quantizer_cls = QUANTIZATION_MODULES.get(mode)
         narrow_range = target_type == TargetType.OPERATION_WITH_WEIGHTS and mode == QuantizationMode.SYMMETRIC

nncf/quantization/algorithms/smooth_quant/torch_fx_backend.py

@@ -105,7 +105,10 @@ def get_weight_value(node_with_weight: NNCFNode, model: torch.fx.GraphModule, nn
 
     @staticmethod
     def weight_update_command(node_with_weight: NNCFNode, weight_value: torch.Tensor) -> OVWeightUpdateCommand:
-        return FXApplyTransformationCommand(constant_update_transformation_builder(node_with_weight, weight_value.data))
+        # TODO(dlyakhov): Use input port id depending on the node metatype/attributes.
+        return FXApplyTransformationCommand(
+            constant_update_transformation_builder(node_with_weight, weight_value.data, input_port_id=1)
+        )
 
     @staticmethod
     def scale_insertion_command(

tests/post_training/pipelines/image_classification_torchvision.py

@@ -92,7 +92,9 @@ def prepare_model(self) -> None:
 
         elif self.backend in [BackendType.OV, BackendType.FP32]:
             with torch.no_grad():
-                self.model = ov.convert_model(model, example_input=self.dummy_tensor, input=self.input_size)
+                with disable_patching():
+                    m = torch.export.export(model, args=(self.dummy_tensor,))
+                self.model = ov.convert_model(m, example_input=self.dummy_tensor, input=self.input_size)
             self.input_name = list(inp.get_any_name() for inp in self.model.inputs)[0]
 
         self._dump_model_fp32()

tests/torch/data/reference_graphs/fx/transformed/constant_update.dot

@@ -0,0 +1,38 @@
+strict digraph  {
+"0 arg0_1" [id=0, type=input];
+"1 _param_constant0" [id=1, type=get_attr];
+"2 _param_constant1" [id=2, type=get_attr];
+"3 conv2d" [id=3, type=conv2d];
+"4 _param_constant2" [id=4, type=get_attr];
+"5 _param_constant3" [id=5, type=get_attr];
+"6 conv2d_1" [id=6, type=conv2d];
+"7 add__updated_constant0" [id=7, type=get_attr];
+"8 add_" [id=8, type=add_];
+"9 _tensor_constant0_1" [id=9, type=get_attr];
+"10 add__1" [id=10, type=add_];
+"11 add" [id=11, type=add];
+"12 _param_constant4" [id=12, type=get_attr];
+"13 _param_constant5" [id=13, type=get_attr];
+"14 conv2d_2" [id=14, type=conv2d];
+"15 _tensor_constant0_2" [id=15, type=get_attr];
+"16 add_1" [id=16, type=add];
+"17 output" [id=17, type=output];
+"0 arg0_1" -> "3 conv2d"  [label="(1, 3, 3, 3)", style=solid];
+"1 _param_constant0" -> "3 conv2d"  [label="(3, 3, 1, 1)", style=solid];
+"2 _param_constant1" -> "3 conv2d"  [label="(3,)", style=solid];
+"3 conv2d" -> "6 conv2d_1"  [label="(1, 3, 3, 3)", style=solid];
+"3 conv2d" -> "8 add_"  [label="(1, 3, 3, 3)", style=solid];
+"4 _param_constant2" -> "6 conv2d_1"  [label="(3, 3, 1, 1)", style=solid];
+"5 _param_constant3" -> "6 conv2d_1"  [label="(3,)", style=solid];
+"6 conv2d_1" -> "10 add__1"  [label="(1, 3, 3, 3)", style=solid];
+"7 add__updated_constant0" -> "8 add_"  [label="(1,)", style=solid];
+"8 add_" -> "11 add"  [label="(1, 3, 3, 3)", style=solid];
+"9 _tensor_constant0_1" -> "10 add__1"  [label="(1,)", style=solid];
+"10 add__1" -> "11 add"  [label="(1, 3, 3, 3)", style=solid];
+"11 add" -> "14 conv2d_2"  [label="(1, 3, 3, 3)", style=solid];
+"12 _param_constant4" -> "14 conv2d_2"  [label="(3, 3, 1, 1)", style=solid];
+"13 _param_constant5" -> "14 conv2d_2"  [label="(3,)", style=solid];
+"14 conv2d_2" -> "16 add_1"  [label="(1, 3, 3, 3)", style=solid];
+"15 _tensor_constant0_2" -> "16 add_1"  [label="(1,)", style=solid];
+"16 add_1" -> "17 output"  [label="(1, 3, 3, 3)", style=solid];
+}

tests/torch/data/reference_graphs/fx/transformed/model_insertion.dot

@@ -0,0 +1,46 @@
+strict digraph  {
+"0 arg0_1" [id=0, type=input];
+"1 _param_constant0" [id=1, type=get_attr];
+"2 _param_constant1" [id=2, type=get_attr];
+"3 TEST_MODULE_0" [id=3, type=call_module];
+"4 TEST_MODULE_1" [id=4, type=call_module];
+"5 conv2d" [id=5, type=conv2d];
+"6 TEST_MODULE_3" [id=6, type=call_module];
+"7 _param_constant2" [id=7, type=get_attr];
+"8 _param_constant3" [id=8, type=get_attr];
+"9 TEST_MODULE_2" [id=9, type=call_module];
+"10 conv2d_1" [id=10, type=conv2d];
+"11 _tensor_constant0" [id=11, type=get_attr];
+"12 add_" [id=12, type=add_];
+"13 _tensor_constant0_1" [id=13, type=get_attr];
+"14 add__1" [id=14, type=add_];
+"15 add" [id=15, type=add];
+"16 _param_constant4" [id=16, type=get_attr];
+"17 _param_constant5" [id=17, type=get_attr];
+"18 conv2d_2" [id=18, type=conv2d];
+"19 _tensor_constant0_2" [id=19, type=get_attr];
+"20 add_1" [id=20, type=add];
+"21 output" [id=21, type=output];
+"0 arg0_1" -> "3 TEST_MODULE_0"  [label="(1, 3, 3, 3)", style=solid];
+"1 _param_constant0" -> "4 TEST_MODULE_1"  [label="(3, 3, 1, 1)", style=solid];
+"2 _param_constant1" -> "5 conv2d"  [label="(3,)", style=solid];
+"3 TEST_MODULE_0" -> "5 conv2d"  [label="(1, 3, 3, 3)", style=solid];
+"4 TEST_MODULE_1" -> "5 conv2d"  [label="(3, 3, 1, 1)", style=solid];
+"5 conv2d" -> "6 TEST_MODULE_3"  [label="(1, 3, 3, 3)", style=solid];
+"6 TEST_MODULE_3" -> "10 conv2d_1"  [label="(1, 3, 3, 3)", style=solid];
+"6 TEST_MODULE_3" -> "12 add_"  [label="(1, 3, 3, 3)", style=solid];
+"7 _param_constant2" -> "9 TEST_MODULE_2"  [label="(3, 3, 1, 1)", style=solid];
+"8 _param_constant3" -> "10 conv2d_1"  [label="(3,)", style=solid];
+"9 TEST_MODULE_2" -> "10 conv2d_1"  [label="(3, 3, 1, 1)", style=solid];
+"10 conv2d_1" -> "14 add__1"  [label="(1, 3, 3, 3)", style=solid];
+"11 _tensor_constant0" -> "12 add_"  [label="(1,)", style=solid];
+"12 add_" -> "15 add"  [label="(1, 3, 3, 3)", style=solid];
+"13 _tensor_constant0_1" -> "14 add__1"  [label="(1,)", style=solid];
+"14 add__1" -> "15 add"  [label="(1, 3, 3, 3)", style=solid];
+"15 add" -> "18 conv2d_2"  [label="(1, 3, 3, 3)", style=solid];
+"16 _param_constant4" -> "18 conv2d_2"  [label="(3, 3, 1, 1)", style=solid];
+"17 _param_constant5" -> "18 conv2d_2"  [label="(3,)", style=solid];
+"18 conv2d_2" -> "20 add_1"  [label="(1, 3, 3, 3)", style=solid];
+"19 _tensor_constant0_2" -> "20 add_1"  [label="(1,)", style=solid];
+"20 add_1" -> "21 output"  [label="(1, 3, 3, 3)", style=solid];
+}

tests/torch/data/reference_graphs/fx/transformed/model_insertion_leaf.dot

@@ -0,0 +1,46 @@
+strict digraph  {
+"0 arg0_1" [id=0, type=input];
+"1 _param_constant0" [id=1, type=get_attr];
+"2 _param_constant1" [id=2, type=get_attr];
+"3 TEST_MODULE_0" [id=3, type=call_module];
+"4 TEST_MODULE_1" [id=4, type=call_module];
+"5 conv2d" [id=5, type=conv2d];
+"6 TEST_MODULE_3" [id=6, type=call_module];
+"7 _param_constant2" [id=7, type=get_attr];
+"8 _param_constant3" [id=8, type=get_attr];
+"9 TEST_MODULE_2" [id=9, type=call_module];
+"10 conv2d_1" [id=10, type=conv2d];
+"11 _tensor_constant0" [id=11, type=get_attr];
+"12 add_" [id=12, type=add_];
+"13 _tensor_constant0_1" [id=13, type=get_attr];
+"14 add__1" [id=14, type=add_];
+"15 add" [id=15, type=add];
+"16 _param_constant4" [id=16, type=get_attr];
+"17 _param_constant5" [id=17, type=get_attr];
+"18 conv2d_2" [id=18, type=conv2d];
+"19 _tensor_constant0_2" [id=19, type=get_attr];
+"20 add_1" [id=20, type=add];
+"21 output" [id=21, type=output];
+"0 arg0_1" -> "3 TEST_MODULE_0"  [label="(1, 3, 3, 3)", style=solid];
+"0 arg0_1" -> "5 conv2d"  [label="(1, 3, 3, 3)", style=solid];
+"1 _param_constant0" -> "4 TEST_MODULE_1"  [label="(3, 3, 1, 1)", style=solid];
+"1 _param_constant0" -> "5 conv2d"  [label="(3, 3, 1, 1)", style=solid];
+"2 _param_constant1" -> "5 conv2d"  [label="(3,)", style=solid];
+"5 conv2d" -> "6 TEST_MODULE_3"  [label="(1, 3, 3, 3)", style=solid];
+"5 conv2d" -> "10 conv2d_1"  [label="(1, 3, 3, 3)", style=solid];
+"5 conv2d" -> "12 add_"  [label="(1, 3, 3, 3)", style=solid];
+"7 _param_constant2" -> "9 TEST_MODULE_2"  [label="(3, 3, 1, 1)", style=solid];
+"7 _param_constant2" -> "10 conv2d_1"  [label="(3, 3, 1, 1)", style=solid];
+"8 _param_constant3" -> "10 conv2d_1"  [label="(3,)", style=solid];
+"10 conv2d_1" -> "14 add__1"  [label="(1, 3, 3, 3)", style=solid];
+"11 _tensor_constant0" -> "12 add_"  [label="(1,)", style=solid];
+"12 add_" -> "15 add"  [label="(1, 3, 3, 3)", style=solid];
+"13 _tensor_constant0_1" -> "14 add__1"  [label="(1,)", style=solid];
+"14 add__1" -> "15 add"  [label="(1, 3, 3, 3)", style=solid];
+"15 add" -> "18 conv2d_2"  [label="(1, 3, 3, 3)", style=solid];
+"16 _param_constant4" -> "18 conv2d_2"  [label="(3, 3, 1, 1)", style=solid];
+"17 _param_constant5" -> "18 conv2d_2"  [label="(3,)", style=solid];
+"18 conv2d_2" -> "20 add_1"  [label="(1, 3, 3, 3)", style=solid];
+"19 _tensor_constant0_2" -> "20 add_1"  [label="(1,)", style=solid];
+"20 add_1" -> "21 output"  [label="(1, 3, 3, 3)", style=solid];
+}

tests/torch/data/reference_graphs/fx/transformed/node_removal_ref.dot

@@ -0,0 +1,32 @@
+strict digraph  {
+"0 arg0_1" [id=0, type=input];
+"1 _param_constant2" [id=1, type=get_attr];
+"2 _param_constant3" [id=2, type=get_attr];
+"3 conv2d_1" [id=3, type=conv2d];
+"4 _tensor_constant0" [id=4, type=get_attr];
+"5 add_" [id=5, type=add_];
+"6 _tensor_constant0_1" [id=6, type=get_attr];
+"7 add__1" [id=7, type=add_];
+"8 add" [id=8, type=add];
+"9 _param_constant4" [id=9, type=get_attr];
+"10 _param_constant5" [id=10, type=get_attr];
+"11 conv2d_2" [id=11, type=conv2d];
+"12 _tensor_constant0_2" [id=12, type=get_attr];
+"13 add_1" [id=13, type=add];
+"14 output" [id=14, type=output];
+"0 arg0_1" -> "3 conv2d_1"  [label="(1, 3, 3, 3)", style=solid];
+"0 arg0_1" -> "5 add_"  [label="(1, 3, 3, 3)", style=solid];
+"1 _param_constant2" -> "3 conv2d_1"  [label="(3, 3, 1, 1)", style=solid];
+"2 _param_constant3" -> "3 conv2d_1"  [label="(3,)", style=solid];
+"3 conv2d_1" -> "7 add__1"  [label="(1, 3, 3, 3)", style=solid];
+"4 _tensor_constant0" -> "5 add_"  [label="(1,)", style=solid];
+"5 add_" -> "8 add"  [label="(1, 3, 3, 3)", style=solid];
+"6 _tensor_constant0_1" -> "7 add__1"  [label="(1,)", style=solid];
+"7 add__1" -> "8 add"  [label="(1, 3, 3, 3)", style=solid];
+"8 add" -> "11 conv2d_2"  [label="(1, 3, 3, 3)", style=solid];
+"9 _param_constant4" -> "11 conv2d_2"  [label="(3, 3, 1, 1)", style=solid];
+"10 _param_constant5" -> "11 conv2d_2"  [label="(3,)", style=solid];
+"11 conv2d_2" -> "13 add_1"  [label="(1, 3, 3, 3)", style=solid];
+"12 _tensor_constant0_2" -> "13 add_1"  [label="(1,)", style=solid];
+"13 add_1" -> "14 output"  [label="(1, 3, 3, 3)", style=solid];
+}

tests/torch/data/reference_graphs/fx/transformed/qdq_insert_conv2d_1_5_1_per_channel.dot

@@ -0,0 +1,48 @@
+strict digraph  {
+"0 arg0_1" [id=0, type=input];
+"1 _param_constant0" [id=1, type=get_attr];
+"2 _param_constant1" [id=2, type=get_attr];
+"3 conv2d" [id=3, type=conv2d];
+"4 _param_constant2" [id=4, type=get_attr];
+"5 _param_constant3" [id=5, type=get_attr];
+"6 conv2d_1_scale_0" [id=6, type=get_attr];
+"7 conv2d_1_zero_point_0" [id=7, type=get_attr];
+"8 quantize_per_channel_default" [id=8, type=quantize_per_channel];
+"9 dequantize_per_channel_default" [id=9, type=dequantize_per_channel];
+"10 conv2d_1" [id=10, type=conv2d];
+"11 _tensor_constant0" [id=11, type=get_attr];
+"12 add_" [id=12, type=add_];
+"13 _tensor_constant0_1" [id=13, type=get_attr];
+"14 add__1" [id=14, type=add_];
+"15 add" [id=15, type=add];
+"16 _param_constant4" [id=16, type=get_attr];
+"17 _param_constant5" [id=17, type=get_attr];
+"18 conv2d_2" [id=18, type=conv2d];
+"19 _tensor_constant0_2" [id=19, type=get_attr];
+"20 add_1" [id=20, type=add];
+"21 output" [id=21, type=output];
+"0 arg0_1" -> "3 conv2d"  [label="(1, 3, 3, 3)", style=solid];
+"1 _param_constant0" -> "3 conv2d"  [label="(3, 3, 1, 1)", style=solid];
+"2 _param_constant1" -> "3 conv2d"  [label="(3,)", style=solid];
+"3 conv2d" -> "10 conv2d_1"  [label="(1, 3, 3, 3)", style=solid];
+"3 conv2d" -> "12 add_"  [label="(1, 3, 3, 3)", style=solid];
+"4 _param_constant2" -> "8 quantize_per_channel_default"  [label="(3, 3, 1, 1)", style=solid];
+"5 _param_constant3" -> "10 conv2d_1"  [label="(3,)", style=solid];
+"6 conv2d_1_scale_0" -> "8 quantize_per_channel_default"  [label="(1,)", style=solid];
+"6 conv2d_1_scale_0" -> "9 dequantize_per_channel_default"  [label="(1,)", style=solid];
+"7 conv2d_1_zero_point_0" -> "8 quantize_per_channel_default"  [label="(1,)", style=solid];
+"7 conv2d_1_zero_point_0" -> "9 dequantize_per_channel_default"  [label="(1,)", style=solid];
+"8 quantize_per_channel_default" -> "9 dequantize_per_channel_default"  [label="(3, 3, 1, 1)", style=solid];
+"9 dequantize_per_channel_default" -> "10 conv2d_1"  [label="(3, 3, 1, 1)", style=solid];
+"10 conv2d_1" -> "14 add__1"  [label="(1, 3, 3, 3)", style=solid];
+"11 _tensor_constant0" -> "12 add_"  [label="(1,)", style=solid];
+"12 add_" -> "15 add"  [label="(1, 3, 3, 3)", style=solid];
+"13 _tensor_constant0_1" -> "14 add__1"  [label="(1,)", style=solid];
+"14 add__1" -> "15 add"  [label="(1, 3, 3, 3)", style=solid];
+"15 add" -> "18 conv2d_2"  [label="(1, 3, 3, 3)", style=solid];
+"16 _param_constant4" -> "18 conv2d_2"  [label="(3, 3, 1, 1)", style=solid];
+"17 _param_constant5" -> "18 conv2d_2"  [label="(3,)", style=solid];
+"18 conv2d_2" -> "20 add_1"  [label="(1, 3, 3, 3)", style=solid];
+"19 _tensor_constant0_2" -> "20 add_1"  [label="(1,)", style=solid];
+"20 add_1" -> "21 output"  [label="(1, 3, 3, 3)", style=solid];
+}

tests/torch/data/reference_graphs/fx/transformed/qdq_insert_conv2d_1_5_1_per_tensor.dot

@@ -0,0 +1,42 @@
+strict digraph  {
+"0 arg0_1" [id=0, type=input];
+"1 _param_constant0" [id=1, type=get_attr];
+"2 _param_constant1" [id=2, type=get_attr];
+"3 conv2d" [id=3, type=conv2d];
+"4 _param_constant2" [id=4, type=get_attr];
+"5 _param_constant3" [id=5, type=get_attr];
+"6 quantize_per_tensor_default" [id=6, type=quantize_per_tensor];
+"7 dequantize_per_tensor_default" [id=7, type=dequantize_per_tensor];
+"8 conv2d_1" [id=8, type=conv2d];
+"9 _tensor_constant0" [id=9, type=get_attr];
+"10 add_" [id=10, type=add_];
+"11 _tensor_constant0_1" [id=11, type=get_attr];
+"12 add__1" [id=12, type=add_];
+"13 add" [id=13, type=add];
+"14 _param_constant4" [id=14, type=get_attr];
+"15 _param_constant5" [id=15, type=get_attr];
+"16 conv2d_2" [id=16, type=conv2d];
+"17 _tensor_constant0_2" [id=17, type=get_attr];
+"18 add_1" [id=18, type=add];
+"19 output" [id=19, type=output];
+"0 arg0_1" -> "3 conv2d"  [label="(1, 3, 3, 3)", style=solid];
+"1 _param_constant0" -> "3 conv2d"  [label="(3, 3, 1, 1)", style=solid];
+"2 _param_constant1" -> "3 conv2d"  [label="(3,)", style=solid];
+"3 conv2d" -> "8 conv2d_1"  [label="(1, 3, 3, 3)", style=solid];
+"3 conv2d" -> "10 add_"  [label="(1, 3, 3, 3)", style=solid];
+"4 _param_constant2" -> "6 quantize_per_tensor_default"  [label="(3, 3, 1, 1)", style=solid];
+"5 _param_constant3" -> "8 conv2d_1"  [label="(3,)", style=solid];
+"6 quantize_per_tensor_default" -> "7 dequantize_per_tensor_default"  [label="(3, 3, 1, 1)", style=solid];
+"7 dequantize_per_tensor_default" -> "8 conv2d_1"  [label="(3, 3, 1, 1)", style=solid];
+"8 conv2d_1" -> "12 add__1"  [label="(1, 3, 3, 3)", style=solid];
+"9 _tensor_constant0" -> "10 add_"  [label="(1,)", style=solid];
+"10 add_" -> "13 add"  [label="(1, 3, 3, 3)", style=solid];
+"11 _tensor_constant0_1" -> "12 add__1"  [label="(1,)", style=solid];
+"12 add__1" -> "13 add"  [label="(1, 3, 3, 3)", style=solid];
+"13 add" -> "16 conv2d_2"  [label="(1, 3, 3, 3)", style=solid];
+"14 _param_constant4" -> "16 conv2d_2"  [label="(3, 3, 1, 1)", style=solid];
+"15 _param_constant5" -> "16 conv2d_2"  [label="(3,)", style=solid];
+"16 conv2d_2" -> "18 add_1"  [label="(1, 3, 3, 3)", style=solid];
+"17 _tensor_constant0_2" -> "18 add_1"  [label="(1,)", style=solid];
+"18 add_1" -> "19 output"  [label="(1, 3, 3, 3)", style=solid];
+}