From 0b3a0bfa38dfbc832b3451d30d234a994a2a7d06 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Fri, 18 Dec 2020 14:47:02 +0300 Subject: [PATCH 01/17] move crop extractor --- model-optimizer/automation/package_BOM.txt | 2 +- .../extractors/crop.py => extensions/front/caffe/crop_ext.py} | 0 .../crop_test.py => extensions/front/caffe/crop_ext_test.py} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename model-optimizer/{mo/front/caffe/extractors/crop.py => extensions/front/caffe/crop_ext.py} (100%) rename model-optimizer/{mo/front/caffe/extractors/crop_test.py => extensions/front/caffe/crop_ext_test.py} (96%) diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index e4080d168e1274..b164165ee01187 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -76,6 +76,7 @@ extensions/front/caffe/binary_conv_ext.py extensions/front/caffe/bn.py extensions/front/caffe/conv_ext.py extensions/front/caffe/correlation_ext.py +extensions/front/caffe/crop_ext.py extensions/front/caffe/ctcgreedydecoder_ext.py extensions/front/caffe/CustomLayersMapping.xml.example extensions/front/caffe/data_augmentation_ext.py @@ -759,7 +760,6 @@ mo/front/caffe/extractor.py mo/front/caffe/extractors/__init__.py mo/front/caffe/extractors/batchnorm.py mo/front/caffe/extractors/concat.py -mo/front/caffe/extractors/crop.py mo/front/caffe/extractors/native_caffe.py mo/front/caffe/extractors/roipooling.py mo/front/caffe/extractors/scale.py diff --git a/model-optimizer/mo/front/caffe/extractors/crop.py b/model-optimizer/extensions/front/caffe/crop_ext.py similarity index 100% rename from model-optimizer/mo/front/caffe/extractors/crop.py rename to model-optimizer/extensions/front/caffe/crop_ext.py diff --git a/model-optimizer/mo/front/caffe/extractors/crop_test.py b/model-optimizer/extensions/front/caffe/crop_ext_test.py similarity index 96% rename from model-optimizer/mo/front/caffe/extractors/crop_test.py rename to model-optimizer/extensions/front/caffe/crop_ext_test.py index 6476aabac3cb28..bfbd7435aa6d77 100644 --- a/model-optimizer/mo/front/caffe/extractors/crop_test.py +++ b/model-optimizer/extensions/front/caffe/crop_ext_test.py @@ -17,7 +17,7 @@ import unittest from unittest.mock import patch -from mo.front.caffe.extractors.crop import CropFrontExtractor +from extensions.front.caffe.crop_ext import CropFrontExtractor from mo.front.common.partial_infer.crop import crop_infer from mo.ops.crop import Crop from mo.ops.op import Op From c2c745f084dfcf8de03ef68ede0d9c38ba639b9b Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Mon, 21 Dec 2020 11:38:33 +0300 Subject: [PATCH 02/17] Add concat_ext.py --- model-optimizer/automation/package_BOM.txt | 2 +- .../front/caffe/concat_ext.py} | 23 ++++++++---- model-optimizer/mo/front/caffe/extractor.py | 4 -- .../mo/front/caffe/extractors/concat_test.py | 37 ------------------- 4 files changed, 16 insertions(+), 50 deletions(-) rename model-optimizer/{mo/front/caffe/extractors/concat.py => extensions/front/caffe/concat_ext.py} (57%) delete mode 100644 model-optimizer/mo/front/caffe/extractors/concat_test.py diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index b164165ee01187..aa40f1eb8eef63 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -74,6 +74,7 @@ extensions/front/caffe/axpy.py extensions/front/caffe/binarization.py extensions/front/caffe/binary_conv_ext.py extensions/front/caffe/bn.py +extensions/front/caffe/concat_ext.py extensions/front/caffe/conv_ext.py extensions/front/caffe/correlation_ext.py extensions/front/caffe/crop_ext.py @@ -759,7 +760,6 @@ mo/front/caffe/custom_layers_mapping.py mo/front/caffe/extractor.py mo/front/caffe/extractors/__init__.py mo/front/caffe/extractors/batchnorm.py -mo/front/caffe/extractors/concat.py mo/front/caffe/extractors/native_caffe.py mo/front/caffe/extractors/roipooling.py mo/front/caffe/extractors/scale.py diff --git a/model-optimizer/mo/front/caffe/extractors/concat.py b/model-optimizer/extensions/front/caffe/concat_ext.py similarity index 57% rename from model-optimizer/mo/front/caffe/extractors/concat.py rename to model-optimizer/extensions/front/caffe/concat_ext.py index 1c7afba15e77e0..f6434b2208d5f6 100644 --- a/model-optimizer/mo/front/caffe/extractors/concat.py +++ b/model-optimizer/extensions/front/caffe/concat_ext.py @@ -14,12 +14,19 @@ limitations under the License. """ -from mo.front.common.partial_infer.concat import concat_infer +from mo.front.onnx.extractors.utils import onnx_attr +from mo.front.extractor import FrontExtractorOp +from mo.ops.concat import Concat + +class ConcatFrontExtractor(FrontExtractorOp): + op = 'concat' + enabled = True - -def concat_ext(pb_layer, pb_model): - return { - 'type': "Concat", - 'axis': pb_layer.concat_param.axis, - 'infer': concat_infer - } + @classmethod + def extract(cls, node): + pb = node.pb + mapping_rule = { + 'axis': pb.concat_param.axis, + } + Concat.update_node_stat(node, mapping_rule) + return cls.enabled diff --git a/model-optimizer/mo/front/caffe/extractor.py b/model-optimizer/mo/front/caffe/extractor.py index d66dac26388de6..eb59473a29eb98 100644 --- a/model-optimizer/mo/front/caffe/extractor.py +++ b/model-optimizer/mo/front/caffe/extractor.py @@ -15,7 +15,6 @@ """ from mo.front.caffe.extractors.batchnorm import batch_norm_ext -from mo.front.caffe.extractors.concat import concat_ext from mo.front.caffe.extractors.native_caffe import native_caffe_node_extractor from mo.front.caffe.extractors.roipooling import roipooling_ext from mo.front.caffe.extractors.scale import scale_ext @@ -46,9 +45,6 @@ def node_pb_arg(pb_extractor): # Activation Layers 'scale': node_pb_arg(scale_ext), - # Utility Layers - 'concat': node_pb_arg(concat_ext), - # Custom, implemented in IE, Fast-RCNN-specific 'roipooling': node_pb_arg(roipooling_ext), } diff --git a/model-optimizer/mo/front/caffe/extractors/concat_test.py b/model-optimizer/mo/front/caffe/extractors/concat_test.py deleted file mode 100644 index 1840c029740455..00000000000000 --- a/model-optimizer/mo/front/caffe/extractors/concat_test.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - Copyright (C) 2018-2020 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -""" - -import unittest - -from mo.front.caffe.extractors.concat import concat_ext -from mo.front.common.partial_infer.concat import concat_infer -from mo.utils.unittest.extractors import FakeParam - - -class FakeProtoLayer: - def __init__(self, axis): - self.concat_param = FakeParam('axis', axis) - - -class TestConcat(unittest.TestCase): - def test_concat(self): - res = concat_ext(FakeProtoLayer(10), None) - exp_res = { - 'axis': 10, - 'infer': concat_infer, - 'type': 'Concat' - } - self.assertEqual(res, exp_res) From b673c70cc2cb97ee7b64a898a3fea5440fc3da80 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Tue, 22 Dec 2020 11:51:38 +0300 Subject: [PATCH 03/17] Add roipooling_ext.py --- model-optimizer/automation/package_BOM.txt | 2 +- .../extensions/front/caffe/concat_ext.py | 3 +- .../front/caffe/roipooling_ext.py} | 28 ++++++++++++------- model-optimizer/mo/front/caffe/extractor.py | 4 --- 4 files changed, 21 insertions(+), 16 deletions(-) rename model-optimizer/{mo/front/caffe/extractors/roipooling.py => extensions/front/caffe/roipooling_ext.py} (52%) diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index aa40f1eb8eef63..130db925a5314b 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -108,6 +108,7 @@ extensions/front/caffe/relu_ext.py extensions/front/caffe/reorgyolo_ext.py extensions/front/caffe/resample_ext.py extensions/front/caffe/reshape.py +extensions/front/caffe/roipooling_ext.py extensions/front/caffe/shufflechannel_ext.py extensions/front/caffe/sigmoid.py extensions/front/caffe/simplernms_ext.py @@ -761,7 +762,6 @@ mo/front/caffe/extractor.py mo/front/caffe/extractors/__init__.py mo/front/caffe/extractors/batchnorm.py mo/front/caffe/extractors/native_caffe.py -mo/front/caffe/extractors/roipooling.py mo/front/caffe/extractors/scale.py mo/front/caffe/extractors/tile.py mo/front/caffe/extractors/utils.py diff --git a/model-optimizer/extensions/front/caffe/concat_ext.py b/model-optimizer/extensions/front/caffe/concat_ext.py index f6434b2208d5f6..3dd5742587d537 100644 --- a/model-optimizer/extensions/front/caffe/concat_ext.py +++ b/model-optimizer/extensions/front/caffe/concat_ext.py @@ -17,7 +17,8 @@ from mo.front.onnx.extractors.utils import onnx_attr from mo.front.extractor import FrontExtractorOp from mo.ops.concat import Concat - + + class ConcatFrontExtractor(FrontExtractorOp): op = 'concat' enabled = True diff --git a/model-optimizer/mo/front/caffe/extractors/roipooling.py b/model-optimizer/extensions/front/caffe/roipooling_ext.py similarity index 52% rename from model-optimizer/mo/front/caffe/extractors/roipooling.py rename to model-optimizer/extensions/front/caffe/roipooling_ext.py index fd7fc4edfedd18..7cebab3d5a10b3 100644 --- a/model-optimizer/mo/front/caffe/extractors/roipooling.py +++ b/model-optimizer/extensions/front/caffe/roipooling_ext.py @@ -14,15 +14,23 @@ limitations under the License. """ -from mo.front.common.partial_infer.roipooling import roipooling_infer +from mo.front.ops.roipooling import ROIPooling +from mo.front.extractor import FrontExtractorOp + + +class ROIPoolingFrontExtractor(FrontExtractorOp): + op = 'roipooling' + enabled = True -def roipooling_ext(proto_layer, model_layer): - param = proto_layer.roi_pooling_param - return { - 'type': 'ROIPooling', - 'pooled_h': param.pooled_h, - 'pooled_w': param.pooled_w, - 'spatial_scale': param.spatial_scale, - 'infer': roipooling_infer - } + @classmethod + def extract(cls, node): + param = node.pb.roi_pooling_param + attrs = { + 'pooled_h': param.pooled_h, + 'pooled_w': param.pooled_w, + 'spatial_scale': param.spatial_scale, + } + + ROIPooling.update_node_stat(node, attrs) + return cls.enabled diff --git a/model-optimizer/mo/front/caffe/extractor.py b/model-optimizer/mo/front/caffe/extractor.py index eb59473a29eb98..72f0c2183868ba 100644 --- a/model-optimizer/mo/front/caffe/extractor.py +++ b/model-optimizer/mo/front/caffe/extractor.py @@ -16,7 +16,6 @@ from mo.front.caffe.extractors.batchnorm import batch_norm_ext from mo.front.caffe.extractors.native_caffe import native_caffe_node_extractor -from mo.front.caffe.extractors.roipooling import roipooling_ext from mo.front.caffe.extractors.scale import scale_ext from mo.front.common.partial_infer.elemental import copy_shape_infer from mo.front.common.register_custom_ops import extension_op_extractor @@ -44,9 +43,6 @@ def node_pb_arg(pb_extractor): # Activation Layers 'scale': node_pb_arg(scale_ext), - - # Custom, implemented in IE, Fast-RCNN-specific - 'roipooling': node_pb_arg(roipooling_ext), } From 145c7cf86e2ca1420b9dc576bf992d023ac2c523 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Tue, 22 Dec 2020 12:05:07 +0300 Subject: [PATCH 04/17] Add roipooling_ext --- model-optimizer/extensions/front/caffe/roipooling_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/extensions/front/caffe/roipooling_ext.py b/model-optimizer/extensions/front/caffe/roipooling_ext.py index 7cebab3d5a10b3..1a5dc61f324726 100644 --- a/model-optimizer/extensions/front/caffe/roipooling_ext.py +++ b/model-optimizer/extensions/front/caffe/roipooling_ext.py @@ -14,7 +14,7 @@ limitations under the License. """ -from mo.front.ops.roipooling import ROIPooling +from mo.ops.roipooling import ROIPooling from mo.front.extractor import FrontExtractorOp From 93bed546c82e36b7d728dacc2415bb2cefe51042 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Tue, 22 Dec 2020 22:26:19 +0300 Subject: [PATCH 05/17] Add scale extractor --- model-optimizer/automation/package_BOM.txt | 2 +- .../extensions/front/caffe/scale_ext.py | 54 +++++++ model-optimizer/mo/front/caffe/extractor.py | 4 - .../mo/front/caffe/extractors/scale.py | 47 ------ .../mo/front/caffe/extractors/scale_test.py | 144 ------------------ 5 files changed, 55 insertions(+), 196 deletions(-) create mode 100644 model-optimizer/extensions/front/caffe/scale_ext.py delete mode 100644 model-optimizer/mo/front/caffe/extractors/scale.py delete mode 100644 model-optimizer/mo/front/caffe/extractors/scale_test.py diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 130db925a5314b..e6ec2e87e07bfd 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -109,6 +109,7 @@ extensions/front/caffe/reorgyolo_ext.py extensions/front/caffe/resample_ext.py extensions/front/caffe/reshape.py extensions/front/caffe/roipooling_ext.py +extensions/front/caffe/scale_ext.py extensions/front/caffe/shufflechannel_ext.py extensions/front/caffe/sigmoid.py extensions/front/caffe/simplernms_ext.py @@ -762,7 +763,6 @@ mo/front/caffe/extractor.py mo/front/caffe/extractors/__init__.py mo/front/caffe/extractors/batchnorm.py mo/front/caffe/extractors/native_caffe.py -mo/front/caffe/extractors/scale.py mo/front/caffe/extractors/tile.py mo/front/caffe/extractors/utils.py mo/front/caffe/loader.py diff --git a/model-optimizer/extensions/front/caffe/scale_ext.py b/model-optimizer/extensions/front/caffe/scale_ext.py new file mode 100644 index 00000000000000..e799100d9a3c2e --- /dev/null +++ b/model-optimizer/extensions/front/caffe/scale_ext.py @@ -0,0 +1,54 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import numpy as np + +from mo.front.caffe.extractors.utils import embed_input, weights_biases +from mo.front.common.partial_infer.elemental import copy_shape_infer +from mo.utils.utils import NamedAttrsClass +from mo.ops.scale_shift import ScaleShiftOp + + +class ScaleFrontExtractor(FrontExtractorOp): + op = 'scale' + enabled = True + + @classmethod + def extract(cls, node): + pb = node.pb + model = node.model_pb + param = pb.scale_param + attrs = { + 'axis': param.axis, + } + + if model is None and len(pb.bottom) == 1: + # default weights and biases for scale layer if the caffemodel file doesn't contain them + model = NamedAttrsClass({'blobs': np.array([NamedAttrsClass({'data': np.array([1])}), + NamedAttrsClass({'data': np.array([0])})])}) + # scale with 1 input and 1 or 2 blobs + if model and len(model.blobs) != 0 and len(pb.bottom) == 1: + attrs.update(weights_biases(param.bias_term, model)) + # 2 inputs + bias + elif len(pb.bottom) == 2 and param.bias_term: + if model is None or len(model.blobs) == 0: + # default bias for scale layer with 2 inputs if the caffemodel file doesn't contain them + model = NamedAttrsClass({'blobs': np.array([NamedAttrsClass({'data': np.array([0])})])}) + + embed_input(attrs, 1, 'biases', model.blobs[0].data) + ScaleShiftOp.update_node_stat(node, attrs) + return cls.enabled + diff --git a/model-optimizer/mo/front/caffe/extractor.py b/model-optimizer/mo/front/caffe/extractor.py index 72f0c2183868ba..4da78b70140907 100644 --- a/model-optimizer/mo/front/caffe/extractor.py +++ b/model-optimizer/mo/front/caffe/extractor.py @@ -16,7 +16,6 @@ from mo.front.caffe.extractors.batchnorm import batch_norm_ext from mo.front.caffe.extractors.native_caffe import native_caffe_node_extractor -from mo.front.caffe.extractors.scale import scale_ext from mo.front.common.partial_infer.elemental import copy_shape_infer from mo.front.common.register_custom_ops import extension_op_extractor from mo.front.extractor import CaffePythonFrontExtractorOp @@ -40,9 +39,6 @@ def node_pb_arg(pb_extractor): # Normalization Layers 'batchnorm': node_pb_arg(batch_norm_ext), - - # Activation Layers - 'scale': node_pb_arg(scale_ext), } diff --git a/model-optimizer/mo/front/caffe/extractors/scale.py b/model-optimizer/mo/front/caffe/extractors/scale.py deleted file mode 100644 index 59a2efd3a4b8b7..00000000000000 --- a/model-optimizer/mo/front/caffe/extractors/scale.py +++ /dev/null @@ -1,47 +0,0 @@ -""" - Copyright (C) 2018-2020 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -""" - -import numpy as np - -from mo.front.caffe.extractors.utils import embed_input, weights_biases -from mo.front.common.partial_infer.elemental import copy_shape_infer -from mo.utils.utils import NamedAttrsClass - - -def scale_ext(pl, ml): - param = pl.scale_param - attrs = { - 'op': 'ScaleShift', - 'type': 'ScaleShift', - 'axis': param.axis, - 'infer': copy_shape_infer - } - if ml is None and len(pl.bottom) == 1: - # default weights and biases for scale layer if the caffemodel file doesn't contain them - ml = NamedAttrsClass({'blobs': np.array([NamedAttrsClass({'data': np.array([1])}), - NamedAttrsClass({'data': np.array([0])})])}) - # scale with 1 input and 1 or 2 blobs - if ml and len(ml.blobs) != 0 and len(pl.bottom) == 1: - attrs.update(weights_biases(param.bias_term, ml)) - # 2 inputs + bias - elif len(pl.bottom) == 2 and param.bias_term: - if ml is None or len(ml.blobs) == 0: - # default bias for scale layer with 2 inputs if the caffemodel file doesn't contain them - ml = NamedAttrsClass({'blobs': np.array([NamedAttrsClass({'data': np.array([0])})])}) - - embed_input(attrs, 1, 'biases', ml.blobs[0].data) - - return attrs diff --git a/model-optimizer/mo/front/caffe/extractors/scale_test.py b/model-optimizer/mo/front/caffe/extractors/scale_test.py deleted file mode 100644 index f1c1baf84c69c9..00000000000000 --- a/model-optimizer/mo/front/caffe/extractors/scale_test.py +++ /dev/null @@ -1,144 +0,0 @@ -""" - Copyright (C) 2018-2020 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -""" - -import unittest - -import numpy as np - -from mo.front.caffe.extractors.scale import scale_ext -from mo.front.common.partial_infer.elemental import copy_shape_infer -from mo.utils.unittest.extractors import FakeMultiParam, FakeModelLayer - - -class FakeProtoLayer: - def __init__(self, val, bottom2=False): - self.scale_param = val - if bottom2: - self.bottom = {"bottom1", "bottom2"} - else: - self.bottom = {"bottom1"} - - -class TestScale(unittest.TestCase): - def test_scale_ext(self): - mean_blob = np.array([1., 2.]) - variance_blob = np.array([3., 4.]) - blobs = [mean_blob, variance_blob] - params = { - 'type': 'Scale', - 'axis': 0, - 'bias_term': True - } - - res = scale_ext(FakeProtoLayer(FakeMultiParam(params)), FakeModelLayer(blobs)) - exp_res = { - 'op': 'ScaleShift', - 'type': 'ScaleShift', - 'axis': 0, - 'infer': copy_shape_infer, - 'weights': mean_blob, - 'biases': variance_blob, - 'embedded_inputs': [ - (1, 'weights', { - 'bin': 'weights' - }), - (2, 'biases', { - 'bin': 'biases' - }) - ] - } - for i in exp_res: - if i in ('weights', 'biases'): - np.testing.assert_array_equal(res[i], exp_res[i]) - else: - self.assertEqual(res[i], exp_res[i]) - - def test_scale_2inputs_ext(self): - params = { - 'type': 'Scale', - 'axis': 0, - 'bias_term': False - } - - res = scale_ext(FakeProtoLayer(FakeMultiParam(params), True), None) - exp_res = { - 'op': 'ScaleShift', - 'type': 'ScaleShift', - 'axis': 0, - 'infer': copy_shape_infer, - } - for i in exp_res: - self.assertEqual(res[i], exp_res[i]) - - def test_scale_2inputs_bias_ext(self): - variance_blob = np.array([3., 4.]) - blobs = [variance_blob] - - params = { - 'type': 'Scale', - 'axis': 0, - 'bias_term': True - } - - res = scale_ext(FakeProtoLayer(FakeMultiParam(params), True), FakeModelLayer(blobs)) - exp_res = { - 'op': 'ScaleShift', - 'type': 'ScaleShift', - 'axis': 0, - 'infer': copy_shape_infer, - 'biases': variance_blob, - 'embedded_inputs': [ - (1, 'biases', { - 'bin': 'biases' - })] - } - for i in exp_res: - if i in ('biases'): - np.testing.assert_array_equal(res[i], exp_res[i]) - else: - self.assertEqual(res[i], exp_res[i]) - - def test_create_default_weights(self): - """ - There are situations when scale layer doesn't have weights and biases. This test checks that if they are not - available in the caffemodel file then default values [1] and [0] are generated. - """ - scale_blob = np.array([1]) - bias_blob = np.array([0]) - params = { - 'type': 'Scale', - 'axis': 0, - 'bias_term': True - } - - res = scale_ext(FakeProtoLayer(FakeMultiParam(params)), None) - exp_res = { - 'op': 'ScaleShift', - 'type': 'ScaleShift', - 'axis': 0, - 'infer': copy_shape_infer, - 'weights': scale_blob, - 'biases': bias_blob, - 'embedded_inputs': [ - (1, 'weights', { - 'bin': 'weights' - }), - (2, 'biases', { - 'bin': 'biases' - }) - ] - } - self.assertDictEqual(exp_res, res) From 655c616bfd92a1ef44f4066af431ed99fd148cac Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Tue, 22 Dec 2020 22:47:54 +0300 Subject: [PATCH 06/17] Add scale extractor --- model-optimizer/extensions/front/caffe/scale_ext.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model-optimizer/extensions/front/caffe/scale_ext.py b/model-optimizer/extensions/front/caffe/scale_ext.py index e799100d9a3c2e..20738867eee6fe 100644 --- a/model-optimizer/extensions/front/caffe/scale_ext.py +++ b/model-optimizer/extensions/front/caffe/scale_ext.py @@ -17,6 +17,7 @@ import numpy as np from mo.front.caffe.extractors.utils import embed_input, weights_biases +from mo.front.extractor import FrontExtractorOp from mo.front.common.partial_infer.elemental import copy_shape_infer from mo.utils.utils import NamedAttrsClass from mo.ops.scale_shift import ScaleShiftOp From cca70ea98f93d341a522371dbde7da3c80a96e74 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Fri, 25 Dec 2020 01:14:26 +0300 Subject: [PATCH 07/17] Add bn_ext.py and dropout_ext.py --- model-optimizer/automation/package_BOM.txt | 3 +- .../extensions/front/caffe/bn_ext.py | 35 +++++ .../extensions/front/caffe/dropout_ext.py | 29 ++++ model-optimizer/mo/front/caffe/extractor.py | 9 +- .../mo/front/caffe/extractors/batchnorm.py | 63 -------- .../front/caffe/extractors/batchnorm_test.py | 147 ------------------ 6 files changed, 67 insertions(+), 219 deletions(-) create mode 100644 model-optimizer/extensions/front/caffe/bn_ext.py create mode 100644 model-optimizer/extensions/front/caffe/dropout_ext.py delete mode 100644 model-optimizer/mo/front/caffe/extractors/batchnorm.py delete mode 100644 model-optimizer/mo/front/caffe/extractors/batchnorm_test.py diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index e6ec2e87e07bfd..9e59cc975f53bd 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -74,6 +74,7 @@ extensions/front/caffe/axpy.py extensions/front/caffe/binarization.py extensions/front/caffe/binary_conv_ext.py extensions/front/caffe/bn.py +extensions/front/caffe/bn_ext.py extensions/front/caffe/concat_ext.py extensions/front/caffe/conv_ext.py extensions/front/caffe/correlation_ext.py @@ -82,6 +83,7 @@ extensions/front/caffe/ctcgreedydecoder_ext.py extensions/front/caffe/CustomLayersMapping.xml.example extensions/front/caffe/data_augmentation_ext.py extensions/front/caffe/detection_output.py +extensions/front/caffe/dropout_ext.py extensions/front/caffe/elementwise_ext.py extensions/front/caffe/eltwise_add_normalize.py extensions/front/caffe/elu.py @@ -761,7 +763,6 @@ mo/front/caffe/collect_attributes.py mo/front/caffe/custom_layers_mapping.py mo/front/caffe/extractor.py mo/front/caffe/extractors/__init__.py -mo/front/caffe/extractors/batchnorm.py mo/front/caffe/extractors/native_caffe.py mo/front/caffe/extractors/tile.py mo/front/caffe/extractors/utils.py diff --git a/model-optimizer/extensions/front/caffe/bn_ext.py b/model-optimizer/extensions/front/caffe/bn_ext.py new file mode 100644 index 00000000000000..77701fe20cf162 --- /dev/null +++ b/model-optimizer/extensions/front/caffe/bn_ext.py @@ -0,0 +1,35 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import logging as log + +from extensions.ops.BatchNormInference import BatchNormInference +from mo.front.extractor import FrontExtractorOp +from mo.front.onnx.extractors.utils import onnx_attr + + + +class BatchNormalizationExtractor(FrontExtractorOp): + op = 'batchnorm' + enabled = True + + @classmethod + def extract(cls, node): + eps = node.pb.batch_norm_param.eps + attr_dict = { + 'eps': eps, + } + BatchNormInference.update_node_stat(node, attr_dict) + return cls.enabled diff --git a/model-optimizer/extensions/front/caffe/dropout_ext.py b/model-optimizer/extensions/front/caffe/dropout_ext.py new file mode 100644 index 00000000000000..ede9a9160209c5 --- /dev/null +++ b/model-optimizer/extensions/front/caffe/dropout_ext.py @@ -0,0 +1,29 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from extensions.ops.identity import Identity +from mo.front.extractor import FrontExtractorOp +from mo.graph.graph import Node + + +class DropoutFrontExtractor(FrontExtractorOp): + op = 'dropout' + enabled = True + + @classmethod + def extract(cls, node: Node): + Identity.update_node_stat(node, {}) + return cls.enabled diff --git a/model-optimizer/mo/front/caffe/extractor.py b/model-optimizer/mo/front/caffe/extractor.py index 4da78b70140907..dc5cf4ea025d5c 100644 --- a/model-optimizer/mo/front/caffe/extractor.py +++ b/model-optimizer/mo/front/caffe/extractor.py @@ -14,7 +14,6 @@ limitations under the License. """ -from mo.front.caffe.extractors.batchnorm import batch_norm_ext from mo.front.caffe.extractors.native_caffe import native_caffe_node_extractor from mo.front.common.partial_infer.elemental import copy_shape_infer from mo.front.common.register_custom_ops import extension_op_extractor @@ -33,13 +32,7 @@ def node_pb_arg(pb_extractor): Keys are names that appear as layer names in .prototxt. Full list is available here: http://caffe.berkeleyvision.org/tutorial/layers.html """ -caffe_type_extractors = { - # Common Layers - 'dropout': node_pb_arg(lambda _, __: dict(op='Dropout', infer=copy_shape_infer)), - - # Normalization Layers - 'batchnorm': node_pb_arg(batch_norm_ext), -} +caffe_type_extractors = {} def common_caffe_fields(node: Node) -> dict: diff --git a/model-optimizer/mo/front/caffe/extractors/batchnorm.py b/model-optimizer/mo/front/caffe/extractors/batchnorm.py deleted file mode 100644 index 02bb833aa0d6e9..00000000000000 --- a/model-optimizer/mo/front/caffe/extractors/batchnorm.py +++ /dev/null @@ -1,63 +0,0 @@ -""" - Copyright (C) 2018-2020 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -""" - -import numpy as np - -from mo.front.caffe.extractors.utils import embed_input -from mo.front.common.partial_infer.elemental import copy_shape_infer - - -def batch_norm_ext(pb_layer, pb_model): - """ - Extracts properties of the BatchNorm layer. - In case of scale, scale is merged into mean and variance - Args: - pb_layer: proto layer, contains own properties of the layer, i.e epsilon - pb_model: caffemodel layer, contains blobs with 0: mean, 1: variance, (opt)2: scale - - Returns: - attrs object with type, partial inference function and mean/variance properties. - """ - assert pb_layer, 'Protobuf layer can not be empty' - param = pb_layer.batch_norm_param - attrs = { - 'op': 'BatchNormalization', - 'type': 'BatchNormalization', - 'eps': param.eps, - 'infer': copy_shape_infer - } - - if not pb_model: - return attrs - - blobs = pb_model.blobs - assert len(blobs) >= 2, 'BatchNorm accepts not less then two input blobs' - mean = np.array(blobs[0].data) - variance = np.array(blobs[1].data) - - if len(blobs) == 3: - scale = blobs[2].data[0] - if scale != 0: - scale = 1.0 / scale - mean *= scale - variance *= scale - - embed_input(attrs, 1, 'gamma', np.ones(mean.shape), 'gamma') - embed_input(attrs, 2, 'beta', np.zeros(variance.shape), 'beta') - embed_input(attrs, 3, 'mean', mean, 'biases') - embed_input(attrs, 4, 'variance', variance, 'weights') - - return attrs diff --git a/model-optimizer/mo/front/caffe/extractors/batchnorm_test.py b/model-optimizer/mo/front/caffe/extractors/batchnorm_test.py deleted file mode 100644 index b852cdeff9c6b9..00000000000000 --- a/model-optimizer/mo/front/caffe/extractors/batchnorm_test.py +++ /dev/null @@ -1,147 +0,0 @@ -""" - Copyright (C) 2018-2020 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -""" - -import unittest - -import numpy as np - -from mo.front.caffe.extractors.batchnorm import batch_norm_ext -from mo.front.common.partial_infer.elemental import copy_shape_infer -from mo.utils.unittest.extractors import FakeParam, FakeModelLayer - - -class FakeBNProtoLayer: - def __init__(self, eps): - self.batch_norm_param = FakeParam('eps', eps) - - -class TestShapesParsing(unittest.TestCase): - def test_bn_ext_no_ml_no_pb(self): - self.assertRaises(AssertionError, batch_norm_ext, None, None) - - def test_bn_ext_no_ml(self): - res = batch_norm_ext(FakeBNProtoLayer(10), None) - exp_res = { - 'op': 'BatchNormalization', - 'type': 'BatchNormalization', - 'eps': 10, - 'infer': copy_shape_infer - } - self.assertEqual(res, exp_res) - - def test_bn_ext_ml_one_blob(self): - self.assertRaises(AssertionError, batch_norm_ext, FakeBNProtoLayer(10), FakeModelLayer([np.array([1, 2])])) - - def test_bn_ext_ml_two_blobs(self): - mean_blob = np.array([1., 2.]) - variance_blob = np.array([3., 4.]) - blobs = [mean_blob, variance_blob] - res = batch_norm_ext(FakeBNProtoLayer(10), - FakeModelLayer(blobs)) - exp_res = { - 'type': 'BatchNormalization', - 'eps': 10, - 'infer': copy_shape_infer, - 'mean': mean_blob, - 'variance': variance_blob, - 'embedded_inputs': [ - (1, 'gamma', { - 'bin': 'gamma' - }), - (2, 'beta', { - 'bin': 'beta' - }), - (3, 'mean', { - 'bin': 'biases' - }), - (4, 'variance', { - 'bin': 'weights' - }) - ] - } - for i in exp_res: - if i in ('mean', 'variance'): - np.testing.assert_array_equal(res[i], exp_res[i]) - else: - self.assertEqual(res[i], exp_res[i]) - - def test_bn_ext_ml_three_blobs(self): - mean_blob = np.array([1., 2.]) - variance_blob = np.array([3., 4.]) - scale_blob = np.array([5., ]) - blobs = [mean_blob, variance_blob, scale_blob] - res = batch_norm_ext(FakeBNProtoLayer(10), - FakeModelLayer(blobs)) - exp_res = { - 'type': 'BatchNormalization', - 'eps': 10, - 'infer': copy_shape_infer, - 'mean': mean_blob * 0.2, - 'variance': variance_blob * 0.2, - 'embedded_inputs': [ - (1, 'gamma', { - 'bin': 'gamma' - }), - (2, 'beta', { - 'bin': 'beta' - }), - (3, 'mean', { - 'bin': 'biases' - }), - (4, 'variance', { - 'bin': 'weights' - }) - ] - } - for i in exp_res: - if i in ('mean', 'variance'): - np.testing.assert_array_equal(res[i], exp_res[i]) - else: - self.assertEqual(res[i], exp_res[i]) - - def test_bn_ext_ml_three_blobs_zero_scale(self): - mean_blob = np.array([1., 2.]) - variance_blob = np.array([3., 4.]) - scale_blob = np.array([0., ]) - blobs = [mean_blob, variance_blob, scale_blob] - res = batch_norm_ext(FakeBNProtoLayer(10), - FakeModelLayer(blobs)) - exp_res = { - 'type': 'BatchNormalization', - 'eps': 10, - 'infer': copy_shape_infer, - 'mean': mean_blob * 0., - 'variance': variance_blob * 0., - 'embedded_inputs': [ - (1, 'gamma', { - 'bin': 'gamma' - }), - (2, 'beta', { - 'bin': 'beta' - }), - (3, 'mean', { - 'bin': 'biases' - }), - (4, 'variance', { - 'bin': 'weights' - }) - ] - } - for i in exp_res: - if i in ('mean', 'variance'): - np.testing.assert_array_equal(res[i], exp_res[i]) - else: - self.assertEqual(res[i], exp_res[i]) \ No newline at end of file From b6263df2f398fe8c41d5e019e007fd40e94acea3 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Fri, 25 Dec 2020 10:15:37 +0300 Subject: [PATCH 08/17] Add bn_ext.py and dropout_ext.py --- .../extensions/front/caffe/bn_ext.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/model-optimizer/extensions/front/caffe/bn_ext.py b/model-optimizer/extensions/front/caffe/bn_ext.py index 77701fe20cf162..ce1570ad5bc55f 100644 --- a/model-optimizer/extensions/front/caffe/bn_ext.py +++ b/model-optimizer/extensions/front/caffe/bn_ext.py @@ -17,7 +17,7 @@ from extensions.ops.BatchNormInference import BatchNormInference from mo.front.extractor import FrontExtractorOp -from mo.front.onnx.extractors.utils import onnx_attr +from mo.front.caffe.extractors.utils import embed_input @@ -31,5 +31,24 @@ def extract(cls, node): attr_dict = { 'eps': eps, } + if pb_model: + + blobs = pb_model.blobs + assert len(blobs) >= 2, 'BatchNorm accepts not less then two input blobs' + mean = np.array(blobs[0].data) + variance = np.array(blobs[1].data) + + if len(blobs) == 3: + scale = blobs[2].data[0] + if scale != 0: + scale = 1.0 / scale + mean *= scale + variance *= scale + + embed_input(attrs, 1, 'gamma', np.ones(mean.shape), 'gamma') + embed_input(attrs, 2, 'beta', np.zeros(variance.shape), 'beta') + embed_input(attrs, 3, 'mean', mean, 'biases') + embed_input(attrs, 4, 'variance', variance, 'weights') + BatchNormInference.update_node_stat(node, attr_dict) return cls.enabled From ca75f7332397432656506d4a50de829d646f8cbf Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Fri, 25 Dec 2020 10:27:30 +0300 Subject: [PATCH 09/17] Add bn_ext.py and dropout_ext.py --- model-optimizer/extensions/front/caffe/bn_ext.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/model-optimizer/extensions/front/caffe/bn_ext.py b/model-optimizer/extensions/front/caffe/bn_ext.py index ce1570ad5bc55f..10d9ecca79d195 100644 --- a/model-optimizer/extensions/front/caffe/bn_ext.py +++ b/model-optimizer/extensions/front/caffe/bn_ext.py @@ -18,7 +18,7 @@ from extensions.ops.BatchNormInference import BatchNormInference from mo.front.extractor import FrontExtractorOp from mo.front.caffe.extractors.utils import embed_input - +import numpy as np class BatchNormalizationExtractor(FrontExtractorOp): @@ -28,9 +28,10 @@ class BatchNormalizationExtractor(FrontExtractorOp): @classmethod def extract(cls, node): eps = node.pb.batch_norm_param.eps - attr_dict = { + attrs= { 'eps': eps, } + pb_model = None if not node.has('pb_model') else node.pb_model if pb_model: blobs = pb_model.blobs @@ -50,5 +51,5 @@ def extract(cls, node): embed_input(attrs, 3, 'mean', mean, 'biases') embed_input(attrs, 4, 'variance', variance, 'weights') - BatchNormInference.update_node_stat(node, attr_dict) + BatchNormInference.update_node_stat(node, attrs) return cls.enabled From f46007dc6c7c95f37426bb6b80d863327ca898a9 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Mon, 18 Jan 2021 11:44:37 +0300 Subject: [PATCH 10/17] Fix bn.ext.py --- model-optimizer/extensions/front/caffe/bn_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/extensions/front/caffe/bn_ext.py b/model-optimizer/extensions/front/caffe/bn_ext.py index 10d9ecca79d195..51707109785928 100644 --- a/model-optimizer/extensions/front/caffe/bn_ext.py +++ b/model-optimizer/extensions/front/caffe/bn_ext.py @@ -28,7 +28,7 @@ class BatchNormalizationExtractor(FrontExtractorOp): @classmethod def extract(cls, node): eps = node.pb.batch_norm_param.eps - attrs= { + attrs = { 'eps': eps, } pb_model = None if not node.has('pb_model') else node.pb_model From 3cf1fbad9a82a3848b69e8eca9d38670ff4f098d Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Mon, 25 Jan 2021 10:21:12 +0300 Subject: [PATCH 11/17] Sort fix --- model-optimizer/extensions/front/caffe/bn.py | 5 +++-- model-optimizer/extensions/front/caffe/bn_ext.py | 4 ++-- model-optimizer/extensions/front/caffe/concat_ext.py | 2 +- model-optimizer/extensions/front/caffe/crop_ext_test.py | 2 +- model-optimizer/extensions/front/caffe/dropout_ext.py | 2 +- model-optimizer/extensions/front/caffe/roipooling_ext.py | 2 +- model-optimizer/extensions/front/caffe/scale_ext.py | 2 +- model-optimizer/mo/front/caffe/extractor.py | 4 +++- 8 files changed, 13 insertions(+), 10 deletions(-) diff --git a/model-optimizer/extensions/front/caffe/bn.py b/model-optimizer/extensions/front/caffe/bn.py index 3ad77c441c512e..4aa3cdc34323fa 100644 --- a/model-optimizer/extensions/front/caffe/bn.py +++ b/model-optimizer/extensions/front/caffe/bn.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ class BNToScaleShift(FrontReplacementOp): """ Replaces BN layer with ScaleShift. """ - op = "batchNormInference" + op = "BN" enabled = True def replace_op(self, graph: Graph, node: Node): @@ -35,6 +35,7 @@ def replace_op(self, graph: Graph, node: Node): param = graph.node[node.id]['pb'].bn_param pb_model = graph.node[node.id]['model_pb'] + blobs = pb_model.blobs if len(blobs) != 4: diff --git a/model-optimizer/extensions/front/caffe/bn_ext.py b/model-optimizer/extensions/front/caffe/bn_ext.py index 51707109785928..c9b6735b664dc5 100644 --- a/model-optimizer/extensions/front/caffe/bn_ext.py +++ b/model-optimizer/extensions/front/caffe/bn_ext.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ class BatchNormalizationExtractor(FrontExtractorOp): def extract(cls, node): eps = node.pb.batch_norm_param.eps attrs = { - 'eps': eps, + 'eps': eps } pb_model = None if not node.has('pb_model') else node.pb_model if pb_model: diff --git a/model-optimizer/extensions/front/caffe/concat_ext.py b/model-optimizer/extensions/front/caffe/concat_ext.py index 3dd5742587d537..41a004f2e3e35d 100644 --- a/model-optimizer/extensions/front/caffe/concat_ext.py +++ b/model-optimizer/extensions/front/caffe/concat_ext.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/model-optimizer/extensions/front/caffe/crop_ext_test.py b/model-optimizer/extensions/front/caffe/crop_ext_test.py index bfbd7435aa6d77..755c0c066d1ae2 100644 --- a/model-optimizer/extensions/front/caffe/crop_ext_test.py +++ b/model-optimizer/extensions/front/caffe/crop_ext_test.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/model-optimizer/extensions/front/caffe/dropout_ext.py b/model-optimizer/extensions/front/caffe/dropout_ext.py index ede9a9160209c5..2737986e83155b 100644 --- a/model-optimizer/extensions/front/caffe/dropout_ext.py +++ b/model-optimizer/extensions/front/caffe/dropout_ext.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/model-optimizer/extensions/front/caffe/roipooling_ext.py b/model-optimizer/extensions/front/caffe/roipooling_ext.py index 1a5dc61f324726..ac2846781cd8c2 100644 --- a/model-optimizer/extensions/front/caffe/roipooling_ext.py +++ b/model-optimizer/extensions/front/caffe/roipooling_ext.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/model-optimizer/extensions/front/caffe/scale_ext.py b/model-optimizer/extensions/front/caffe/scale_ext.py index 20738867eee6fe..29c586b9640e91 100644 --- a/model-optimizer/extensions/front/caffe/scale_ext.py +++ b/model-optimizer/extensions/front/caffe/scale_ext.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/model-optimizer/mo/front/caffe/extractor.py b/model-optimizer/mo/front/caffe/extractor.py index dc5cf4ea025d5c..5570efd443206c 100644 --- a/model-optimizer/mo/front/caffe/extractor.py +++ b/model-optimizer/mo/front/caffe/extractor.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ def node_pb_arg(pb_extractor): Keys are names that appear as layer names in .prototxt. Full list is available here: http://caffe.berkeleyvision.org/tutorial/layers.html """ + caffe_type_extractors = {} @@ -43,6 +44,7 @@ def common_caffe_fields(node: Node) -> dict: if isinstance(layer_type, int): layer_type = pb.LayerType.DESCRIPTOR.values_by_number[layer_type].name layer_type = str(layer_type) + return { 'kind': 'op', 'name': pb.name, From 9e9fdabc64d772131f536d1674ce913978d39c6c Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Mon, 25 Jan 2021 10:34:26 +0300 Subject: [PATCH 12/17] Fix bn_test.py --- model-optimizer/extensions/front/caffe/bn_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/extensions/front/caffe/bn_test.py b/model-optimizer/extensions/front/caffe/bn_test.py index 37c0c3fa5b5cd4..3c82b59e24572f 100644 --- a/model-optimizer/extensions/front/caffe/bn_test.py +++ b/model-optimizer/extensions/front/caffe/bn_test.py @@ -47,7 +47,7 @@ def test_bn(self): FakeParam('data', shift)]) nodes = [ ('input', {'kind': 'op', 'type': 'Identity', 'op': 'Identity'}), - ('bn', {'type': None, 'kind': 'op', 'op': 'batchNormInference', 'pb': bn_pb, 'model_pb': bn_bin}), + ('bn', {'type': None, 'kind': 'op', 'op': 'BN', 'pb': bn_pb, 'model_pb': bn_bin}), ('output', {'kind': 'op', 'type': 'Identity', 'op': 'Identity'}), ] edges = [ From 1a77463bddabb2f4fe43d87b3f2a5f04634d3a65 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Mon, 25 Jan 2021 13:51:15 +0300 Subject: [PATCH 13/17] rename to batchnorm_ext --- .../extensions/front/caffe/batchnorm_ext.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 model-optimizer/extensions/front/caffe/batchnorm_ext.py diff --git a/model-optimizer/extensions/front/caffe/batchnorm_ext.py b/model-optimizer/extensions/front/caffe/batchnorm_ext.py new file mode 100644 index 00000000000000..c9b6735b664dc5 --- /dev/null +++ b/model-optimizer/extensions/front/caffe/batchnorm_ext.py @@ -0,0 +1,55 @@ +""" + Copyright (C) 2018-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import logging as log + +from extensions.ops.BatchNormInference import BatchNormInference +from mo.front.extractor import FrontExtractorOp +from mo.front.caffe.extractors.utils import embed_input +import numpy as np + + +class BatchNormalizationExtractor(FrontExtractorOp): + op = 'batchnorm' + enabled = True + + @classmethod + def extract(cls, node): + eps = node.pb.batch_norm_param.eps + attrs = { + 'eps': eps + } + pb_model = None if not node.has('pb_model') else node.pb_model + if pb_model: + + blobs = pb_model.blobs + assert len(blobs) >= 2, 'BatchNorm accepts not less then two input blobs' + mean = np.array(blobs[0].data) + variance = np.array(blobs[1].data) + + if len(blobs) == 3: + scale = blobs[2].data[0] + if scale != 0: + scale = 1.0 / scale + mean *= scale + variance *= scale + + embed_input(attrs, 1, 'gamma', np.ones(mean.shape), 'gamma') + embed_input(attrs, 2, 'beta', np.zeros(variance.shape), 'beta') + embed_input(attrs, 3, 'mean', mean, 'biases') + embed_input(attrs, 4, 'variance', variance, 'weights') + + BatchNormInference.update_node_stat(node, attrs) + return cls.enabled From a81aadb94e0a2210bd50a302e359c7faa7d2d39f Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Wed, 27 Jan 2021 12:58:51 +0300 Subject: [PATCH 14/17] Add bn_ext --- model-optimizer/automation/package_BOM.txt | 2 + model-optimizer/extensions/front/caffe/bn.py | 1 + .../extensions/front/caffe/bn_ext.py | 32 ++-------------- model-optimizer/extensions/ops/BN.py | 38 +++++++++++++++++++ 4 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 model-optimizer/extensions/ops/BN.py diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 9e59cc975f53bd..1f168d8ab8483f 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -71,6 +71,7 @@ extensions/front/caffe/accum_ext.py extensions/front/caffe/argmax_ext.py extensions/front/caffe/ArgMaxFlatten.py extensions/front/caffe/axpy.py +extensions/front/caffe/batchnorm_ext.py extensions/front/caffe/binarization.py extensions/front/caffe/binary_conv_ext.py extensions/front/caffe/bn.py @@ -623,6 +624,7 @@ extensions/ops/axpy.py extensions/ops/BatchNormInference.py extensions/ops/binarization.py extensions/ops/BlockLSTM.py +extensions/ops/BN.py extensions/ops/box_nms.py extensions/ops/bucketize.py extensions/ops/Cast.py diff --git a/model-optimizer/extensions/front/caffe/bn.py b/model-optimizer/extensions/front/caffe/bn.py index 4aa3cdc34323fa..58cba644e0fe73 100644 --- a/model-optimizer/extensions/front/caffe/bn.py +++ b/model-optimizer/extensions/front/caffe/bn.py @@ -31,6 +31,7 @@ class BNToScaleShift(FrontReplacementOp): enabled = True def replace_op(self, graph: Graph, node: Node): + # This transformation does not work!!! attrs = {'name': node.id + "/ScaleShift_"} param = graph.node[node.id]['pb'].bn_param diff --git a/model-optimizer/extensions/front/caffe/bn_ext.py b/model-optimizer/extensions/front/caffe/bn_ext.py index c9b6735b664dc5..a18692228c160d 100644 --- a/model-optimizer/extensions/front/caffe/bn_ext.py +++ b/model-optimizer/extensions/front/caffe/bn_ext.py @@ -15,41 +15,17 @@ """ import logging as log -from extensions.ops.BatchNormInference import BatchNormInference +from extensions.ops.BN import BN from mo.front.extractor import FrontExtractorOp from mo.front.caffe.extractors.utils import embed_input import numpy as np -class BatchNormalizationExtractor(FrontExtractorOp): - op = 'batchnorm' +class BNExtractor(FrontExtractorOp): + op = 'BN' enabled = True @classmethod def extract(cls, node): - eps = node.pb.batch_norm_param.eps - attrs = { - 'eps': eps - } - pb_model = None if not node.has('pb_model') else node.pb_model - if pb_model: - - blobs = pb_model.blobs - assert len(blobs) >= 2, 'BatchNorm accepts not less then two input blobs' - mean = np.array(blobs[0].data) - variance = np.array(blobs[1].data) - - if len(blobs) == 3: - scale = blobs[2].data[0] - if scale != 0: - scale = 1.0 / scale - mean *= scale - variance *= scale - - embed_input(attrs, 1, 'gamma', np.ones(mean.shape), 'gamma') - embed_input(attrs, 2, 'beta', np.zeros(variance.shape), 'beta') - embed_input(attrs, 3, 'mean', mean, 'biases') - embed_input(attrs, 4, 'variance', variance, 'weights') - - BatchNormInference.update_node_stat(node, attrs) + BN.update_node_stat(node, {}) return cls.enabled diff --git a/model-optimizer/extensions/ops/BN.py b/model-optimizer/extensions/ops/BN.py new file mode 100644 index 00000000000000..0f25b94f43272e --- /dev/null +++ b/model-optimizer/extensions/ops/BN.py @@ -0,0 +1,38 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from mo.graph.graph import Graph +from mo.ops.op import Op + + +class BN(Op): + """ + BN operation will be replaced by BNToScaleShift FrontReplacer. + """ + op = 'BN' + enabled = False + + def __init__(self, graph: Graph, attrs: dict): + super().__init__(graph, { + 'type': None, + 'op': self.op, + 'in_ports_count': 5, + 'out_ports_count': 1, + 'infer': self.infer + }, attrs) + @staticmethod + def infer(node): + node.out_port(0).data.set_shape(node.in_port(0).data.get_shape()) From e708c92a70927084b57d0078ea219f2e2d471e02 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Wed, 27 Jan 2021 19:30:10 +0300 Subject: [PATCH 15/17] Fix batchnorm_ext.py --- model-optimizer/extensions/front/caffe/batchnorm_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/extensions/front/caffe/batchnorm_ext.py b/model-optimizer/extensions/front/caffe/batchnorm_ext.py index c9b6735b664dc5..489a518c85b64c 100644 --- a/model-optimizer/extensions/front/caffe/batchnorm_ext.py +++ b/model-optimizer/extensions/front/caffe/batchnorm_ext.py @@ -31,7 +31,7 @@ def extract(cls, node): attrs = { 'eps': eps } - pb_model = None if not node.has('pb_model') else node.pb_model + pb_model = None if not node.has('model_pb') else node.model_pb if pb_model: blobs = pb_model.blobs From fbf406848b17d302e41d8132dbb635c64926efda Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Thu, 28 Jan 2021 11:38:14 +0300 Subject: [PATCH 16/17] small fix --- model-optimizer/extensions/front/caffe/batchnorm_ext.py | 7 +++---- model-optimizer/extensions/front/caffe/bn.py | 1 - model-optimizer/extensions/front/caffe/bn_ext.py | 4 ++-- model-optimizer/extensions/front/caffe/bn_test.py | 5 ++--- model-optimizer/extensions/front/caffe/concat_ext.py | 2 +- model-optimizer/extensions/front/caffe/crop_ext.py | 2 +- model-optimizer/extensions/front/caffe/roipooling_ext.py | 5 ++--- model-optimizer/extensions/front/caffe/scale_ext.py | 4 ++-- model-optimizer/extensions/ops/BN.py | 9 +++------ 9 files changed, 16 insertions(+), 23 deletions(-) diff --git a/model-optimizer/extensions/front/caffe/batchnorm_ext.py b/model-optimizer/extensions/front/caffe/batchnorm_ext.py index 489a518c85b64c..7cd50839b72ff6 100644 --- a/model-optimizer/extensions/front/caffe/batchnorm_ext.py +++ b/model-optimizer/extensions/front/caffe/batchnorm_ext.py @@ -14,11 +14,11 @@ limitations under the License. """ import logging as log +import numpy as np from extensions.ops.BatchNormInference import BatchNormInference -from mo.front.extractor import FrontExtractorOp from mo.front.caffe.extractors.utils import embed_input -import numpy as np +from mo.front.extractor import FrontExtractorOp class BatchNormalizationExtractor(FrontExtractorOp): @@ -31,9 +31,8 @@ def extract(cls, node): attrs = { 'eps': eps } - pb_model = None if not node.has('model_pb') else node.model_pb + pb_model = None if not node.soft_get('model_pb', None) else node.model_pb if pb_model: - blobs = pb_model.blobs assert len(blobs) >= 2, 'BatchNorm accepts not less then two input blobs' mean = np.array(blobs[0].data) diff --git a/model-optimizer/extensions/front/caffe/bn.py b/model-optimizer/extensions/front/caffe/bn.py index 58cba644e0fe73..4aa3cdc34323fa 100644 --- a/model-optimizer/extensions/front/caffe/bn.py +++ b/model-optimizer/extensions/front/caffe/bn.py @@ -31,7 +31,6 @@ class BNToScaleShift(FrontReplacementOp): enabled = True def replace_op(self, graph: Graph, node: Node): - # This transformation does not work!!! attrs = {'name': node.id + "/ScaleShift_"} param = graph.node[node.id]['pb'].bn_param diff --git a/model-optimizer/extensions/front/caffe/bn_ext.py b/model-optimizer/extensions/front/caffe/bn_ext.py index a18692228c160d..053f695106d267 100644 --- a/model-optimizer/extensions/front/caffe/bn_ext.py +++ b/model-optimizer/extensions/front/caffe/bn_ext.py @@ -14,11 +14,11 @@ limitations under the License. """ import logging as log +import numpy as np from extensions.ops.BN import BN -from mo.front.extractor import FrontExtractorOp from mo.front.caffe.extractors.utils import embed_input -import numpy as np +from mo.front.extractor import FrontExtractorOp class BNExtractor(FrontExtractorOp): diff --git a/model-optimizer/extensions/front/caffe/bn_test.py b/model-optimizer/extensions/front/caffe/bn_test.py index 3c82b59e24572f..c6be7929e05463 100644 --- a/model-optimizer/extensions/front/caffe/bn_test.py +++ b/model-optimizer/extensions/front/caffe/bn_test.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,9 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. """ -import unittest - import numpy as np +import unittest from extensions.front.caffe.bn import BNToScaleShift from mo.graph.graph import Node diff --git a/model-optimizer/extensions/front/caffe/concat_ext.py b/model-optimizer/extensions/front/caffe/concat_ext.py index 41a004f2e3e35d..b45b53bff8e415 100644 --- a/model-optimizer/extensions/front/caffe/concat_ext.py +++ b/model-optimizer/extensions/front/caffe/concat_ext.py @@ -14,8 +14,8 @@ limitations under the License. """ -from mo.front.onnx.extractors.utils import onnx_attr from mo.front.extractor import FrontExtractorOp +from mo.front.onnx.extractors.utils import onnx_attr from mo.ops.concat import Concat diff --git a/model-optimizer/extensions/front/caffe/crop_ext.py b/model-optimizer/extensions/front/caffe/crop_ext.py index 55957e56869525..073e286b4330f0 100644 --- a/model-optimizer/extensions/front/caffe/crop_ext.py +++ b/model-optimizer/extensions/front/caffe/crop_ext.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/model-optimizer/extensions/front/caffe/roipooling_ext.py b/model-optimizer/extensions/front/caffe/roipooling_ext.py index ac2846781cd8c2..76cbd4fd3925b0 100644 --- a/model-optimizer/extensions/front/caffe/roipooling_ext.py +++ b/model-optimizer/extensions/front/caffe/roipooling_ext.py @@ -14,11 +14,10 @@ limitations under the License. """ +from mo.front.extractor import FrontExtractorOp from mo.ops.roipooling import ROIPooling -from mo.front.extractor import FrontExtractorOp - - + class ROIPoolingFrontExtractor(FrontExtractorOp): op = 'roipooling' enabled = True diff --git a/model-optimizer/extensions/front/caffe/scale_ext.py b/model-optimizer/extensions/front/caffe/scale_ext.py index 29c586b9640e91..53c5a1a323b396 100644 --- a/model-optimizer/extensions/front/caffe/scale_ext.py +++ b/model-optimizer/extensions/front/caffe/scale_ext.py @@ -17,10 +17,10 @@ import numpy as np from mo.front.caffe.extractors.utils import embed_input, weights_biases -from mo.front.extractor import FrontExtractorOp from mo.front.common.partial_infer.elemental import copy_shape_infer -from mo.utils.utils import NamedAttrsClass +from mo.front.extractor import FrontExtractorOp from mo.ops.scale_shift import ScaleShiftOp +from mo.utils.utils import NamedAttrsClass class ScaleFrontExtractor(FrontExtractorOp): diff --git a/model-optimizer/extensions/ops/BN.py b/model-optimizer/extensions/ops/BN.py index 0f25b94f43272e..2e3114d4fcb408 100644 --- a/model-optimizer/extensions/ops/BN.py +++ b/model-optimizer/extensions/ops/BN.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ class BN(Op): """ - BN operation will be replaced by BNToScaleShift FrontReplacer. + BN operation comes from caffe and will be replaced by BNToScaleShift FrontReplacer. """ op = 'BN' enabled = False @@ -31,8 +31,5 @@ def __init__(self, graph: Graph, attrs: dict): 'op': self.op, 'in_ports_count': 5, 'out_ports_count': 1, - 'infer': self.infer + 'infer': None }, attrs) - @staticmethod - def infer(node): - node.out_port(0).data.set_shape(node.in_port(0).data.get_shape()) From 4212dccd3081e2b0fd2a84a797b5c9fecd52f8f0 Mon Sep 17 00:00:00 2001 From: Volosenkov Date: Fri, 29 Jan 2021 10:19:29 +0300 Subject: [PATCH 17/17] Small fix --- model-optimizer/extensions/front/caffe/batchnorm_ext.py | 1 - model-optimizer/extensions/front/caffe/bn_ext.py | 3 --- model-optimizer/extensions/front/caffe/concat_ext.py | 1 - 3 files changed, 5 deletions(-) diff --git a/model-optimizer/extensions/front/caffe/batchnorm_ext.py b/model-optimizer/extensions/front/caffe/batchnorm_ext.py index 7cd50839b72ff6..419fb738f1ae32 100644 --- a/model-optimizer/extensions/front/caffe/batchnorm_ext.py +++ b/model-optimizer/extensions/front/caffe/batchnorm_ext.py @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. """ -import logging as log import numpy as np from extensions.ops.BatchNormInference import BatchNormInference diff --git a/model-optimizer/extensions/front/caffe/bn_ext.py b/model-optimizer/extensions/front/caffe/bn_ext.py index 053f695106d267..39f83be46065f7 100644 --- a/model-optimizer/extensions/front/caffe/bn_ext.py +++ b/model-optimizer/extensions/front/caffe/bn_ext.py @@ -13,11 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. """ -import logging as log -import numpy as np from extensions.ops.BN import BN -from mo.front.caffe.extractors.utils import embed_input from mo.front.extractor import FrontExtractorOp diff --git a/model-optimizer/extensions/front/caffe/concat_ext.py b/model-optimizer/extensions/front/caffe/concat_ext.py index b45b53bff8e415..abbc85cd390f99 100644 --- a/model-optimizer/extensions/front/caffe/concat_ext.py +++ b/model-optimizer/extensions/front/caffe/concat_ext.py @@ -15,7 +15,6 @@ """ from mo.front.extractor import FrontExtractorOp -from mo.front.onnx.extractors.utils import onnx_attr from mo.ops.concat import Concat