From 2110a29b7c66f1cef9842e26ea1fbc24bc2bedbf Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Mon, 12 Oct 2020 14:10:27 +0300 Subject: [PATCH] [MO] [Kaldi] Add TDNN Component (#1870) * [MO] [Kaldi] Added TDNN Component * TdnnComponent replacer graphical comment updated * Added SpecAugmentTimeMaskComponent * some refactor of memoryoffset shape_infer * moved memoryoffset splitting to the middle stage * some corrections - set `need_shape_inferenc`=False in split_memoryoffset - use cycle instead of pattern in tdnn_replacer * separated splitting of MemoryOffsets in LSTM and TDNN blocks * set transpose_weights=True in TdnnComponent * Corrected Supported_Frameworks_Layers * corrected comments * separate naming for tdnn and lstm memoryoffset splits * corrected BOM file * corrected generaldropout_ext.py and removed 'has_default' for tdnn_component * corrections after PR review * renamed LSTM -> recurrent; added setting element_size for paired nodes of tdnn_memoffset and othe minor changes * Update split_tdnn_memoryoffset.py * corrected partial infer with new API in elemental.py and split_tdnn_memoryoffset.py --- .../Supported_Frameworks_Layers.md | 6 +- model-optimizer/automation/package_BOM.txt | 10 +- .../front/kaldi/memory_offset_adjustment.py | 4 +- .../front/kaldi/split_memoryoffsets.py | 49 ---------- .../kaldi/split_recurrent_memoryoffset.py | 62 ++++++++++++ .../front/kaldi/tdnn_component_replacer.py | 95 +++++++++++++++++++ .../kaldi/tdnn_component_replacer_test.py | 87 +++++++++++++++++ .../middle/split_tdnn_memoryoffset.py | 48 ++++++++++ .../front/common/partial_infer/elemental.py | 4 +- .../mo/front/kaldi/extractors/const_ext.py | 32 +++++++ .../kaldi/extractors/generaldropout_ext.py | 46 +++++++++ .../mo/front/kaldi/extractors/mul_ext.py | 28 ++++++ .../extractors/specaugment_component_ext.py | 28 ++++++ .../kaldi/extractors/tdnncomponent_ext.py | 71 ++++++++++++++ .../mo/front/kaldi/loader/loader.py | 41 ++++++-- .../mo/front/kaldi/loader/utils.py | 4 + model-optimizer/mo/middle/passes/infer.py | 2 +- model-optimizer/mo/ops/memoryoffset.py | 40 ++------ model-optimizer/mo/ops/tdnncomponent.py | 32 +++++++ 19 files changed, 592 insertions(+), 97 deletions(-) delete mode 100644 model-optimizer/extensions/front/kaldi/split_memoryoffsets.py create mode 100644 model-optimizer/extensions/front/kaldi/split_recurrent_memoryoffset.py create mode 100644 model-optimizer/extensions/front/kaldi/tdnn_component_replacer.py create mode 100644 model-optimizer/extensions/front/kaldi/tdnn_component_replacer_test.py create mode 100644 model-optimizer/extensions/middle/split_tdnn_memoryoffset.py create mode 100644 model-optimizer/mo/front/kaldi/extractors/const_ext.py create mode 100644 model-optimizer/mo/front/kaldi/extractors/generaldropout_ext.py create mode 100644 model-optimizer/mo/front/kaldi/extractors/mul_ext.py create mode 100644 model-optimizer/mo/front/kaldi/extractors/specaugment_component_ext.py create mode 100644 model-optimizer/mo/front/kaldi/extractors/tdnncomponent_ext.py create mode 100644 model-optimizer/mo/ops/tdnncomponent.py diff --git a/docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md b/docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md index e7bb8788972af0..50920f6e4c0cfa 100644 --- a/docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md +++ b/docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md @@ -264,6 +264,9 @@ Standard Kaldi\* Layers: | Crop | No | | elementwiseproductcomponent | No | | fixedaffinecomponent | No | +| fixedbiascomponent | No | +| fixedscalecomponent | No | +| generaldropoutcomponent| Not needed for inference | | linearcomponent | No | | logsoftmaxcomponent | No | | lstmnonlinearitycomponent | No | @@ -279,12 +282,13 @@ Standard Kaldi\* Layers: | rectifiedlinearcomponent | No | | rescale | No | | sigmoid | No | -| slice | No | | softmax | No | | softmaxComponent | No | | softsign | No | +| specaugmenttimemaskcomponent | Not needed for inference | | splicecomponent | No | | tanhcomponent | No | +| tdnncomponent | No | ## ONNX\* Supported Operators diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index ae7f4dee408fac..1ef8e97494eaf8 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -145,8 +145,9 @@ extensions/front/kaldi/replace_lstm_node_pattern.py extensions/front/kaldi/replace_lstm_nonlinearity.py extensions/front/kaldi/set_ports.py extensions/front/kaldi/sigmoid_ext.py -extensions/front/kaldi/split_memoryoffsets.py +extensions/front/kaldi/split_recurrent_memoryoffset.py extensions/front/kaldi/tanh_component_ext.py +extensions/front/kaldi/tdnn_component_replacer.py extensions/front/LayerNorm.py extensions/front/Log1p.py extensions/front/LogSoftmax.py @@ -567,6 +568,7 @@ extensions/middle/SharedWeightsDuplication.py extensions/middle/SliceConverter.py extensions/middle/SliceLikeToStridedSlice.py extensions/middle/sparse_reshape.py +extensions/middle/split_tdnn_memoryoffset.py extensions/middle/SplitConcatPairToInterpolate.py extensions/middle/SwapAxesMiddleReplacer.py extensions/middle/TensorIterator_utils.py @@ -781,17 +783,20 @@ mo/front/kaldi/extractors/batchnorm_component_ext.py mo/front/kaldi/extractors/bias_component_ext.py mo/front/kaldi/extractors/clip_ext.py mo/front/kaldi/extractors/concat_ext.py +mo/front/kaldi/extractors/const_ext.py mo/front/kaldi/extractors/convolutional_1d_component_ext.py mo/front/kaldi/extractors/convolutional_component_ext.py mo/front/kaldi/extractors/copy_ext.py mo/front/kaldi/extractors/crop_ext.py mo/front/kaldi/extractors/elementwise_component_ext.py mo/front/kaldi/extractors/fixed_affine_component_ext.py +mo/front/kaldi/extractors/generaldropout_ext.py mo/front/kaldi/extractors/linear_component_ext.py mo/front/kaldi/extractors/lstm_nonlinearity_ext.py mo/front/kaldi/extractors/lstm_projected_streams_ext.py mo/front/kaldi/extractors/max_pooling_ext.py mo/front/kaldi/extractors/memoryoffset_ext.py +mo/front/kaldi/extractors/mul_ext.py mo/front/kaldi/extractors/naturalgradient_affine_component_ext.py mo/front/kaldi/extractors/noop_ext.py mo/front/kaldi/extractors/normalize_component_ext.py @@ -800,7 +805,9 @@ mo/front/kaldi/extractors/rectified_linear_component_ext.py mo/front/kaldi/extractors/rescale_ext.py mo/front/kaldi/extractors/scale_component_ext.py mo/front/kaldi/extractors/softmax_ext.py +mo/front/kaldi/extractors/specaugment_component_ext.py mo/front/kaldi/extractors/splice_component_ext.py +mo/front/kaldi/extractors/tdnncomponent_ext.py mo/front/kaldi/loader/__init__.py mo/front/kaldi/loader/loader.py mo/front/kaldi/loader/utils.py @@ -915,6 +922,7 @@ mo/ops/softmax.py mo/ops/space_to_batch.py mo/ops/squeeze.py mo/ops/strided_slice.py +mo/ops/tdnncomponent.py mo/ops/tile.py mo/ops/unsqueeze.py mo/pipeline/__init__.py diff --git a/model-optimizer/extensions/front/kaldi/memory_offset_adjustment.py b/model-optimizer/extensions/front/kaldi/memory_offset_adjustment.py index b1f7d1e0e2bb3f..565adfc2658fdc 100644 --- a/model-optimizer/extensions/front/kaldi/memory_offset_adjustment.py +++ b/model-optimizer/extensions/front/kaldi/memory_offset_adjustment.py @@ -100,8 +100,8 @@ class MemoryOffsetAdjustment(FrontReplacementSubgraph): graph_condition = [lambda graph: graph.graph['fw'] == 'kaldi'] def run_before(self): - from extensions.front.kaldi.split_memoryoffsets import SplitMemoryOffsets - return [SplitMemoryOffsets] + from extensions.front.kaldi.split_recurrent_memoryoffset import SplitRecurrentMemoryOffset + return [SplitRecurrentMemoryOffset] def find_and_replace_pattern(self, graph: Graph): should_continue = False diff --git a/model-optimizer/extensions/front/kaldi/split_memoryoffsets.py b/model-optimizer/extensions/front/kaldi/split_memoryoffsets.py deleted file mode 100644 index 6d5cd9b4a37c69..00000000000000 --- a/model-optimizer/extensions/front/kaldi/split_memoryoffsets.py +++ /dev/null @@ -1,49 +0,0 @@ -""" - Copyright (C) 2018-2020 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -""" -from mo.front.common.replacement import FrontReplacementPattern -from mo.graph.graph import Graph -from mo.ops.memoryoffset import MemoryOffset -from mo.ops.result import Result - - -class SplitMemoryOffsets(FrontReplacementPattern): - ''' - Split MemoryOffsets in 2 parts to cut cycles - ''' - enabled = True - run_not_recursively = True - - def run_after(self): - from extensions.front.restore_ports import RestorePorts - return [RestorePorts] - - def pattern(self): - return dict( - nodes=[ - ('mem_offset', dict(kind='op', op='MemoryOffset', splitted=False)) - ], - edges=[] - ) - - @staticmethod - def replace_pattern(graph: Graph, match: dict): - offset_node = match['mem_offset'] - paired_node = MemoryOffset(graph, {'name': offset_node.pair_name, 'splitted': True, 'pair_name': offset_node.id, - 't': offset_node.t, 'has_default': offset_node.has_default}).create_node() - offset_node['splitted'] = True - offset_node.out_port(0).get_connection().set_source(paired_node.out_port(0)) - res_node = Result(graph, {'name': offset_node.id+"_output"}).create_node() - offset_node.out_port(0).connect(res_node.in_port(0)) diff --git a/model-optimizer/extensions/front/kaldi/split_recurrent_memoryoffset.py b/model-optimizer/extensions/front/kaldi/split_recurrent_memoryoffset.py new file mode 100644 index 00000000000000..78f91a9aeb178d --- /dev/null +++ b/model-optimizer/extensions/front/kaldi/split_recurrent_memoryoffset.py @@ -0,0 +1,62 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import networkx as nx + +from mo.front.common.replacement import FrontReplacementSubgraph +from mo.graph.graph import Graph +from mo.ops.memoryoffset import MemoryOffset +from mo.ops.result import Result +from mo.utils.error import Error +from mo.utils.graph import Node + + +class SplitRecurrentMemoryOffset(FrontReplacementSubgraph): + """ + Splits MemoryOffsets in recurrent blocks (typically LSTM blocks) into 2 parts. + + These parts then will be converted to ReadValue and Assign. Splitting complicates shape inference but + MemoryOffsets in recurrent blocks are cycled and, in order to make topological sort possible + during shape inference, they are splitted earlier on the front phase. In contrast, + MemoryOffsets in TDNN blocks are not cycled, so they will be splitted after shape infer on the middle. + Now only LSTM blocks with MemoryOffset are present. + """ + enabled = True + graph_condition = [lambda graph: graph.graph['fw'] == 'kaldi'] + + @staticmethod + def split_offset(offset_node: Node): + paired_node = MemoryOffset(offset_node.graph, {'name': offset_node.pair_name, 'splitted': True, + 'pair_name': offset_node.id, + 'element_size': offset_node['element_size'], + 't': offset_node.t, + 'has_default': offset_node.has_default}).create_node() + offset_node['splitted'] = True + offset_node.out_port(0).get_connection().set_source(paired_node.out_port(0)) + res_node = Result(offset_node.graph, {'name': offset_node.id + '_output'}).create_node() + offset_node.out_port(0).connect(res_node.in_port(0)) + + def find_and_replace_pattern(self, graph: Graph): + for offset_node in graph.get_op_nodes(op='MemoryOffset', splitted=False): + try: + # if graph contains recurrent block -> split MemoryOffset to enable shape infer + nx.find_cycle(graph, offset_node.id) + except nx.NetworkXNoCycle as e: + # MemoryOffset node is not in a recurrent block -- no splitting is needed + return + + if not offset_node.has_valid('element_size'): + raise Error("In a recurrent block 'element_size' for node {} is not set".format(offset_node.id)) + SplitRecurrentMemoryOffset.split_offset(offset_node) diff --git a/model-optimizer/extensions/front/kaldi/tdnn_component_replacer.py b/model-optimizer/extensions/front/kaldi/tdnn_component_replacer.py new file mode 100644 index 00000000000000..02413c63b70a72 --- /dev/null +++ b/model-optimizer/extensions/front/kaldi/tdnn_component_replacer.py @@ -0,0 +1,95 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from extensions.ops.MatMul import FullyConnected +from mo.front.common.replacement import FrontReplacementPattern +from mo.front.tf.graph_utils import create_op_with_const_inputs +from mo.graph.graph import Graph, Node +from mo.graph.graph import rename_nodes +from mo.ops.concat import Concat +from mo.ops.memoryoffset import MemoryOffset + + +class TdnnComponentReplacer(FrontReplacementPattern): + ''' + Expand TdnnComponent into MemoryOffsets, Concat and FullyConected nodes + + BEFORE: + placeholder + | + TdnnComponent('time_offsets': t1, t2,... tk) + | + _______________________________________________________________ + + AFTER: + placeholder + __________________|___________________________ + / | \ \ + MemoryOffset(t1) MemoryOffset(t2) ... MemoryOffset(tk) + \_____________ _____|______________/____________/ + Concat + | + FullyConnected + | + ''' + enabled = True + run_not_recursively = True + + def run_before(self): + from extensions.front.kaldi.memory_offset_adjustment import MemoryOffsetAdjustment + return [MemoryOffsetAdjustment] + + def find_and_replace_pattern(self, graph: Graph): + for node in graph.get_op_nodes(op='tdnncomponent'): + self.replace_tdnn(graph, node) + + def replace_tdnn(self, graph: Graph, tdnn_node: Node): + tdnn_name = tdnn_node.soft_get('name', tdnn_node.id) + + concat_node = Concat(graph, {'axis': 1}).create_node() + rename_nodes([(tdnn_node, tdnn_name + '/to_be_removed'), (concat_node, tdnn_name)]) + + for offset_ind, t in enumerate(tdnn_node['time_offsets']): + concat_node.add_input_port(offset_ind) + if t != 0: + memory_name = tdnn_name + '/MemoryOffset/' + str(abs(t)) + memoryoffset_node = MemoryOffset(graph, {'name': memory_name, 't': t, + 'pair_name': memory_name + '_out', + 'has_default': False, 'splitted': False}).create_node() + + tdnn_node.in_port(0).get_source().connect(memoryoffset_node.in_port(0)) + memoryoffset_node.out_port(0).connect(concat_node.in_port(offset_ind)) + else: + # 0 time delay is not allowed in IE, it's meaningless + # if time offset is 0 then connect input of tdnncomponent directly to Concat without memoryoffset + tdnn_node.in_port(0).get_source().connect(concat_node.in_port(offset_ind)) + + weights = tdnn_node['weights'] + fc_inputs = {1: weights} + + bias_term = False + if tdnn_node.has_valid('biases'): + assert len(tdnn_node['biases']) == weights.shape[0] + fc_inputs.update({2: tdnn_node['biases']}) + bias_term = True + + fc_node = create_op_with_const_inputs(graph, FullyConnected, fc_inputs, + {'name': tdnn_name + '/FC', 'out-size': weights.shape[0], + 'transpose_weights': True, 'bias_term': bias_term}) + + concat_node.out_port(0).connect(fc_node.in_port(0)) + tdnn_node.in_port(0).disconnect() + tdnn_node.out_port(0).get_connection().set_source(fc_node.out_port(0)) diff --git a/model-optimizer/extensions/front/kaldi/tdnn_component_replacer_test.py b/model-optimizer/extensions/front/kaldi/tdnn_component_replacer_test.py new file mode 100644 index 00000000000000..e5ffc3edf4b73e --- /dev/null +++ b/model-optimizer/extensions/front/kaldi/tdnn_component_replacer_test.py @@ -0,0 +1,87 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import unittest + +import numpy as np +from generator import generator, generate + +from extensions.front.kaldi.tdnn_component_replacer import TdnnComponentReplacer +from mo.utils.ir_engine.compare_graphs import compare_graphs +from mo.utils.unittest.graph import build_graph, regular_op, result, connect_front, const + + +@generator +class TdnnComponentReplacerTest(unittest.TestCase): + + @generate(*[ + ([[1, 1, 1], [4, 4, 4]], [1, 2], [-1, 1],), + ([[1, 1, 1], [4, 4, 4]], [1, 2], [-1, 1, 2, 10, 1000],), + ([[1, 1, 1], [4, 4, 4]], [1, 2], [-1, 0]), + ]) + def test_tdnnreplacer(self, weights, biases, time_offsets): + def generate_offsets(): + offset_edges = [] + offset_nodes = {} + + for i, t in enumerate(time_offsets): + offset_nodes.update(**regular_op('memoryoffset_' + str(i), {'type': None})) + + if t != 0: + offset_edges.append(('placeholder', 'memoryoffset_' + str(i), {'out': 0, 'in': 0})) + offset_edges.append(('memoryoffset_' + str(i), 'concat', {'out': 0, 'in': i})) + else: + offset_edges.append(('placeholder', 'concat', {'out': 0, 'in': i})) + + return offset_nodes, offset_edges + + offset_nodes, ref_offset_edges = generate_offsets() + + nodes = { + **offset_nodes, + **regular_op('placeholder', {'type': 'Parameter'}), + **regular_op('tdnncomponent', {'op': 'tdnncomponent', + 'weights': np.array(weights), + 'biases': np.array(biases), + 'time_offsets': np.array(time_offsets)}), + **const('weights', np.array(weights)), + **const('biases', np.array(biases)), + **regular_op('concat', {'type': 'Concat', 'axis': 1}), + **regular_op('memoryoffset_0', {'type': None}), + **regular_op('memoryoffset_1', {'type': None}), + **regular_op('memoryoffset_2', {'type': None}), + **regular_op('fully_connected', {'type': 'FullyConnected'}), + **result('result'), + } + + graph = build_graph(nodes, [ + *connect_front('placeholder', 'tdnncomponent'), + *connect_front('tdnncomponent', 'result') + ], nodes_with_edges_only=True) + + graph.stage = 'front' + + ref_graph = build_graph(nodes, [ + *ref_offset_edges, + *connect_front('concat', '0:fully_connected'), + *connect_front('weights', '1:fully_connected'), + *connect_front('biases', '2:fully_connected'), + *connect_front('fully_connected', 'result') + ], nodes_with_edges_only=True) + + TdnnComponentReplacer().find_and_replace_pattern(graph) + + (flag, resp) = compare_graphs(graph, ref_graph, 'result', check_op_attrs=True) + self.assertTrue(flag, resp) diff --git a/model-optimizer/extensions/middle/split_tdnn_memoryoffset.py b/model-optimizer/extensions/middle/split_tdnn_memoryoffset.py new file mode 100644 index 00000000000000..73e5239551453a --- /dev/null +++ b/model-optimizer/extensions/middle/split_tdnn_memoryoffset.py @@ -0,0 +1,48 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from mo.graph.graph import Graph +from mo.middle.replacement import MiddleReplacementPattern +from mo.ops.memoryoffset import MemoryOffset +from mo.ops.result import Result + + +class SplitTdnnMemoryOffset(MiddleReplacementPattern): + ''' + Splits MemoryOffsets in TDNN blocks into 2 parts. These parts then will be converted to ReadValue and Assign. + ''' + enabled = True + run_not_recursively = True + + def run_before(self): + from extensions.middle.ReplaceMemoryOffsetWithSplice import ReplaceMemoryOffsetWithMemoryNodePattern, ReplaceMemoryOffsetNodePattern + return [ReplaceMemoryOffsetNodePattern, ReplaceMemoryOffsetWithMemoryNodePattern] + + def find_and_replace_pattern(self, graph: Graph): + for offset_node in graph.get_op_nodes(op='MemoryOffset', splitted=False): + paired_node = MemoryOffset(graph, {'name': offset_node.pair_name, 'splitted': True, 'pair_name': offset_node.id, + 't': offset_node.t, 'has_default': offset_node.has_default}).create_node() + offset_node['splitted'] = True + offset_node.out_port(0).get_connection().set_source(paired_node.out_port(0)) + res_node = Result(graph, {'name': offset_node.id + "_output"}).create_node() + offset_node.out_port(0).connect(res_node.in_port(0)) + + # If 'element_size' is previously copied from Parameter of from node with defined dim + if offset_node.has_valid('element_size'): + paired_node['element_size'] = offset_node['element_size'] + # Copy shape from previous node. Typically (but not always) for TDNN blocks this is the case + else: + paired_node['element_size'] = offset_node.in_port(0).data.get_shape()[1] diff --git a/model-optimizer/mo/front/common/partial_infer/elemental.py b/model-optimizer/mo/front/common/partial_infer/elemental.py index 89618251aebb56..0a050adc4d016d 100644 --- a/model-optimizer/mo/front/common/partial_infer/elemental.py +++ b/model-optimizer/mo/front/common/partial_infer/elemental.py @@ -30,8 +30,8 @@ def copy_shape_infer(node, value_infer=None): Args: node: graph node """ - single_output_infer(node, lambda n: n.in_node().shape, value_infer) + single_output_infer(node, lambda n: n.in_port(0).data.get_shape(), value_infer) def copy_value(node): - return None if node.in_node().value is None else node.in_node().value.copy() + return None if node.in_node().value is None else node.in_port(0).data.get_value() diff --git a/model-optimizer/mo/front/kaldi/extractors/const_ext.py b/model-optimizer/mo/front/kaldi/extractors/const_ext.py new file mode 100644 index 00000000000000..39a0e7b13987c9 --- /dev/null +++ b/model-optimizer/mo/front/kaldi/extractors/const_ext.py @@ -0,0 +1,32 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from mo.front.extractor import FrontExtractorOp +from mo.ops.const import Const + + +class ConstantExtractor(FrontExtractorOp): + op = 'Const' + enabled = True + + @classmethod + def extract(cls, node): + attrs = { + 'data_type': node.value.dtype, + 'value': node.value, + } + Const.update_node_stat(node, attrs) + return cls.enabled diff --git a/model-optimizer/mo/front/kaldi/extractors/generaldropout_ext.py b/model-optimizer/mo/front/kaldi/extractors/generaldropout_ext.py new file mode 100644 index 00000000000000..1e4ed01343ecef --- /dev/null +++ b/model-optimizer/mo/front/kaldi/extractors/generaldropout_ext.py @@ -0,0 +1,46 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from mo.front.extractor import FrontExtractorOp +from mo.front.kaldi.loader.utils import read_binary_integer32_token, collect_until_token, \ + read_binary_float_token +from extensions.ops.identity import Identity + + +class GeneralDropoutComponentFrontExtractor(FrontExtractorOp): + op = 'generaldropoutcomponent' + enabled = True + + @classmethod + def extract(cls, node): + pb = node.parameters + + collect_until_token(pb, b'') + dim = read_binary_integer32_token(pb) + + collect_until_token(pb, b'') + block_dim = read_binary_integer32_token(pb) + + collect_until_token(pb, b'') + time_period = read_binary_integer32_token(pb) + + collect_until_token(pb, b'') + dropout_proporion = read_binary_float_token(pb) + + # collect_until_token(pb, b'') + Identity.update_node_stat(node, {}) + + return cls.enabled diff --git a/model-optimizer/mo/front/kaldi/extractors/mul_ext.py b/model-optimizer/mo/front/kaldi/extractors/mul_ext.py new file mode 100644 index 00000000000000..d8047d4403599c --- /dev/null +++ b/model-optimizer/mo/front/kaldi/extractors/mul_ext.py @@ -0,0 +1,28 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from extensions.ops.elementwise import Mul +from mo.front.extractor import FrontExtractorOp + + +class MulFrontExtractor(FrontExtractorOp): + op = 'Mul' + enabled = True + + @classmethod + def extract(cls, node): + Mul.update_node_stat(node, {}) + return cls.enabled diff --git a/model-optimizer/mo/front/kaldi/extractors/specaugment_component_ext.py b/model-optimizer/mo/front/kaldi/extractors/specaugment_component_ext.py new file mode 100644 index 00000000000000..9210f9a548d08a --- /dev/null +++ b/model-optimizer/mo/front/kaldi/extractors/specaugment_component_ext.py @@ -0,0 +1,28 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from extensions.ops.identity import Identity +from mo.front.extractor import FrontExtractorOp + + +class SpecAugmentComponentFrontExtractor(FrontExtractorOp): + op = 'specaugmenttimemaskcomponent' + enabled = True + + @classmethod + def extract(cls, node): + Identity.update_node_stat(node, {}) + return cls.enabled diff --git a/model-optimizer/mo/front/kaldi/extractors/tdnncomponent_ext.py b/model-optimizer/mo/front/kaldi/extractors/tdnncomponent_ext.py new file mode 100644 index 00000000000000..441eac0131252c --- /dev/null +++ b/model-optimizer/mo/front/kaldi/extractors/tdnncomponent_ext.py @@ -0,0 +1,71 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import numpy as np + +from mo.front.extractor import FrontExtractorOp +from mo.front.kaldi.loader.utils import read_binary_bool_token, read_binary_integer32_token, collect_until_token, \ + read_binary_float_token +from mo.front.kaldi.utils import read_binary_vector, read_binary_matrix +from mo.ops.tdnncomponent import TdnnComponent + + +class TdnnComponentFrontExtractor(FrontExtractorOp): + op = 'tdnncomponent' + enabled = True + + @classmethod + def extract(cls, node): + pb = node.parameters + + collect_until_token(pb, b'') + max_change = read_binary_float_token(pb) + + collect_until_token(pb, b'') + collect_until_token(pb, b'') + + collect_until_token(pb, b'') + time_offsets = read_binary_vector(pb, False, np.int32) + + collect_until_token(pb, b'') + weights, weights_shape = read_binary_matrix(pb) + collect_until_token(pb, b'') + bias_params = read_binary_vector(pb) + + collect_until_token(pb, b'') + orthonormal_constraint = read_binary_float_token(pb) # used only on training + + collect_until_token(pb, b'') + use_natural_grad = read_binary_bool_token(pb) # used only on training + collect_until_token(pb, b'') + num_samples_hist = read_binary_float_token(pb) + + collect_until_token(pb, b'') + alpha_in_out = read_binary_float_token(pb), read_binary_float_token(pb) # for training, usually (4, 4) + + # according to Kaldi documentation http://kaldi-asr.org/doc/classkaldi_1_1nnet3_1_1TdnnComponent.html#details + # it looks like it's used only during training (but not 100% sure) + collect_until_token(pb, b'') + rank_in_out = read_binary_integer32_token(pb), read_binary_integer32_token(pb) + + biases = np.array(bias_params) if len(bias_params) != 0 else None + attrs = { + 'weights': np.reshape(weights, weights_shape), + 'biases': biases, + 'time_offsets': time_offsets, + } + TdnnComponent.update_node_stat(node, attrs) + return cls.enabled diff --git a/model-optimizer/mo/front/kaldi/loader/loader.py b/model-optimizer/mo/front/kaldi/loader/loader.py index dcbc01bc25a511..5d3cbaee02d068 100644 --- a/model-optimizer/mo/front/kaldi/loader/loader.py +++ b/model-optimizer/mo/front/kaldi/loader/loader.py @@ -13,17 +13,21 @@ See the License for the specific language governing permissions and limitations under the License. """ + import logging as log from io import IOBase import networkx as nx import numpy as np +from extensions.ops.elementwise import Mul from extensions.ops.split import AttributedVariadicSplit +from mo.front.common.partial_infer.utils import float_array from mo.front.kaldi.loader.utils import find_next_tag, read_placeholder, find_next_component, get_name_from_path, \ find_end_of_component, end_of_nnet_tag, read_binary_integer32_token, get_parameters, read_token_value, \ collect_until_token, collect_until_token_and_read, create_edge_attrs, get_args_for_specifier from mo.graph.graph import Node, Graph +from mo.ops.const import Const from mo.utils.error import Error from mo.utils.utils import refer_to_faq_msg @@ -74,7 +78,7 @@ def load_parallel_component(file_descr, graph: Graph, prev_layer_id): split_id = graph.unique_id(prefix='NestedNets/VariadicSplit') attrs = {'out_ports_count': nnet_count, 'size_splits': split_points, 'axis': 1, 'name': split_id} variadic_split_node = AttributedVariadicSplit(graph, attrs).create_node() - prev_layer_node = Node(graph, prev_layer_id) + prev_layer_node = Node(graph, prev_layer_id) prev_layer_node.add_output_port(0) graph.create_edge(prev_layer_node, variadic_split_node, 0, 0) @@ -247,9 +251,7 @@ def load_components(file_descr, graph, component_layer_map=None): node = Node(graph, layer) node['parameters'] = get_parameters(file_descr, start_index, end_index) node['op'] = component_type - # read dim info where possible to simplify shape calculation for MemoryOffset - # shape calculation for MemoryOffset can't be done through shape of previous layer because - # it is separated in 2 parts to remove cycle from graph + # Read dim info where possible to simplify shape calculation for MemoryOffset for o_n_name, params in node.get_outputs(): o_n = Node(graph, o_n_name) if o_n['op'] == 'MemoryOffset' and dim != 0: @@ -459,7 +461,7 @@ def parse_specifier(string, graph, layer_node_map): out_port = len(Node(graph, node).out_nodes()) in_port = len(Node(graph, memory_name).in_nodes()) Node(graph, memory_name).add_input_port(in_port) - Node(graph, node).add_output_port(out_port) + Node(graph, node).add_output_port(out_port, skip_if_exist=True) graph.create_edge(Node(graph, node), Node(graph, memory_name), out_port, in_port) else: memory_name = layer_node_map[layer_name] @@ -480,13 +482,12 @@ def parse_specifier(string, graph, layer_node_map): else: sum_name = layer_node_map[layer_name] - i = 0 - for node in nodes: + for i, node in enumerate(nodes): out_port = len(Node(graph, node).out_nodes()) - Node(graph, node).add_output_port(out_port) + Node(graph, node).add_output_port(out_port, skip_if_exist=True) Node(graph, sum_name).add_input_port(i) graph.add_edge(node, sum_name, **create_edge_attrs(node, sum_name, i)) - i = i + 1 + return sum_name elif spec == b'IfDefined': node_id = parse_specifier(args[0], graph, layer_node_map) @@ -497,3 +498,25 @@ def parse_specifier(string, graph, layer_node_map): elif spec == b'ReplaceIndex': node = parse_specifier(args[0], graph, layer_node_map) return node + elif spec == b'Scale': + node_name = parse_specifier(args[1], graph, layer_node_map) + scale_value = float(args[0]) + layer_name = '{}/Mul/{}'.format(node_name, scale_value) + + if layer_name not in layer_node_map: + scale_name = graph.unique_id(prefix=layer_name) + scale_node = Mul(graph, {'name': scale_name}).create_node() + + layer_node_map[layer_name] = scale_name + + scale_const_name = 'Const_{}'.format(scale_value) + const_node = Const(graph, {'name': scale_const_name, 'value': float_array([scale_value])}).create_node() + + node = Node(graph, node_name) + graph.create_edge(const_node, scale_node, 0, 0) + out_port = len(node.out_nodes()) + graph.create_edge(node, scale_node, out_port, 1) + else: + scale_name = layer_node_map[layer_name] + + return scale_name diff --git a/model-optimizer/mo/front/kaldi/loader/utils.py b/model-optimizer/mo/front/kaldi/loader/utils.py index e27c54193ac141..7ee1447bc79e24 100644 --- a/model-optimizer/mo/front/kaldi/loader/utils.py +++ b/model-optimizer/mo/front/kaldi/loader/utils.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import io import os import struct @@ -40,6 +41,7 @@ 'fixedaffinecomponent', 'fixedscalecomponent', 'fixedbiascomponent', + 'generaldropoutcomponent', 'linearcomponent', 'logsoftmaxcomponent', 'lstmnonlinearitycomponent', @@ -58,9 +60,11 @@ 'sigmoidcomponent', 'softmax', 'softmaxcomponent', + 'specaugmenttimemaskcomponent', 'splicecomponent', 'sumgroupcomponent', 'tanhcomponent', + 'tdnncomponent', ] diff --git a/model-optimizer/mo/middle/passes/infer.py b/model-optimizer/mo/middle/passes/infer.py index 26097728ed6d61..af6d628e32a286 100644 --- a/model-optimizer/mo/middle/passes/infer.py +++ b/model-optimizer/mo/middle/passes/infer.py @@ -108,7 +108,7 @@ def partial_infer(graph: Graph, start_node: str = None): graph.strict_mode = True # Mark all nodes as not inferred yet - if not start_node is None: + if start_node is not None: start_index = nodes.index(start_node) nx.set_node_attributes(G=graph.subgraph(nodes[start_index:]), name='is_partial_inferred', values=False) else: diff --git a/model-optimizer/mo/ops/memoryoffset.py b/model-optimizer/mo/ops/memoryoffset.py index 05df30c43d91b8..fe8007d9306536 100644 --- a/model-optimizer/mo/ops/memoryoffset.py +++ b/model-optimizer/mo/ops/memoryoffset.py @@ -13,58 +13,34 @@ See the License for the specific language governing permissions and limitations under the License. """ -import numpy as np from mo.front.common.partial_infer.elemental import copy_shape_infer from mo.graph.graph import Graph, Node from mo.ops.op import Op -from mo.utils.error import Error class MemoryOffset(Op): op = 'MemoryOffset' - enabled = True + enabled = False def __init__(self, graph: Graph, attrs: dict): super().__init__(graph, { 'op': 'MemoryOffset', + 'type': None, 'pair_name': None, + 'splitted': False, 'has_default': False, 'infer': __class__.infer, 'in_ports_count': 1, 'out_ports_count': 1, }, attrs) - def supported_attrs(self): - return ['t'] @staticmethod def infer(node: Node): - # MemoryOffset is split into 2 parts to avoid cycle in graph - # Calculate shape from shape of previous layer where possible - # In other cases information about shapes from initial Kaldi model used - if not node.in_port(0).disconnected(): - copy_shape_infer(node) - pair_node = Node(node.graph, node.pair_name) - pair_node.out_port(0).data.set_shape(node.out_port(0).data.get_shape()) + if node.has_valid('element_size'): + # element_size should be set by Kaldi loader or by MemoryOffsetAdjustment + node.out_port(0).data.set_shape([1, node['element_size']]) else: - pair_node = Node(node.graph, node.pair_name) - if pair_node.in_port(0).data.get_shape() is not None: - node.out_port(0).data.set_shape(pair_node.in_port(0).data.get_shape()) - copy_shape_infer(pair_node) - elif pair_node.has_valid('element_size'): - # TODO Add here real batch - node.out_port(0).data.set_shape(np.array([1, pair_node['element_size']])) - elif pair_node.in_port(0).get_source().node.has_valid('out-size'): - out_size = pair_node.in_port(0).get_source().node['out-size'] - node.out_port(0).data.set_shape(np.array([1, out_size])) - elif pair_node.in_port(0).get_source().node.op in ["Add", "ReLU"] and \ - pair_node.in_port(0).get_source().node.in_port(0).get_source().node.has_valid('out-size'): - out_size = pair_node.in_port(0).get_source().node.in_port(0).get_source().node['out-size'] - node.out_port(0).data.set_shape(np.array([1, out_size])) - elif pair_node.in_port(0).get_source().node.has_valid('in_dim'): - out_size = pair_node.in_port(0).get_source().node['in_dim'] - node.out_port(0).data.set_shape(np.array([1, out_size])) - else: - raise Error("Can't calculate MemoryOffset shape for node {}. ".format(node.id) + - "Possibly you need to add shape for it through --input_shape") + # for TDNN blocks + copy_shape_infer(node) diff --git a/model-optimizer/mo/ops/tdnncomponent.py b/model-optimizer/mo/ops/tdnncomponent.py new file mode 100644 index 00000000000000..727b3c93d5d119 --- /dev/null +++ b/model-optimizer/mo/ops/tdnncomponent.py @@ -0,0 +1,32 @@ +""" + Copyright (C) 2018-2020 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from mo.graph.graph import Graph +from mo.ops.op import Op + + +class TdnnComponent(Op): + op = 'tdnncomponent' + enabled = False + + def __init__(self, graph: Graph, attrs: dict): + super().__init__(graph, { + 'type': None, + 'op': self.op, + 'infer': None, + 'in_ports_count': 1, + 'out_ports_count': 1, + }, attrs)