Add json validate

model-optimizer/automation/package_BOM.txt

@@ -1074,6 +1074,7 @@ mo/utils/logger.py
 mo/utils/model_analysis.py
 mo/utils/pipeline_config.py
 mo/utils/replacement_pattern.py
+mo/utils/schema.json
 mo/utils/shape.py
 mo/utils/simple_proto_parser.py
 mo/utils/str_to.py

model-optimizer/mo/utils/custom_replacement_config.py

@@ -6,10 +6,12 @@
 import os
 from re import compile, match
 
+import fastjsonschema as json_validate
+
 from mo.graph.graph import Node, Graph
 from mo.utils.error import Error
 from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes
-from mo.utils.utils import refer_to_faq_msg
+from mo.utils.utils import refer_to_faq_msg, get_mo_root_dir
 
 
 class CustomReplacementDescriptor(object):
@@ -297,12 +299,12 @@ def update_custom_replacement_attributes(self, graph: Graph):
                 log.debug("Node {} doesn't have output edges. Consider it output".format(node_name))
                 output_tensors.add((generate_pattern_for_node(graph, pattern, node_name), 0))
 
-        if not self.has('inputs'):
+        if not self.has('inputs') or len(self._replacement_desc['inputs']) == 0:
             self._replacement_desc['inputs'] = [[{'node': desc[0], 'port': desc[1]} for desc in inp]
                                                 for inp in sorted(input_nodes_mapping.values())]
             log.debug('Updated inputs of sub-graph for instance "{}"'.format(self.instances))
 
-        if not self.has('outputs'):
+        if not self.has('outputs') or len(self._replacement_desc['outputs']) == 0:
             self._replacement_desc['outputs'] = [{'node': node, 'port': port} for node, port in sorted(output_tensors)]
             log.debug('Updated outputs of sub-graph for instance "{}"'.format(self.instances))
 
@@ -342,13 +344,8 @@ def parse_custom_replacement_config_file(file_name: str):
     if not os.path.exists(file_name):
         raise Error("Custom replacements configuration file '{}' does not exist. ".format(file_name) +
                     refer_to_faq_msg(69))
-    try:
-        with open(file_name, 'r') as f:
-            data = json.load(f)
-    except Exception as exc:
-        raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(file_name, exc) +
-                    refer_to_faq_msg(70)) from exc
 
+    data = load_and_validate_json_config(file_name)
     result = list()
     validation_errors = list()
     for attrs in data:
@@ -394,3 +391,30 @@ def generate_pattern_for_node(graph: Graph, sub_graph_pattern: str, node_name: s
 
     raise RuntimeError('The pattern that uniquely identifies node "{}" using sub-graph pattern "{}" has not been found'.
                        format(node_name, sub_graph_pattern))
+
+
+def load_and_validate_json_config(config_file_name: str):
+    """
+    Reads and validate custom replacement configuration file config_file_name.
+    :param config_file_name: name of the file to read from.
+    :return: The dictionary which was serialize from json config file.
+    """
+    try:
+        with open(config_file_name, 'r') as f:
+            json_config = json.load(f)
+    except Exception as exc:
+        raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(config_file_name, exc) +
+                    refer_to_faq_msg(70)) from exc
+
+    try:
+        base_dir = get_mo_root_dir()
+        schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json')
+        with open(schema_file, 'r') as f:
+            schema = json.load(f)
+            validator = json_validate.compile(schema)
+            data = validator(json_config)
+    except Exception as exc:
+        raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(config_file_name, exc) +
+                    refer_to_faq_msg(70)) from exc
+
+    return data

model-optimizer/mo/utils/schema.json

@@ -0,0 +1,128 @@
+{
+	"definitions": {},
+	"$schema": "http://json-schema.org/draft-07/schema#", 
+	"$id": "https://example.com/object1624983563.json", 
+	"title": "Root", 
+	"type": "array",
+	"default": [],
+	"items":{
+		"$id": "#root/items", 
+		"title": "Items", 
+		"type": "object",
+		"required": [
+			"id",
+			"match_kind"
+		],
+		"properties": {
+			"custom_attributes": {
+				"$id": "#root/items/custom_attributes", 
+				"title": "Custom_attributes", 
+				"type": "object",
+				"properties": {
+				}
+			},
+			"id": {
+				"$id": "#root/items/id", 
+				"title": "Id", 
+				"type": "string",
+				"pattern": "^.*$",
+				"minLength": 1
+			},
+			"inputs": {
+				"$id": "#root/items/inputs", 
+				"title": "Inputs", 
+				"type": "array",
+				"default": [],
+				"items":{
+					"$id": "#root/items/inputs/items", 
+					"title": "Items", 
+					"type": "array",
+					"default": [],
+					"items":{
+						"$id": "#root/items/inputs/items/items", 
+						"title": "Items", 
+						"type": "object",
+						"properties": {
+							"node": {
+								"$id": "#root/items/inputs/items/items/node", 
+								"title": "Node", 
+								"type": "string",
+								"default": "",
+								"pattern": "^.*$"
+							},
+							"port": {
+								"$id": "#root/items/inputs/items/items/port", 
+								"title": "Port", 
+								"type": "integer",
+								"default": 0
+							}
+						},
+						"required": ["node", "port"]
+					}
+
+				}
+			},
+			"instances": {
+				"$id": "#root/items/instances", 
+				"title": "Instances", 
+				"type": ["array", "object"],
+				"items":{
+					"$id": "#root/items/instances/items", 
+					"title": "Items", 
+					"type": "string",
+					"default": "",
+					"pattern": "^.*$"
+				}
+			},
+			"match_kind": {
+				"$id": "#root/items/match_kind", 
+				"title": "Match_kind",
+				"type": "string",
+				"enum": ["points", "scope", "general"],
+				"default": "points",
+				"pattern": "^.*$"
+			},
+			"outputs": {
+				"$id": "#root/items/outputs", 
+				"title": "Outputs", 
+				"type": "array",
+				"default": [],
+				"items":{
+					"$id": "#root/items/outputs/items", 
+					"title": "Items", 
+					"type": "object",
+					"properties": {
+						"node": {
+							"$id": "#root/items/outputs/items/node", 
+							"title": "Node", 
+							"type": "string",
+							"default": "",
+							"pattern": "^.*$"
+						},
+						"port": {
+							"$id": "#root/items/outputs/items/port", 
+							"title": "Port", 
+							"type": "integer",
+							"default": 0
+						}
+					},
+					"required": ["node", "port"]
+				}
+
+			},
+			"include_inputs_to_sub_graph": {
+				"$id": "#root/items/include_inputs_to_sub_graph", 
+				"title": "Include_inputs_to_sub_graph",
+				"type": "boolean",
+				"default": false
+			},
+			"include_outputs_to_sub_graph": {
+				"$id": "#root/items/include_outputs_to_sub_graph", 
+				"title": "Include_outputs_to_sub_graph",
+				"type": "boolean",
+				"default": false
+			}
+		}
+	}
+
+}

model-optimizer/mo/utils/utils.py

@@ -5,7 +5,6 @@
 import os
 import re
 import warnings
-
 from typing import Callable
 
 import numpy as np

model-optimizer/requirements.txt

@@ -8,3 +8,4 @@ onnx>=1.8.1
 defusedxml>=0.7.1
 urllib3>=1.26.4
 requests>=2.25.1
+fastjsonschema~=2.15.1

model-optimizer/requirements_caffe.txt

@@ -3,3 +3,4 @@ numpy>=1.16.6,<1.20
 protobuf>=3.15.6
 defusedxml>=0.7.1
 requests>=2.25.1
+fastjsonschema~=2.15.1

model-optimizer/requirements_dev.txt

@@ -6,3 +6,4 @@ test-generator==0.1.1
 defusedxml>=0.5.0
 requests>=2.20.0
 pytest>=6.2.4
+fastjsonschema~=2.15.1

model-optimizer/requirements_kaldi.txt

@@ -2,3 +2,4 @@ networkx~=2.5
 numpy>=1.16.6,<1.20
 defusedxml>=0.7.1
 requests>=2.25.1
+fastjsonschema~=2.15.1

model-optimizer/requirements_mxnet.txt

@@ -5,3 +5,4 @@ numpy>=1.16.6,<1.20
 defusedxml>=0.7.1
 urllib3>=1.26.4
 requests>=2.25.1
+fastjsonschema~=2.15.1

model-optimizer/requirements_onnx.txt

@@ -3,3 +3,4 @@ networkx~=2.5
 numpy>=1.16.6,<1.20
 defusedxml>=0.7.1
 requests>=2.25.1
+fastjsonschema~=2.15.1

model-optimizer/requirements_tf.txt

@@ -4,3 +4,4 @@ networkx~=2.5
 numpy>=1.16.6,<1.19
 defusedxml>=0.7.1
 requests>=2.25.1
+fastjsonschema~=2.15.1

model-optimizer/requirements_tf2.txt

@@ -3,3 +3,4 @@ networkx~=2.5
 numpy>=1.16.6,<1.20
 defusedxml>=0.7.1
 requests>=2.25.1
+fastjsonschema~=2.15.1

model-optimizer/setup.py

@@ -102,6 +102,7 @@ def find_package_modules(self, package, package_dir):
     },
     package_data={
       'mo.mo.front.caffe.proto': ['*.proto'],
+      'mo.mo.utils': ['*.json'],
       'mo.extensions.front.mxnet': ['*.json'],
       'mo.extensions.front.onnx': ['*.json'],
       'mo.extensions.front.tf': ['*.json'],

model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py

@@ -0,0 +1,72 @@
+# Copyright (C) 2018-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import unittest
+
+from generator import generator, generate
+
+from mo.utils.custom_replacement_config import load_and_validate_json_config
+from mo.utils.error import Error
+from mo.utils.utils import get_mo_root_dir
+
+
+@generator
+class TestSchema(unittest.TestCase):
+    base_dir = get_mo_root_dir()
+    path = os.path.join(base_dir, 'extensions', 'front', )
+    schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json')
+
+    test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]'
+    test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]'
+
+    @generate(*[('tf', 'efficient_det_support_api_v2.0.json'),
+                ('tf', 'efficient_det_support_api_v2.4.json'),
+                ('tf', 'faster_rcnn_support_api_v1.7.json'),
+                ('tf', 'faster_rcnn_support_api_v1.10.json'),
+                ('tf', 'faster_rcnn_support_api_v1.13.json'),
+                ('tf', 'faster_rcnn_support_api_v1.14.json'),
+                ('tf', 'faster_rcnn_support_api_v1.15.json'),
+                ('tf', 'faster_rcnn_support_api_v2.0.json'),
+                ('tf', 'faster_rcnn_support_api_v2.4.json'),
+                ('tf', 'mask_rcnn_support.json'),
+                ('tf', 'mask_rcnn_support_api_v1.7.json'),
+                ('tf', 'mask_rcnn_support_api_v1.11.json'),
+                ('tf', 'mask_rcnn_support_api_v1.13.json'),
+                ('tf', 'mask_rcnn_support_api_v1.14.json'),
+                ('tf', 'mask_rcnn_support_api_v1.15.json'),
+                ('tf', 'mask_rcnn_support_api_v2.0.json'),
+                ('tf', 'retinanet.json'),
+                ('tf', 'rfcn_support.json'),
+                ('tf', 'rfcn_support_api_v1.10.json'),
+                ('tf', 'rfcn_support_api_v1.13.json'),
+                ('tf', 'rfcn_support_api_v1.14.json'),
+                ('tf', 'ssd_support.json'),
+                ('tf', 'ssd_support_api_v1.14.json'),
+                ('tf', 'ssd_support_api_v1.15.json'),
+                ('tf', 'ssd_support_api_v2.0.json'),
+                ('tf', 'ssd_support_api_v2.4.json'),
+                ('tf', 'ssd_toolbox_detection_output.json'),
+                ('tf', 'ssd_toolbox_multihead_detection_output.json'),
+                ('tf', 'ssd_v2_support.json'),
+                ('tf', 'yolo_v1.json'),
+                ('tf', 'yolo_v1_tiny.json'),
+                ('tf', 'yolo_v2.json'),
+                ('tf', 'yolo_v2_tiny.json'),
+                ('tf', 'yolo_v2_tiny_voc.json'),
+                ('tf', 'yolo_v3.json'),
+                ('tf', 'yolo_v3_tiny.json'),
+                ('tf', 'yolo_v3_voc.json'),
+                ('onnx', 'faster_rcnn.json'),
+                ('onnx', 'person_detection_crossroad.json'),
+                ('mxnet', 'yolo_v3_mobilenet1_voc.json'),
+                ])
+    def test_schema_file(self, config_path, transformation_config):
+        transformation_file = os.path.join(self.path, config_path, transformation_config)
+        self.assertTrue(load_and_validate_json_config(transformation_file))
+
+    def test_schema_id_empty(self):
+        self.assertRaises(Error, load_and_validate_json_config, self.test_json1)
+
+    def test_schema_match_kind_wrong(self):
+        self.assertRaises(Error, load_and_validate_json_config, self.test_json2)

model-optimizer/unit_tests/mo/utils/utils_test.py

@@ -1,10 +1,12 @@
 # Copyright (C) 2018-2021 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+import json
 import unittest
 
 import numpy as np
 
+from mo.utils.error import Error
 from mo.utils.utils import match_shapes
 
 

tests/stress_tests/scripts/requirements.txt

@@ -1,5 +1,6 @@
 pymongo
 Jinja2
 PyYAML
+fastjsonschema~=2.15.1
 
 h5py<3.0.0 # WA for OMZ Keras models. Details: https://github.com/openvinotoolkit/open_model_zoo/issues/1806