From da41f57fcfe5ca76d959ba308f35417c5ce65b42 Mon Sep 17 00:00:00 2001 From: iimironov Date: Tue, 29 Jun 2021 19:48:22 +0300 Subject: [PATCH 01/28] Add json validate --- .../mo/utils/custom_replacement_config.py | 11 ++ model-optimizer/mo/utils/schema.json | 111 ++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 model-optimizer/mo/utils/schema.json diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 32f7b6808f563b..8c1b074cc55b7f 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -6,6 +6,8 @@ import os from re import compile, match +import fastjsonschema as json_validate + from mo.graph.graph import Node, Graph from mo.utils.error import Error from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes @@ -349,6 +351,15 @@ def parse_custom_replacement_config_file(file_name: str): raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(file_name, exc) + refer_to_faq_msg(70)) from exc + try: + with open('schema.json', 'r') as f: + schema = json.load(f) + validator = json_validate.compile(schema) + validator(json) + except Exception as exc: + raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(file_name, exc) + + refer_to_faq_msg(70)) from exc + result = list() validation_errors = list() for attrs in data: diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json new file mode 100644 index 00000000000000..e78b37a69fa557 --- /dev/null +++ b/model-optimizer/mo/utils/schema.json @@ -0,0 +1,111 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1624983563.json", + "title": "Root", + "type": "array", + "default": [], + "items":{ + "$id": "#root/items", + "title": "Items", + "type": "object", + "properties": { + "custom_attributes": { + "$id": "#root/items/custom_attributes", + "title": "Custom_attributes", + "type": "object", + "properties": { + } + } +, + "id": { + "$id": "#root/items/id", + "title": "Id", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "inputs": { + "$id": "#root/items/inputs", + "title": "Inputs", + "type": "array", + "default": [], + "items":{ + "$id": "#root/items/inputs/items", + "title": "Items", + "type": "array", + "default": [], + "items":{ + "$id": "#root/items/inputs/items/items", + "title": "Items", + "type": "object", + "properties": { + "node": { + "$id": "#root/items/inputs/items/items/node", + "title": "Node", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "port": { + "$id": "#root/items/inputs/items/items/port", + "title": "Port", + "type": "integer", + "default": 0 + } + } + } + + } + }, + "instances": { + "$id": "#root/items/instances", + "title": "Instances", + "type": ["array", "object"], + "default": [], + "items":{ + "$id": "#root/items/instances/items", + "title": "Items", + "type": "string", + "default": "", + "pattern": "^.*$" + } + }, + "match_kind": { + "$id": "#root/items/match_kind", + "title": "Match_kind", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "outputs": { + "$id": "#root/items/outputs", + "title": "Outputs", + "type": "array", + "default": [], + "items":{ + "$id": "#root/items/outputs/items", + "title": "Items", + "type": "object", + "properties": { + "node": { + "$id": "#root/items/outputs/items/node", + "title": "Node", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "port": { + "$id": "#root/items/outputs/items/port", + "title": "Port", + "type": "integer", + "default": 0 + } + } + } + + } + } + } + +} \ No newline at end of file From b029aca2d7052ce64fed714d47646eee2a7b4ee9 Mon Sep 17 00:00:00 2001 From: iimironov Date: Tue, 29 Jun 2021 19:57:47 +0300 Subject: [PATCH 02/28] Fix json schema --- model-optimizer/mo/utils/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json index e78b37a69fa557..e1b1f364249602 100644 --- a/model-optimizer/mo/utils/schema.json +++ b/model-optimizer/mo/utils/schema.json @@ -108,4 +108,4 @@ } } -} \ No newline at end of file +} From 1be9dfcad2c37b9d75d87b969abda4652b46e473 Mon Sep 17 00:00:00 2001 From: iimironov Date: Wed, 30 Jun 2021 14:15:16 +0300 Subject: [PATCH 03/28] Fix schema loader --- model-optimizer/mo/utils/custom_replacement_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 8c1b074cc55b7f..13a38604351b62 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -352,10 +352,12 @@ def parse_custom_replacement_config_file(file_name: str): refer_to_faq_msg(70)) from exc try: - with open('schema.json', 'r') as f: + base_dir = os.path.dirname(__file__) + schema_file = os.path.join(base_dir, "schema.json") + with open(schema_file, 'r') as f: schema = json.load(f) validator = json_validate.compile(schema) - validator(json) + validator(data) except Exception as exc: raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(file_name, exc) + refer_to_faq_msg(70)) from exc From e8149573cc20824b66ffef631eb5784265c502ac Mon Sep 17 00:00:00 2001 From: iimironov Date: Wed, 30 Jun 2021 19:48:16 +0300 Subject: [PATCH 04/28] Add unit test --- model-optimizer/requirements.txt | 1 + .../unit_tests/mo/utils/schema_test.py | 65 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 model-optimizer/unit_tests/mo/utils/schema_test.py diff --git a/model-optimizer/requirements.txt b/model-optimizer/requirements.txt index cc0ec363dc3e19..b54ca23344b580 100644 --- a/model-optimizer/requirements.txt +++ b/model-optimizer/requirements.txt @@ -8,3 +8,4 @@ onnx>=1.8.1 defusedxml>=0.7.1 urllib3>=1.26.4 requests>=2.25.1 +fastjsonschema~=2.15.1 diff --git a/model-optimizer/unit_tests/mo/utils/schema_test.py b/model-optimizer/unit_tests/mo/utils/schema_test.py new file mode 100644 index 00000000000000..f2190e03b2f70a --- /dev/null +++ b/model-optimizer/unit_tests/mo/utils/schema_test.py @@ -0,0 +1,65 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import unittest + +import fastjsonschema as json_validate +from generator import generator, generate + +path = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'extensions', 'front',) +schema_file = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'mo', 'utils', 'schema.json') + +@generator +class TestSchema(unittest.TestCase): + @generate(*[('tf', 'efficient_det_support_api_v2.0.json'), + ('tf', 'efficient_det_support_api_v2.4.json'), + ('tf', 'faster_rcnn_support_api_v1.7.json'), + ('tf', 'faster_rcnn_support_api_v1.10.json'), + ('tf', 'faster_rcnn_support_api_v1.13.json'), + ('tf', 'faster_rcnn_support_api_v1.14.json'), + ('tf', 'faster_rcnn_support_api_v1.15.json'), + ('tf', 'faster_rcnn_support_api_v2.0.json'), + ('tf', 'faster_rcnn_support_api_v2.4.json'), + ('tf', 'mask_rcnn_support.json'), + ('tf', 'mask_rcnn_support_api_v1.7.json'), + ('tf', 'mask_rcnn_support_api_v1.11.json'), + ('tf', 'mask_rcnn_support_api_v1.13.json'), + ('tf', 'mask_rcnn_support_api_v1.14.json'), + ('tf', 'mask_rcnn_support_api_v1.15.json'), + ('tf', 'mask_rcnn_support_api_v2.0.json'), + ('tf', 'retinanet.json'), + ('tf', 'rfcn_support.json'), + ('tf', 'rfcn_support_api_v1.10.json'), + ('tf', 'rfcn_support_api_v1.13.json'), + ('tf', 'rfcn_support_api_v1.14.json'), + ('tf', 'ssd_support.json'), + ('tf', 'ssd_support_api_v1.14.json'), + ('tf', 'ssd_support_api_v1.15.json'), + ('tf', 'ssd_support_api_v2.0.json'), + ('tf', 'ssd_support_api_v2.4.json'), + ('tf', 'ssd_toolbox_detection_output.json'), + ('tf', 'ssd_toolbox_multihead_detection_output.json'), + ('tf', 'ssd_v2_support.json'), + ('tf', 'yolo_v1.json'), + ('tf', 'yolo_v1_tiny.json'), + ('tf', 'yolo_v2.json'), + ('tf', 'yolo_v2_tiny.json'), + ('tf', 'yolo_v2_tiny_voc.json'), + ('tf', 'yolo_v3.json'), + ('tf', 'yolo_v3_tiny.json'), + ('tf', 'yolo_v3_voc.json'), + ('onnx', 'faster_rcnn.json'), + ('onnx', 'person_detection_crossroad.json'), + ('mxnet', 'yolo_v3_mobilenet1_voc.json'), + ]) + def test_schema_file(self, config_path, transformation_config): + transformation_file = os.path.join(path, config_path, transformation_config) + with open(transformation_file, 'r') as f: + data = json.load(f) + + with open(schema_file, 'r') as f: + schema = json.load(f) + validator = json_validate.compile(schema) + validator(data) From 28dccecce44473e087104a5870b2497a3a99b9cc Mon Sep 17 00:00:00 2001 From: iimironov Date: Thu, 1 Jul 2021 12:03:31 +0300 Subject: [PATCH 05/28] Update bom file --- model-optimizer/automation/package_BOM.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 6f04528816844a..0f4dcc3eac78e3 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -1077,6 +1077,7 @@ mo/utils/logger.py mo/utils/model_analysis.py mo/utils/pipeline_config.py mo/utils/replacement_pattern.py +mo/utils/schema.json mo/utils/shape.py mo/utils/simple_proto_parser.py mo/utils/str_to.py From b3fd7f85bb4d6dc693129705353af4dbbcd80801 Mon Sep 17 00:00:00 2001 From: iimironov Date: Thu, 1 Jul 2021 12:42:02 +0300 Subject: [PATCH 06/28] Update all requarments --- model-optimizer/requirements_caffe.txt | 1 + model-optimizer/requirements_kaldi.txt | 1 + model-optimizer/requirements_mxnet.txt | 1 + model-optimizer/requirements_onnx.txt | 1 + model-optimizer/requirements_tf.txt | 1 + model-optimizer/requirements_tf2.txt | 1 + 6 files changed, 6 insertions(+) diff --git a/model-optimizer/requirements_caffe.txt b/model-optimizer/requirements_caffe.txt index 6dd20d0ec83707..d1eef645f44490 100644 --- a/model-optimizer/requirements_caffe.txt +++ b/model-optimizer/requirements_caffe.txt @@ -3,3 +3,4 @@ numpy>=1.16.6,<1.20 protobuf>=3.15.6 defusedxml>=0.7.1 requests>=2.25.1 +fastjsonschema~=2.15.1 diff --git a/model-optimizer/requirements_kaldi.txt b/model-optimizer/requirements_kaldi.txt index dfbdea1a50a2b0..1068d95240cb7b 100644 --- a/model-optimizer/requirements_kaldi.txt +++ b/model-optimizer/requirements_kaldi.txt @@ -2,3 +2,4 @@ networkx~=2.5 numpy>=1.16.6,<1.20 defusedxml>=0.7.1 requests>=2.25.1 +fastjsonschema~=2.15.1 diff --git a/model-optimizer/requirements_mxnet.txt b/model-optimizer/requirements_mxnet.txt index 1fc809413292f1..61897faa5003da 100644 --- a/model-optimizer/requirements_mxnet.txt +++ b/model-optimizer/requirements_mxnet.txt @@ -5,3 +5,4 @@ numpy>=1.16.6,<1.20 defusedxml>=0.7.1 urllib3>=1.26.4 requests>=2.25.1 +fastjsonschema~=2.15.1 diff --git a/model-optimizer/requirements_onnx.txt b/model-optimizer/requirements_onnx.txt index 0997041cfcc4b0..a6415939ccda25 100644 --- a/model-optimizer/requirements_onnx.txt +++ b/model-optimizer/requirements_onnx.txt @@ -3,3 +3,4 @@ networkx~=2.5 numpy>=1.16.6,<1.20 defusedxml>=0.7.1 requests>=2.25.1 +fastjsonschema~=2.15.1 diff --git a/model-optimizer/requirements_tf.txt b/model-optimizer/requirements_tf.txt index 6d75cff9fa95e7..11eee1b8af1abe 100644 --- a/model-optimizer/requirements_tf.txt +++ b/model-optimizer/requirements_tf.txt @@ -4,3 +4,4 @@ networkx~=2.5 numpy>=1.16.6,<1.19 defusedxml>=0.7.1 requests>=2.25.1 +fastjsonschema~=2.15.1 diff --git a/model-optimizer/requirements_tf2.txt b/model-optimizer/requirements_tf2.txt index 02d71d2585d0e3..760999011abc68 100644 --- a/model-optimizer/requirements_tf2.txt +++ b/model-optimizer/requirements_tf2.txt @@ -3,3 +3,4 @@ networkx~=2.5 numpy>=1.16.6,<1.20 defusedxml>=0.7.1 requests>=2.25.1 +fastjsonschema~=2.15.1 From b6dcbf41ae0d4a227f50a00d7b61c298de81523d Mon Sep 17 00:00:00 2001 From: iimironov Date: Thu, 1 Jul 2021 16:21:16 +0300 Subject: [PATCH 07/28] Update dev requarments --- model-optimizer/requirements_dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/model-optimizer/requirements_dev.txt b/model-optimizer/requirements_dev.txt index a1c72e7eeb0ab5..67640a482294cd 100644 --- a/model-optimizer/requirements_dev.txt +++ b/model-optimizer/requirements_dev.txt @@ -6,3 +6,4 @@ test-generator==0.1.1 defusedxml>=0.5.0 requests>=2.20.0 pytest>=6.2.4 +fastjsonschema~=2.15.1 From 6e37852291073c9a3161172b85f21dcbb532d0d3 Mon Sep 17 00:00:00 2001 From: iimironov Date: Thu, 1 Jul 2021 20:39:13 +0300 Subject: [PATCH 08/28] Update requrments --- tests/stress_tests/scripts/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/stress_tests/scripts/requirements.txt b/tests/stress_tests/scripts/requirements.txt index 8fe0104b0ae4e0..5d067c9229777f 100644 --- a/tests/stress_tests/scripts/requirements.txt +++ b/tests/stress_tests/scripts/requirements.txt @@ -1,5 +1,6 @@ pymongo Jinja2 PyYAML +fastjsonschema~=2.15.1 h5py<3.0.0 # WA for OMZ Keras models. Details: https://github.com/openvinotoolkit/open_model_zoo/issues/1806 \ No newline at end of file From a6d768c56a48bbb39738629e783bb3f6cb933168 Mon Sep 17 00:00:00 2001 From: iimironov Date: Tue, 6 Jul 2021 18:17:04 +0300 Subject: [PATCH 09/28] Update path to schema --- model-optimizer/mo/utils/custom_replacement_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 13a38604351b62..2c2e864bffe4a3 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -11,6 +11,7 @@ from mo.graph.graph import Node, Graph from mo.utils.error import Error from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes +from mo.utils.utils import get_mo_root_dir from mo.utils.utils import refer_to_faq_msg @@ -352,8 +353,8 @@ def parse_custom_replacement_config_file(file_name: str): refer_to_faq_msg(70)) from exc try: - base_dir = os.path.dirname(__file__) - schema_file = os.path.join(base_dir, "schema.json") + base_dir = get_mo_root_dir()#os.path.dirname(__file__) + schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') with open(schema_file, 'r') as f: schema = json.load(f) validator = json_validate.compile(schema) From 74b22e11536712ac64113204b2fe121d7ebb88fd Mon Sep 17 00:00:00 2001 From: iimironov Date: Wed, 7 Jul 2021 19:02:07 +0300 Subject: [PATCH 10/28] Update schema --- model-optimizer/mo/utils/schema.json | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json index e1b1f364249602..39fcd29ae6905b 100644 --- a/model-optimizer/mo/utils/schema.json +++ b/model-optimizer/mo/utils/schema.json @@ -9,6 +9,10 @@ "$id": "#root/items", "title": "Items", "type": "object", + "required": [ + "id", + "match_kind" + ], "properties": { "custom_attributes": { "$id": "#root/items/custom_attributes", @@ -16,14 +20,13 @@ "type": "object", "properties": { } - } -, + }, "id": { "$id": "#root/items/id", "title": "Id", "type": "string", - "default": "", - "pattern": "^.*$" + "pattern": "^.*$", + "minLength": 1 }, "inputs": { "$id": "#root/items/inputs", @@ -62,7 +65,6 @@ "$id": "#root/items/instances", "title": "Instances", "type": ["array", "object"], - "default": [], "items":{ "$id": "#root/items/instances/items", "title": "Items", @@ -73,9 +75,10 @@ }, "match_kind": { "$id": "#root/items/match_kind", - "title": "Match_kind", + "title": "Match_kind", "type": "string", - "default": "", + "enum": ["points", "scope", "general"], + "default": "points", "pattern": "^.*$" }, "outputs": { From d9943a6995dde2ba3e35de2879abeed185e3e145 Mon Sep 17 00:00:00 2001 From: iimironov Date: Thu, 8 Jul 2021 16:32:59 +0300 Subject: [PATCH 11/28] Add some unit tests --- model-optimizer/mo/utils/schema.json | 12 ++++++++++ .../unit_tests/mo/utils/schema_test.py | 23 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json index 39fcd29ae6905b..357c87aaf553b6 100644 --- a/model-optimizer/mo/utils/schema.json +++ b/model-optimizer/mo/utils/schema.json @@ -107,6 +107,18 @@ } } + }, + "include_inputs_to_sub_graph": { + "$id": "#root/items/include_inputs_to_sub_graph", + "title": "Include_inputs_to_sub_graph", + "type": "boolean", + "default": false + }, + "include_outputs_to_sub_graph": { + "$id": "#root/items/include_outputs_to_sub_graph", + "title": "Include_outputs_to_sub_graph", + "type": "boolean", + "default": false } } } diff --git a/model-optimizer/unit_tests/mo/utils/schema_test.py b/model-optimizer/unit_tests/mo/utils/schema_test.py index f2190e03b2f70a..a31400167001a2 100644 --- a/model-optimizer/unit_tests/mo/utils/schema_test.py +++ b/model-optimizer/unit_tests/mo/utils/schema_test.py @@ -11,6 +11,9 @@ path = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'extensions', 'front',) schema_file = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'mo', 'utils', 'schema.json') +test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' +test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' + @generator class TestSchema(unittest.TestCase): @generate(*[('tf', 'efficient_det_support_api_v2.0.json'), @@ -63,3 +66,23 @@ def test_schema_file(self, config_path, transformation_config): schema = json.load(f) validator = json_validate.compile(schema) validator(data) + + def test_schema_id_empty(self): + data = json.loads(test_json1) + try: + with open(schema_file, 'r') as f: + schema = json.load(f) + validator = json_validate.compile(schema) + validator(data) + except Exception as e: + assert e.message == "data[0].id must be longer than or equal to 1 characters" + + def test_schema_match_kind_wrong(self): + data = json.loads(test_json2) + try: + with open(schema_file, 'r') as f: + schema = json.load(f) + validator = json_validate.compile(schema) + validator(data) + except Exception as e: + assert e.message == "data[0].match_kind must be one of ['points', 'scope', 'general']" From 8702ec503182eb101bfa8dde84a6b63af9b59402 Mon Sep 17 00:00:00 2001 From: iimironov Date: Fri, 9 Jul 2021 12:54:39 +0300 Subject: [PATCH 12/28] Move schema to root dir --- .../mo/utils/custom_replacement_config.py | 4 +- model-optimizer/mo/utils/schema.json | 126 ------------------ model-optimizer/setup.py | 4 + .../unit_tests/mo/utils/schema_test.py | 2 +- 4 files changed, 7 insertions(+), 129 deletions(-) delete mode 100644 model-optimizer/mo/utils/schema.json diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 2c2e864bffe4a3..0d0f6d32ddac72 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -353,8 +353,8 @@ def parse_custom_replacement_config_file(file_name: str): refer_to_faq_msg(70)) from exc try: - base_dir = get_mo_root_dir()#os.path.dirname(__file__) - schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') + base_dir = get_mo_root_dir() + schema_file = os.path.join(base_dir, 'schema.json') with open(schema_file, 'r') as f: schema = json.load(f) validator = json_validate.compile(schema) diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json deleted file mode 100644 index 357c87aaf553b6..00000000000000 --- a/model-optimizer/mo/utils/schema.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "definitions": {}, - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://example.com/object1624983563.json", - "title": "Root", - "type": "array", - "default": [], - "items":{ - "$id": "#root/items", - "title": "Items", - "type": "object", - "required": [ - "id", - "match_kind" - ], - "properties": { - "custom_attributes": { - "$id": "#root/items/custom_attributes", - "title": "Custom_attributes", - "type": "object", - "properties": { - } - }, - "id": { - "$id": "#root/items/id", - "title": "Id", - "type": "string", - "pattern": "^.*$", - "minLength": 1 - }, - "inputs": { - "$id": "#root/items/inputs", - "title": "Inputs", - "type": "array", - "default": [], - "items":{ - "$id": "#root/items/inputs/items", - "title": "Items", - "type": "array", - "default": [], - "items":{ - "$id": "#root/items/inputs/items/items", - "title": "Items", - "type": "object", - "properties": { - "node": { - "$id": "#root/items/inputs/items/items/node", - "title": "Node", - "type": "string", - "default": "", - "pattern": "^.*$" - }, - "port": { - "$id": "#root/items/inputs/items/items/port", - "title": "Port", - "type": "integer", - "default": 0 - } - } - } - - } - }, - "instances": { - "$id": "#root/items/instances", - "title": "Instances", - "type": ["array", "object"], - "items":{ - "$id": "#root/items/instances/items", - "title": "Items", - "type": "string", - "default": "", - "pattern": "^.*$" - } - }, - "match_kind": { - "$id": "#root/items/match_kind", - "title": "Match_kind", - "type": "string", - "enum": ["points", "scope", "general"], - "default": "points", - "pattern": "^.*$" - }, - "outputs": { - "$id": "#root/items/outputs", - "title": "Outputs", - "type": "array", - "default": [], - "items":{ - "$id": "#root/items/outputs/items", - "title": "Items", - "type": "object", - "properties": { - "node": { - "$id": "#root/items/outputs/items/node", - "title": "Node", - "type": "string", - "default": "", - "pattern": "^.*$" - }, - "port": { - "$id": "#root/items/outputs/items/port", - "title": "Port", - "type": "integer", - "default": 0 - } - } - } - - }, - "include_inputs_to_sub_graph": { - "$id": "#root/items/include_inputs_to_sub_graph", - "title": "Include_inputs_to_sub_graph", - "type": "boolean", - "default": false - }, - "include_outputs_to_sub_graph": { - "$id": "#root/items/include_outputs_to_sub_graph", - "title": "Include_outputs_to_sub_graph", - "type": "boolean", - "default": false - } - } - } - -} diff --git a/model-optimizer/setup.py b/model-optimizer/setup.py index c18c1d11947f69..796af418a3f0d0 100644 --- a/model-optimizer/setup.py +++ b/model-optimizer/setup.py @@ -28,11 +28,14 @@ def read_text(path): # Detect all the framework specific requirements_*.txt files. requirements_txt = [] py_modules = [] +schema_json = [] for item in os.listdir(): if re.match(r'requirements(.*)\.txt', item): requirements_txt.append(item) if re.match(r'mo(.*)\.py', item): py_modules.append(item.split('.')[0]) + if re.match(r'schema\.json', item): + schema_json.append(item) # Minimal set of dependencies deps_whitelist = ('networkx', 'defusedxml', 'numpy') @@ -91,6 +94,7 @@ def find_package_modules(self, package, package_dir): packages=packages, package_dir={PACKAGE_NAME: '.'}, py_modules=py_modules, + schema_json=schema_json, cmdclass={ 'install': InstallCmd, 'build_py': BuildCmd, diff --git a/model-optimizer/unit_tests/mo/utils/schema_test.py b/model-optimizer/unit_tests/mo/utils/schema_test.py index a31400167001a2..1c11dfaa13437a 100644 --- a/model-optimizer/unit_tests/mo/utils/schema_test.py +++ b/model-optimizer/unit_tests/mo/utils/schema_test.py @@ -9,7 +9,7 @@ from generator import generator, generate path = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'extensions', 'front',) -schema_file = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'mo', 'utils', 'schema.json') +schema_file = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'schema.json') test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' From aa3ddc32ef08ccfa97fd52c4da2d4e9e1b742824 Mon Sep 17 00:00:00 2001 From: iimironov Date: Fri, 9 Jul 2021 13:33:20 +0300 Subject: [PATCH 13/28] Update schema path in bom file --- model-optimizer/automation/package_BOM.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index a0a32f6fbf9d6f..4399938ef957dc 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -1072,7 +1072,6 @@ mo/utils/logger.py mo/utils/model_analysis.py mo/utils/pipeline_config.py mo/utils/replacement_pattern.py -mo/utils/schema.json mo/utils/shape.py mo/utils/simple_proto_parser.py mo/utils/str_to.py @@ -1096,3 +1095,4 @@ requirements_mxnet.txt requirements_onnx.txt requirements_tf.txt requirements_tf2.txt +schema.json From 4dd8b73d3148e44d555bd2e9ac694975384c849c Mon Sep 17 00:00:00 2001 From: iimironov Date: Fri, 9 Jul 2021 14:30:33 +0300 Subject: [PATCH 14/28] Fix unit test --- model-optimizer/unit_tests/mo/utils/schema_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/model-optimizer/unit_tests/mo/utils/schema_test.py b/model-optimizer/unit_tests/mo/utils/schema_test.py index 1c11dfaa13437a..528848407d55aa 100644 --- a/model-optimizer/unit_tests/mo/utils/schema_test.py +++ b/model-optimizer/unit_tests/mo/utils/schema_test.py @@ -8,8 +8,11 @@ import fastjsonschema as json_validate from generator import generator, generate -path = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'extensions', 'front',) -schema_file = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'schema.json') +from mo.utils.utils import get_mo_root_dir + +base_dir = get_mo_root_dir() +path = os.path.join(base_dir, 'extensions', 'front',) +schema_file = os.path.join(base_dir, 'schema.json') test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' From e1e180697efdde4e49d0b7269cf3b4382cb62aee Mon Sep 17 00:00:00 2001 From: iimironov Date: Fri, 9 Jul 2021 14:51:55 +0300 Subject: [PATCH 15/28] Fix bom --- model-optimizer/automation/package_BOM.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 4399938ef957dc..061893b66accc9 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -1095,4 +1095,4 @@ requirements_mxnet.txt requirements_onnx.txt requirements_tf.txt requirements_tf2.txt -schema.json +schema.json \ No newline at end of file From 50812f7b6eeda10d3ed07ed1188db6d8d3c01d49 Mon Sep 17 00:00:00 2001 From: iimironov Date: Fri, 9 Jul 2021 15:34:32 +0300 Subject: [PATCH 16/28] Change path to schema --- model-optimizer/automation/package_BOM.txt | 2 +- .../mo/utils/custom_replacement_config.py | 2 +- model-optimizer/mo/utils/schema.json | 126 ++++++++++++++++++ model-optimizer/setup.py | 4 +- .../unit_tests/mo/utils/schema_test.py | 2 +- 5 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 model-optimizer/mo/utils/schema.json diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 061893b66accc9..a0a32f6fbf9d6f 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -1072,6 +1072,7 @@ mo/utils/logger.py mo/utils/model_analysis.py mo/utils/pipeline_config.py mo/utils/replacement_pattern.py +mo/utils/schema.json mo/utils/shape.py mo/utils/simple_proto_parser.py mo/utils/str_to.py @@ -1095,4 +1096,3 @@ requirements_mxnet.txt requirements_onnx.txt requirements_tf.txt requirements_tf2.txt -schema.json \ No newline at end of file diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 0d0f6d32ddac72..54d842570e32ba 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -354,7 +354,7 @@ def parse_custom_replacement_config_file(file_name: str): try: base_dir = get_mo_root_dir() - schema_file = os.path.join(base_dir, 'schema.json') + schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') with open(schema_file, 'r') as f: schema = json.load(f) validator = json_validate.compile(schema) diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json new file mode 100644 index 00000000000000..357c87aaf553b6 --- /dev/null +++ b/model-optimizer/mo/utils/schema.json @@ -0,0 +1,126 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1624983563.json", + "title": "Root", + "type": "array", + "default": [], + "items":{ + "$id": "#root/items", + "title": "Items", + "type": "object", + "required": [ + "id", + "match_kind" + ], + "properties": { + "custom_attributes": { + "$id": "#root/items/custom_attributes", + "title": "Custom_attributes", + "type": "object", + "properties": { + } + }, + "id": { + "$id": "#root/items/id", + "title": "Id", + "type": "string", + "pattern": "^.*$", + "minLength": 1 + }, + "inputs": { + "$id": "#root/items/inputs", + "title": "Inputs", + "type": "array", + "default": [], + "items":{ + "$id": "#root/items/inputs/items", + "title": "Items", + "type": "array", + "default": [], + "items":{ + "$id": "#root/items/inputs/items/items", + "title": "Items", + "type": "object", + "properties": { + "node": { + "$id": "#root/items/inputs/items/items/node", + "title": "Node", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "port": { + "$id": "#root/items/inputs/items/items/port", + "title": "Port", + "type": "integer", + "default": 0 + } + } + } + + } + }, + "instances": { + "$id": "#root/items/instances", + "title": "Instances", + "type": ["array", "object"], + "items":{ + "$id": "#root/items/instances/items", + "title": "Items", + "type": "string", + "default": "", + "pattern": "^.*$" + } + }, + "match_kind": { + "$id": "#root/items/match_kind", + "title": "Match_kind", + "type": "string", + "enum": ["points", "scope", "general"], + "default": "points", + "pattern": "^.*$" + }, + "outputs": { + "$id": "#root/items/outputs", + "title": "Outputs", + "type": "array", + "default": [], + "items":{ + "$id": "#root/items/outputs/items", + "title": "Items", + "type": "object", + "properties": { + "node": { + "$id": "#root/items/outputs/items/node", + "title": "Node", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "port": { + "$id": "#root/items/outputs/items/port", + "title": "Port", + "type": "integer", + "default": 0 + } + } + } + + }, + "include_inputs_to_sub_graph": { + "$id": "#root/items/include_inputs_to_sub_graph", + "title": "Include_inputs_to_sub_graph", + "type": "boolean", + "default": false + }, + "include_outputs_to_sub_graph": { + "$id": "#root/items/include_outputs_to_sub_graph", + "title": "Include_outputs_to_sub_graph", + "type": "boolean", + "default": false + } + } + } + +} diff --git a/model-optimizer/setup.py b/model-optimizer/setup.py index 796af418a3f0d0..8cce3abe9bed77 100644 --- a/model-optimizer/setup.py +++ b/model-optimizer/setup.py @@ -28,14 +28,12 @@ def read_text(path): # Detect all the framework specific requirements_*.txt files. requirements_txt = [] py_modules = [] -schema_json = [] +schema_json = [os.path.join(SETUP_DIR, 'mo', 'utils', 'schema.json')] for item in os.listdir(): if re.match(r'requirements(.*)\.txt', item): requirements_txt.append(item) if re.match(r'mo(.*)\.py', item): py_modules.append(item.split('.')[0]) - if re.match(r'schema\.json', item): - schema_json.append(item) # Minimal set of dependencies deps_whitelist = ('networkx', 'defusedxml', 'numpy') diff --git a/model-optimizer/unit_tests/mo/utils/schema_test.py b/model-optimizer/unit_tests/mo/utils/schema_test.py index 528848407d55aa..f4121b5d616593 100644 --- a/model-optimizer/unit_tests/mo/utils/schema_test.py +++ b/model-optimizer/unit_tests/mo/utils/schema_test.py @@ -12,7 +12,7 @@ base_dir = get_mo_root_dir() path = os.path.join(base_dir, 'extensions', 'front',) -schema_file = os.path.join(base_dir, 'schema.json') +schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' From 1f226e5a69ca427d9f4fac80ad8d26b485d0c09b Mon Sep 17 00:00:00 2001 From: iimironov Date: Fri, 9 Jul 2021 22:04:37 +0300 Subject: [PATCH 17/28] update setup --- model-optimizer/setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model-optimizer/setup.py b/model-optimizer/setup.py index 8cce3abe9bed77..62a39e3521a085 100644 --- a/model-optimizer/setup.py +++ b/model-optimizer/setup.py @@ -28,7 +28,6 @@ def read_text(path): # Detect all the framework specific requirements_*.txt files. requirements_txt = [] py_modules = [] -schema_json = [os.path.join(SETUP_DIR, 'mo', 'utils', 'schema.json')] for item in os.listdir(): if re.match(r'requirements(.*)\.txt', item): requirements_txt.append(item) @@ -92,7 +91,6 @@ def find_package_modules(self, package, package_dir): packages=packages, package_dir={PACKAGE_NAME: '.'}, py_modules=py_modules, - schema_json=schema_json, cmdclass={ 'install': InstallCmd, 'build_py': BuildCmd, @@ -107,6 +105,7 @@ def find_package_modules(self, package, package_dir): 'mo.extensions.front.mxnet': ['*.json'], 'mo.extensions.front.onnx': ['*.json'], 'mo.extensions.front.tf': ['*.json'], + 'mo.utils': ['*.json'], }, extras_require={ 'caffe': read_text('requirements_caffe.txt'), From 0b05608b1b40dddc50b8a79053f2b37255256121 Mon Sep 17 00:00:00 2001 From: iimironov Date: Mon, 12 Jul 2021 12:24:39 +0300 Subject: [PATCH 18/28] Fix setup --- model-optimizer/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/setup.py b/model-optimizer/setup.py index 62a39e3521a085..d441e02ee5f61e 100644 --- a/model-optimizer/setup.py +++ b/model-optimizer/setup.py @@ -102,10 +102,10 @@ def find_package_modules(self, package, package_dir): }, package_data={ 'mo.mo.front.caffe.proto': ['*.proto'], + 'mo.mo.utils': ['*.json'], 'mo.extensions.front.mxnet': ['*.json'], 'mo.extensions.front.onnx': ['*.json'], 'mo.extensions.front.tf': ['*.json'], - 'mo.utils': ['*.json'], }, extras_require={ 'caffe': read_text('requirements_caffe.txt'), From 85cf10e373bb45ce2ad3414738d95748c6fe69a2 Mon Sep 17 00:00:00 2001 From: iimironov Date: Mon, 12 Jul 2021 21:21:07 +0300 Subject: [PATCH 19/28] Fix mo args test --- model-optimizer/mo/utils/custom_replacement_config.py | 4 ++-- model-optimizer/mo/utils/schema.json | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 54d842570e32ba..a14f31fed8a8ab 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -300,12 +300,12 @@ def update_custom_replacement_attributes(self, graph: Graph): log.debug("Node {} doesn't have output edges. Consider it output".format(node_name)) output_tensors.add((generate_pattern_for_node(graph, pattern, node_name), 0)) - if not self.has('inputs'): + if not self.has('inputs') or len(self._replacement_desc['inputs']) == 0: self._replacement_desc['inputs'] = [[{'node': desc[0], 'port': desc[1]} for desc in inp] for inp in sorted(input_nodes_mapping.values())] log.debug('Updated inputs of sub-graph for instance "{}"'.format(self.instances)) - if not self.has('outputs'): + if not self.has('outputs') or len(self._replacement_desc['outputs']) == 0: self._replacement_desc['outputs'] = [{'node': node, 'port': port} for node, port in sorted(output_tensors)] log.debug('Updated outputs of sub-graph for instance "{}"'.format(self.instances)) diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json index 357c87aaf553b6..9494071528f901 100644 --- a/model-optimizer/mo/utils/schema.json +++ b/model-optimizer/mo/utils/schema.json @@ -56,7 +56,8 @@ "type": "integer", "default": 0 } - } + }, + "required": ["node", "port"] } } @@ -104,7 +105,8 @@ "type": "integer", "default": 0 } - } + }, + "required": ["node", "port"] } }, From ccd66dbe598381899c06bc2bd7f2a741c65c5083 Mon Sep 17 00:00:00 2001 From: iimironov Date: Fri, 16 Jul 2021 13:03:19 +0300 Subject: [PATCH 20/28] Refactoring some code --- .../mo/utils/custom_replacement_config.py | 16 +--- model-optimizer/mo/utils/utils.py | 17 +++- .../unit_tests/mo/utils/schema_test.py | 91 ------------------- .../unit_tests/mo/utils/utils_test.py | 73 ++++++++++++++- 4 files changed, 90 insertions(+), 107 deletions(-) delete mode 100644 model-optimizer/unit_tests/mo/utils/schema_test.py diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index a14f31fed8a8ab..8710939137ddff 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -6,13 +6,11 @@ import os from re import compile, match -import fastjsonschema as json_validate - from mo.graph.graph import Node, Graph from mo.utils.error import Error from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes -from mo.utils.utils import get_mo_root_dir from mo.utils.utils import refer_to_faq_msg +from mo.utils.utils import validate_json_config class CustomReplacementDescriptor(object): @@ -348,21 +346,11 @@ def parse_custom_replacement_config_file(file_name: str): try: with open(file_name, 'r') as f: data = json.load(f) + validate_json_config(data, file_name) except Exception as exc: raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(file_name, exc) + refer_to_faq_msg(70)) from exc - try: - base_dir = get_mo_root_dir() - schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') - with open(schema_file, 'r') as f: - schema = json.load(f) - validator = json_validate.compile(schema) - validator(data) - except Exception as exc: - raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(file_name, exc) + - refer_to_faq_msg(70)) from exc - result = list() validation_errors = list() for attrs in data: diff --git a/model-optimizer/mo/utils/utils.py b/model-optimizer/mo/utils/utils.py index 0c153f80dedb6e..3dd6c474491733 100644 --- a/model-optimizer/mo/utils/utils.py +++ b/model-optimizer/mo/utils/utils.py @@ -2,12 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 import functools +import json import os import re import warnings - from typing import Callable +from mo.utils.error import Error +import fastjsonschema as json_validate import numpy as np @@ -133,3 +135,16 @@ def unique_by(xs: list, predicate: Callable) -> list: """ groups = group_by_with_binary_predicate(xs, predicate) return [group[0] for group in groups] + + +def validate_json_config(json_config: list, config_file_name: str): + try: + base_dir = get_mo_root_dir() + schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') + with open(schema_file, 'r') as f: + schema = json.load(f) + validator = json_validate.compile(schema) + validator(json_config) + except Exception as exc: + raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + + refer_to_faq_msg(70)) from exc diff --git a/model-optimizer/unit_tests/mo/utils/schema_test.py b/model-optimizer/unit_tests/mo/utils/schema_test.py deleted file mode 100644 index f4121b5d616593..00000000000000 --- a/model-optimizer/unit_tests/mo/utils/schema_test.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import json -import os -import unittest - -import fastjsonschema as json_validate -from generator import generator, generate - -from mo.utils.utils import get_mo_root_dir - -base_dir = get_mo_root_dir() -path = os.path.join(base_dir, 'extensions', 'front',) -schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') - -test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' -test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' - -@generator -class TestSchema(unittest.TestCase): - @generate(*[('tf', 'efficient_det_support_api_v2.0.json'), - ('tf', 'efficient_det_support_api_v2.4.json'), - ('tf', 'faster_rcnn_support_api_v1.7.json'), - ('tf', 'faster_rcnn_support_api_v1.10.json'), - ('tf', 'faster_rcnn_support_api_v1.13.json'), - ('tf', 'faster_rcnn_support_api_v1.14.json'), - ('tf', 'faster_rcnn_support_api_v1.15.json'), - ('tf', 'faster_rcnn_support_api_v2.0.json'), - ('tf', 'faster_rcnn_support_api_v2.4.json'), - ('tf', 'mask_rcnn_support.json'), - ('tf', 'mask_rcnn_support_api_v1.7.json'), - ('tf', 'mask_rcnn_support_api_v1.11.json'), - ('tf', 'mask_rcnn_support_api_v1.13.json'), - ('tf', 'mask_rcnn_support_api_v1.14.json'), - ('tf', 'mask_rcnn_support_api_v1.15.json'), - ('tf', 'mask_rcnn_support_api_v2.0.json'), - ('tf', 'retinanet.json'), - ('tf', 'rfcn_support.json'), - ('tf', 'rfcn_support_api_v1.10.json'), - ('tf', 'rfcn_support_api_v1.13.json'), - ('tf', 'rfcn_support_api_v1.14.json'), - ('tf', 'ssd_support.json'), - ('tf', 'ssd_support_api_v1.14.json'), - ('tf', 'ssd_support_api_v1.15.json'), - ('tf', 'ssd_support_api_v2.0.json'), - ('tf', 'ssd_support_api_v2.4.json'), - ('tf', 'ssd_toolbox_detection_output.json'), - ('tf', 'ssd_toolbox_multihead_detection_output.json'), - ('tf', 'ssd_v2_support.json'), - ('tf', 'yolo_v1.json'), - ('tf', 'yolo_v1_tiny.json'), - ('tf', 'yolo_v2.json'), - ('tf', 'yolo_v2_tiny.json'), - ('tf', 'yolo_v2_tiny_voc.json'), - ('tf', 'yolo_v3.json'), - ('tf', 'yolo_v3_tiny.json'), - ('tf', 'yolo_v3_voc.json'), - ('onnx', 'faster_rcnn.json'), - ('onnx', 'person_detection_crossroad.json'), - ('mxnet', 'yolo_v3_mobilenet1_voc.json'), - ]) - def test_schema_file(self, config_path, transformation_config): - transformation_file = os.path.join(path, config_path, transformation_config) - with open(transformation_file, 'r') as f: - data = json.load(f) - - with open(schema_file, 'r') as f: - schema = json.load(f) - validator = json_validate.compile(schema) - validator(data) - - def test_schema_id_empty(self): - data = json.loads(test_json1) - try: - with open(schema_file, 'r') as f: - schema = json.load(f) - validator = json_validate.compile(schema) - validator(data) - except Exception as e: - assert e.message == "data[0].id must be longer than or equal to 1 characters" - - def test_schema_match_kind_wrong(self): - data = json.loads(test_json2) - try: - with open(schema_file, 'r') as f: - schema = json.load(f) - validator = json_validate.compile(schema) - validator(data) - except Exception as e: - assert e.message == "data[0].match_kind must be one of ['points', 'scope', 'general']" diff --git a/model-optimizer/unit_tests/mo/utils/utils_test.py b/model-optimizer/unit_tests/mo/utils/utils_test.py index afc493adfc35f9..43e11edd8380a2 100644 --- a/model-optimizer/unit_tests/mo/utils/utils_test.py +++ b/model-optimizer/unit_tests/mo/utils/utils_test.py @@ -1,11 +1,15 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import json +import os import unittest import numpy as np +from generator import generator, generate -from mo.utils.utils import match_shapes +from mo.utils.error import Error +from mo.utils.utils import match_shapes, validate_json_config, get_mo_root_dir class TestMatchShapes(unittest.TestCase): @@ -27,3 +31,70 @@ def test_negative(self): self.assertFalse(self.run_match_shapes([-1,2,3], [1,3,3])) self.assertFalse(self.run_match_shapes([1,-1,3], [2,2])) self.assertFalse(self.run_match_shapes([-1, -1, -1], [2, 3, 4, 5])) + + + + +@generator +class TestSchema(unittest.TestCase): + base_dir = get_mo_root_dir() + path = os.path.join(base_dir, 'extensions', 'front', ) + schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') + + test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' + test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' + + @generate(*[('tf', 'efficient_det_support_api_v2.0.json'), + ('tf', 'efficient_det_support_api_v2.4.json'), + ('tf', 'faster_rcnn_support_api_v1.7.json'), + ('tf', 'faster_rcnn_support_api_v1.10.json'), + ('tf', 'faster_rcnn_support_api_v1.13.json'), + ('tf', 'faster_rcnn_support_api_v1.14.json'), + ('tf', 'faster_rcnn_support_api_v1.15.json'), + ('tf', 'faster_rcnn_support_api_v2.0.json'), + ('tf', 'faster_rcnn_support_api_v2.4.json'), + ('tf', 'mask_rcnn_support.json'), + ('tf', 'mask_rcnn_support_api_v1.7.json'), + ('tf', 'mask_rcnn_support_api_v1.11.json'), + ('tf', 'mask_rcnn_support_api_v1.13.json'), + ('tf', 'mask_rcnn_support_api_v1.14.json'), + ('tf', 'mask_rcnn_support_api_v1.15.json'), + ('tf', 'mask_rcnn_support_api_v2.0.json'), + ('tf', 'retinanet.json'), + ('tf', 'rfcn_support.json'), + ('tf', 'rfcn_support_api_v1.10.json'), + ('tf', 'rfcn_support_api_v1.13.json'), + ('tf', 'rfcn_support_api_v1.14.json'), + ('tf', 'ssd_support.json'), + ('tf', 'ssd_support_api_v1.14.json'), + ('tf', 'ssd_support_api_v1.15.json'), + ('tf', 'ssd_support_api_v2.0.json'), + ('tf', 'ssd_support_api_v2.4.json'), + ('tf', 'ssd_toolbox_detection_output.json'), + ('tf', 'ssd_toolbox_multihead_detection_output.json'), + ('tf', 'ssd_v2_support.json'), + ('tf', 'yolo_v1.json'), + ('tf', 'yolo_v1_tiny.json'), + ('tf', 'yolo_v2.json'), + ('tf', 'yolo_v2_tiny.json'), + ('tf', 'yolo_v2_tiny_voc.json'), + ('tf', 'yolo_v3.json'), + ('tf', 'yolo_v3_tiny.json'), + ('tf', 'yolo_v3_voc.json'), + ('onnx', 'faster_rcnn.json'), + ('onnx', 'person_detection_crossroad.json'), + ('mxnet', 'yolo_v3_mobilenet1_voc.json'), + ]) + def test_schema_file(self, config_path, transformation_config): + transformation_file = os.path.join(self.path, config_path, transformation_config) + with open(transformation_file, 'r') as f: + data = json.load(f) + validate_json_config(data, transformation_file) + + def test_schema_id_empty(self): + data = json.loads(self.test_json1) + self.assertRaises(Error, validate_json_config, data, self.test_json1) + + def test_schema_match_kind_wrong(self): + data = json.loads(self.test_json2) + self.assertRaises(Error, validate_json_config, data, self.test_json2) From 76093fa9bf93edf03d7d4459ad9841af56e9aa2b Mon Sep 17 00:00:00 2001 From: iimironov Date: Mon, 19 Jul 2021 12:15:58 +0300 Subject: [PATCH 21/28] Refactoring according to review --- .../mo/utils/custom_replacement_config.py | 40 ++++++++--- model-optimizer/mo/utils/utils.py | 16 ----- .../utils/custom_replacement_config_test.py | 72 +++++++++++++++++++ .../unit_tests/mo/utils/utils_test.py | 71 +----------------- 4 files changed, 104 insertions(+), 95 deletions(-) create mode 100644 model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 8710939137ddff..d58842ff703a21 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -6,11 +6,12 @@ import os from re import compile, match +import fastjsonschema as json_validate + from mo.graph.graph import Node, Graph from mo.utils.error import Error from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes -from mo.utils.utils import refer_to_faq_msg -from mo.utils.utils import validate_json_config +from mo.utils.utils import refer_to_faq_msg, get_mo_root_dir class CustomReplacementDescriptor(object): @@ -343,14 +344,8 @@ def parse_custom_replacement_config_file(file_name: str): if not os.path.exists(file_name): raise Error("Custom replacements configuration file '{}' does not exist. ".format(file_name) + refer_to_faq_msg(69)) - try: - with open(file_name, 'r') as f: - data = json.load(f) - validate_json_config(data, file_name) - except Exception as exc: - raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(file_name, exc) + - refer_to_faq_msg(70)) from exc + data = load_and_validate_json_config(file_name) result = list() validation_errors = list() for attrs in data: @@ -396,3 +391,30 @@ def generate_pattern_for_node(graph: Graph, sub_graph_pattern: str, node_name: s raise RuntimeError('The pattern that uniquely identifies node "{}" using sub-graph pattern "{}" has not been found'. format(node_name, sub_graph_pattern)) + + +def load_and_validate_json_config(config_file_name: str): + """ + Reads and validate custom replacement configuration file config_file_name. + :param config_file_name: name of the file to read from. + :return: The dictionary which was serialize from json config file. + """ + try: + with open(config_file_name, 'r') as f: + json_config = json.load(f) + except Exception as exc: + raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + + refer_to_faq_msg(70)) from exc + + try: + base_dir = get_mo_root_dir() + schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') + with open(schema_file, 'r') as f: + schema = json.load(f) + validator = json_validate.compile(schema) + data = validator(json_config) + except Exception as exc: + raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + + refer_to_faq_msg(70)) from exc + + return data diff --git a/model-optimizer/mo/utils/utils.py b/model-optimizer/mo/utils/utils.py index 3dd6c474491733..ebce897d18fb00 100644 --- a/model-optimizer/mo/utils/utils.py +++ b/model-optimizer/mo/utils/utils.py @@ -2,14 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 import functools -import json import os import re import warnings from typing import Callable -from mo.utils.error import Error -import fastjsonschema as json_validate import numpy as np @@ -135,16 +132,3 @@ def unique_by(xs: list, predicate: Callable) -> list: """ groups = group_by_with_binary_predicate(xs, predicate) return [group[0] for group in groups] - - -def validate_json_config(json_config: list, config_file_name: str): - try: - base_dir = get_mo_root_dir() - schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') - with open(schema_file, 'r') as f: - schema = json.load(f) - validator = json_validate.compile(schema) - validator(json_config) - except Exception as exc: - raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + - refer_to_faq_msg(70)) from exc diff --git a/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py b/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py new file mode 100644 index 00000000000000..452273ecba6b6a --- /dev/null +++ b/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py @@ -0,0 +1,72 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os +import unittest + +from generator import generator, generate + +from mo.utils.custom_replacement_config import load_and_validate_json_config +from mo.utils.error import Error +from mo.utils.utils import get_mo_root_dir + + +@generator +class TestSchema(unittest.TestCase): + base_dir = get_mo_root_dir() + path = os.path.join(base_dir, 'extensions', 'front', ) + schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') + + test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' + test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' + + @generate(*[('tf', 'efficient_det_support_api_v2.0.json'), + ('tf', 'efficient_det_support_api_v2.4.json'), + ('tf', 'faster_rcnn_support_api_v1.7.json'), + ('tf', 'faster_rcnn_support_api_v1.10.json'), + ('tf', 'faster_rcnn_support_api_v1.13.json'), + ('tf', 'faster_rcnn_support_api_v1.14.json'), + ('tf', 'faster_rcnn_support_api_v1.15.json'), + ('tf', 'faster_rcnn_support_api_v2.0.json'), + ('tf', 'faster_rcnn_support_api_v2.4.json'), + ('tf', 'mask_rcnn_support.json'), + ('tf', 'mask_rcnn_support_api_v1.7.json'), + ('tf', 'mask_rcnn_support_api_v1.11.json'), + ('tf', 'mask_rcnn_support_api_v1.13.json'), + ('tf', 'mask_rcnn_support_api_v1.14.json'), + ('tf', 'mask_rcnn_support_api_v1.15.json'), + ('tf', 'mask_rcnn_support_api_v2.0.json'), + ('tf', 'retinanet.json'), + ('tf', 'rfcn_support.json'), + ('tf', 'rfcn_support_api_v1.10.json'), + ('tf', 'rfcn_support_api_v1.13.json'), + ('tf', 'rfcn_support_api_v1.14.json'), + ('tf', 'ssd_support.json'), + ('tf', 'ssd_support_api_v1.14.json'), + ('tf', 'ssd_support_api_v1.15.json'), + ('tf', 'ssd_support_api_v2.0.json'), + ('tf', 'ssd_support_api_v2.4.json'), + ('tf', 'ssd_toolbox_detection_output.json'), + ('tf', 'ssd_toolbox_multihead_detection_output.json'), + ('tf', 'ssd_v2_support.json'), + ('tf', 'yolo_v1.json'), + ('tf', 'yolo_v1_tiny.json'), + ('tf', 'yolo_v2.json'), + ('tf', 'yolo_v2_tiny.json'), + ('tf', 'yolo_v2_tiny_voc.json'), + ('tf', 'yolo_v3.json'), + ('tf', 'yolo_v3_tiny.json'), + ('tf', 'yolo_v3_voc.json'), + ('onnx', 'faster_rcnn.json'), + ('onnx', 'person_detection_crossroad.json'), + ('mxnet', 'yolo_v3_mobilenet1_voc.json'), + ]) + def test_schema_file(self, config_path, transformation_config): + transformation_file = os.path.join(self.path, config_path, transformation_config) + self.assertTrue(load_and_validate_json_config(transformation_file)) + + def test_schema_id_empty(self): + self.assertRaises(Error, load_and_validate_json_config, self.test_json1) + + def test_schema_match_kind_wrong(self): + self.assertRaises(Error, load_and_validate_json_config, self.test_json2) diff --git a/model-optimizer/unit_tests/mo/utils/utils_test.py b/model-optimizer/unit_tests/mo/utils/utils_test.py index 43e11edd8380a2..4fe98f8f58ded1 100644 --- a/model-optimizer/unit_tests/mo/utils/utils_test.py +++ b/model-optimizer/unit_tests/mo/utils/utils_test.py @@ -2,14 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 import json -import os import unittest import numpy as np -from generator import generator, generate from mo.utils.error import Error -from mo.utils.utils import match_shapes, validate_json_config, get_mo_root_dir +from mo.utils.utils import match_shapes class TestMatchShapes(unittest.TestCase): @@ -31,70 +29,3 @@ def test_negative(self): self.assertFalse(self.run_match_shapes([-1,2,3], [1,3,3])) self.assertFalse(self.run_match_shapes([1,-1,3], [2,2])) self.assertFalse(self.run_match_shapes([-1, -1, -1], [2, 3, 4, 5])) - - - - -@generator -class TestSchema(unittest.TestCase): - base_dir = get_mo_root_dir() - path = os.path.join(base_dir, 'extensions', 'front', ) - schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') - - test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' - test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' - - @generate(*[('tf', 'efficient_det_support_api_v2.0.json'), - ('tf', 'efficient_det_support_api_v2.4.json'), - ('tf', 'faster_rcnn_support_api_v1.7.json'), - ('tf', 'faster_rcnn_support_api_v1.10.json'), - ('tf', 'faster_rcnn_support_api_v1.13.json'), - ('tf', 'faster_rcnn_support_api_v1.14.json'), - ('tf', 'faster_rcnn_support_api_v1.15.json'), - ('tf', 'faster_rcnn_support_api_v2.0.json'), - ('tf', 'faster_rcnn_support_api_v2.4.json'), - ('tf', 'mask_rcnn_support.json'), - ('tf', 'mask_rcnn_support_api_v1.7.json'), - ('tf', 'mask_rcnn_support_api_v1.11.json'), - ('tf', 'mask_rcnn_support_api_v1.13.json'), - ('tf', 'mask_rcnn_support_api_v1.14.json'), - ('tf', 'mask_rcnn_support_api_v1.15.json'), - ('tf', 'mask_rcnn_support_api_v2.0.json'), - ('tf', 'retinanet.json'), - ('tf', 'rfcn_support.json'), - ('tf', 'rfcn_support_api_v1.10.json'), - ('tf', 'rfcn_support_api_v1.13.json'), - ('tf', 'rfcn_support_api_v1.14.json'), - ('tf', 'ssd_support.json'), - ('tf', 'ssd_support_api_v1.14.json'), - ('tf', 'ssd_support_api_v1.15.json'), - ('tf', 'ssd_support_api_v2.0.json'), - ('tf', 'ssd_support_api_v2.4.json'), - ('tf', 'ssd_toolbox_detection_output.json'), - ('tf', 'ssd_toolbox_multihead_detection_output.json'), - ('tf', 'ssd_v2_support.json'), - ('tf', 'yolo_v1.json'), - ('tf', 'yolo_v1_tiny.json'), - ('tf', 'yolo_v2.json'), - ('tf', 'yolo_v2_tiny.json'), - ('tf', 'yolo_v2_tiny_voc.json'), - ('tf', 'yolo_v3.json'), - ('tf', 'yolo_v3_tiny.json'), - ('tf', 'yolo_v3_voc.json'), - ('onnx', 'faster_rcnn.json'), - ('onnx', 'person_detection_crossroad.json'), - ('mxnet', 'yolo_v3_mobilenet1_voc.json'), - ]) - def test_schema_file(self, config_path, transformation_config): - transformation_file = os.path.join(self.path, config_path, transformation_config) - with open(transformation_file, 'r') as f: - data = json.load(f) - validate_json_config(data, transformation_file) - - def test_schema_id_empty(self): - data = json.loads(self.test_json1) - self.assertRaises(Error, validate_json_config, data, self.test_json1) - - def test_schema_match_kind_wrong(self): - data = json.loads(self.test_json2) - self.assertRaises(Error, validate_json_config, data, self.test_json2) From 036989b48a277ceb38380ed70ec5afe64cd7af53 Mon Sep 17 00:00:00 2001 From: iimironov Date: Tue, 20 Jul 2021 11:28:49 +0300 Subject: [PATCH 22/28] Update sort imports --- model-optimizer/mo/utils/custom_replacement_config.py | 3 +-- .../unit_tests/mo/utils/custom_replacement_config_test.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index d58842ff703a21..a999ed710ec672 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -1,13 +1,12 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import fastjsonschema as json_validate import json import logging as log import os from re import compile, match -import fastjsonschema as json_validate - from mo.graph.graph import Node, Graph from mo.utils.error import Error from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes diff --git a/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py b/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py index 452273ecba6b6a..bb83d86ecbd680 100644 --- a/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py +++ b/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py @@ -3,7 +3,6 @@ import os import unittest - from generator import generator, generate from mo.utils.custom_replacement_config import load_and_validate_json_config From 7a0392b77c59531bec8d26990f106131c405e1c8 Mon Sep 17 00:00:00 2001 From: iimironov Date: Tue, 20 Jul 2021 11:50:47 +0300 Subject: [PATCH 23/28] Remove id attribute from schema --- model-optimizer/mo/utils/schema.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json index 9494071528f901..0f135202a70f4e 100644 --- a/model-optimizer/mo/utils/schema.json +++ b/model-optimizer/mo/utils/schema.json @@ -1,7 +1,6 @@ { "definitions": {}, - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://example.com/object1624983563.json", + "$schema": "http://json-schema.org/draft-07/schema#", "title": "Root", "type": "array", "default": [], From 30889a9e57abcc9f6882b5a00e782827f92ab5c9 Mon Sep 17 00:00:00 2001 From: iimironov Date: Tue, 20 Jul 2021 12:22:43 +0300 Subject: [PATCH 24/28] Refactoring validator --- model-optimizer/mo/utils/custom_replacement_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index a999ed710ec672..466789c1575f0f 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -411,9 +411,9 @@ def load_and_validate_json_config(config_file_name: str): with open(schema_file, 'r') as f: schema = json.load(f) validator = json_validate.compile(schema) - data = validator(json_config) + validator(json_config) except Exception as exc: raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + refer_to_faq_msg(70)) from exc - return data + return json_config From 776fefdfcd42ab0de38b18c842c516078454e42a Mon Sep 17 00:00:00 2001 From: iimironov Date: Wed, 21 Jul 2021 14:40:40 +0300 Subject: [PATCH 25/28] Fix according to review --- model-optimizer/mo/utils/custom_replacement_config.py | 4 ++-- model-optimizer/mo/utils/utils.py | 1 + model-optimizer/unit_tests/mo/utils/utils_test.py | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 466789c1575f0f..35586f90464d93 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -10,7 +10,7 @@ from mo.graph.graph import Node, Graph from mo.utils.error import Error from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes -from mo.utils.utils import refer_to_faq_msg, get_mo_root_dir +from mo.utils.utils import get_mo_root_dir, refer_to_faq_msg class CustomReplacementDescriptor(object): @@ -396,7 +396,7 @@ def load_and_validate_json_config(config_file_name: str): """ Reads and validate custom replacement configuration file config_file_name. :param config_file_name: name of the file to read from. - :return: The dictionary which was serialize from json config file. + :return: A dictionary serialized from json config file. """ try: with open(config_file_name, 'r') as f: diff --git a/model-optimizer/mo/utils/utils.py b/model-optimizer/mo/utils/utils.py index ebce897d18fb00..0c153f80dedb6e 100644 --- a/model-optimizer/mo/utils/utils.py +++ b/model-optimizer/mo/utils/utils.py @@ -5,6 +5,7 @@ import os import re import warnings + from typing import Callable import numpy as np diff --git a/model-optimizer/unit_tests/mo/utils/utils_test.py b/model-optimizer/unit_tests/mo/utils/utils_test.py index 4fe98f8f58ded1..afc493adfc35f9 100644 --- a/model-optimizer/unit_tests/mo/utils/utils_test.py +++ b/model-optimizer/unit_tests/mo/utils/utils_test.py @@ -1,12 +1,10 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import json import unittest import numpy as np -from mo.utils.error import Error from mo.utils.utils import match_shapes From bdc77e4ef97a9382d1c5dba8eae6698b6755fd1e Mon Sep 17 00:00:00 2001 From: iimironov Date: Thu, 22 Jul 2021 12:24:22 +0300 Subject: [PATCH 26/28] Move schema from json to dict. Update unit tests. --- .../mo/utils/custom_replacement_config.py | 15 +- model-optimizer/mo/utils/json_schema.py | 129 ++++++++++++++++++ model-optimizer/mo/utils/schema.json | 127 ----------------- model-optimizer/setup.py | 1 - .../utils/custom_replacement_config_test.py | 61 ++------- 5 files changed, 148 insertions(+), 185 deletions(-) create mode 100644 model-optimizer/mo/utils/json_schema.py delete mode 100644 model-optimizer/mo/utils/schema.json diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 35586f90464d93..75389aa33a7928 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -10,6 +10,7 @@ from mo.graph.graph import Node, Graph from mo.utils.error import Error from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes +from mo.utils.json_schema import schema_dict from mo.utils.utils import get_mo_root_dir, refer_to_faq_msg @@ -398,22 +399,14 @@ def load_and_validate_json_config(config_file_name: str): :param config_file_name: name of the file to read from. :return: A dictionary serialized from json config file. """ + try: with open(config_file_name, 'r') as f: json_config = json.load(f) - except Exception as exc: - raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + - refer_to_faq_msg(70)) from exc - - try: - base_dir = get_mo_root_dir() - schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') - with open(schema_file, 'r') as f: - schema = json.load(f) - validator = json_validate.compile(schema) + validator = json_validate.compile(schema_dict) validator(json_config) except Exception as exc: - raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + + raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + refer_to_faq_msg(70)) from exc return json_config diff --git a/model-optimizer/mo/utils/json_schema.py b/model-optimizer/mo/utils/json_schema.py new file mode 100644 index 00000000000000..a201818eb72719 --- /dev/null +++ b/model-optimizer/mo/utils/json_schema.py @@ -0,0 +1,129 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +schema_dict = { + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Root", + "type": "array", + "default": [], + "items": { + "$id": "#root/items", + "title": "Items", + "type": "object", + "required": [ + "id", + "match_kind" + ], + "properties": { + "custom_attributes": { + "$id": "#root/items/custom_attributes", + "title": "Custom_attributes", + "type": "object", + "properties": { + } + }, + "id": { + "$id": "#root/items/id", + "title": "Id", + "type": "string", + "pattern": "^.*$", + "minLength": 1 + }, + "inputs": { + "$id": "#root/items/inputs", + "title": "Inputs", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/inputs/items", + "title": "Items", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/inputs/items/items", + "title": "Items", + "type": "object", + "properties": { + "node": { + "$id": "#root/items/inputs/items/items/node", + "title": "Node", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "port": { + "$id": "#root/items/inputs/items/items/port", + "title": "Port", + "type": "integer", + "default": 0 + } + }, + "required": ["node", "port"] + } + + } + }, + "instances": { + "$id": "#root/items/instances", + "title": "Instances", + "type": ["array", "object"], + "items": { + "$id": "#root/items/instances/items", + "title": "Items", + "type": "string", + "default": "", + "pattern": "^.*$" + } + }, + "match_kind": { + "$id": "#root/items/match_kind", + "title": "Match_kind", + "type": "string", + "enum": ["points", "scope", "general"], + "default": "points", + "pattern": "^.*$" + }, + "outputs": { + "$id": "#root/items/outputs", + "title": "Outputs", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/outputs/items", + "title": "Items", + "type": "object", + "properties": { + "node": { + "$id": "#root/items/outputs/items/node", + "title": "Node", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "port": { + "$id": "#root/items/outputs/items/port", + "title": "Port", + "type": "integer", + "default": 0 + } + }, + "required": ["node", "port"] + } + + }, + "include_inputs_to_sub_graph": { + "$id": "#root/items/include_inputs_to_sub_graph", + "title": "Include_inputs_to_sub_graph", + "type": "boolean", + "default": False + }, + "include_outputs_to_sub_graph": { + "$id": "#root/items/include_outputs_to_sub_graph", + "title": "Include_outputs_to_sub_graph", + "type": "boolean", + "default": False + } + } + } +} diff --git a/model-optimizer/mo/utils/schema.json b/model-optimizer/mo/utils/schema.json deleted file mode 100644 index 0f135202a70f4e..00000000000000 --- a/model-optimizer/mo/utils/schema.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "definitions": {}, - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Root", - "type": "array", - "default": [], - "items":{ - "$id": "#root/items", - "title": "Items", - "type": "object", - "required": [ - "id", - "match_kind" - ], - "properties": { - "custom_attributes": { - "$id": "#root/items/custom_attributes", - "title": "Custom_attributes", - "type": "object", - "properties": { - } - }, - "id": { - "$id": "#root/items/id", - "title": "Id", - "type": "string", - "pattern": "^.*$", - "minLength": 1 - }, - "inputs": { - "$id": "#root/items/inputs", - "title": "Inputs", - "type": "array", - "default": [], - "items":{ - "$id": "#root/items/inputs/items", - "title": "Items", - "type": "array", - "default": [], - "items":{ - "$id": "#root/items/inputs/items/items", - "title": "Items", - "type": "object", - "properties": { - "node": { - "$id": "#root/items/inputs/items/items/node", - "title": "Node", - "type": "string", - "default": "", - "pattern": "^.*$" - }, - "port": { - "$id": "#root/items/inputs/items/items/port", - "title": "Port", - "type": "integer", - "default": 0 - } - }, - "required": ["node", "port"] - } - - } - }, - "instances": { - "$id": "#root/items/instances", - "title": "Instances", - "type": ["array", "object"], - "items":{ - "$id": "#root/items/instances/items", - "title": "Items", - "type": "string", - "default": "", - "pattern": "^.*$" - } - }, - "match_kind": { - "$id": "#root/items/match_kind", - "title": "Match_kind", - "type": "string", - "enum": ["points", "scope", "general"], - "default": "points", - "pattern": "^.*$" - }, - "outputs": { - "$id": "#root/items/outputs", - "title": "Outputs", - "type": "array", - "default": [], - "items":{ - "$id": "#root/items/outputs/items", - "title": "Items", - "type": "object", - "properties": { - "node": { - "$id": "#root/items/outputs/items/node", - "title": "Node", - "type": "string", - "default": "", - "pattern": "^.*$" - }, - "port": { - "$id": "#root/items/outputs/items/port", - "title": "Port", - "type": "integer", - "default": 0 - } - }, - "required": ["node", "port"] - } - - }, - "include_inputs_to_sub_graph": { - "$id": "#root/items/include_inputs_to_sub_graph", - "title": "Include_inputs_to_sub_graph", - "type": "boolean", - "default": false - }, - "include_outputs_to_sub_graph": { - "$id": "#root/items/include_outputs_to_sub_graph", - "title": "Include_outputs_to_sub_graph", - "type": "boolean", - "default": false - } - } - } - -} diff --git a/model-optimizer/setup.py b/model-optimizer/setup.py index d441e02ee5f61e..c18c1d11947f69 100644 --- a/model-optimizer/setup.py +++ b/model-optimizer/setup.py @@ -102,7 +102,6 @@ def find_package_modules(self, package, package_dir): }, package_data={ 'mo.mo.front.caffe.proto': ['*.proto'], - 'mo.mo.utils': ['*.json'], 'mo.extensions.front.mxnet': ['*.json'], 'mo.extensions.front.onnx': ['*.json'], 'mo.extensions.front.tf': ['*.json'], diff --git a/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py b/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py index bb83d86ecbd680..3407ca5816e1b9 100644 --- a/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py +++ b/model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py @@ -3,6 +3,7 @@ import os import unittest +from fnmatch import fnmatch from generator import generator, generate from mo.utils.custom_replacement_config import load_and_validate_json_config @@ -10,59 +11,27 @@ from mo.utils.utils import get_mo_root_dir +def get_json_configs(mo_root_dir): + config_path = os.path.join(mo_root_dir, 'extensions', 'front') + pattern = "*.json" + config_files_list = [] + for path, subdirs, files in os.walk(config_path): + for name in files: + if fnmatch(name, pattern): + config_files_list.append((os.path.join(path, name),)) + return config_files_list + @generator class TestSchema(unittest.TestCase): base_dir = get_mo_root_dir() - path = os.path.join(base_dir, 'extensions', 'front', ) schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json') - + transformation_configs = get_json_configs(base_dir) test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]' test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]' - @generate(*[('tf', 'efficient_det_support_api_v2.0.json'), - ('tf', 'efficient_det_support_api_v2.4.json'), - ('tf', 'faster_rcnn_support_api_v1.7.json'), - ('tf', 'faster_rcnn_support_api_v1.10.json'), - ('tf', 'faster_rcnn_support_api_v1.13.json'), - ('tf', 'faster_rcnn_support_api_v1.14.json'), - ('tf', 'faster_rcnn_support_api_v1.15.json'), - ('tf', 'faster_rcnn_support_api_v2.0.json'), - ('tf', 'faster_rcnn_support_api_v2.4.json'), - ('tf', 'mask_rcnn_support.json'), - ('tf', 'mask_rcnn_support_api_v1.7.json'), - ('tf', 'mask_rcnn_support_api_v1.11.json'), - ('tf', 'mask_rcnn_support_api_v1.13.json'), - ('tf', 'mask_rcnn_support_api_v1.14.json'), - ('tf', 'mask_rcnn_support_api_v1.15.json'), - ('tf', 'mask_rcnn_support_api_v2.0.json'), - ('tf', 'retinanet.json'), - ('tf', 'rfcn_support.json'), - ('tf', 'rfcn_support_api_v1.10.json'), - ('tf', 'rfcn_support_api_v1.13.json'), - ('tf', 'rfcn_support_api_v1.14.json'), - ('tf', 'ssd_support.json'), - ('tf', 'ssd_support_api_v1.14.json'), - ('tf', 'ssd_support_api_v1.15.json'), - ('tf', 'ssd_support_api_v2.0.json'), - ('tf', 'ssd_support_api_v2.4.json'), - ('tf', 'ssd_toolbox_detection_output.json'), - ('tf', 'ssd_toolbox_multihead_detection_output.json'), - ('tf', 'ssd_v2_support.json'), - ('tf', 'yolo_v1.json'), - ('tf', 'yolo_v1_tiny.json'), - ('tf', 'yolo_v2.json'), - ('tf', 'yolo_v2_tiny.json'), - ('tf', 'yolo_v2_tiny_voc.json'), - ('tf', 'yolo_v3.json'), - ('tf', 'yolo_v3_tiny.json'), - ('tf', 'yolo_v3_voc.json'), - ('onnx', 'faster_rcnn.json'), - ('onnx', 'person_detection_crossroad.json'), - ('mxnet', 'yolo_v3_mobilenet1_voc.json'), - ]) - def test_schema_file(self, config_path, transformation_config): - transformation_file = os.path.join(self.path, config_path, transformation_config) - self.assertTrue(load_and_validate_json_config(transformation_file)) + @generate(*transformation_configs) + def test_schema_file(self, transformation_config): + self.assertTrue(load_and_validate_json_config(transformation_config)) def test_schema_id_empty(self): self.assertRaises(Error, load_and_validate_json_config, self.test_json1) From 1b14a8329290cff6e71f19101df38df999f2b518 Mon Sep 17 00:00:00 2001 From: iimironov Date: Thu, 22 Jul 2021 16:26:06 +0300 Subject: [PATCH 27/28] Fix BOM file --- model-optimizer/automation/package_BOM.txt | 2 +- model-optimizer/mo/utils/custom_replacement_config.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 41c7c796291edb..6a686717179232 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -1032,6 +1032,7 @@ mo/utils/graph.py mo/utils/guess_framework.py mo/utils/ie_version.py mo/utils/import_extensions.py +mo/utils/json_schema.py mo/utils/ir_engine/__init__.py mo/utils/ir_engine/compare_graphs.py mo/utils/ir_engine/ir_engine.py @@ -1074,7 +1075,6 @@ mo/utils/logger.py mo/utils/model_analysis.py mo/utils/pipeline_config.py mo/utils/replacement_pattern.py -mo/utils/schema.json mo/utils/shape.py mo/utils/simple_proto_parser.py mo/utils/str_to.py diff --git a/model-optimizer/mo/utils/custom_replacement_config.py b/model-optimizer/mo/utils/custom_replacement_config.py index 75389aa33a7928..47176e36e4ac18 100644 --- a/model-optimizer/mo/utils/custom_replacement_config.py +++ b/model-optimizer/mo/utils/custom_replacement_config.py @@ -405,8 +405,8 @@ def load_and_validate_json_config(config_file_name: str): json_config = json.load(f) validator = json_validate.compile(schema_dict) validator(json_config) - except Exception as exc: - raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(config_file_name, exc) + - refer_to_faq_msg(70)) from exc + except Exception as e: + raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(config_file_name, e) + + refer_to_faq_msg(70)) from e return json_config From 28866b04a69a18a6e286ab42097ed46ddba9c247 Mon Sep 17 00:00:00 2001 From: iimironov Date: Thu, 22 Jul 2021 17:57:32 +0300 Subject: [PATCH 28/28] Update bom file --- model-optimizer/automation/package_BOM.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 6a686717179232..939f75b685d292 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -1032,7 +1032,6 @@ mo/utils/graph.py mo/utils/guess_framework.py mo/utils/ie_version.py mo/utils/import_extensions.py -mo/utils/json_schema.py mo/utils/ir_engine/__init__.py mo/utils/ir_engine/compare_graphs.py mo/utils/ir_engine/ir_engine.py @@ -1071,6 +1070,7 @@ mo/utils/ir_reader/extenders/topk_extender.py mo/utils/ir_reader/extenders/variadic_split_extender.py mo/utils/ir_reader/layer_to_class.py mo/utils/ir_reader/restore_graph.py +mo/utils/json_schema.py mo/utils/logger.py mo/utils/model_analysis.py mo/utils/pipeline_config.py