Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add json validate #6449

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
da41f57
Add json validate
iimironov Jun 29, 2021
b029aca
Fix json schema
iimironov Jun 29, 2021
1be9dfc
Fix schema loader
iimironov Jun 30, 2021
e814957
Add unit test
iimironov Jun 30, 2021
28dccec
Update bom file
iimironov Jul 1, 2021
b3fd7f8
Update all requarments
iimironov Jul 1, 2021
ae769dd
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 1, 2021
b6dcbf4
Update dev requarments
iimironov Jul 1, 2021
6e37852
Update requrments
iimironov Jul 1, 2021
09e68ff
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 6, 2021
a6d768c
Update path to schema
iimironov Jul 6, 2021
74b22e1
Update schema
iimironov Jul 7, 2021
02d555c
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 7, 2021
d9943a6
Add some unit tests
iimironov Jul 8, 2021
9adef2f
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 8, 2021
8702ec5
Move schema to root dir
iimironov Jul 9, 2021
979b61e
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 9, 2021
aa3ddc3
Update schema path in bom file
iimironov Jul 9, 2021
4dd8b73
Fix unit test
iimironov Jul 9, 2021
e1e1806
Fix bom
iimironov Jul 9, 2021
50812f7
Change path to schema
iimironov Jul 9, 2021
1f226e5
update setup
iimironov Jul 9, 2021
03c28db
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 9, 2021
2c7a00a
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 12, 2021
0b05608
Fix setup
iimironov Jul 12, 2021
9da6f7a
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 12, 2021
85cf10e
Fix mo args test
iimironov Jul 12, 2021
b35a6df
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 12, 2021
ccd66db
Refactoring some code
iimironov Jul 16, 2021
c1995f0
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 16, 2021
e270017
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 16, 2021
76093fa
Refactoring according to review
iimironov Jul 19, 2021
f641c9f
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 19, 2021
358f336
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 19, 2021
036989b
Update sort imports
iimironov Jul 20, 2021
39647c6
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 20, 2021
7a0392b
Remove id attribute from schema
iimironov Jul 20, 2021
30889a9
Refactoring validator
iimironov Jul 20, 2021
776fefd
Fix according to review
iimironov Jul 21, 2021
5626241
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 21, 2021
bdc77e4
Move schema from json to dict. Update unit tests.
iimironov Jul 22, 2021
1b14a83
Fix BOM file
iimironov Jul 22, 2021
28866b0
Update bom file
iimironov Jul 22, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions model-optimizer/automation/package_BOM.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1070,6 +1070,7 @@ mo/utils/ir_reader/extenders/topk_extender.py
mo/utils/ir_reader/extenders/variadic_split_extender.py
mo/utils/ir_reader/layer_to_class.py
mo/utils/ir_reader/restore_graph.py
mo/utils/json_schema.py
mo/utils/logger.py
mo/utils/model_analysis.py
mo/utils/pipeline_config.py
Expand Down
34 changes: 25 additions & 9 deletions model-optimizer/mo/utils/custom_replacement_config.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

import fastjsonschema as json_validate
import json
import logging as log
import os
Expand All @@ -9,7 +10,8 @@
from mo.graph.graph import Node, Graph
from mo.utils.error import Error
from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes
from mo.utils.utils import refer_to_faq_msg
from mo.utils.json_schema import schema_dict
from mo.utils.utils import get_mo_root_dir, refer_to_faq_msg


class CustomReplacementDescriptor(object):
Expand Down Expand Up @@ -297,12 +299,12 @@ def update_custom_replacement_attributes(self, graph: Graph):
log.debug("Node {} doesn't have output edges. Consider it output".format(node_name))
output_tensors.add((generate_pattern_for_node(graph, pattern, node_name), 0))

if not self.has('inputs'):
if not self.has('inputs') or len(self._replacement_desc['inputs']) == 0:
iimironov marked this conversation as resolved.
Show resolved Hide resolved
self._replacement_desc['inputs'] = [[{'node': desc[0], 'port': desc[1]} for desc in inp]
for inp in sorted(input_nodes_mapping.values())]
log.debug('Updated inputs of sub-graph for instance "{}"'.format(self.instances))

if not self.has('outputs'):
if not self.has('outputs') or len(self._replacement_desc['outputs']) == 0:
self._replacement_desc['outputs'] = [{'node': node, 'port': port} for node, port in sorted(output_tensors)]
log.debug('Updated outputs of sub-graph for instance "{}"'.format(self.instances))

Expand Down Expand Up @@ -342,13 +344,8 @@ def parse_custom_replacement_config_file(file_name: str):
if not os.path.exists(file_name):
raise Error("Custom replacements configuration file '{}' does not exist. ".format(file_name) +
refer_to_faq_msg(69))
try:
with open(file_name, 'r') as f:
data = json.load(f)
except Exception as exc:
raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(file_name, exc) +
refer_to_faq_msg(70)) from exc

data = load_and_validate_json_config(file_name)
result = list()
validation_errors = list()
for attrs in data:
Expand Down Expand Up @@ -394,3 +391,22 @@ def generate_pattern_for_node(graph: Graph, sub_graph_pattern: str, node_name: s

raise RuntimeError('The pattern that uniquely identifies node "{}" using sub-graph pattern "{}" has not been found'.
format(node_name, sub_graph_pattern))


def load_and_validate_json_config(config_file_name: str):
"""
Reads and validate custom replacement configuration file config_file_name.
:param config_file_name: name of the file to read from.
:return: A dictionary serialized from json config file.
"""

try:
with open(config_file_name, 'r') as f:
json_config = json.load(f)
validator = json_validate.compile(schema_dict)
validator(json_config)
except Exception as e:
raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(config_file_name, e) +
refer_to_faq_msg(70)) from e

return json_config
129 changes: 129 additions & 0 deletions model-optimizer/mo/utils/json_schema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

schema_dict = {
"definitions": {},
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Root",
"type": "array",
"default": [],
"items": {
"$id": "#root/items",
"title": "Items",
"type": "object",
"required": [
"id",
"match_kind"
],
"properties": {
"custom_attributes": {
"$id": "#root/items/custom_attributes",
"title": "Custom_attributes",
"type": "object",
"properties": {
}
},
"id": {
"$id": "#root/items/id",
"title": "Id",
"type": "string",
"pattern": "^.*$",
"minLength": 1
},
"inputs": {
"$id": "#root/items/inputs",
"title": "Inputs",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/inputs/items",
"title": "Items",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/inputs/items/items",
"title": "Items",
"type": "object",
"properties": {
"node": {
"$id": "#root/items/inputs/items/items/node",
"title": "Node",
"type": "string",
"default": "",
"pattern": "^.*$"
},
"port": {
"$id": "#root/items/inputs/items/items/port",
"title": "Port",
"type": "integer",
"default": 0
}
},
"required": ["node", "port"]
}

}
},
"instances": {
"$id": "#root/items/instances",
"title": "Instances",
"type": ["array", "object"],
"items": {
"$id": "#root/items/instances/items",
"title": "Items",
"type": "string",
"default": "",
"pattern": "^.*$"
}
},
"match_kind": {
"$id": "#root/items/match_kind",
"title": "Match_kind",
"type": "string",
"enum": ["points", "scope", "general"],
"default": "points",
"pattern": "^.*$"
},
"outputs": {
"$id": "#root/items/outputs",
"title": "Outputs",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/outputs/items",
"title": "Items",
"type": "object",
"properties": {
"node": {
"$id": "#root/items/outputs/items/node",
"title": "Node",
"type": "string",
"default": "",
"pattern": "^.*$"
},
"port": {
"$id": "#root/items/outputs/items/port",
"title": "Port",
"type": "integer",
"default": 0
}
},
"required": ["node", "port"]
}

},
"include_inputs_to_sub_graph": {
"$id": "#root/items/include_inputs_to_sub_graph",
"title": "Include_inputs_to_sub_graph",
"type": "boolean",
"default": False
},
"include_outputs_to_sub_graph": {
"$id": "#root/items/include_outputs_to_sub_graph",
"title": "Include_outputs_to_sub_graph",
"type": "boolean",
"default": False
}
}
}
}
1 change: 1 addition & 0 deletions model-optimizer/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ onnx>=1.8.1
defusedxml>=0.7.1
urllib3>=1.26.4
requests>=2.25.1
fastjsonschema~=2.15.1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@generalova-kate , if we introduce new dependency should we update the openvino-dev package requirements as well?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@evgenytalanin-intel , could you suggest someone to answer the question above?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ttroilov , could you take a look if we are ok to add this new dependency to the MO from the legal perspective?

iimironov marked this conversation as resolved.
Show resolved Hide resolved
1 change: 1 addition & 0 deletions model-optimizer/requirements_caffe.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ numpy>=1.16.6,<1.20
protobuf>=3.15.6
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ test-generator==0.1.1
defusedxml>=0.5.0
requests>=2.20.0
pytest>=6.2.4
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_kaldi.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_mxnet.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ numpy>=1.16.6,<1.20
defusedxml>=0.7.1
urllib3>=1.26.4
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_onnx.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_tf.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ networkx~=2.5
numpy>=1.16.6,<1.19
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_tf2.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

import os
import unittest
from fnmatch import fnmatch
from generator import generator, generate

from mo.utils.custom_replacement_config import load_and_validate_json_config
from mo.utils.error import Error
from mo.utils.utils import get_mo_root_dir


def get_json_configs(mo_root_dir):
config_path = os.path.join(mo_root_dir, 'extensions', 'front')
pattern = "*.json"
config_files_list = []
for path, subdirs, files in os.walk(config_path):
for name in files:
if fnmatch(name, pattern):
config_files_list.append((os.path.join(path, name),))
return config_files_list

@generator
class TestSchema(unittest.TestCase):
base_dir = get_mo_root_dir()
schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json')
transformation_configs = get_json_configs(base_dir)
test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]'
test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]'

@generate(*transformation_configs)
def test_schema_file(self, transformation_config):
self.assertTrue(load_and_validate_json_config(transformation_config))

def test_schema_id_empty(self):
self.assertRaises(Error, load_and_validate_json_config, self.test_json1)

def test_schema_match_kind_wrong(self):
self.assertRaises(Error, load_and_validate_json_config, self.test_json2)
1 change: 1 addition & 0 deletions tests/stress_tests/scripts/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
pymongo
Jinja2
PyYAML
fastjsonschema~=2.15.1

h5py<3.0.0 # WA for OMZ Keras models. Details: https://github.com/openvinotoolkit/open_model_zoo/issues/1806