Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add json validate #6449

Merged
Merged
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
da41f57
Add json validate
iimironov Jun 29, 2021
b029aca
Fix json schema
iimironov Jun 29, 2021
1be9dfc
Fix schema loader
iimironov Jun 30, 2021
e814957
Add unit test
iimironov Jun 30, 2021
28dccec
Update bom file
iimironov Jul 1, 2021
b3fd7f8
Update all requarments
iimironov Jul 1, 2021
ae769dd
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 1, 2021
b6dcbf4
Update dev requarments
iimironov Jul 1, 2021
6e37852
Update requrments
iimironov Jul 1, 2021
09e68ff
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 6, 2021
a6d768c
Update path to schema
iimironov Jul 6, 2021
74b22e1
Update schema
iimironov Jul 7, 2021
02d555c
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 7, 2021
d9943a6
Add some unit tests
iimironov Jul 8, 2021
9adef2f
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 8, 2021
8702ec5
Move schema to root dir
iimironov Jul 9, 2021
979b61e
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 9, 2021
aa3ddc3
Update schema path in bom file
iimironov Jul 9, 2021
4dd8b73
Fix unit test
iimironov Jul 9, 2021
e1e1806
Fix bom
iimironov Jul 9, 2021
50812f7
Change path to schema
iimironov Jul 9, 2021
1f226e5
update setup
iimironov Jul 9, 2021
03c28db
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 9, 2021
2c7a00a
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 12, 2021
0b05608
Fix setup
iimironov Jul 12, 2021
9da6f7a
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 12, 2021
85cf10e
Fix mo args test
iimironov Jul 12, 2021
b35a6df
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 12, 2021
ccd66db
Refactoring some code
iimironov Jul 16, 2021
c1995f0
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 16, 2021
e270017
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 16, 2021
76093fa
Refactoring according to review
iimironov Jul 19, 2021
f641c9f
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 19, 2021
358f336
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 19, 2021
036989b
Update sort imports
iimironov Jul 20, 2021
39647c6
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 20, 2021
7a0392b
Remove id attribute from schema
iimironov Jul 20, 2021
30889a9
Refactoring validator
iimironov Jul 20, 2021
776fefd
Fix according to review
iimironov Jul 21, 2021
5626241
Merge branch 'master' into imironov/add-json-validator
iimironov Jul 21, 2021
bdc77e4
Move schema from json to dict. Update unit tests.
iimironov Jul 22, 2021
1b14a83
Fix BOM file
iimironov Jul 22, 2021
28866b0
Update bom file
iimironov Jul 22, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions model-optimizer/automation/package_BOM.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1074,6 +1074,7 @@ mo/utils/logger.py
mo/utils/model_analysis.py
mo/utils/pipeline_config.py
mo/utils/replacement_pattern.py
mo/utils/schema.json
mo/utils/shape.py
mo/utils/simple_proto_parser.py
mo/utils/str_to.py
Expand Down
18 changes: 16 additions & 2 deletions model-optimizer/mo/utils/custom_replacement_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@
import os
from re import compile, match

import fastjsonschema as json_validate
iimironov marked this conversation as resolved.
Show resolved Hide resolved

from mo.graph.graph import Node, Graph
from mo.utils.error import Error
from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes
from mo.utils.utils import get_mo_root_dir
from mo.utils.utils import refer_to_faq_msg


Expand Down Expand Up @@ -297,12 +300,12 @@ def update_custom_replacement_attributes(self, graph: Graph):
log.debug("Node {} doesn't have output edges. Consider it output".format(node_name))
output_tensors.add((generate_pattern_for_node(graph, pattern, node_name), 0))

if not self.has('inputs'):
if not self.has('inputs') or len(self._replacement_desc['inputs']) == 0:
iimironov marked this conversation as resolved.
Show resolved Hide resolved
self._replacement_desc['inputs'] = [[{'node': desc[0], 'port': desc[1]} for desc in inp]
for inp in sorted(input_nodes_mapping.values())]
log.debug('Updated inputs of sub-graph for instance "{}"'.format(self.instances))

if not self.has('outputs'):
if not self.has('outputs') or len(self._replacement_desc['outputs']) == 0:
self._replacement_desc['outputs'] = [{'node': node, 'port': port} for node, port in sorted(output_tensors)]
log.debug('Updated outputs of sub-graph for instance "{}"'.format(self.instances))

Expand Down Expand Up @@ -349,6 +352,17 @@ def parse_custom_replacement_config_file(file_name: str):
raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(file_name, exc) +
refer_to_faq_msg(70)) from exc

try:
base_dir = get_mo_root_dir()
schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json')
with open(schema_file, 'r') as f:
schema = json.load(f)
validator = json_validate.compile(schema)
validator(data)
except Exception as exc:
raise Error("Failed to validate custom replacements configuration file '{}': {}. ".format(file_name, exc) +
refer_to_faq_msg(70)) from exc
iimironov marked this conversation as resolved.
Show resolved Hide resolved

result = list()
validation_errors = list()
for attrs in data:
Expand Down
128 changes: 128 additions & 0 deletions model-optimizer/mo/utils/schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
{
"definitions": {},
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://example.com/object1624983563.json",
iimironov marked this conversation as resolved.
Show resolved Hide resolved
"title": "Root",
"type": "array",
"default": [],
"items":{
"$id": "#root/items",
"title": "Items",
"type": "object",
"required": [
"id",
"match_kind"
],
"properties": {
"custom_attributes": {
"$id": "#root/items/custom_attributes",
"title": "Custom_attributes",
"type": "object",
"properties": {
}
},
"id": {
"$id": "#root/items/id",
"title": "Id",
"type": "string",
"pattern": "^.*$",
"minLength": 1
},
"inputs": {
"$id": "#root/items/inputs",
"title": "Inputs",
"type": "array",
"default": [],
"items":{
"$id": "#root/items/inputs/items",
"title": "Items",
"type": "array",
"default": [],
"items":{
"$id": "#root/items/inputs/items/items",
"title": "Items",
"type": "object",
"properties": {
"node": {
"$id": "#root/items/inputs/items/items/node",
"title": "Node",
"type": "string",
"default": "",
"pattern": "^.*$"
iimironov marked this conversation as resolved.
Show resolved Hide resolved
},
"port": {
"$id": "#root/items/inputs/items/items/port",
"title": "Port",
"type": "integer",
iimironov marked this conversation as resolved.
Show resolved Hide resolved
"default": 0
}
},
"required": ["node", "port"]
}

}
},
"instances": {
"$id": "#root/items/instances",
"title": "Instances",
"type": ["array", "object"],
"items":{
"$id": "#root/items/instances/items",
"title": "Items",
"type": "string",
"default": "",
"pattern": "^.*$"
iimironov marked this conversation as resolved.
Show resolved Hide resolved
}
},
"match_kind": {
"$id": "#root/items/match_kind",
"title": "Match_kind",
"type": "string",
"enum": ["points", "scope", "general"],
"default": "points",
"pattern": "^.*$"
},
"outputs": {
"$id": "#root/items/outputs",
"title": "Outputs",
"type": "array",
"default": [],
"items":{
"$id": "#root/items/outputs/items",
"title": "Items",
"type": "object",
"properties": {
"node": {
"$id": "#root/items/outputs/items/node",
"title": "Node",
"type": "string",
"default": "",
"pattern": "^.*$"
},
"port": {
"$id": "#root/items/outputs/items/port",
"title": "Port",
"type": "integer",
"default": 0
}
},
"required": ["node", "port"]
}

},
"include_inputs_to_sub_graph": {
"$id": "#root/items/include_inputs_to_sub_graph",
"title": "Include_inputs_to_sub_graph",
"type": "boolean",
"default": false
},
"include_outputs_to_sub_graph": {
"$id": "#root/items/include_outputs_to_sub_graph",
"title": "Include_outputs_to_sub_graph",
"type": "boolean",
"default": false
}
}
}

}
1 change: 1 addition & 0 deletions model-optimizer/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ onnx>=1.8.1
defusedxml>=0.7.1
urllib3>=1.26.4
requests>=2.25.1
fastjsonschema~=2.15.1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@generalova-kate , if we introduce new dependency should we update the openvino-dev package requirements as well?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@evgenytalanin-intel , could you suggest someone to answer the question above?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ttroilov , could you take a look if we are ok to add this new dependency to the MO from the legal perspective?

iimironov marked this conversation as resolved.
Show resolved Hide resolved
1 change: 1 addition & 0 deletions model-optimizer/requirements_caffe.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ numpy>=1.16.6,<1.20
protobuf>=3.15.6
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ test-generator==0.1.1
defusedxml>=0.5.0
requests>=2.20.0
pytest>=6.2.4
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_kaldi.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_mxnet.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ numpy>=1.16.6,<1.20
defusedxml>=0.7.1
urllib3>=1.26.4
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_onnx.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_tf.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ networkx~=2.5
numpy>=1.16.6,<1.19
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/requirements_tf2.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1
1 change: 1 addition & 0 deletions model-optimizer/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ def find_package_modules(self, package, package_dir):
},
package_data={
'mo.mo.front.caffe.proto': ['*.proto'],
'mo.mo.utils': ['*.json'],
'mo.extensions.front.mxnet': ['*.json'],
'mo.extensions.front.onnx': ['*.json'],
'mo.extensions.front.tf': ['*.json'],
Expand Down
91 changes: 91 additions & 0 deletions model-optimizer/unit_tests/mo/utils/schema_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

import json
import os
import unittest

import fastjsonschema as json_validate
from generator import generator, generate

from mo.utils.utils import get_mo_root_dir

base_dir = get_mo_root_dir()
path = os.path.join(base_dir, 'extensions', 'front',)
schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json')

test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]'
test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]'

@generator
class TestSchema(unittest.TestCase):
@generate(*[('tf', 'efficient_det_support_api_v2.0.json'),
('tf', 'efficient_det_support_api_v2.4.json'),
('tf', 'faster_rcnn_support_api_v1.7.json'),
('tf', 'faster_rcnn_support_api_v1.10.json'),
('tf', 'faster_rcnn_support_api_v1.13.json'),
('tf', 'faster_rcnn_support_api_v1.14.json'),
('tf', 'faster_rcnn_support_api_v1.15.json'),
('tf', 'faster_rcnn_support_api_v2.0.json'),
('tf', 'faster_rcnn_support_api_v2.4.json'),
('tf', 'mask_rcnn_support.json'),
('tf', 'mask_rcnn_support_api_v1.7.json'),
('tf', 'mask_rcnn_support_api_v1.11.json'),
('tf', 'mask_rcnn_support_api_v1.13.json'),
('tf', 'mask_rcnn_support_api_v1.14.json'),
('tf', 'mask_rcnn_support_api_v1.15.json'),
('tf', 'mask_rcnn_support_api_v2.0.json'),
('tf', 'retinanet.json'),
iimironov marked this conversation as resolved.
Show resolved Hide resolved
('tf', 'rfcn_support.json'),
('tf', 'rfcn_support_api_v1.10.json'),
('tf', 'rfcn_support_api_v1.13.json'),
('tf', 'rfcn_support_api_v1.14.json'),
('tf', 'ssd_support.json'),
('tf', 'ssd_support_api_v1.14.json'),
('tf', 'ssd_support_api_v1.15.json'),
('tf', 'ssd_support_api_v2.0.json'),
('tf', 'ssd_support_api_v2.4.json'),
('tf', 'ssd_toolbox_detection_output.json'),
('tf', 'ssd_toolbox_multihead_detection_output.json'),
('tf', 'ssd_v2_support.json'),
('tf', 'yolo_v1.json'),
('tf', 'yolo_v1_tiny.json'),
('tf', 'yolo_v2.json'),
('tf', 'yolo_v2_tiny.json'),
('tf', 'yolo_v2_tiny_voc.json'),
('tf', 'yolo_v3.json'),
('tf', 'yolo_v3_tiny.json'),
('tf', 'yolo_v3_voc.json'),
('onnx', 'faster_rcnn.json'),
('onnx', 'person_detection_crossroad.json'),
('mxnet', 'yolo_v3_mobilenet1_voc.json'),
])
def test_schema_file(self, config_path, transformation_config):
transformation_file = os.path.join(path, config_path, transformation_config)
with open(transformation_file, 'r') as f:
data = json.load(f)

with open(schema_file, 'r') as f:
schema = json.load(f)
validator = json_validate.compile(schema)
validator(data)

def test_schema_id_empty(self):
data = json.loads(test_json1)
try:
with open(schema_file, 'r') as f:
schema = json.load(f)
validator = json_validate.compile(schema)
validator(data)
except Exception as e:
assert e.message == "data[0].id must be longer than or equal to 1 characters"

def test_schema_match_kind_wrong(self):
data = json.loads(test_json2)
try:
with open(schema_file, 'r') as f:
schema = json.load(f)
validator = json_validate.compile(schema)
validator(data)
except Exception as e:
iimironov marked this conversation as resolved.
Show resolved Hide resolved
assert e.message == "data[0].match_kind must be one of ['points', 'scope', 'general']"
1 change: 1 addition & 0 deletions tests/stress_tests/scripts/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
pymongo
Jinja2
PyYAML
fastjsonschema~=2.15.1

h5py<3.0.0 # WA for OMZ Keras models. Details: https://github.com/openvinotoolkit/open_model_zoo/issues/1806