E2E open-sourced (#20429)

Co-authored-by: Roman Kazantsev <[email protected]>

tests/CMakeLists.txt

@@ -5,3 +5,4 @@
 add_subdirectory(layer_tests)
 add_subdirectory(model_hub_tests)
 add_subdirectory(samples_tests)
+add_subdirectory(e2e_tests)

tests/e2e_tests/CMakeLists.txt

@@ -0,0 +1,9 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+cmake_minimum_required(VERSION 3.13)
+
+project(e2e_tests)
+
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL)

tests/e2e_tests/README.md

@@ -0,0 +1,50 @@
+# End-to-end Tests User Documentation
+This folder contains a code to run end-to-end validation of OpenVINO on real models of different frameworks (PyTorch, TensorFlow, and ONNX)
+
+The documentation provides necessary information about environment setup for e2e validation run, adding new model to the validation, and instructions to launch validation.
+
+> The following steps assume that your current working directory is:
+> `tests/e2e_tests`
+
+### Environment preparation:
+   * Install Python modules required for tests:
+       ```bash
+       pip3 install -r requirements.txt 
+       ```
+
+### Add model from TensorFlow Hub repo to end-to-end validation:
+To add new test for model from TF Hub repo just add new line into pipelines/production/tf_hub/precommit.yml
+This line should contain comma separated model name and its link
+```
+movenet/singlepose/lightning,https://www.kaggle.com/models/google/movenet/frameworks/tensorFlow2/variations/singlepose-lightning/versions/4
+```
+
+### Main entry-point 
+
+There is one main testing entry-point which is responsible for test run - test_base.py. This script performs the
+following actions:
+1. Loads model from its source 
+2. Infers original model through framework
+3. Converts original model through OVC convert model
+4. Infers converted model through OpenVINO
+5. Provides results of element-wise comparison of framework and OpenVINO inference
+
+#### Launch tests
+
+[test_base.py](https://github.com/openvinotoolkit/openvino/tree/master/tests/e2e_tests/test_base.py) is the main script to run end-to-end tests.
+Run all end-to-end tests in `pipelines/`:
+```bash
+pytest test_base.py
+```
+`test_base.py` options:  
+
+- `--modules=MODULES [MODULES ...]` - Paths to tests.  
+- `-k TESTNAME [TESTNAME ...]`- Test names.
+- `-s` - Step-by-step logging.
+
+Example:
+```bash
+pytest test_base.py -s --modules=pipelines/production/tf_hub
+```
+
+> For full information on pytest options, run `pytest --help` or see the [documentation](https://docs.pytest.org)

tests/e2e_tests/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2018-2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+

tests/e2e_tests/collect_refs.py

@@ -0,0 +1,62 @@
+# Copyright (C) 2018-2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Main entry-point to collect references for E2E tests.
+
+Default run:
+$ pytest collect_refs.py
+
+Options[*]:
+--modules       Paths to references
+--env_conf      Path to environment config
+--dry_run       Disable reference saving
+
+[*] For more information see conftest.py
+"""
+# pylint:disable=invalid-name
+import numpy as np
+import logging as log
+import os
+from e2e_tests.common.parsers import pipeline_cfg_to_string
+from e2e_tests.common.common.pipeline import Pipeline
+
+pytest_plugins = ('e2e_tests.common.plugins.ref_collect.conftest', )
+
+
+def save_reference(refs, path, use_torch_to_save):
+    log.info("saving reference results to {path}".format(path=path))
+    os.makedirs(os.path.dirname(path), mode=0o755, exist_ok=True)
+    if use_torch_to_save:
+        import torch
+        torch.save(refs, path)
+    else:
+        np.savez(path, **refs)
+
+
+def test_collect_reference(reference, dry_run):
+    """Parameterized reference collection.
+
+    :param reference: reference collection instance
+
+    :param dry_run: dry-run flag. if True, disables saving reference result to
+                    filesystem
+    """
+    for attr in ['pipeline', 'store_path']:
+        if attr not in reference:
+            raise ValueError(
+                'obligatory attribute is missing: {attr}'.format(attr=attr))
+    pipeline = Pipeline(reference['pipeline'])
+    log.debug("Reference Pipeline:\n{}".format(pipeline_cfg_to_string(pipeline._config)))
+    pipeline.run()
+    refs = pipeline.fetch_results()
+    if not dry_run:
+        save_reference(refs, reference['store_path'], reference.get('use_torch_to_save', False))
+        # Always save to `store_path_for_ref_save` (it points to share in automatics)
+        if 'store_path_for_ref_save' in reference and reference['store_path'] != reference['store_path_for_ref_save']:
+            save_reference(refs, reference['store_path_for_ref_save'], reference.get('use_torch_to_save', False))
+    else:
+        log.info("dry run option is used. reference results are not saved")
+
+
+
+

tests/e2e_tests/common/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (C) 2018-2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0

tests/e2e_tests/common/common/base_provider.py

@@ -0,0 +1,64 @@
+# Copyright (C) 2018-2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+class BaseProviderMeta(type):
+    def __new__(mcs, name, bases, attrs, **kwargs):
+        cls = super().__new__(mcs, name, bases, attrs)
+        # do not create container for abstract provider
+        if '_is_base_provider' in attrs:
+            return cls
+        assert issubclass(cls, BaseProvider), "Do not use metaclass directly"
+        cls.register(cls)
+        return cls
+
+
+class BaseProvider(metaclass=BaseProviderMeta):
+    _is_base_provider = True
+    registry = {}
+    __action_name__ = None
+
+    @classmethod
+    def register(cls, provider):
+        provider_name = getattr(cls, '__action_name__')
+        if not provider_name:
+            return
+        cls.registry[provider_name] = provider
+
+    @classmethod
+    def provide(cls, provider, *args, **kwargs):
+        if provider not in cls.registry:
+            raise ValueError("Requested provider {} not registered".format(provider))
+        root_provider = cls.registry[provider]
+        root_provider.validate()
+        return root_provider(*args, **kwargs)
+
+
+class StepProviderMeta(type):
+    def __new__(mcs, name, bases, attrs, **kwargs):
+        cls = super().__new__(mcs, name, bases, attrs)
+        # do not create container for abstract provider
+        if '_is_base_provider' in attrs:
+            return cls
+        assert issubclass(cls, BaseStepProvider), "Do not use metaclass directly"
+        cls.register(cls)
+        return cls
+
+
+class BaseStepProvider(metaclass=StepProviderMeta):
+    _is_base_provider = True
+    registry = {}
+    __step_name__ = None
+
+    @classmethod
+    def register(cls, provider):
+        provider_name = getattr(cls, '__step_name__', None)
+        if not provider_name:
+            return
+        cls.registry[provider_name] = provider
+
+    @classmethod
+    def provide(cls, provider, *args, **kwargs):
+        if provider not in cls.registry:
+            raise ValueError("Requested provider {} not registered".format(provider))
+        root_provider = cls.registry[provider]
+        return root_provider(*args, **kwargs)