From 63ff14a5ca3818fe07e6d3f27d50413bc7194b05 Mon Sep 17 00:00:00 2001 From: Kate Generalova Date: Wed, 27 Jan 2021 12:09:07 +0300 Subject: [PATCH] test: add GPL check for PyPi packages for Linux (#40) * test: add GPL check for PyPi packages for Linux * test: fix tests for data_runtime images (#38) * test: fix mount_root for data_runtime images * test: add skip mark if mount_root folder was not removed completely * test: mark test_detection_ssd_python for data_runtime images * fix: PEP8 and bandit issues Co-authored-by: Ilya Naumov --- .github/workflows/codestyle.yml | 9 +-- setup.cfg | 2 +- .../demos/test_demos_linux_runtime.py | 2 - .../samples/test_samples_linux_runtime.py | 2 - tests/functional/test_pypi_deps.py | 71 +++++++++++++++++++ tests/resources/pypi_deps/get_gpl_packages.py | 53 ++++++++++++++ 6 files changed, 128 insertions(+), 11 deletions(-) create mode 100644 tests/functional/test_pypi_deps.py create mode 100644 tests/resources/pypi_deps/get_gpl_packages.py diff --git a/.github/workflows/codestyle.yml b/.github/workflows/codestyle.yml index d5ede041..4cbd2c2e 100644 --- a/.github/workflows/codestyle.yml +++ b/.github/workflows/codestyle.yml @@ -24,15 +24,15 @@ jobs: - name: Running Bandit if: ${{ matrix.os == 'ubuntu-18.04' }} - run: python -m bandit -r ./ -f screen |& tee bandit.log + run: python -m bandit -r ./ -f screen - name: Running PEP checks if: ${{ matrix.os == 'ubuntu-18.04' }} - run: python -m flake8 ./ --config=setup.cfg --show-source |& tee flake8.log + run: python -m flake8 ./ --config=setup.cfg --show-source - name: Running MyPy checks if: ${{ matrix.os == 'ubuntu-18.04' }} - run: python -m mypy ./ --config-file ./setup.cfg --show-error-context --show-column-numbers --pretty |& tee mypy.log + run: python -m mypy ./ --config-file ./setup.cfg --show-error-context --show-column-numbers --pretty - name: Running pytest (not Docker image tests) if: ${{ always() }} @@ -44,7 +44,4 @@ jobs: with: name: codestyle_checks path: | - ./bandit.log - ./flake8.log - ./mypy.log ./utils_unittests.html \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index c60466a3..2d6d31a3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ docstring-convention = google ignore = DAR101, DAR201, DAR401, D107, D415, I201, T001, S404, S603, G004, B009, E800 enable-extensions=G per-file-ignores = - tests/*: D100,D101,D102,D104 + tests/*: D100,D101,D102,D104,S108 tests/conftest.py: D100,D101,D102,D103,D104 [pydocstyle] diff --git a/tests/functional/demos/test_demos_linux_runtime.py b/tests/functional/demos/test_demos_linux_runtime.py index 2f794665..1c5059a4 100644 --- a/tests/functional/demos/test_demos_linux_runtime.py +++ b/tests/functional/demos/test_demos_linux_runtime.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (C) 2019-2020 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import pathlib - import pytest diff --git a/tests/functional/samples/test_samples_linux_runtime.py b/tests/functional/samples/test_samples_linux_runtime.py index 11aa0c09..24203734 100644 --- a/tests/functional/samples/test_samples_linux_runtime.py +++ b/tests/functional/samples/test_samples_linux_runtime.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (C) 2019-2020 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import pathlib - import pytest from utils.exceptions import FailedTest diff --git a/tests/functional/test_pypi_deps.py b/tests/functional/test_pypi_deps.py new file mode 100644 index 00000000..cf8c9e5d --- /dev/null +++ b/tests/functional/test_pypi_deps.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2019-2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +import os +import pathlib + +import pytest + + +class TestPyPiDependencies: + @pytest.mark.usefixtures('_is_image_os') + @pytest.mark.parametrize('_is_image_os', ['ubuntu18', 'ubuntu20', 'centos7', 'centos8'], indirect=True) + @pytest.mark.xfail(reason='47558 GPL Unidecode PyPi package as dependency for OMZ text_to_speech_demo') + def test_gpl_pypi_deps(self, tester, image): + root = pathlib.Path(os.path.realpath(__name__)).parent + image_folder = image.replace('/', '_').replace(':', '_') + pypi_log_folder = root / 'logs' / image_folder / 'pypi_deps' + if not pypi_log_folder.exists(): + pypi_log_folder.mkdir() + kwargs = { + 'volumes': { + root / 'tests' / 'resources' / 'pypi_deps': {'bind': '/tmp/pypi_deps', 'mode': 'rw'}, # nosec + pypi_log_folder: {'bind': '/tmp/logs', 'mode': 'rw'}, # nosec + }, + } + tester.test_docker_image( + image, + ['/bin/bash -ac "python3 -m pip freeze 2>&1 | tee /tmp/logs/pypi_deps.log"', + '/bin/bash -ac "python3 -m pip check 2>&1 | tee /tmp/logs/pypi_deps_check.log"', + 'python3 -m pip install pipdeptree', + '/bin/bash -ac "python3 -m pipdeptree -e PyGObject 2>&1 | tee /tmp/logs/pypi_deps_tree.log"', + 'python3 -m pip install pip-licenses', + 'pip-licenses --output-file /tmp/logs/pypi_licenses.log', + 'pip-licenses -f json --output-file /tmp/logs/pypi_licenses.json', + 'python3 /tmp/pypi_deps/get_gpl_packages.py -f /tmp/logs/pypi_licenses.json ' + '-l /tmp/logs/pypi_licenses_gpl.json', + ], + self.test_gpl_pypi_deps.__name__, **kwargs, + ) + + @pytest.mark.usefixtures('_is_image_os', '_is_distribution') + @pytest.mark.parametrize('_is_image_os', ['ubuntu18', 'ubuntu20', 'centos7', 'centos8'], indirect=True) + @pytest.mark.parametrize('_is_distribution', ['dev', 'proprietary'], indirect=True) + def test_gpl_pypi_deps_venv_tf2(self, tester, image): + root = pathlib.Path(os.path.realpath(__name__)).parent + image_folder = image.replace('/', '_').replace(':', '_') + pypi_log_folder = root / 'logs' / image_folder / 'pypi_deps' + if not pypi_log_folder.exists(): + pypi_log_folder.mkdir() + kwargs = { + 'volumes': { + root / 'tests' / 'resources' / 'pypi_deps': {'bind': '/tmp/pypi_deps', 'mode': 'rw'}, # nosec + pypi_log_folder: {'bind': '/tmp/logs', 'mode': 'rw'}, # nosec + }, + } + tester.test_docker_image( + image, + ['/bin/bash -ac "cd /opt/intel/venv_tf2 && . ./bin/activate && ' + 'python3 -m pip freeze 2>&1 | tee /tmp/logs/pypi_deps_tf2.log"', + '/bin/bash -ac "cd /opt/intel/venv_tf2 && . ./bin/activate && ' + 'python3 -m pip check 2>&1 | tee /tmp/logs/pypi_deps_check_tf2.log"', + '/bin/bash -ac "cd /opt/intel/venv_tf2 && . ./bin/activate && python3 -m pip install pipdeptree && ' + 'python3 -m pipdeptree -e PyGObject 2>&1 | tee /tmp/logs/pypi_deps_tree_tf2.log"', + '/bin/bash -ac "cd /opt/intel/venv_tf2 && . ./bin/activate && python3 -m pip install pip-licenses && ' + 'pip-licenses --output-file /tmp/logs/pypi_licenses_tf2.log && ' + 'pip-licenses -f json --output-file /tmp/logs/pypi_licenses_tf2.json"', + 'python3 /tmp/pypi_deps/get_gpl_packages.py -f /tmp/logs/pypi_licenses_tf2.json ' + '-l /tmp/logs/pypi_licenses_gpl_tf2.json', + ], + self.test_gpl_pypi_deps_venv_tf2.__name__, **kwargs, + ) diff --git a/tests/resources/pypi_deps/get_gpl_packages.py b/tests/resources/pypi_deps/get_gpl_packages.py new file mode 100644 index 00000000..19b194ca --- /dev/null +++ b/tests/resources/pypi_deps/get_gpl_packages.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2019-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""Check GPL/LGPL license for the installed PyPi packages +""" +import argparse +import json +import logging +import sys + +parser = argparse.ArgumentParser(description='This is GPl/LGPL licenses checker for PyPi packages') +parser.add_argument( + '-f', + '--file', + metavar='PATH', + required=True, + help='JSON file with packages meta', +) +parser.add_argument( + '-l', + '--logs', + metavar='PATH', + required=False, + default='pypi_licenses_gpl.json', + help='Log file in json format', +) + +logging.basicConfig(level='INFO') +log = logging.getLogger(__name__) +log.info('Start searching GPl/LGPL licenses in the installed PyPi packages ...') +args = parser.parse_args() +with open(args.file) as licenses_file: + pkg_licenses = json.load(licenses_file) + +exit_code = 0 +gpl_pkgs = [] +for pkg in pkg_licenses: + if 'GPL' in pkg['License']: + gpl_pkgs.append(pkg) + if 'LGPL' not in pkg['License']: + log.error(f'GPL package was found in PyPi environment: {pkg}') + exit_code = 1 +log.debug(f'Found GPL/LGPL packages: {gpl_pkgs}') +with open(args.logs, 'w') as gpl_licenses_file: + json.dump(gpl_pkgs, gpl_licenses_file) +log.info(f'See GPL/LGPL licenses in the json log: {args.logs}') + +if exit_code != 0: + log.info('FAILED') +else: + log.info('PASSED') + +sys.exit(exit_code)