From 2991408924d5a54cb9b30e0d5764294513866daf Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 14 Oct 2022 16:56:03 +0200 Subject: [PATCH] Is1884/using integration library (#40) --- .github/CODEOWNERS | 4 - .github/PULL_REQUEST_TEMPLATE.md | 41 +- .github/workflows/github-ci.yml | 4 +- .gitignore | 7 +- .vscode/extensions.json | 7 + .vscode/launch.template.json | 23 + .vscode/settings.template.json | 39 ++ LICENSE | 2 +- Makefile | 13 +- README.md | 37 +- VERSION | 2 +- cookiecutter.json | 70 +-- setup.cfg | 3 +- tests/test_bake_project.py | 75 +-- {{cookiecutter.project_slug}}/.cookiecutterrc | 2 + .../.github/workflows/github-ci.yml | 4 +- .../delete-image-from-registry-portus.bash | 0 .../delete-image-from-registry.bash | 0 .../{ci => .gitlab}/docker-registry-curl.bash | 0 .../{ci => .gitlab}/gitlab-ci.yml | 2 +- {{cookiecutter.project_slug}}/.pylintrc | 549 ------------------ {{cookiecutter.project_slug}}/Makefile | 74 ++- {{cookiecutter.project_slug}}/VERSION | 1 - .../VERSION_INTEGRATION | 1 - .../docker-compose-meta.yml | 10 - {{cookiecutter.project_slug}}/requirements.in | 13 +- .../tests/conftest.py | 57 +- .../tests/integration/conftest.py | 45 -- .../integration/test_docker_container.py | 149 +---- .../tests/integration/test_docker_image.py | 74 +-- .../tests/unit/test_folder_structure.py | 38 +- .../tests/unit/test_validation_data.py | 83 +-- .../tools/run_creator.py | 94 --- .../tools/update_compose_labels.py | 88 --- .../versioning/integration.cfg | 11 - .../versioning/service.cfg | 15 - 36 files changed, 320 insertions(+), 1317 deletions(-) delete mode 100644 .github/CODEOWNERS create mode 100644 .vscode/extensions.json create mode 100644 .vscode/launch.template.json create mode 100644 .vscode/settings.template.json rename {{cookiecutter.project_slug}}/{ci => .gitlab}/delete-image-from-registry-portus.bash (100%) rename {{cookiecutter.project_slug}}/{ci => .gitlab}/delete-image-from-registry.bash (100%) rename {{cookiecutter.project_slug}}/{ci => .gitlab}/docker-registry-curl.bash (100%) rename {{cookiecutter.project_slug}}/{ci => .gitlab}/gitlab-ci.yml (97%) delete mode 100644 {{cookiecutter.project_slug}}/.pylintrc delete mode 100644 {{cookiecutter.project_slug}}/VERSION delete mode 100644 {{cookiecutter.project_slug}}/VERSION_INTEGRATION delete mode 100644 {{cookiecutter.project_slug}}/docker-compose-meta.yml delete mode 100644 {{cookiecutter.project_slug}}/tests/integration/conftest.py delete mode 100644 {{cookiecutter.project_slug}}/tools/run_creator.py delete mode 100644 {{cookiecutter.project_slug}}/tools/update_compose_labels.py delete mode 100644 {{cookiecutter.project_slug}}/versioning/integration.cfg delete mode 100644 {{cookiecutter.project_slug}}/versioning/service.cfg diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index e8a104e7..00000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# Order is important. The last matching pattern has the most precedence. - -# files and folders recursively -/ @sanderegg diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index da24effc..6c97bfd4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,15 +1,48 @@ - + ## What do these changes do? - + + + +## Related issue/s + + + + +## How to test + + -## Related issue number ## Checklist -- [ ] I think the code is well written + diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 75150b40..54318c43 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -13,9 +13,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python: [3.8] + python: [3.9] os: [ubuntu-20.04] - docker_buildx: [v0.5.1] + docker_buildx: [v0.8.2] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index 25f8e7c5..a3b20f62 100644 --- a/.gitignore +++ b/.gitignore @@ -127,7 +127,6 @@ tmp/ *ignore* .tmp* -# vscode configuration -.vscode - -TODO.md \ No newline at end of file +# IDEs config +.vscode/launch.json +.vscode/settings.json \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..330d346c --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "eamodio.gitlens", + "ms-python.python", + "samuelcolvin.jinjahtml", + ] +} diff --git a/.vscode/launch.template.json b/.vscode/launch.template.json new file mode 100644 index 00000000..f2f58cbc --- /dev/null +++ b/.vscode/launch.template.json @@ -0,0 +1,23 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Run Test", + "type": "python", + "request": "launch", + "module": "pytest", + "args": [ + "--ff", + "--log-cli-level=INFO", + "--pdb", + "--setup-show", + "-sx", + "-vv", + "${file}" + ], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal", + "justMyCode": false + } + ] +} diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json new file mode 100644 index 00000000..bc3daee3 --- /dev/null +++ b/.vscode/settings.template.json @@ -0,0 +1,39 @@ +// This is a template. Clone and replace extension ".template.json" by ".json" +{ + "editor.tabSize": 2, + "editor.insertSpaces": true, + "editor.detectIndentation": false, + "eslint.alwaysShowStatus": true, + "files.associations": { + ".*rc": "ini", + ".env*": "ini", + "Dockerfile*": "dockerfile", + "**/requirements/*.txt": "pip-requirements", + "**/requirements/*.in": "pip-requirements", + "*Makefile": "makefile", + "*.cwl": "yaml" + }, + "files.eol": "\n", + "files.exclude": { + "**/__pycache__": true + }, + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "files.trimTrailingWhitespace": true, + "python.formatting.autopep8Args": [ + "--max-line-length 140" + ], + "python.analysis.extraPaths": [ + "./tests" + ], + "python.linting.pylintEnabled": true, + "python.linting.enabled": true, + "[python]": { + "editor.detectIndentation": false, + "editor.tabSize": 4 + }, + "[makefile]": { + "editor.insertSpaces": false + }, + "python.testing.pytestEnabled": true +} diff --git a/LICENSE b/LICENSE index 51f80cac..daf0bb90 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License -Copyright 2019 Sylvain Anderegg <35365065+sanderegg@users.noreply.github.com>. +Copyright 2019 IT'IS Foundation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 732bf93f..6667fa3b 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,9 @@ TEMPLATE = $(CURDIR) wheel \ setuptools +requirements.txt: requirements.in + # freezes requirements + .venv/bin/pip-compile --upgrade --build-isolation --output-file $@ $(word2, $^) devenv: .venv ## create a python virtual environment with tools to dev, run and tests cookie-cutter # installing extra tools @@ -84,7 +87,7 @@ help: ## this colorful help @awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) @echo "" -git_clean_args = -dxf --exclude=.vscode/ +git_clean_args = -dxf --exclude=.vscode/ --exclude=.venv/ --exclude=.python .PHONY: clean clean-force clean: ## cleans all unversioned files in project and temp files create by this makefile @@ -98,3 +101,11 @@ clean: ## cleans all unversioned files in project and temp files create by this clean-force: clean # removing .venv -@rm -rf .venv + + +.PHONY: info +info: ## displays info about the scope + # python + @echo $(shell which python) + @python --version + @echo $(shell which pip) diff --git a/README.md b/README.md index 2100a023..c873840d 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,29 @@ -cookiecutter-osparc-service -========================================== +# cookiecutter-osparc-service -Cookicutter to generate an oSparc compatible service for the oSparc simcore platform. +Status: ![Build Status](https://github.com/ITISFoundation/cookiecutter-osparc-service/workflows/Github-CI%20Push/PR/badge.svg) -Status: -------- -Currently only for **computational services**. +Cookiecutter to generate an oSparc compatible service for the oSparc simcore platform. Currently only for **computational services**. -![Build Status](https://github.com/ITISFoundation/cookiecutter-osparc-service/workflows/Github-CI%20Push/PR/badge.svg) -Requirements ------------- +## Requirements + GNU Make Python3 Python3-venv cookiecutter python package ```console -sudo apt-get update +sudo apt-get update sudo apt-get install -y make python3-venv # install GNU Make, python3-venv (python3 is usually already installed) python3 -m venv .venv # create a python virtual environment source .venv/bin/activate # activate the python virtual environment pip install cookiecutter # install the cookicutter package ``` -Usage ------ +## Usage -Generate a new Cookiecutter template layout: +Generate a new Cookiecutter template layout: ```console python3 -m venv .venv # create a python virtual environment source .venv/bin/activate # activate the python virtual environment @@ -38,8 +33,7 @@ cookiecutter gh:ITISFoundation/cookiecutter-osparc-service # generate a cook -Development ------------- +## Development ```console git clone https://github.com/ITISFoundation/cookiecutter-osparc-service.git @@ -49,8 +43,7 @@ source .venv/bin/activate make play ``` -Testing ------------- +## Testing ```console git clone https://github.com/ITISFoundation/cookiecutter-osparc-service.git @@ -63,7 +56,13 @@ make tests -License -------- +## License This project is licensed under the terms of the [MIT License](/LICENSE) + + +--- + +

+ +

diff --git a/VERSION b/VERSION index 341cf11f..60a2d3e9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.0 \ No newline at end of file +0.4.0 \ No newline at end of file diff --git a/cookiecutter.json b/cookiecutter.json index 04f8eb79..5b075ad0 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -1,36 +1,36 @@ { - "author_name": "Your full name", - "author_email": "Your address email (eq. you@example.com)", - "author_affiliation": "University of Anywhere, Department of something", - "contact_email": "{{ cookiecutter.author_email }}", - "project_name": "Name of the project", - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '-') }}", - "project_package_name": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "project_short_description": "{{ cookiecutter.project_name }}", - "project_type": [ - "computational" - ], - "docker_base": [ - "alpine:3.7", - "alpine:3.8", - "centos:7", - "custom:special-image", - "python:3.6", - "python:3.7", - "python:3.8", - "python:3.6-slim", - "python:3.7-slim", - "python:3.8-slim", - "ubuntu:18.04" - ], - "number_of_inputs": 2, - "number_of_outputs": 1, - "project_git_repo": "https://github.com/ITISFoundation/{{ cookiecutter.project_slug }}", - "git_username": "Yourusername", - "default_docker_registry": "itisfoundation", - "release_date": "{% now 'utc', '%Y' %}", - "version": "0.1.0", - "_extensions": [ - "jinja2_time.TimeExtension" - ] -} \ No newline at end of file + "author_name": "Your full name", + "author_email": "Your address email (eq. you@example.com)", + "author_affiliation": "University of Anywhere, Department of something", + "contact_email": "{{ cookiecutter.author_email }}", + "project_name": "Name of the project", + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '-') }}", + "project_package_name": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", + "project_short_description": "{{ cookiecutter.project_name }}", + "project_type": [ + "computational" + ], + "docker_base": [ + "alpine:3.7", + "alpine:3.8", + "centos:7", + "custom:special-image", + "python:3.6", + "python:3.7", + "python:3.8", + "python:3.6-slim", + "python:3.7-slim", + "python:3.8-slim", + "ubuntu:18.04" + ], + "number_of_inputs": 2, + "number_of_outputs": 1, + "project_git_repo": "https://github.com/ITISFoundation/{{ cookiecutter.project_slug }}", + "git_username": "Yourusername", + "default_docker_registry": "itisfoundation", + "release_date": "{% now 'utc', '%Y' %}", + "version": "0.1.0", + "_extensions": [ + "jinja2_time.TimeExtension" + ] +} diff --git a/setup.cfg b/setup.cfg index 2fca96a8..b3752bde 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.0 +current_version = 0.4.0 commit = True message = cookiecutter version: {current_version} → {new_version} tag = False @@ -8,4 +8,3 @@ tag = False [tool:pytest] testpaths = tests/ - diff --git a/tests/test_bake_project.py b/tests/test_bake_project.py index 33017f77..97c7c1f7 100644 --- a/tests/test_bake_project.py +++ b/tests/test_bake_project.py @@ -1,56 +1,59 @@ -#pylint: disable=W0621 +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable import json -import logging -import os + import subprocess import sys from pathlib import Path -from typing import Dict +from pytest_cookies.plugin import Cookies, Result import pytest -# current directory -current_dir = Path(sys.argv[0] if __name__ == - "__main__" else __file__).resolve().parent +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +repo_basedir =current_dir.parent +cookiecutter_json = repo_basedir / "cookiecutter.json" -logger = logging.getLogger(__name__) -def test_project_tree(cookies): - result = cookies.bake(extra_context={'project_slug': 'test_project'}) +def test_minimal_config_to_bake(cookies: Cookies): + result = cookies.bake(extra_context={"project_slug": "test_project"}) assert result.exit_code == 0 assert result.exception is None - assert result.project.basename == 'test_project' - + assert result.project.basename == "test_project" -def _get_cookiecutter_config() -> Dict: - cookiecutter_config_file = current_dir / "../cookiecutter.json" - with cookiecutter_config_file.open() as fp: - return json.load(fp) + print(f"{result}", f"{result.context=}") -flavors = _get_cookiecutter_config()["docker_base"] - +@pytest.fixture( + params=json.loads(cookiecutter_json.read_text())["docker_base"] +) +def baked_project(cookies: Cookies, request) -> Result: + result = cookies.bake( + extra_context={ + "project_slug": "DummyProject", + "project_name": "dummy-project", + "default_docker_registry": "test.test.com", + "docker_base": request.param, + } + ) -@pytest.fixture(params=flavors) -def baked_project(cookies, request): - return cookies.bake(extra_context={'project_slug': 'dummy-project', 'default_docker_registry': 'test.test.com', 'docker_base': request.param}) + assert result.exception is None + assert result.exit_code == 0 + return result -commands = ( - "ls -la .", - "make help", - "make devenv", - "make devenv build up", - "make devenv build-devel up-devel", - "make info-build", - "make devenv build tests", +@pytest.mark.parametrize( + "commands_on_baked_project", + ( + "ls -la .; make help", + # TODO: cannot use `source` to activate venvs ... not sure how to proceed here. Suggestions? + ## "make devenv; source .venv/bin/activate && make build info-build test", + ), ) - - -@pytest.mark.parametrize("command", commands) -def test_run_tests(baked_project, command: str): - working_dir = Path(baked_project.project) - assert subprocess.run(command.split(), cwd=working_dir, - check=True).returncode == 0 +def test_make_workflows(baked_project: Result, commands_on_baked_project: str): + working_dir = baked_project.project_path + subprocess.run( + ["/bin/bash", "-c", commands_on_baked_project], cwd=working_dir, check=True + ) diff --git a/{{cookiecutter.project_slug}}/.cookiecutterrc b/{{cookiecutter.project_slug}}/.cookiecutterrc index 59feeb40..6d6f580e 100644 --- a/{{cookiecutter.project_slug}}/.cookiecutterrc +++ b/{{cookiecutter.project_slug}}/.cookiecutterrc @@ -17,5 +17,7 @@ default_context: {% for key, value in cookiecutter.items()|sort %} + {%- if key!='_extensions' %} {{ "{0:26}".format(key + ":") }} {{ "{0!r}".format(value).strip("u") }} + {%- endif %} {%- endfor %} diff --git a/{{cookiecutter.project_slug}}/.github/workflows/github-ci.yml b/{{cookiecutter.project_slug}}/.github/workflows/github-ci.yml index e4e27bac..a4c0df61 100644 --- a/{{cookiecutter.project_slug}}/.github/workflows/github-ci.yml +++ b/{{cookiecutter.project_slug}}/.github/workflows/github-ci.yml @@ -37,10 +37,10 @@ jobs: restore-keys: | {{ "${{ runner.os }}" }}-pip- - name: set owner variable - run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*} + run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV - name: set docker image tag if: github.ref != 'refs/heads/master' - run: echo ::set-env name=DOCKER_IMAGE_TAG::${GITHUB_REF##*/} + run: echo "DOCKER_IMAGE_TAG=${GITHUB_REF##*/} >> $GITHUB_ENV - name: set dev environs run: make devenv - name: get current image if available diff --git a/{{cookiecutter.project_slug}}/ci/delete-image-from-registry-portus.bash b/{{cookiecutter.project_slug}}/.gitlab/delete-image-from-registry-portus.bash similarity index 100% rename from {{cookiecutter.project_slug}}/ci/delete-image-from-registry-portus.bash rename to {{cookiecutter.project_slug}}/.gitlab/delete-image-from-registry-portus.bash diff --git a/{{cookiecutter.project_slug}}/ci/delete-image-from-registry.bash b/{{cookiecutter.project_slug}}/.gitlab/delete-image-from-registry.bash similarity index 100% rename from {{cookiecutter.project_slug}}/ci/delete-image-from-registry.bash rename to {{cookiecutter.project_slug}}/.gitlab/delete-image-from-registry.bash diff --git a/{{cookiecutter.project_slug}}/ci/docker-registry-curl.bash b/{{cookiecutter.project_slug}}/.gitlab/docker-registry-curl.bash similarity index 100% rename from {{cookiecutter.project_slug}}/ci/docker-registry-curl.bash rename to {{cookiecutter.project_slug}}/.gitlab/docker-registry-curl.bash diff --git a/{{cookiecutter.project_slug}}/ci/gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab/gitlab-ci.yml similarity index 97% rename from {{cookiecutter.project_slug}}/ci/gitlab-ci.yml rename to {{cookiecutter.project_slug}}/.gitlab/gitlab-ci.yml index 39ef5b86..2a46c96a 100644 --- a/{{cookiecutter.project_slug}}/ci/gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab/gitlab-ci.yml @@ -53,7 +53,7 @@ remove_{{ cookiecutter.project_slug }}_builds: - export DOCKER_PROJECT=$SC_CI_PROJECT_PATH_NAME/simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/{{ cookiecutter.project_name.lower().replace(' ', '-') }} - export API_USER=$SC_CI_TESTING_REGISTRY_USER - export API_TOKEN=$SC_CI_TESTING_REGISTRY_PORTUS_TOKEN - - bash $SC_CI_{{ cookiecutter.project_package_name.upper() }}_LOCATION/ci/delete-image-from-registry-portus.bash # this will fail if registry is not available through Portus + - bash $SC_CI_{{ cookiecutter.project_package_name.upper() }}_LOCATION/.gitlab/delete-image-from-registry-portus.bash # this will fail if registry is not available through Portus when: manual environment: name: $CI_PROJECT_PATH_SLUG/$CI_COMMIT_REF_SLUG/{{ cookiecutter.project_slug }} diff --git a/{{cookiecutter.project_slug}}/.pylintrc b/{{cookiecutter.project_slug}}/.pylintrc deleted file mode 100644 index 141c39de..00000000 --- a/{{cookiecutter.project_slug}}/.pylintrc +++ /dev/null @@ -1,549 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - locally-enabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - too-few-public-methods, - trailing-whitespace, - too-few-public-methods, - duplicate-code, - too-many-locals, - C, - fixme, - import-error - - - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=optparse.Values,sys.exit - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=150 - -# Maximum number of lines in a module -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[BASIC] - -# Naming style matching correct argument names -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style -#argument-rgx= - -# Naming style matching correct attribute names -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style -#class-attribute-rgx= - -# Naming style matching correct class names -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming-style -#class-rgx= - -# Naming style matching correct constant names -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma -good-names=i, - j, - k, - ex, - Run, - _ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming style matching correct inline iteration names -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style -#inlinevar-rgx= - -# Naming style matching correct method names -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style -#method-rgx= - -# Naming style matching correct module names -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style -#variable-rgx= - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/{{cookiecutter.project_slug}}/Makefile b/{{cookiecutter.project_slug}}/Makefile index 236b4614..aeed9a54 100644 --- a/{{cookiecutter.project_slug}}/Makefile +++ b/{{cookiecutter.project_slug}}/Makefile @@ -11,14 +11,14 @@ SHELL = /bin/sh .DEFAULT_GOAL := help -export VCS_URL := $(shell git config --get remote.origin.url || echo unversioned) -export VCS_REF := $(shell git rev-parse --short HEAD || echo unversioned) -export VCS_STATUS := $(if $(shell git status -s || echo unversioned),'modified/untracked','clean') +export VCS_URL := $(shell git config --get remote.origin.url 2> /dev/null || echo unversioned repo) +export VCS_REF := $(shell git rev-parse --short HEAD 2> /dev/null || echo unversioned repo) +export VCS_STATUS := $(if $(shell git status -s 2> /dev/null || echo unversioned repo),'modified/untracked','clean') export BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") export DOCKER_REGISTRY ?= {{ cookiecutter.default_docker_registry }} export DOCKER_IMAGE_NAME ?= {{ cookiecutter.project_name.lower().replace(' ', '-') }} -export DOCKER_IMAGE_TAG ?= $(shell cat VERSION) +export DOCKER_IMAGE_TAG ?= $(shell cat VERSION 2> /dev/null || echo undefined) export COMPOSE_INPUT_DIR := ./validation/input export COMPOSE_OUTPUT_DIR := .tmp/output @@ -39,13 +39,14 @@ APP_NAME := {{ cookiecutter.project_slug }} $@/bin/pip3 install \ pip-tools + requirements.txt: .venv requirements.in # freezes requirements - $ $@ + +service.cli/run: $(METADATA) ## generates run from metadata # Updates adapter script from metadata in $< - @.venv/bin/python3 tools/run_creator.py --metadata $< --runscript $@ + @ooil run-creator --metadata $< --runscript $@ -docker-compose-meta.yml: $(metatada) +docker-compose-meta.yml: $(METADATA) ## generates docker-copose from metadata # Injects metadata from $< as labels - @.venv/bin/python3 tools/update_compose_labels.py --compose $@ --metadata $< + @ooil compose --to-spec-file $@ --metadata $< define _docker_compose_build export DOCKER_BUILD_TARGET=$(if $(findstring -devel,$@),development,$(if $(findstring -cache,$@),cache,production)); \ $(if $(findstring -x,$@),\ - docker buildx > /dev/null; export DOCKER_CLI_EXPERIMENTAL=enabled; docker buildx bake --file docker-compose-build.yml --file docker-compose-meta.yml $(if $(findstring -nc,$@),--no-cache,);,\ + docker buildx > /dev/null; export DOCKER_CLI_EXPERIMENTAL=enabled; docker buildx bake --file docker-compose-meta.yml --file docker-compose-build.yml $(if $(findstring -nc,$@),--no-cache,);,\ $(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \ - docker-compose --file docker-compose-build.yml --file docker-compose-meta.yml build $(if $(findstring -nc,$@),--no-cache,) --parallel;\ + docker-compose --file docker-compose-meta.yml --file docker-compose-build.yml build $(if $(findstring -nc,$@),--no-cache,) --parallel;\ ) endef @@ -112,14 +119,19 @@ tests: tests-unit tests-integration ## runs unit and integration tests # PUBLISHING ----------------------------------- -define _bumpversion - # upgrades as $(subst $(1),,$@) version, commits and tags - @bump2version --verbose --list --config-file $(1) $(subst $(2),,$@) -endef .PHONY: version-service-patch version-service-minor version-service-major -version-service-patch version-service-minor version-service-major: versioning/service.cfg ## kernel/service versioning as patch - @$(call _bumpversion,$<,version-service-) +version-service-patch version-service-minor version-service-major: $(METADATA) ## kernel/service versioning as patch + ooil bump-version --metadata-file $< --upgrade $(subst version-service-,,$@) + # syncing metadata upstream + @$(MAKE) VERSION + @$(MAKE) docker-compose-meta.yml + +.PHONY: version-integration-patch version-integration-minor version-integration-major +version-integration-patch version-integration-minor version-integration-major: $(METADATA) ## integration versioning as patch (bug fixes not affecting API/handling), minor/major (backwards-compatible/INcompatible API changes) + ooil bump-version --metadata-file $< --upgrade $(subst version-integration-,,$@) integration-version + # syncing metadata upstream + @$(MAKE) docker-compose-meta.yml .PHONY: tag-local tag-local: @@ -149,12 +161,6 @@ pull-latest pull-version: ## pull service from registry @docker pull $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(if $(findstring version,$@),$(DOCKER_IMAGE_TAG),latest) -.PHONY: version-integration-patch version-integration-minor version-integration-major -version-integration-patch version-integration-minor version-integration-major: versioning/integration.cfg ## integration versioning as patch (bug fixes not affecting API/handling), minor/major (backwards-compatible/INcompatible API changes) - @$(call _bumpversion,$<,version-integration-) - - - # DEVELOPMENT ---------------------------------------- @@ -218,10 +224,26 @@ help: ## this colorful help .PHONY: clean clean-force -git_clean_args = -dxf -e .vscode/ -e .venv +git_clean_args = -dxf --exclude=.vscode/ --exclude=.venv --exclude=.python-version clean: ## cleans all unversioned files in project and temp files create by this makefile # Cleaning unversioned @git clean -n $(git_clean_args) @echo -n "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ] @echo -n "$(shell whoami), are you REALLY sure? [y/N] " && read ans && [ $${ans:-N} = y ] @git clean $(git_clean_args) + +.PHONY: info +info: + # integration tools + @ooil --version + # variables + @echo "VCS_URL : $(VCS_URL)" + @echo "VCS_REF : $(VCS_REF)" + @echo "VCS_STATUS : $(VCS_STATUS)" + @echo "BUILD_DATE : $(BUILD_DATE)" + @echo "METADATA : $(METADATA)" + @echo "DOCKER_REGISTRY : $(DOCKER_REGISTRY)" + @echo "DOCKER_IMAGE_NAME : $(DOCKER_IMAGE_NAME)" + @echo "DOCKER_IMAGE_TAG : $(DOCKER_IMAGE_TAG)" + @echo "COMPOSE_INPUT_DIR : $(COMPOSE_INPUT_DIR)" + @echo "COMPOSE_OUTPUT_DIR : $(COMPOSE_OUTPUT_DIR)" diff --git a/{{cookiecutter.project_slug}}/VERSION b/{{cookiecutter.project_slug}}/VERSION deleted file mode 100644 index 94fc7796..00000000 --- a/{{cookiecutter.project_slug}}/VERSION +++ /dev/null @@ -1 +0,0 @@ -{{ cookiecutter.version }} diff --git a/{{cookiecutter.project_slug}}/VERSION_INTEGRATION b/{{cookiecutter.project_slug}}/VERSION_INTEGRATION deleted file mode 100644 index 3eefcb9d..00000000 --- a/{{cookiecutter.project_slug}}/VERSION_INTEGRATION +++ /dev/null @@ -1 +0,0 @@ -1.0.0 diff --git a/{{cookiecutter.project_slug}}/docker-compose-meta.yml b/{{cookiecutter.project_slug}}/docker-compose-meta.yml deleted file mode 100644 index ba04b4d4..00000000 --- a/{{cookiecutter.project_slug}}/docker-compose-meta.yml +++ /dev/null @@ -1,10 +0,0 @@ -# NOTE: DO NOT MODIFY MANUALLY. Execute make docker-compose-meta.yml instead -version: '3.7' -services: - {{ cookiecutter.project_slug }}: - build: - labels: - org.label-schema.schema-version: "1.0" - org.label-schema.build-date: "${BUILD_DATE}" - org.label-schema.vcs-url: {{ cookiecutter.project_git_repo }} - org.label-schema.vcs-ref: "${VCS_REF}" diff --git a/{{cookiecutter.project_slug}}/requirements.in b/{{cookiecutter.project_slug}}/requirements.in index 52a87842..412bb8fd 100644 --- a/{{cookiecutter.project_slug}}/requirements.in +++ b/{{cookiecutter.project_slug}}/requirements.in @@ -1,5 +1,12 @@ -# tools -pyyaml + +git+https://github.com/ITISFoundation/osparc-simcore.git@master#egg=simcore-models-library&subdirectory=packages/models-library +git+https://github.com/ITISFoundation/osparc-simcore.git@master#egg=simcore-service-integration&subdirectory=packages/service-integration + +# -e ../osparc-simcore/packages/models-library +# -e ../osparc-simcore/packages/service-integration + +# formatter +black # tests coverage @@ -9,6 +16,4 @@ pytest pytest-cookies pytest-cov pytest-instafail -pytest-mock pytest-sugar -pyyaml diff --git a/{{cookiecutter.project_slug}}/tests/conftest.py b/{{cookiecutter.project_slug}}/tests/conftest.py index 26ee8955..6cac9b93 100644 --- a/{{cookiecutter.project_slug}}/tests/conftest.py +++ b/{{cookiecutter.project_slug}}/tests/conftest.py @@ -7,63 +7,24 @@ import pytest -current_dir = Path(sys.argv[0] if __name__ == - "__main__" else __file__).resolve().parent - - -@pytest.fixture(scope='session') -def tests_dir() -> Path: - assert current_dir.exists() - return current_dir +pytest_plugins = [ + "service_integration.pytest_plugin.folder_structure", + "service_integration.pytest_plugin.validation_data", + "service_integration.pytest_plugin.docker_integration", +] -@pytest.fixture(scope='session') -def validation_dir(project_slug_dir: Path) -> Path: - validation_dir = project_slug_dir / "validation" - assert validation_dir.exists() - return validation_dir +current_dir = Path(sys.argv[0] if __name__ == + "__main__" else __file__).resolve().parent @pytest.fixture(scope='session') -def project_slug_dir(tests_dir: Path) -> Path: - project_slug_dir = tests_dir.parent +def project_slug_dir() -> Path: + project_slug_dir = current_dir.parent assert project_slug_dir.exists() return project_slug_dir -@pytest.fixture(scope='session') -def src_dir(project_slug_dir: Path) -> Path: - src_dir = project_slug_dir / "src" - assert src_dir.exists() - return src_dir - - -@pytest.fixture(scope='session') -def tools_dir(project_slug_dir: Path) -> Path: - tools_dir = project_slug_dir / "tools" - assert tools_dir.exists() - return tools_dir - - -@pytest.fixture(scope='session') -def docker_dir(project_slug_dir: Path) -> Path: - docker_dir = project_slug_dir / "docker" - assert docker_dir.exists() - return docker_dir - - -@pytest.fixture(scope='session') -def package_dir(src_dir: Path) -> Path: - package_dir = src_dir / "name_of_the_project" - assert package_dir.exists() - return package_dir - -@pytest.fixture(scope='session') -def metadata_file(project_slug_dir: Path) -> Path: - metadata_file = project_slug_dir / "metadata" / "metadata.yml" - assert metadata_file.exists() - return metadata_file - @pytest.fixture(scope='session') def git_root_dir() -> Path: # finds where is .git diff --git a/{{cookiecutter.project_slug}}/tests/integration/conftest.py b/{{cookiecutter.project_slug}}/tests/integration/conftest.py deleted file mode 100644 index 21701890..00000000 --- a/{{cookiecutter.project_slug}}/tests/integration/conftest.py +++ /dev/null @@ -1,45 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -import os -from pathlib import Path - -import pytest - -import docker - - -@pytest.fixture -def docker_client() -> docker.DockerClient: - return docker.from_env() - - -@pytest.fixture -def docker_image_key(docker_client: docker.DockerClient) -> str: - image_key = "{{ cookiecutter.project_name.lower().replace(' ', '-') }}:" - docker_images = [image for image in docker_client.images.list() if any( - image_key in tag for tag in image.tags)] - return docker_images[0].tags[0] - - -@pytest.fixture -def docker_image(docker_client: docker.DockerClient, docker_image_key: str) -> docker.models.images.Image: - docker_image = docker_client.images.get(docker_image_key) - assert docker_image - return docker_image - - -def _is_gitlab_executor() -> bool: - return "GITLAB_CI" in os.environ - - -@pytest.fixture -def temporary_path(tmp_path: Path) -> Path: - if _is_gitlab_executor(): - # /builds is a path that is shared between the docker in docker container and the job builder container - shared_path = Path( - "/builds/{}/tmp".format(os.environ["CI_PROJECT_PATH"])) - shared_path.mkdir(parents=True, exist_ok=True) - return shared_path - return tmp_path diff --git a/{{cookiecutter.project_slug}}/tests/integration/test_docker_container.py b/{{cookiecutter.project_slug}}/tests/integration/test_docker_container.py index 262a4d73..2724a60a 100644 --- a/{{cookiecutter.project_slug}}/tests/integration/test_docker_container.py +++ b/{{cookiecutter.project_slug}}/tests/integration/test_docker_container.py @@ -2,151 +2,16 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import filecmp -import json -import os -import shutil -from pathlib import Path -from pprint import pformat -from typing import Dict -import pytest +from typing import Dict import docker - -_FOLDER_NAMES = ["input", "output"] -_CONTAINER_FOLDER = Path("/home/scu/data") - - -@pytest.fixture -def host_folders(temporary_path: Path) -> Dict: - tmp_dir = temporary_path - - host_folders = {} - for folder in _FOLDER_NAMES: - path = tmp_dir / folder - if path.exists(): - shutil.rmtree(path) - path.mkdir() - # we need to ensure the path is writable for the docker container (Gitlab-CI case) - os.chmod(str(path), 0o775) - assert path.exists() - host_folders[folder] = path - - return host_folders - - -@pytest.fixture -def container_variables() -> Dict: - # of type INPUT_FOLDER=/home/scu/data/input - env = {"{}_FOLDER".format(str(folder).upper()): ( - _CONTAINER_FOLDER / folder).as_posix() for folder in _FOLDER_NAMES} - return env - - -@pytest.fixture -def validation_folders(validation_dir: Path) -> Dict: - return {folder: (validation_dir / folder) for folder in _FOLDER_NAMES} - - -@pytest.fixture -def docker_container(validation_folders: Dict, host_folders: Dict, docker_client: docker.DockerClient, docker_image_key: str, container_variables: Dict) -> docker.models.containers.Container: - # copy files to input folder, copytree needs to not have the input folder around. - host_folders["input"].rmdir() - shutil.copytree(validation_folders["input"], host_folders["input"]) - assert Path(host_folders["input"]).exists() - # run the container (this may take some time) - container = None - try: - volumes = {host_folders[folder]: {"bind": container_variables["{}_FOLDER".format( - str(folder).upper())]} for folder in _FOLDER_NAMES} - container = docker_client.containers.run(docker_image_key, - "run", detach=True, remove=False, volumes=volumes, environment=container_variables) - response = container.wait() - if response["StatusCode"] > 0: - logs = container.logs(timestamps=True) - pytest.fail("The container stopped with exit code {}\n\n\ncommand:\n {}, \n\n\nlog:\n{}".format(response["StatusCode"], - "run", pformat( - (container.logs(timestamps=True).decode("UTF-8")).split("\n"), width=200 - ))) - else: - yield container - except docker.errors.ContainerError as exc: - # the container did not run correctly - pytest.fail("The container stopped with exit code {}\n\n\ncommand:\n {}, \n\n\nlog:\n{}".format(exc.exit_status, - exc.command, pformat( - (container.logs(timestamps=True).decode("UTF-8")).split("\n"), width=200 - ) if container else "")) - finally: - # cleanup - if container: - container.remove() - - -def _convert_to_simcore_labels(image_labels: Dict) -> Dict: - io_simcore_labels = {} - for key, value in image_labels.items(): - if str(key).startswith("io.simcore."): - simcore_label = json.loads(value) - simcore_keys = list(simcore_label.keys()) - assert len(simcore_keys) == 1 - simcore_key = simcore_keys[0] - simcore_value = simcore_label[simcore_key] - io_simcore_labels[simcore_key] = simcore_value - assert len(io_simcore_labels) > 0 - return io_simcore_labels +from service_integration.pytest_plugin.docker_integration import assert_container_runs def test_run_container(validation_folders: Dict, host_folders: Dict, docker_container: docker.models.containers.Container): - for folder in _FOLDER_NAMES: - # test if the files that should be there are actually there and correct - list_of_files = [ - x.name for x in validation_folders[folder].iterdir() if not ".gitkeep" in x.name] - for file_name in list_of_files: - assert Path(host_folders[folder] / file_name).exists( - ), f"{file_name} is missing from {host_folders[folder]}" - - # we look for missing files only. contents is the responsibility of the service creator - _, _, errors = filecmp.cmpfiles( - host_folders[folder], validation_folders[folder], list_of_files, shallow=True) - assert not errors, f"{errors} are missing in {host_folders[folder]}" - - if folder == "input": - continue - # test if the generated files are the ones expected - list_of_files = [ - x.name for x in host_folders[folder].iterdir() if not ".gitkeep" in x.name] - for file_name in list_of_files: - assert Path(validation_folders[folder] / file_name).exists( - ), "{} is not present in {}".format(file_name, validation_folders[folder]) - _, _, errors = filecmp.cmpfiles( - host_folders[folder], validation_folders[folder], list_of_files, shallow=False) - # assert not mismatch, "wrong/incorrect generated files in {}".format(host_folders[folder]) - assert not errors, f"{errors} should not be available in {host_folders[folder]}" - - # check the output is correct based on container labels - output_cfg = {} - output_cfg_file = Path(host_folders["output"] / "outputs.json") - if output_cfg_file.exists(): - with output_cfg_file.open() as fp: - output_cfg = json.load(fp) - - container_labels = docker_container.labels - io_simcore_labels = _convert_to_simcore_labels(container_labels) - assert "outputs" in io_simcore_labels - for key, value in io_simcore_labels["outputs"].items(): - assert "type" in value - # rationale: files are on their own and other types are in inputs.json - if not "data:" in value["type"]: - # check that keys are available - assert key in output_cfg - else: - # it's a file and it should be in the folder as well using key as the filename - filename_to_look_for = key - if "fileToKeyMap" in value: - # ...or there is a mapping - assert len(value["fileToKeyMap"]) > 0 - for filename, mapped_value in value["fileToKeyMap"].items(): - assert mapped_value == key - filename_to_look_for = filename - assert (host_folders["output"] / filename_to_look_for).exists() + assert_container_runs( + validation_folders, + host_folders, + docker_container + ) diff --git a/{{cookiecutter.project_slug}}/tests/integration/test_docker_image.py b/{{cookiecutter.project_slug}}/tests/integration/test_docker_image.py index ca01a0b0..cb54f77a 100644 --- a/{{cookiecutter.project_slug}}/tests/integration/test_docker_image.py +++ b/{{cookiecutter.project_slug}}/tests/integration/test_docker_image.py @@ -2,79 +2,19 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import json -import shutil -import urllib.request -from pathlib import Path from typing import Dict -import pytest - import docker -import jsonschema -import yaml - - -# HELPERS -def _download_url(url: str, file: Path): - # Download the file from `url` and save it locally under `file_name`: - with urllib.request.urlopen(url) as response, file.open('wb') as out_file: - shutil.copyfileobj(response, out_file) - assert file.exists() - - -def _convert_to_simcore_labels(image_labels: Dict) -> Dict: - io_simcore_labels = {} - for key, value in image_labels.items(): - if str(key).startswith("io.simcore."): - simcore_label = json.loads(value) - simcore_keys = list(simcore_label.keys()) - assert len(simcore_keys) == 1 - simcore_key = simcore_keys[0] - simcore_value = simcore_label[simcore_key] - io_simcore_labels[simcore_key] = simcore_value - assert len(io_simcore_labels) > 0 - return io_simcore_labels - -# FIXTURES -@pytest.fixture -def osparc_service_labels_jsonschema(tmp_path) -> Dict: - url = "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/api/specs/common/schemas/node-meta-v0.0.1.json" - file_name = tmp_path / "service_label.json" - _download_url(url, file_name) - with file_name.open() as fp: - json_schema = json.load(fp) - return json_schema - - -@pytest.fixture(scope='session') -def metadata_labels(metadata_file: Path) -> Dict: - with metadata_file.open() as fp: - metadata = yaml.safe_load(fp) - return metadata - -# TESTS +from service_integration.pytest_plugin.docker_integration import ( + assert_docker_io_simcore_labels_against_files, + assert_validate_docker_io_simcore_labels) def test_docker_io_simcore_labels_against_files(docker_image: docker.models.images.Image, metadata_labels: Dict): - image_labels = docker_image.labels - io_simcore_labels = _convert_to_simcore_labels(image_labels) - # check files are identical - for key, value in io_simcore_labels.items(): - assert key in metadata_labels - assert value == metadata_labels[key] + assert_docker_io_simcore_labels_against_files( + docker_image, metadata_labels) def test_validate_docker_io_simcore_labels(docker_image: docker.models.images.Image, osparc_service_labels_jsonschema: Dict): - image_labels = docker_image.labels - # get io labels - io_simcore_labels = _convert_to_simcore_labels(image_labels) - # validate schema - try: - jsonschema.validate(io_simcore_labels, - osparc_service_labels_jsonschema) - except jsonschema.SchemaError: - pytest.fail("Schema {} contains errors".format( - osparc_service_labels_jsonschema)) - except jsonschema.ValidationError: - pytest.fail("Failed to validate docker image io labels against schema") + assert_validate_docker_io_simcore_labels( + docker_image, osparc_service_labels_jsonschema) diff --git a/{{cookiecutter.project_slug}}/tests/unit/test_folder_structure.py b/{{cookiecutter.project_slug}}/tests/unit/test_folder_structure.py index e865319b..69a24bad 100644 --- a/{{cookiecutter.project_slug}}/tests/unit/test_folder_structure.py +++ b/{{cookiecutter.project_slug}}/tests/unit/test_folder_structure.py @@ -5,40 +5,10 @@ from pathlib import Path import pytest +from service_integration.pytest_plugin.folder_structure import ( + assert_path_in_repo, get_expected_files) -expected_files = ( - ".cookiecutterrc", - ".dockerignore", - ".gitignore", - ".pylintrc", - "metadata:metadata.yml", - "docker/{{ cookiecutter.docker_base.split(":")[0] }}:entrypoint.sh", - "docker/{{ cookiecutter.docker_base.split(":")[0] }}:Dockerfile", - "service.cli:execute.sh", - "tools:run_creator.py", - "tools:update_compose_labels.py", - "versioning:integration.cfg", - "versioning:service.cfg", - "requirements.in", - "requirements.txt", - "Makefile", - "VERSION", - "README.md", - "docker-compose-build.yml", - "docker-compose-meta.yml", - "docker-compose.devel.yml", - "docker-compose.yml", -) - -@pytest.mark.parametrize("expected_path", expected_files) +@pytest.mark.parametrize("expected_path", get_expected_files("{{ cookiecutter.docker_base.split(":")[0] }}")) def test_path_in_repo(expected_path: str, project_slug_dir: Path): - - if ":" in expected_path: - folder, glob = expected_path.split(":") - folder_path = project_slug_dir / folder - assert folder_path.exists(), f"folder {folder_path} is missing!" - assert any(folder_path.glob(glob)), f"no {glob} in {folder_path}" - else: - assert (project_slug_dir/expected_path).exists( - ), f"{expected_path} is missing from {project_slug_dir}" + assert_path_in_repo(expected_path, project_slug_dir) diff --git a/{{cookiecutter.project_slug}}/tests/unit/test_validation_data.py b/{{cookiecutter.project_slug}}/tests/unit/test_validation_data.py index 5ee83b84..261c0700 100644 --- a/{{cookiecutter.project_slug}}/tests/unit/test_validation_data.py +++ b/{{cookiecutter.project_slug}}/tests/unit/test_validation_data.py @@ -1,70 +1,26 @@ # pylint:disable=unused-variable # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import json -from pathlib import Path -from typing import Dict, Iterator, Optional - -import pytest - -import yaml - - -@pytest.fixture -def port_type() -> str: - return "" - - -@pytest.fixture -def label_cfg(metadata_file: Path, port_type: str) -> Dict: - ports_type = f"{port_type}s" - with metadata_file.open() as fp: - cfg = yaml.safe_load(fp) - assert ports_type in cfg - return cfg[ports_type] - - -@pytest.fixture -def validation_folder(validation_dir: Path, port_type: str) -> Path: - return validation_dir / port_type +from pathlib import Path +from typing import Dict -@pytest.fixture -def validation_cfg(validation_dir: Path, port_type: str) -> Optional[Dict]: - validation_file = validation_dir / \ - port_type / (f"{port_type}s.json") - if validation_file.exists(): - with validation_file.open() as fp: - return json.load(fp) - # it may not exist if only files are required - return None +## from service_integration.pytest_plugin.validation_data import assert_validation_data_follows_definition -def _find_key_in_cfg(filename: str, value: Dict) -> Iterator[str]: - for k, v in value.items(): - if k == filename: - if isinstance(v, dict): - assert "data:" in v["type"] - yield k - else: - yield v - elif isinstance(v, dict): - for result in _find_key_in_cfg(filename, v): - yield result +def test_validation_data_follows_definition( + label_cfg: Dict, validation_cfg: Dict, validation_folder: Path +): + # assert_validation_data_follows_definition(label_cfg, validation_cfg, validation_folder) -@pytest.mark.parametrize("port_type", [ - "input", - "output" -]) -def test_validation_data_follows_definition(label_cfg: Dict, validation_cfg: Dict, validation_folder: Path): for key, value in label_cfg.items(): assert "type" in value # rationale: files are on their own and other types are in inputs.json if not "data:" in value["type"]: # check that keys are available - assert key in validation_cfg, f"missing {key} in validation config file" + assert key in validation_cfg else: # it's a file and it should be in the folder as well using key as the filename filename_to_look_for = key @@ -72,13 +28,11 @@ def test_validation_data_follows_definition(label_cfg: Dict, validation_cfg: Dic # ...or there is a mapping assert len(value["fileToKeyMap"]) > 0 for filename, mapped_value in value["fileToKeyMap"].items(): - assert mapped_value == key, f"file to key map for {key} has an incorrectly set {mapped_value}, it should be equal to {key}" + assert mapped_value == key filename_to_look_for = filename - assert (validation_folder / filename_to_look_for).exists( - ), f"{filename_to_look_for} is missing from {validation_folder}" + assert (validation_folder / filename_to_look_for).exists() else: - assert (validation_folder / filename_to_look_for).exists( - ), f"{filename_to_look_for} is missing from {validation_folder}" + assert (validation_folder / filename_to_look_for).exists() if validation_cfg: for key, value in validation_cfg.items(): @@ -88,19 +42,8 @@ def test_validation_data_follows_definition(label_cfg: Dict, validation_cfg: Dic "number": (float, int), "integer": int, "boolean": bool, - "string": str + "string": str, } if not "data:" in label_cfg[key]["type"]: # check the type is correct - assert isinstance(value, label2types[label_cfg[key]["type"]] - ), f"{value} has not the expected type {label2types[label_cfg[key]['type']]}" - - for path in validation_folder.glob("**/*"): - if path.name in ["inputs.json", "outputs.json", ".gitkeep"]: - continue - assert path.is_file(), f"{path} is not a file!" - filename = path.name - # this filename shall be available as a key in the labels somewhere - key = next(_find_key_in_cfg(str(filename), label_cfg)) - - assert key in label_cfg, f"{key} was not found in {label_cfg}" + assert isinstance(value, label2types[label_cfg[key]["type"]]) diff --git a/{{cookiecutter.project_slug}}/tools/run_creator.py b/{{cookiecutter.project_slug}}/tools/run_creator.py deleted file mode 100644 index 3ab7c6c6..00000000 --- a/{{cookiecutter.project_slug}}/tools/run_creator.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/python - -""" Creates a sh script that uses jq tool to retrieve variables - to use in sh from a json file for use in an osparc service. - - Usage python run_creator --folder path/to/inputs.json --runscript path/to/put/the/script -:return: error code -""" - - -import argparse -import logging -import stat -import sys -from enum import IntEnum -from pathlib import Path -from typing import Dict - -import yaml - -log = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - -class ExitCode(IntEnum): - SUCCESS = 0 - FAIL = 1 - - -def get_input_config(metadata_file: Path) -> Dict: - inputs = {} - with metadata_file.open() as fp: - metadata = yaml.safe_load(fp) - if "inputs" in metadata: - inputs = metadata["inputs"] - return inputs - - -def main(args=None) -> int: - try: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--metadata", help="The metadata yaml of the node", - type=Path, required=False, default="/metadata/metadata.yml") - parser.add_argument( - "--runscript", help="The run script", type=Path, required=True) - options = parser.parse_args(args) - - # generate variables for input - input_script = [""" -#!/bin/sh -#--------------------------------------------------------------- -# AUTO-GENERATED CODE, do not modify this will be overwritten!!! -#--------------------------------------------------------------- -# shell strict mode: -set -o errexit -set -o nounset -IFS=$(printf '\\n\\t') -cd "$(dirname "$0")" -json_input=$INPUT_FOLDER/inputs.json - """ - ] - input_config = get_input_config(options.metadata) - for input_key, input_value in input_config.items(): - if "data:" in input_value["type"]: - filename = input_key - if "fileToKeyMap" in input_value and len(input_value["fileToKeyMap"]) > 0: - filename, _ = next( - iter(input_value["fileToKeyMap"].items())) - input_script.append( - f"{str(input_key).upper()}=$INPUT_FOLDER/{str(filename)}") - input_script.append(f"export {str(input_key).upper()}") - else: - input_script.append( - f"{str(input_key).upper()}=$(< \"$json_input\" jq '.{input_key}')") - input_script.append(f"export {str(input_key).upper()}") - - input_script.extend([""" -exec execute.sh - """ - ]) - - # write shell script - shell_script = str("\n").join(input_script) - options.runscript.write_text(shell_script) - st = options.runscript.stat() - options.runscript.chmod(st.st_mode | stat.S_IEXEC) - return ExitCode.SUCCESS - except: # pylint: disable=bare-except - log.exception("Unexpected error:") - return ExitCode.FAIL - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/{{cookiecutter.project_slug}}/tools/update_compose_labels.py b/{{cookiecutter.project_slug}}/tools/update_compose_labels.py deleted file mode 100644 index 62343f40..00000000 --- a/{{cookiecutter.project_slug}}/tools/update_compose_labels.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/python - -""" Update a docker-compose file with json files in a path - - Usage: python update_compose_labels --c docker-compose.yml -f folder/path - -:return: error code -""" - -import argparse -import json -import logging -import sys -from enum import IntEnum -from pathlib import Path -from typing import Dict - -import yaml - -log = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - -class ExitCode(IntEnum): - SUCCESS = 0 - FAIL = 1 - - -def get_compose_file(compose_file: Path) -> Dict: - with compose_file.open() as filep: - return yaml.safe_load(filep) - - -def get_metadata_file(metadata_file: Path) -> Dict: - with metadata_file.open() as fp: - return yaml.safe_load(fp) - - -def stringify_metadata(metadata: Dict) -> Dict[str, str]: - jsons = {} - for key, value in metadata.items(): - jsons[f"io.simcore.{key}"] = json.dumps({key: value}) - return jsons - - -def update_compose_labels(compose_cfg: Dict, metadata: Dict[str, str]) -> bool: - compose_labels = compose_cfg["services"]["{{ cookiecutter.project_slug }}"]["build"]["labels"] - changed = False - for key, value in metadata.items(): - if key in compose_labels: - if compose_labels[key] == value: - continue - compose_labels[key] = value - changed = True - return changed - - -def main(args=None) -> int: - try: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--compose", help="The compose file where labels shall be updated", type=Path, required=True) - parser.add_argument("--metadata", help="The metadata yaml file", - type=Path, required=False, default="metadata/metadata.yml") - options = parser.parse_args(args) - log.info("Testing if %s needs updates using labels in %s", - options.compose, options.metadata) - # get available jsons - compose_cfg = get_compose_file(options.compose) - metadata = get_metadata_file(options.metadata) - json_metadata = stringify_metadata(metadata) - if update_compose_labels(compose_cfg, json_metadata): - log.info("Updating %s using labels in %s", - options.compose, options.metadata) - # write the file back - with options.compose.open('w') as fp: - yaml.safe_dump(compose_cfg, fp, default_flow_style=False) - log.info("Update completed") - else: - log.info("No update necessary") - return ExitCode.SUCCESS - except: # pylint: disable=bare-except - log.exception("Unexpected error:") - return ExitCode.FAIL - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/{{cookiecutter.project_slug}}/versioning/integration.cfg b/{{cookiecutter.project_slug}}/versioning/integration.cfg deleted file mode 100644 index bed29672..00000000 --- a/{{cookiecutter.project_slug}}/versioning/integration.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[bumpversion] -current_version = 1.0.0 -commit = False -message = integration version: {current_version} → {new_version} -tag = False - -[bumpversion:file:VERSION_INTEGRATION] - -[bumpversion:file:metadata/metadata.yml] -search = integration-version: {current_version} -replace = integration-version: {new_version} diff --git a/{{cookiecutter.project_slug}}/versioning/service.cfg b/{{cookiecutter.project_slug}}/versioning/service.cfg deleted file mode 100644 index 7ad16480..00000000 --- a/{{cookiecutter.project_slug}}/versioning/service.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[bumpversion] -current_version = {{ cookiecutter.version }} -commit = False -message = service/kernel version: {current_version} → {new_version} -tag = False - -[bumpversion:file:VERSION] - -[bumpversion:file:metadata/metadata.yml] -search = version: {current_version} -replace = version: {new_version} - -[bumpversion:file:.cookiecutterrc] -search = version: '{current_version}' -replace = version: '{new_version}'