From 56cf0c59f3644e154796bc38d76d252c9ec294ca Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:09:31 +0100 Subject: [PATCH 01/78] added skeleton --- packages/aws-library/Makefile | 49 ++++++++ packages/aws-library/README.md | 22 ++++ packages/aws-library/VERSION | 1 + packages/aws-library/requirements/Makefile | 6 + packages/aws-library/requirements/_base.in | 7 ++ packages/aws-library/requirements/_base.txt | 73 ++++++++++++ packages/aws-library/requirements/_test.in | 23 ++++ packages/aws-library/requirements/_test.txt | 115 +++++++++++++++++++ packages/aws-library/requirements/_tools.in | 5 + packages/aws-library/requirements/_tools.txt | 89 ++++++++++++++ packages/aws-library/requirements/ci.txt | 18 +++ packages/aws-library/requirements/dev.txt | 18 +++ packages/aws-library/setup.cfg | 17 +++ packages/aws-library/setup.py | 61 ++++++++++ packages/aws-library/tests/conftest.py | 20 ++++ 15 files changed, 524 insertions(+) create mode 100644 packages/aws-library/Makefile create mode 100644 packages/aws-library/README.md create mode 100644 packages/aws-library/VERSION create mode 100644 packages/aws-library/requirements/Makefile create mode 100644 packages/aws-library/requirements/_base.in create mode 100644 packages/aws-library/requirements/_base.txt create mode 100644 packages/aws-library/requirements/_test.in create mode 100644 packages/aws-library/requirements/_test.txt create mode 100644 packages/aws-library/requirements/_tools.in create mode 100644 packages/aws-library/requirements/_tools.txt create mode 100644 packages/aws-library/requirements/ci.txt create mode 100644 packages/aws-library/requirements/dev.txt create mode 100644 packages/aws-library/setup.cfg create mode 100644 packages/aws-library/setup.py create mode 100644 packages/aws-library/tests/conftest.py diff --git a/packages/aws-library/Makefile b/packages/aws-library/Makefile new file mode 100644 index 00000000000..6ba1a1e4919 --- /dev/null +++ b/packages/aws-library/Makefile @@ -0,0 +1,49 @@ +# +# Targets for DEVELOPMENT of aws Library +# +include ../../scripts/common.Makefile +include ../../scripts/common-package.Makefile + +.PHONY: requirements +requirements: ## compiles pip requirements (.in -> .txt) + @$(MAKE_C) requirements reqs + + +.PHONY: install-dev install-prod install-ci +install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode + # installing in $(subst install-,,$@) mode + pip-sync requirements/$(subst install-,,$@).txt + + +.PHONY: tests tests-ci +tests: ## runs unit tests + # running unit tests + @pytest \ + --asyncio-mode=auto \ + --color=yes \ + --cov-config=../../.coveragerc \ + --cov-report=term-missing \ + --cov=dask_task_models_library \ + --durations=10 \ + --exitfirst \ + --failed-first \ + --pdb \ + -vv \ + $(CURDIR)/tests + +tests-ci: ## runs unit tests + # running unit tests + @pytest \ + --asyncio-mode=auto \ + --color=yes \ + --cov-append \ + --cov-config=../../.coveragerc \ + --cov-report=term-missing \ + --cov-report=xml \ + --cov=dask_task_models_library \ + --durations=10 \ + --log-date-format="%Y-%m-%d %H:%M:%S" \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --verbose \ + -m "not heavy_load" \ + $(CURDIR)/tests diff --git a/packages/aws-library/README.md b/packages/aws-library/README.md new file mode 100644 index 00000000000..c7df3095401 --- /dev/null +++ b/packages/aws-library/README.md @@ -0,0 +1,22 @@ +# simcore AWS library + +Provides a wrapper around AWS python libraries. + +Requirements to be compatible with the library: + +- only AWS-related code + + +## Installation + +```console +make help +make install-dev +``` + +## Test + +```console +make help +make test-dev +``` diff --git a/packages/aws-library/VERSION b/packages/aws-library/VERSION new file mode 100644 index 00000000000..6e8bf73aa55 --- /dev/null +++ b/packages/aws-library/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/packages/aws-library/requirements/Makefile b/packages/aws-library/requirements/Makefile new file mode 100644 index 00000000000..3f25442b790 --- /dev/null +++ b/packages/aws-library/requirements/Makefile @@ -0,0 +1,6 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in new file mode 100644 index 00000000000..062459a0a33 --- /dev/null +++ b/packages/aws-library/requirements/_base.in @@ -0,0 +1,7 @@ +# +# Specifies third-party dependencies for 'dask-task-models-library' +# +--constraint ../../../requirements/constraints.txt + +aioboto3 +types-aiobotocore[ec2] diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt new file mode 100644 index 00000000000..ddc7b204d4e --- /dev/null +++ b/packages/aws-library/requirements/_base.txt @@ -0,0 +1,73 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/_base.txt --strip-extras requirements/_base.in +# +aioboto3==12.0.0 + # via -r requirements/_base.in +aiobotocore==2.7.0 + # via + # aioboto3 + # aiobotocore +aiohttp==3.8.6 + # via + # -c requirements/../../../requirements/constraints.txt + # aiobotocore +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp +async-timeout==4.0.3 + # via aiohttp +attrs==23.1.0 + # via aiohttp +boto3==1.28.64 + # via aiobotocore +botocore==1.31.64 + # via + # aiobotocore + # boto3 + # s3transfer +botocore-stubs==1.31.79 + # via types-aiobotocore +charset-normalizer==3.3.2 + # via aiohttp +frozenlist==1.4.0 + # via + # aiohttp + # aiosignal +idna==3.4 + # via yarl +jmespath==1.0.1 + # via + # boto3 + # botocore +multidict==6.0.4 + # via + # aiohttp + # yarl +python-dateutil==2.8.2 + # via botocore +s3transfer==0.7.0 + # via boto3 +six==1.16.0 + # via python-dateutil +types-aiobotocore==2.7.0 + # via -r requirements/_base.in +types-aiobotocore-ec2==2.7.0 + # via types-aiobotocore +types-awscrt==0.19.8 + # via botocore-stubs +typing-extensions==4.8.0 + # via + # types-aiobotocore + # types-aiobotocore-ec2 +urllib3==2.0.7 + # via + # -c requirements/../../../requirements/constraints.txt + # botocore +wrapt==1.15.0 + # via aiobotocore +yarl==1.9.2 + # via aiohttp diff --git a/packages/aws-library/requirements/_test.in b/packages/aws-library/requirements/_test.in new file mode 100644 index 00000000000..83367d97185 --- /dev/null +++ b/packages/aws-library/requirements/_test.in @@ -0,0 +1,23 @@ +# +# Specifies dependencies required to run 'models-library' +# +--constraint ../../../requirements/constraints.txt + +# Adds base AS CONSTRAINT specs, not requirement. +# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt +# +--constraint _base.txt + +# testing +coverage +faker +pint +pytest +pytest-aiohttp # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76 +pytest-cov +pytest-icdiff +pytest-instafail +pytest-mock +pytest-runner +pytest-sugar +pyyaml diff --git a/packages/aws-library/requirements/_test.txt b/packages/aws-library/requirements/_test.txt new file mode 100644 index 00000000000..7284cc793c2 --- /dev/null +++ b/packages/aws-library/requirements/_test.txt @@ -0,0 +1,115 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in +# +aiohttp==3.8.6 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # pytest-aiohttp +aiosignal==1.3.1 + # via + # -c requirements/_base.txt + # aiohttp +async-timeout==4.0.3 + # via + # -c requirements/_base.txt + # aiohttp +attrs==23.1.0 + # via + # -c requirements/_base.txt + # aiohttp +charset-normalizer==3.3.2 + # via + # -c requirements/_base.txt + # aiohttp +coverage==7.3.2 + # via + # -r requirements/_test.in + # pytest-cov +exceptiongroup==1.1.3 + # via pytest +faker==19.13.0 + # via -r requirements/_test.in +frozenlist==1.4.0 + # via + # -c requirements/_base.txt + # aiohttp + # aiosignal +icdiff==2.0.7 + # via pytest-icdiff +idna==3.4 + # via + # -c requirements/_base.txt + # yarl +iniconfig==2.0.0 + # via pytest +multidict==6.0.4 + # via + # -c requirements/_base.txt + # aiohttp + # yarl +packaging==23.2 + # via + # pytest + # pytest-sugar +pint==0.22 + # via -r requirements/_test.in +pluggy==1.3.0 + # via pytest +pprintpp==0.4.0 + # via pytest-icdiff +pytest==7.4.3 + # via + # -r requirements/_test.in + # pytest-aiohttp + # pytest-asyncio + # pytest-cov + # pytest-icdiff + # pytest-instafail + # pytest-mock + # pytest-sugar +pytest-aiohttp==1.0.5 + # via -r requirements/_test.in +pytest-asyncio==0.21.1 + # via pytest-aiohttp +pytest-cov==4.1.0 + # via -r requirements/_test.in +pytest-icdiff==0.8 + # via -r requirements/_test.in +pytest-instafail==0.5.0 + # via -r requirements/_test.in +pytest-mock==3.12.0 + # via -r requirements/_test.in +pytest-runner==6.0.0 + # via -r requirements/_test.in +pytest-sugar==0.9.7 + # via -r requirements/_test.in +python-dateutil==2.8.2 + # via + # -c requirements/_base.txt + # faker +pyyaml==6.0.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in +six==1.16.0 + # via + # -c requirements/_base.txt + # python-dateutil +termcolor==2.3.0 + # via pytest-sugar +tomli==2.0.1 + # via + # coverage + # pytest +typing-extensions==4.8.0 + # via + # -c requirements/_base.txt + # pint +yarl==1.9.2 + # via + # -c requirements/_base.txt + # aiohttp diff --git a/packages/aws-library/requirements/_tools.in b/packages/aws-library/requirements/_tools.in new file mode 100644 index 00000000000..1def82c12a3 --- /dev/null +++ b/packages/aws-library/requirements/_tools.in @@ -0,0 +1,5 @@ +--constraint ../../../requirements/constraints.txt +--constraint _base.txt +--constraint _test.txt + +--requirement ../../../requirements/devenv.txt diff --git a/packages/aws-library/requirements/_tools.txt b/packages/aws-library/requirements/_tools.txt new file mode 100644 index 00000000000..643e487286b --- /dev/null +++ b/packages/aws-library/requirements/_tools.txt @@ -0,0 +1,89 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in +# +astroid==3.0.1 + # via pylint +black==23.10.1 + # via -r requirements/../../../requirements/devenv.txt +build==1.0.3 + # via pip-tools +bump2version==1.0.1 + # via -r requirements/../../../requirements/devenv.txt +cfgv==3.4.0 + # via pre-commit +click==8.1.7 + # via + # black + # pip-tools +dill==0.3.7 + # via pylint +distlib==0.3.7 + # via virtualenv +filelock==3.13.1 + # via virtualenv +identify==2.5.31 + # via pre-commit +isort==5.12.0 + # via + # -r requirements/../../../requirements/devenv.txt + # pylint +mccabe==0.7.0 + # via pylint +mypy-extensions==1.0.0 + # via black +nodeenv==1.8.0 + # via pre-commit +packaging==23.2 + # via + # -c requirements/_test.txt + # black + # build +pathspec==0.11.2 + # via black +pip-tools==7.3.0 + # via -r requirements/../../../requirements/devenv.txt +platformdirs==3.11.0 + # via + # black + # pylint + # virtualenv +pre-commit==3.5.0 + # via -r requirements/../../../requirements/devenv.txt +pylint==3.0.2 + # via -r requirements/../../../requirements/devenv.txt +pyproject-hooks==1.0.0 + # via build +pyyaml==6.0.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_test.txt + # pre-commit +ruff==0.1.4 + # via -r requirements/../../../requirements/devenv.txt +tomli==2.0.1 + # via + # -c requirements/_test.txt + # black + # build + # pip-tools + # pylint + # pyproject-hooks +tomlkit==0.12.2 + # via pylint +typing-extensions==4.8.0 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # astroid + # black +virtualenv==20.24.6 + # via pre-commit +wheel==0.41.3 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/packages/aws-library/requirements/ci.txt b/packages/aws-library/requirements/ci.txt new file mode 100644 index 00000000000..d03f5794f0e --- /dev/null +++ b/packages/aws-library/requirements/ci.txt @@ -0,0 +1,18 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'models-library' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +../pytest-simcore/ + +# current module +. diff --git a/packages/aws-library/requirements/dev.txt b/packages/aws-library/requirements/dev.txt new file mode 100644 index 00000000000..32d383e9ccc --- /dev/null +++ b/packages/aws-library/requirements/dev.txt @@ -0,0 +1,18 @@ +# Shortcut to install all packages needed to develop 'models-library' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +--editable ../pytest-simcore/ + +# current module +--editable . diff --git a/packages/aws-library/setup.cfg b/packages/aws-library/setup.cfg new file mode 100644 index 00000000000..e138d387a7d --- /dev/null +++ b/packages/aws-library/setup.cfg @@ -0,0 +1,17 @@ +[bumpversion] +current_version = 0.1.0 +commit = True +message = packages/aws-library version: {current_version} → {new_version} +tag = False +commit_args = --no-verify + +[bumpversion:file:VERSION] + +[bdist_wheel] +universal = 1 + +[aliases] +test = pytest + +[tool:pytest] +asyncio_mode = auto diff --git a/packages/aws-library/setup.py b/packages/aws-library/setup.py new file mode 100644 index 00000000000..a0ecf2fc04a --- /dev/null +++ b/packages/aws-library/setup.py @@ -0,0 +1,61 @@ +import re +import sys +from pathlib import Path + +from setuptools import find_packages, setup + + +def read_reqs(reqs_path: Path) -> set[str]: + return { + r + for r in re.findall( + r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", + reqs_path.read_text(), + re.MULTILINE, + ) + if isinstance(r, str) + } + + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +INSTALL_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_base.in") +) # WEAK requirements + +TEST_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_test.txt") + | { + "simcore-models-library", + } +) # STRICT requirements + + +SETUP = { + "name": "simcore-aws-library", + "version": Path(CURRENT_DIR / "VERSION").read_text().strip(), + "author": "Sylvain Anderegg (sanderegg)", + "description": "Core service library for AWS APIs", + "python_requires": "~=3.10", + "classifiers": [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3.10", + ], + "long_description": Path(CURRENT_DIR / "README.md").read_text(), + "license": "MIT license", + "install_requires": INSTALL_REQUIREMENTS, + "packages": find_packages(where="src"), + "package_dir": {"": "src"}, + "include_package_data": True, + "test_suite": "tests", + "tests_require": TEST_REQUIREMENTS, + "extras_require": {"test": TEST_REQUIREMENTS}, + "zip_safe": False, +} + + +if __name__ == "__main__": + setup(**SETUP) diff --git a/packages/aws-library/tests/conftest.py b/packages/aws-library/tests/conftest.py new file mode 100644 index 00000000000..d004b25966f --- /dev/null +++ b/packages/aws-library/tests/conftest.py @@ -0,0 +1,20 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import +from pathlib import Path + +import aws_library +import pytest + +pytest_plugins = [ + "pytest_simcore.repository_paths", + "pytest_simcore.pydantic_models", + "pytest_simcore.pytest_global_environs", +] + + +@pytest.fixture(scope="session") +def package_dir() -> Path: + pdir = Path(aws_library.__file__).resolve().parent + assert pdir.exists() + return pdir From 17c4741a21f966833a7c9ca26b6a5b2df2a1a360 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:22:14 +0100 Subject: [PATCH 02/78] preparing new library --- .github/codeql/codeql-config.yml | 1 + .github/workflows/ci-testing-deploy.yml | 47 +++++++++++++++++++ ci/github/unit-testing/aws-library.bash | 41 ++++++++++++++++ packages/aws-library/requirements/_base.in | 2 +- packages/aws-library/setup.py | 3 -- .../aws-library/src/aws_library/__init__.py | 3 ++ 6 files changed, 93 insertions(+), 4 deletions(-) create mode 100755 ci/github/unit-testing/aws-library.bash create mode 100644 packages/aws-library/src/aws_library/__init__.py diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index ace67457635..adac3b13795 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -3,6 +3,7 @@ name: "ospac-simcore CodeQL config" disable-default-queries: false paths: + - packages/aws-library/src - packages/dask-task-models-library/src - packages/models-library/src/models_library - packages/postgres-database/src/simcore_postgres_database diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 4f001b7fcb6..6e184986192 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -50,6 +50,7 @@ jobs: runs-on: ubuntu-latest # Set job outputs to values from filter step outputs: + aws-library: ${{ steps.filter.outputs.aws-library }} dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }} models-library: ${{ steps.filter.outputs.models-library }} postgres-database: ${{ steps.filter.outputs.postgres-database }} @@ -87,6 +88,12 @@ jobs: id: filter with: filters: | + aws-library: + - 'packages/aws-library/**' + - 'packages/pytest-simcore/**' + - 'services/docker-compose*' + - 'scripts/mypy/*' + - 'mypy.ini' dask-task-models-library: - 'packages/dask-task-models-library/**' - 'packages/pytest-simcore/**' @@ -804,6 +811,45 @@ jobs: with: flags: unittests #optional + unit-test-aws-library: + needs: changes + if: ${{ needs.changes.outputs.aws-library == 'true' || github.event_name == 'push' }} + timeout-minutes: 18 # if this timeout gets too small, then split the tests + name: "[unit] aws-library" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.10"] + os: [ubuntu-22.04] + docker_buildx: [v0.10.4] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + version: ${{ matrix.docker_buildx }} + driver: docker-container + - name: setup python environment + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: "pip" + cache-dependency-path: "packages/aws-library/requirements/ci.txt" + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - name: install + run: ./ci/github/unit-testing/aws-library.bash install + - name: typecheck + run: ./ci/github/unit-testing/aws-library.bash typecheck + - name: test + if: always() + run: ./ci/github/unit-testing/aws-library.bash test + - uses: codecov/codecov-action@v3.1.4 + with: + flags: unittests #optional + unit-test-dask-task-models-library: needs: changes if: ${{ needs.changes.outputs.dask-task-models-library == 'true' || github.event_name == 'push' }} @@ -1447,6 +1493,7 @@ jobs: unit-test-catalog, unit-test-clusters-keeper, unit-test-dask-sidecar, + unit-test-aws-library, unit-test-dask-task-models-library, unit-test-datcore-adapter, unit-test-director-v2, diff --git a/ci/github/unit-testing/aws-library.bash b/ci/github/unit-testing/aws-library.bash new file mode 100755 index 00000000000..6250656e18f --- /dev/null +++ b/ci/github/unit-testing/aws-library.bash @@ -0,0 +1,41 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +install() { + bash ci/helpers/ensure_python_pip.bash + make devenv + # shellcheck source=/dev/null + source .venv/bin/activate + pushd packages/aws-library + make install-ci + popd + .venv/bin/pip list --verbose +} + +test() { + # shellcheck source=/dev/null + source .venv/bin/activate + pushd packages/aws-library + make tests-ci + popd +} + +typecheck() { + pushd packages/aws-library + make mypy + popd +} + +# Check if the function exists (bash specific) +if declare -f "$1" >/dev/null; then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in index 062459a0a33..8c0f5b2213d 100644 --- a/packages/aws-library/requirements/_base.in +++ b/packages/aws-library/requirements/_base.in @@ -1,5 +1,5 @@ # -# Specifies third-party dependencies for 'dask-task-models-library' +# Specifies third-party dependencies for 'aws-library' # --constraint ../../../requirements/constraints.txt diff --git a/packages/aws-library/setup.py b/packages/aws-library/setup.py index a0ecf2fc04a..d4ffa1a620d 100644 --- a/packages/aws-library/setup.py +++ b/packages/aws-library/setup.py @@ -25,9 +25,6 @@ def read_reqs(reqs_path: Path) -> set[str]: TEST_REQUIREMENTS = tuple( read_reqs(CURRENT_DIR / "requirements" / "_test.txt") - | { - "simcore-models-library", - } ) # STRICT requirements diff --git a/packages/aws-library/src/aws_library/__init__.py b/packages/aws-library/src/aws_library/__init__.py new file mode 100644 index 00000000000..8abb76e0ad2 --- /dev/null +++ b/packages/aws-library/src/aws_library/__init__.py @@ -0,0 +1,3 @@ +import pkg_resources + +__version__: str = pkg_resources.get_distribution("simcore-aws-library").version From a5480630c52d8a87bcd63e7b195bfc8b38bb299f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:52:54 +0100 Subject: [PATCH 03/78] fixes tests --- packages/aws-library/requirements/_base.in | 1 + packages/aws-library/requirements/_base.txt | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in index 8c0f5b2213d..b0d869925e2 100644 --- a/packages/aws-library/requirements/_base.in +++ b/packages/aws-library/requirements/_base.in @@ -4,4 +4,5 @@ --constraint ../../../requirements/constraints.txt aioboto3 +pydantic[email] types-aiobotocore[ec2] diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index ddc7b204d4e..6b5158e2195 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -33,12 +33,18 @@ botocore-stubs==1.31.79 # via types-aiobotocore charset-normalizer==3.3.2 # via aiohttp +dnspython==2.4.2 + # via email-validator +email-validator==2.1.0.post1 + # via pydantic frozenlist==1.4.0 # via # aiohttp # aiosignal idna==3.4 - # via yarl + # via + # email-validator + # yarl jmespath==1.0.1 # via # boto3 @@ -47,6 +53,10 @@ multidict==6.0.4 # via # aiohttp # yarl +pydantic==1.10.13 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_base.in python-dateutil==2.8.2 # via botocore s3transfer==0.7.0 @@ -61,6 +71,7 @@ types-awscrt==0.19.8 # via botocore-stubs typing-extensions==4.8.0 # via + # pydantic # types-aiobotocore # types-aiobotocore-ec2 urllib3==2.0.7 From 97fffa982cff7a38b60cd9db39c47fb8c49b0d54 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 08:44:58 +0100 Subject: [PATCH 04/78] add ec2 folder --- packages/aws-library/src/aws_library/ec2/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/aws-library/src/aws_library/ec2/__init__.py diff --git a/packages/aws-library/src/aws_library/ec2/__init__.py b/packages/aws-library/src/aws_library/ec2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 90ec3633dc239a3f51c85779d8fb5faf189dd10e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 08:49:42 +0100 Subject: [PATCH 05/78] adding dependency to aws-library --- services/autoscaling/requirements/ci.txt | 3 ++- services/autoscaling/requirements/dev.txt | 1 + services/autoscaling/requirements/prod.txt | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/services/autoscaling/requirements/ci.txt b/services/autoscaling/requirements/ci.txt index d2b1c3498e4..4adf7f199a7 100644 --- a/services/autoscaling/requirements/ci.txt +++ b/services/autoscaling/requirements/ci.txt @@ -11,11 +11,12 @@ --requirement _test.txt # installs this repo's packages +../../packages/aws-library +../../packages/dask-task-models-library ../../packages/models-library ../../packages/pytest-simcore ../../packages/service-library[fastapi] ../../packages/settings-library -../../packages/dask-task-models-library # installs current package . diff --git a/services/autoscaling/requirements/dev.txt b/services/autoscaling/requirements/dev.txt index 8cbfdbe8e3f..432e7ef62e9 100644 --- a/services/autoscaling/requirements/dev.txt +++ b/services/autoscaling/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/aws-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/autoscaling/requirements/prod.txt b/services/autoscaling/requirements/prod.txt index 7d635391e63..9452305726e 100644 --- a/services/autoscaling/requirements/prod.txt +++ b/services/autoscaling/requirements/prod.txt @@ -10,6 +10,7 @@ --requirement _base.txt # installs this repo's packages +../../packages/aws-library ../../packages/models-library ../../packages/service-library[fastapi] ../../packages/settings-library From affac84973dcd68beb9fa7fba1b05715d93856cc Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 08:49:54 +0100 Subject: [PATCH 06/78] moving models --- .../aws-library/src/aws_library/ec2/models.py | 60 +++++++++++++++++++ .../src/simcore_service_autoscaling/models.py | 59 +----------------- 2 files changed, 61 insertions(+), 58 deletions(-) create mode 100644 packages/aws-library/src/aws_library/ec2/models.py diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py new file mode 100644 index 00000000000..d166bb28603 --- /dev/null +++ b/packages/aws-library/src/aws_library/ec2/models.py @@ -0,0 +1,60 @@ +import datetime +from dataclasses import dataclass + +from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt +from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType + + +class Resources(BaseModel): + cpus: NonNegativeFloat + ram: ByteSize + + @classmethod + def create_as_empty(cls) -> "Resources": + return cls(cpus=0, ram=ByteSize(0)) + + def __ge__(self, other: "Resources") -> bool: + return self.cpus >= other.cpus and self.ram >= other.ram + + def __gt__(self, other: "Resources") -> bool: + return self.cpus > other.cpus or self.ram > other.ram + + def __add__(self, other: "Resources") -> "Resources": + return Resources.construct( + **{ + key: a + b + for (key, a), b in zip( + self.dict().items(), other.dict().values(), strict=True + ) + } + ) + + def __sub__(self, other: "Resources") -> "Resources": + return Resources.construct( + **{ + key: a - b + for (key, a), b in zip( + self.dict().items(), other.dict().values(), strict=True + ) + } + ) + + +@dataclass(frozen=True) +class EC2InstanceType: + name: InstanceTypeType + cpus: PositiveInt + ram: ByteSize + + +InstancePrivateDNSName = str + + +@dataclass(frozen=True) +class EC2InstanceData: + launch_time: datetime.datetime + id: str + aws_private_dns: InstancePrivateDNSName + type: InstanceTypeType + state: InstanceStateNameType + resources: Resources diff --git a/services/autoscaling/src/simcore_service_autoscaling/models.py b/services/autoscaling/src/simcore_service_autoscaling/models.py index 8be0f953fe8..9e3ffec2df9 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/models.py +++ b/services/autoscaling/src/simcore_service_autoscaling/models.py @@ -1,65 +1,8 @@ -import datetime from dataclasses import dataclass, field from typing import Any, TypeAlias +from aws_library.ec2.models import EC2InstanceData from models_library.generated_models.docker_rest_api import Node -from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt -from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType - - -class Resources(BaseModel): - cpus: NonNegativeFloat - ram: ByteSize - - @classmethod - def create_as_empty(cls) -> "Resources": - return cls(cpus=0, ram=ByteSize(0)) - - def __ge__(self, other: "Resources") -> bool: - return self.cpus >= other.cpus and self.ram >= other.ram - - def __gt__(self, other: "Resources") -> bool: - return self.cpus > other.cpus or self.ram > other.ram - - def __add__(self, other: "Resources") -> "Resources": - return Resources.construct( - **{ - key: a + b - for (key, a), b in zip( - self.dict().items(), other.dict().values(), strict=True - ) - } - ) - - def __sub__(self, other: "Resources") -> "Resources": - return Resources.construct( - **{ - key: a - b - for (key, a), b in zip( - self.dict().items(), other.dict().values(), strict=True - ) - } - ) - - -@dataclass(frozen=True) -class EC2InstanceType: - name: InstanceTypeType - cpus: PositiveInt - ram: ByteSize - - -InstancePrivateDNSName = str - - -@dataclass(frozen=True) -class EC2InstanceData: - launch_time: datetime.datetime - id: str - aws_private_dns: InstancePrivateDNSName - type: InstanceTypeType - state: InstanceStateNameType - resources: Resources @dataclass(frozen=True) From c047cdd768e909443f9f15aa4eff6c22befbd7f4 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 08:59:19 +0100 Subject: [PATCH 07/78] moved models to aws-library --- .../modules/auto_scaling_core.py | 9 ++------- .../modules/auto_scaling_mode_base.py | 3 ++- .../modules/auto_scaling_mode_computational.py | 9 ++------- .../modules/auto_scaling_mode_dynamic.py | 3 ++- .../src/simcore_service_autoscaling/modules/dask.py | 2 +- .../src/simcore_service_autoscaling/modules/ec2.py | 3 ++- .../utils/auto_scaling_core.py | 3 ++- .../utils/computational_scaling.py | 9 ++------- .../simcore_service_autoscaling/utils/dynamic_scaling.py | 3 ++- .../src/simcore_service_autoscaling/utils/rabbitmq.py | 3 ++- .../simcore_service_autoscaling/utils/utils_docker.py | 3 ++- .../src/simcore_service_autoscaling/utils/utils_ec2.py | 3 ++- services/autoscaling/tests/unit/conftest.py | 3 ++- services/autoscaling/tests/unit/test_models.py | 5 +++-- .../unit/test_modules_auto_scaling_computational.py | 2 +- .../tests/unit/test_modules_auto_scaling_dynamic.py | 3 ++- services/autoscaling/tests/unit/test_modules_dask.py | 2 +- services/autoscaling/tests/unit/test_modules_ec2.py | 4 ++-- .../tests/unit/test_utils_computational_scaling.py | 3 +-- services/autoscaling/tests/unit/test_utils_docker.py | 6 +++--- .../autoscaling/tests/unit/test_utils_dynamic_scaling.py | 2 +- services/autoscaling/tests/unit/test_utils_ec2.py | 3 +-- 22 files changed, 40 insertions(+), 46 deletions(-) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py index 5dcd7de627c..ef2c5ab5a8b 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py @@ -7,6 +7,7 @@ from typing import cast import arrow +from aws_library.ec2.models import EC2InstanceType, Resources from fastapi import FastAPI from models_library.generated_models.docker_rest_api import ( Availability, @@ -24,13 +25,7 @@ Ec2TooManyInstancesError, ) from ..core.settings import ApplicationSettings, get_application_settings -from ..models import ( - AssociatedInstance, - Cluster, - EC2InstanceData, - EC2InstanceType, - Resources, -) +from ..models import AssociatedInstance, Cluster, EC2InstanceData from ..utils import utils_docker, utils_ec2 from ..utils.auto_scaling_core import ( associate_ec2_instances_with_nodes, diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py index 2adf6268e8c..9a2cb291dd6 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py @@ -2,13 +2,14 @@ from collections.abc import Iterable from dataclasses import dataclass +from aws_library.ec2.models import EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import DockerLabelKey from models_library.generated_models.docker_rest_api import Node as DockerNode from servicelib.logging_utils import LogLevelInt from types_aiobotocore_ec2.literals import InstanceTypeType -from ..models import AssociatedInstance, EC2InstanceData, EC2InstanceType, Resources +from ..models import AssociatedInstance, EC2InstanceData @dataclass diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py index 0fe64471056..8a4f8e7e810 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py @@ -2,6 +2,7 @@ import logging from collections.abc import Iterable +from aws_library.ec2.models import EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import ( DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY, @@ -14,13 +15,7 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import ( - AssociatedInstance, - DaskTask, - EC2InstanceData, - EC2InstanceType, - Resources, -) +from ..models import AssociatedInstance, DaskTask, EC2InstanceData from ..utils import computational_scaling as utils from ..utils import utils_docker, utils_ec2 from . import dask diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py index 6a2d5814c85..3d5c673fbab 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py @@ -1,5 +1,6 @@ from collections.abc import Iterable +from aws_library.ec2.models import EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import DockerLabelKey from models_library.generated_models.docker_rest_api import Node, Task @@ -7,7 +8,7 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import AssociatedInstance, EC2InstanceData, EC2InstanceType, Resources +from ..models import AssociatedInstance, EC2InstanceData from ..utils import dynamic_scaling as utils from ..utils import utils_docker, utils_ec2 from ..utils.rabbitmq import log_tasks_message, progress_tasks_message diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py b/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py index d94c855a60d..07716fbbd26 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py @@ -4,6 +4,7 @@ from typing import Any, Final, TypeAlias import distributed +from aws_library.ec2.models import Resources from pydantic import AnyUrl, ByteSize, parse_obj_as from ..core.errors import ( @@ -17,7 +18,6 @@ DaskTaskId, DaskTaskResources, EC2InstanceData, - Resources, ) from ..utils.auto_scaling_core import ( node_host_name_from_ec2_private_dns, diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py b/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py index f8c0eb595e1..a97baf83b70 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py @@ -7,6 +7,7 @@ import botocore.exceptions from aiobotocore.session import ClientCreatorContext from aiocache import cached +from aws_library.ec2.models import EC2InstanceType, Resources from fastapi import FastAPI from pydantic import ByteSize, parse_obj_as from servicelib.logging_utils import log_context @@ -25,7 +26,7 @@ Ec2TooManyInstancesError, ) from ..core.settings import EC2InstancesSettings, EC2Settings -from ..models import EC2InstanceData, EC2InstanceType, Resources +from ..models import EC2InstanceData from ..utils.utils_ec2 import compose_user_data logger = logging.getLogger(__name__) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py index 7d9ab4f310d..0c0dd33cb00 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py @@ -3,12 +3,13 @@ import re from typing import Final +from aws_library.ec2.models import EC2InstanceType, Resources from models_library.generated_models.docker_rest_api import Node from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.errors import Ec2InstanceInvalidError, Ec2InvalidDnsNameError from ..core.settings import ApplicationSettings -from ..models import AssociatedInstance, EC2InstanceData, EC2InstanceType, Resources +from ..models import AssociatedInstance, EC2InstanceData from ..modules.auto_scaling_mode_base import BaseAutoscaling from . import utils_docker diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py b/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py index f5cb43c4acb..7425914875e 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py @@ -3,19 +3,14 @@ from collections.abc import Iterable from typing import Final +from aws_library.ec2.models import EC2InstanceType, Resources from dask_task_models_library.constants import DASK_TASK_EC2_RESOURCE_RESTRICTION_KEY from fastapi import FastAPI from servicelib.utils_formatting import timedelta_as_minute_second from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import ( - AssociatedInstance, - DaskTask, - EC2InstanceData, - EC2InstanceType, - Resources, -) +from ..models import AssociatedInstance, DaskTask, EC2InstanceData _logger = logging.getLogger(__name__) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py b/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py index 549b59bb38c..7f82b07d062 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py @@ -2,12 +2,13 @@ import logging from collections.abc import Iterable +from aws_library.ec2.models import EC2InstanceType, Resources from fastapi import FastAPI from models_library.generated_models.docker_rest_api import Task from servicelib.utils_formatting import timedelta_as_minute_second from ..core.settings import get_application_settings -from ..models import AssociatedInstance, EC2InstanceData, EC2InstanceType, Resources +from ..models import AssociatedInstance, EC2InstanceData from . import utils_docker from .rabbitmq import log_tasks_message, progress_tasks_message diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py b/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py index e8bce0ba5aa..d8c88c50a86 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py @@ -1,6 +1,7 @@ import asyncio import logging +from aws_library.ec2.models import Resources from fastapi import FastAPI from models_library.docker import StandardSimcoreDockerLabels from models_library.generated_models.docker_rest_api import Task @@ -13,7 +14,7 @@ from servicelib.logging_utils import log_catch from ..core.settings import ApplicationSettings, get_application_settings -from ..models import Cluster, Resources +from ..models import Cluster from ..modules.rabbitmq import post_message logger = logging.getLogger(__name__) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py index 65b8ce8af05..61e8c64be44 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py @@ -13,6 +13,7 @@ from typing import Final, cast import yaml +from aws_library.ec2.models import Resources from models_library.docker import ( DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY, DockerGenericTag, @@ -33,7 +34,7 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import ApplicationSettings -from ..models import EC2InstanceData, Resources +from ..models import EC2InstanceData from ..modules.docker import AutoscalingDocker logger = logging.getLogger(__name__) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py index 672a8c76733..88c18314460 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py @@ -8,10 +8,11 @@ from collections.abc import Callable from textwrap import dedent +from aws_library.ec2.models import EC2InstanceType, Resources + from .._meta import VERSION from ..core.errors import ConfigurationError, Ec2InstanceNotFoundError from ..core.settings import ApplicationSettings -from ..models import EC2InstanceType, Resources logger = logging.getLogger(__name__) diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index e388acba225..2e32db769fd 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -23,6 +23,7 @@ import simcore_service_autoscaling from aiohttp.test_utils import unused_port from asgi_lifespan import LifespanManager +from aws_library.ec2.models import Resources from deepdiff import DeepDiff from faker import Faker from fakeredis.aioredis import FakeRedis @@ -45,7 +46,7 @@ from settings_library.rabbit import RabbitSettings from simcore_service_autoscaling.core.application import create_app from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings -from simcore_service_autoscaling.models import Cluster, DaskTaskResources, Resources +from simcore_service_autoscaling.models import Cluster, DaskTaskResources from simcore_service_autoscaling.modules.docker import AutoscalingDocker from simcore_service_autoscaling.modules.ec2 import AutoscalingEC2, EC2InstanceData from tenacity import retry diff --git a/services/autoscaling/tests/unit/test_models.py b/services/autoscaling/tests/unit/test_models.py index fdb9362591a..2525377823b 100644 --- a/services/autoscaling/tests/unit/test_models.py +++ b/services/autoscaling/tests/unit/test_models.py @@ -3,14 +3,15 @@ # pylint: disable=unused-variable -from typing import Any, Awaitable, Callable +from collections.abc import Awaitable, Callable +from typing import Any import aiodocker import pytest +from aws_library.ec2.models import Resources from models_library.docker import DockerLabelKey, StandardSimcoreDockerLabels from models_library.generated_models.docker_rest_api import Service, Task from pydantic import ByteSize, ValidationError, parse_obj_as -from simcore_service_autoscaling.models import Resources @pytest.mark.parametrize( diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index 0482437bb5d..cea344f7e03 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -18,6 +18,7 @@ import distributed import pytest +from aws_library.ec2.models import Resources from dask_task_models_library.constants import DASK_TASK_EC2_RESOURCE_RESTRICTION_KEY from faker import Faker from fastapi import FastAPI @@ -34,7 +35,6 @@ AssociatedInstance, Cluster, EC2InstanceData, - Resources, ) from simcore_service_autoscaling.modules.auto_scaling_core import ( _deactivate_empty_nodes, diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index cb41d2d3278..c7ec52b6237 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -16,6 +16,7 @@ import aiodocker import pytest +from aws_library.ec2.models import Resources from faker import Faker from fastapi import FastAPI from models_library.docker import ( @@ -35,7 +36,7 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_autoscaling.core.settings import ApplicationSettings -from simcore_service_autoscaling.models import AssociatedInstance, Cluster, Resources +from simcore_service_autoscaling.models import AssociatedInstance, Cluster from simcore_service_autoscaling.modules.auto_scaling_core import ( _activate_drained_nodes, _deactivate_empty_nodes, diff --git a/services/autoscaling/tests/unit/test_modules_dask.py b/services/autoscaling/tests/unit/test_modules_dask.py index 561ed1755d3..09b23f84b33 100644 --- a/services/autoscaling/tests/unit/test_modules_dask.py +++ b/services/autoscaling/tests/unit/test_modules_dask.py @@ -9,6 +9,7 @@ import distributed import pytest +from aws_library.ec2.models import Resources from faker import Faker from pydantic import AnyUrl, ByteSize, parse_obj_as from simcore_service_autoscaling.core.errors import ( @@ -21,7 +22,6 @@ DaskTaskId, DaskTaskResources, EC2InstanceData, - Resources, ) from simcore_service_autoscaling.modules.dask import ( DaskTask, diff --git a/services/autoscaling/tests/unit/test_modules_ec2.py b/services/autoscaling/tests/unit/test_modules_ec2.py index b64b8517f5c..727501adec5 100644 --- a/services/autoscaling/tests/unit/test_modules_ec2.py +++ b/services/autoscaling/tests/unit/test_modules_ec2.py @@ -7,6 +7,7 @@ import botocore.exceptions import pytest +from aws_library.ec2.models import EC2InstanceType from faker import Faker from fastapi import FastAPI from moto.server import ThreadedMotoServer @@ -18,7 +19,6 @@ Ec2TooManyInstancesError, ) from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings -from simcore_service_autoscaling.models import EC2InstanceType from simcore_service_autoscaling.modules.ec2 import ( AutoscalingEC2, EC2InstanceData, @@ -73,7 +73,7 @@ async def test_ec2_does_not_initialize_if_deactivated( initialized_app: FastAPI, ): assert hasattr(initialized_app.state, "ec2_client") - assert initialized_app.state.ec2_client == None + assert initialized_app.state.ec2_client is None with pytest.raises(ConfigurationError): get_ec2_client(initialized_app) diff --git a/services/autoscaling/tests/unit/test_utils_computational_scaling.py b/services/autoscaling/tests/unit/test_utils_computational_scaling.py index 17c3e5f5d9d..945b883187e 100644 --- a/services/autoscaling/tests/unit/test_utils_computational_scaling.py +++ b/services/autoscaling/tests/unit/test_utils_computational_scaling.py @@ -8,6 +8,7 @@ from unittest import mock import pytest +from aws_library.ec2.models import EC2InstanceType, Resources from faker import Faker from models_library.generated_models.docker_rest_api import Node as DockerNode from pydantic import ByteSize, parse_obj_as @@ -17,8 +18,6 @@ DaskTask, DaskTaskResources, EC2InstanceData, - EC2InstanceType, - Resources, ) from simcore_service_autoscaling.utils.computational_scaling import ( _DEFAULT_MAX_CPU, diff --git a/services/autoscaling/tests/unit/test_utils_docker.py b/services/autoscaling/tests/unit/test_utils_docker.py index 3a5985becf0..8a0a3a7c3af 100644 --- a/services/autoscaling/tests/unit/test_utils_docker.py +++ b/services/autoscaling/tests/unit/test_utils_docker.py @@ -6,11 +6,13 @@ import datetime import itertools import random +from collections.abc import AsyncIterator, Awaitable, Callable from copy import deepcopy -from typing import Any, AsyncIterator, Awaitable, Callable +from typing import Any import aiodocker import pytest +from aws_library.ec2.models import Resources from deepdiff import DeepDiff from faker import Faker from models_library.docker import DockerGenericTag, DockerLabelKey @@ -24,7 +26,6 @@ from pydantic import ByteSize, parse_obj_as from pytest_mock.plugin import MockerFixture from servicelib.docker_utils import to_datetime -from simcore_service_autoscaling.models import Resources from simcore_service_autoscaling.modules.docker import AutoscalingDocker from simcore_service_autoscaling.utils.utils_docker import ( Node, @@ -72,7 +73,6 @@ async def _creator(labels: list[str]) -> None: "Labels": {f"{label}": "true" for label in labels}, }, ) - return yield _creator # revert labels diff --git a/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py b/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py index ccc283144b3..16eee35cd5d 100644 --- a/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py +++ b/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py @@ -8,11 +8,11 @@ from datetime import timedelta import pytest +from aws_library.ec2.models import EC2InstanceType from faker import Faker from models_library.generated_models.docker_rest_api import Task from pydantic import ByteSize from pytest_mock import MockerFixture -from simcore_service_autoscaling.models import EC2InstanceType from simcore_service_autoscaling.modules.ec2 import EC2InstanceData from simcore_service_autoscaling.utils.dynamic_scaling import ( try_assigning_task_to_pending_instances, diff --git a/services/autoscaling/tests/unit/test_utils_ec2.py b/services/autoscaling/tests/unit/test_utils_ec2.py index dc9ea43d8be..94ac0e16e9a 100644 --- a/services/autoscaling/tests/unit/test_utils_ec2.py +++ b/services/autoscaling/tests/unit/test_utils_ec2.py @@ -6,15 +6,14 @@ import random import pytest +from aws_library.ec2.models import EC2InstanceType, Resources from faker import Faker from pydantic import ByteSize from simcore_service_autoscaling.core.errors import ( ConfigurationError, Ec2InstanceNotFoundError, ) -from simcore_service_autoscaling.models import Resources from simcore_service_autoscaling.utils.utils_ec2 import ( - EC2InstanceType, closest_instance_policy, compose_user_data, find_best_fitting_ec2_instance, From c00fe58a4c46bde52e7f30bf2802890a534583a6 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 09:01:42 +0100 Subject: [PATCH 08/78] added error --- packages/aws-library/src/aws_library/ec2/errors.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 packages/aws-library/src/aws_library/ec2/errors.py diff --git a/packages/aws-library/src/aws_library/ec2/errors.py b/packages/aws-library/src/aws_library/ec2/errors.py new file mode 100644 index 00000000000..c6879567d77 --- /dev/null +++ b/packages/aws-library/src/aws_library/ec2/errors.py @@ -0,0 +1,5 @@ +from pydantic.errors import PydanticErrorMixin + + +class EC2RuntimeError(PydanticErrorMixin, RuntimeError): + msg_template: str = "EC2 client unexpected error" From 5ddedbdb69fc164a020da8a16d0715dccc7f9c70 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 09:06:46 +0100 Subject: [PATCH 09/78] add empty shell for client --- packages/aws-library/src/aws_library/ec2/client.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/aws-library/src/aws_library/ec2/client.py diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py new file mode 100644 index 00000000000..e69de29bb2d From ae601768e46d02338b66c829714a3f703c29ac3d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:53:57 +0100 Subject: [PATCH 10/78] added aiocache --- packages/aws-library/requirements/_base.in | 1 + packages/aws-library/requirements/_base.txt | 8 +++++--- packages/aws-library/requirements/_tools.txt | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in index b0d869925e2..efdc1a7b6e2 100644 --- a/packages/aws-library/requirements/_base.in +++ b/packages/aws-library/requirements/_base.in @@ -4,5 +4,6 @@ --constraint ../../../requirements/constraints.txt aioboto3 +aiocache pydantic[email] types-aiobotocore[ec2] diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index 6b5158e2195..f54fc3ff41b 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -10,6 +10,8 @@ aiobotocore==2.7.0 # via # aioboto3 # aiobotocore +aiocache==0.12.2 + # via -r requirements/_base.in aiohttp==3.8.6 # via # -c requirements/../../../requirements/constraints.txt @@ -29,7 +31,7 @@ botocore==1.31.64 # aiobotocore # boto3 # s3transfer -botocore-stubs==1.31.79 +botocore-stubs==1.31.83 # via types-aiobotocore charset-normalizer==3.3.2 # via aiohttp @@ -67,7 +69,7 @@ types-aiobotocore==2.7.0 # via -r requirements/_base.in types-aiobotocore-ec2==2.7.0 # via types-aiobotocore -types-awscrt==0.19.8 +types-awscrt==0.19.10 # via botocore-stubs typing-extensions==4.8.0 # via @@ -78,7 +80,7 @@ urllib3==2.0.7 # via # -c requirements/../../../requirements/constraints.txt # botocore -wrapt==1.15.0 +wrapt==1.16.0 # via aiobotocore yarl==1.9.2 # via aiohttp diff --git a/packages/aws-library/requirements/_tools.txt b/packages/aws-library/requirements/_tools.txt index 643e487286b..f1e127151e7 100644 --- a/packages/aws-library/requirements/_tools.txt +++ b/packages/aws-library/requirements/_tools.txt @@ -6,7 +6,7 @@ # astroid==3.0.1 # via pylint -black==23.10.1 +black==23.11.0 # via -r requirements/../../../requirements/devenv.txt build==1.0.3 # via pip-tools @@ -61,7 +61,7 @@ pyyaml==6.0.1 # -c requirements/../../../requirements/constraints.txt # -c requirements/_test.txt # pre-commit -ruff==0.1.4 +ruff==0.1.5 # via -r requirements/../../../requirements/devenv.txt tomli==2.0.1 # via From d7b2ed146ec8f4f73334767ffbfac0a7f1fdcdfc Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:28:57 +0100 Subject: [PATCH 11/78] wrong coverage name --- packages/aws-library/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/aws-library/Makefile b/packages/aws-library/Makefile index 6ba1a1e4919..d7bf9225383 100644 --- a/packages/aws-library/Makefile +++ b/packages/aws-library/Makefile @@ -23,7 +23,7 @@ tests: ## runs unit tests --color=yes \ --cov-config=../../.coveragerc \ --cov-report=term-missing \ - --cov=dask_task_models_library \ + --cov=aws_library \ --durations=10 \ --exitfirst \ --failed-first \ @@ -40,7 +40,7 @@ tests-ci: ## runs unit tests --cov-config=../../.coveragerc \ --cov-report=term-missing \ --cov-report=xml \ - --cov=dask_task_models_library \ + --cov=aws_library \ --durations=10 \ --log-date-format="%Y-%m-%d %H:%M:%S" \ --log-format="%(asctime)s %(levelname)s %(message)s" \ From 3e4c833809e3e0c88d1e97f96d5cd79f03e18cdf Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:29:09 +0100 Subject: [PATCH 12/78] add dependency --- packages/aws-library/requirements/ci.txt | 1 + packages/aws-library/requirements/dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/aws-library/requirements/ci.txt b/packages/aws-library/requirements/ci.txt index d03f5794f0e..db14400c671 100644 --- a/packages/aws-library/requirements/ci.txt +++ b/packages/aws-library/requirements/ci.txt @@ -13,6 +13,7 @@ # installs this repo's packages ../pytest-simcore/ +../service-library/ # current module . diff --git a/packages/aws-library/requirements/dev.txt b/packages/aws-library/requirements/dev.txt index 32d383e9ccc..b125999a475 100644 --- a/packages/aws-library/requirements/dev.txt +++ b/packages/aws-library/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../pytest-simcore/ +--editable ../service-library/ # current module --editable . From 6ea6c1fd3498401be810417aee1f1fb992566bfc Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:29:43 +0100 Subject: [PATCH 13/78] add dependency --- packages/aws-library/requirements/ci.txt | 1 + packages/aws-library/requirements/dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/aws-library/requirements/ci.txt b/packages/aws-library/requirements/ci.txt index db14400c671..604f51d486e 100644 --- a/packages/aws-library/requirements/ci.txt +++ b/packages/aws-library/requirements/ci.txt @@ -14,6 +14,7 @@ # installs this repo's packages ../pytest-simcore/ ../service-library/ +../settings-library/ # current module . diff --git a/packages/aws-library/requirements/dev.txt b/packages/aws-library/requirements/dev.txt index b125999a475..b183acca92c 100644 --- a/packages/aws-library/requirements/dev.txt +++ b/packages/aws-library/requirements/dev.txt @@ -14,6 +14,7 @@ # installs this repo's packages --editable ../pytest-simcore/ --editable ../service-library/ +--editable ../settings-library/ # current module --editable . From 38bea1744e6594501460791800925a0df634b7bf Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:30:14 +0100 Subject: [PATCH 14/78] bring in all --- .../aws-library/src/aws_library/ec2/client.py | 244 ++++++++++++++++++ .../aws-library/src/aws_library/ec2/errors.py | 10 + .../aws-library/src/aws_library/ec2/models.py | 8 +- .../aws-library/src/aws_library/ec2/utils.py | 10 + .../src/settings_library/ec2.py | 67 +++++ 5 files changed, 336 insertions(+), 3 deletions(-) create mode 100644 packages/aws-library/src/aws_library/ec2/utils.py create mode 100644 packages/settings-library/src/settings_library/ec2.py diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py index e69de29bb2d..1d2af87f045 100644 --- a/packages/aws-library/src/aws_library/ec2/client.py +++ b/packages/aws-library/src/aws_library/ec2/client.py @@ -0,0 +1,244 @@ +import contextlib +import logging +from dataclasses import dataclass +from typing import cast + +import aioboto3 +import botocore.exceptions +from aiobotocore.session import ClientCreatorContext +from aiocache import cached +from pydantic import ByteSize +from servicelib.logging_utils import log_context +from settings_library.ec2 import EC2InstancesSettings, EC2Settings +from types_aiobotocore_ec2 import EC2Client +from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType +from types_aiobotocore_ec2.type_defs import FilterTypeDef + +from .errors import EC2InstanceNotFoundError, EC2TooManyInstancesError +from .models import EC2InstanceData, EC2InstanceType, EC2Tags, Resources +from .utils import compose_user_data + +_logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class AutoscalingEC2: + client: EC2Client + session: aioboto3.Session + exit_stack: contextlib.AsyncExitStack + + @classmethod + async def create(cls, settings: EC2Settings) -> "AutoscalingEC2": + session = aioboto3.Session() + session_client = session.client( + "ec2", + endpoint_url=settings.EC2_ENDPOINT, + aws_access_key_id=settings.EC2_ACCESS_KEY_ID, + aws_secret_access_key=settings.EC2_SECRET_ACCESS_KEY, + region_name=settings.EC2_REGION_NAME, + ) + assert isinstance(session_client, ClientCreatorContext) # nosec + exit_stack = contextlib.AsyncExitStack() + ec2_client = cast( + EC2Client, await exit_stack.enter_async_context(session_client) + ) + return cls(ec2_client, session, exit_stack) + + async def close(self) -> None: + await self.exit_stack.aclose() + + async def ping(self) -> bool: + try: + await self.client.describe_account_attributes() + return True + except Exception: # pylint: disable=broad-except + return False + + @cached(noself=True) + async def get_ec2_instance_capabilities( + self, + instance_type_names: set[InstanceTypeType], + ) -> list[EC2InstanceType]: + """instance_type_names must be a set of unique values""" + instance_types = await self.client.describe_instance_types( + InstanceTypes=list(instance_type_names) + ) + list_instances: list[EC2InstanceType] = [] + for instance in instance_types.get("InstanceTypes", []): + with contextlib.suppress(KeyError): + list_instances.append( + EC2InstanceType( + name=instance["InstanceType"], + cpus=instance["VCpuInfo"]["DefaultVCpus"], + ram=ByteSize( + int(instance["MemoryInfo"]["SizeInMiB"]) * 1024 * 1024 + ), + ) + ) + return list_instances + + async def start_aws_instance( + self, + instance_settings: EC2InstancesSettings, + instance_type: EC2InstanceType, + tags: dict[str, str], + startup_script: str, + number_of_instances: int, + ) -> list[EC2InstanceData]: + with log_context( + _logger, + logging.INFO, + msg=f"launching {number_of_instances} AWS instance(s) {instance_type.name} with {tags=}", + ): + # first check the max amount is not already reached + current_instances = await self.get_instances( + key_names=[instance_settings.EC2_INSTANCES_KEY_NAME], tags=tags + ) + if ( + len(current_instances) + number_of_instances + > instance_settings.EC2_INSTANCES_MAX_INSTANCES + ): + raise EC2TooManyInstancesError( + num_instances=instance_settings.EC2_INSTANCES_MAX_INSTANCES + ) + + instances = await self.client.run_instances( + ImageId=instance_settings.EC2_INSTANCES_AMI_ID, + MinCount=number_of_instances, + MaxCount=number_of_instances, + InstanceType=instance_type.name, + InstanceInitiatedShutdownBehavior="terminate", + KeyName=instance_settings.EC2_INSTANCES_KEY_NAME, + SubnetId=instance_settings.EC2_INSTANCES_SUBNET_ID, + TagSpecifications=[ + { + "ResourceType": "instance", + "Tags": [ + {"Key": tag_key, "Value": tag_value} + for tag_key, tag_value in tags.items() + ], + } + ], + UserData=compose_user_data(startup_script), + SecurityGroupIds=instance_settings.EC2_INSTANCES_SECURITY_GROUP_IDS, + ) + instance_ids = [i["InstanceId"] for i in instances["Instances"]] + _logger.info( + "New instances launched: %s, waiting for them to start now...", + instance_ids, + ) + + # wait for the instance to be in a pending state + # NOTE: reference to EC2 states https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html + waiter = self.client.get_waiter("instance_exists") + await waiter.wait(InstanceIds=instance_ids) + _logger.info("instances %s exists now.", instance_ids) + + # get the private IPs + instances = await self.client.describe_instances(InstanceIds=instance_ids) + instance_datas = [ + EC2InstanceData( + launch_time=instance["LaunchTime"], + id=instance["InstanceId"], + aws_private_dns=instance["PrivateDnsName"], + type=instance["InstanceType"], + state=instance["State"]["Name"], + resources=Resources(cpus=instance_type.cpus, ram=instance_type.ram), + ) + for instance in instances["Reservations"][0]["Instances"] + ] + _logger.info( + "%s is available, happy computing!!", + f"{instance_datas=}", + ) + return instance_datas + + async def get_instances( + self, + *, + key_names: list[str], + tags: dict[str, str], + state_names: list[InstanceStateNameType] | None = None, + ) -> list[EC2InstanceData]: + # NOTE: be careful: Name=instance-state-name,Values=["pending", "running"] means pending OR running + # NOTE2: AND is done by repeating Name=instance-state-name,Values=pending Name=instance-state-name,Values=running + if state_names is None: + state_names = ["pending", "running"] + + filters: list[FilterTypeDef] = [ + { + "Name": "key-name", + "Values": key_names, + }, + {"Name": "instance-state-name", "Values": state_names}, + ] + filters.extend( + [{"Name": f"tag:{key}", "Values": [value]} for key, value in tags.items()] + ) + + instances = await self.client.describe_instances(Filters=filters) + all_instances = [] + for reservation in instances["Reservations"]: + assert "Instances" in reservation # nosec + for instance in reservation["Instances"]: + assert "LaunchTime" in instance # nosec + assert "InstanceId" in instance # nosec + assert "PrivateDnsName" in instance # nosec + assert "InstanceType" in instance # nosec + assert "State" in instance # nosec + assert "Name" in instance["State"] # nosec + ec2_instance_types = await self.get_ec2_instance_capabilities( + {instance["InstanceType"]} + ) + assert len(ec2_instance_types) == 1 # nosec + all_instances.append( + EC2InstanceData( + launch_time=instance["LaunchTime"], + id=instance["InstanceId"], + aws_private_dns=instance["PrivateDnsName"], + type=instance["InstanceType"], + state=instance["State"]["Name"], + resources=Resources( + cpus=ec2_instance_types[0].cpus, + ram=ec2_instance_types[0].ram, + ), + ) + ) + _logger.debug( + "received: %s instances with %s", f"{len(all_instances)}", f"{state_names=}" + ) + return all_instances + + async def terminate_instances(self, instance_datas: list[EC2InstanceData]) -> None: + try: + with log_context( + _logger, + logging.INFO, + msg=f"terminating instances {[i.id for i in instance_datas]}", + ): + await self.client.terminate_instances( + InstanceIds=[i.id for i in instance_datas] + ) + except botocore.exceptions.ClientError as exc: + if ( + exc.response.get("Error", {}).get("Code", "") + == "InvalidInstanceID.NotFound" + ): + raise EC2InstanceNotFoundError from exc + raise # pragma: no cover + + async def set_instances_tags( + self, instances: list[EC2InstanceData], *, tags: EC2Tags + ) -> None: + with log_context( + _logger, + logging.DEBUG, + msg=f"setting {tags=} on instances '[{[i.id for i in instances]}]'", + ): + await self.client.create_tags( + Resources=[i.id for i in instances], + Tags=[ + {"Key": tag_key, "Value": tag_value} + for tag_key, tag_value in tags.items() + ], + ) diff --git a/packages/aws-library/src/aws_library/ec2/errors.py b/packages/aws-library/src/aws_library/ec2/errors.py index c6879567d77..4de33f15020 100644 --- a/packages/aws-library/src/aws_library/ec2/errors.py +++ b/packages/aws-library/src/aws_library/ec2/errors.py @@ -3,3 +3,13 @@ class EC2RuntimeError(PydanticErrorMixin, RuntimeError): msg_template: str = "EC2 client unexpected error" + + +class EC2InstanceNotFoundError(EC2RuntimeError): + msg_template: str = "EC2 instance was not found" + + +class EC2TooManyInstancesError(EC2RuntimeError): + msg_template: str = ( + "The maximum amount of instances {num_instances} is already reached!" + ) diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py index d166bb28603..b48a25cbb0d 100644 --- a/packages/aws-library/src/aws_library/ec2/models.py +++ b/packages/aws-library/src/aws_library/ec2/models.py @@ -1,5 +1,6 @@ import datetime from dataclasses import dataclass +from typing import TypeAlias from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType @@ -47,14 +48,15 @@ class EC2InstanceType: ram: ByteSize -InstancePrivateDNSName = str +InstancePrivateDNSName: TypeAlias = str +EC2Tags: TypeAlias = dict[str, str] @dataclass(frozen=True) class EC2InstanceData: launch_time: datetime.datetime - id: str + id: str # noqa: A003 aws_private_dns: InstancePrivateDNSName - type: InstanceTypeType + type: InstanceTypeType # noqa: A003 state: InstanceStateNameType resources: Resources diff --git a/packages/aws-library/src/aws_library/ec2/utils.py b/packages/aws-library/src/aws_library/ec2/utils.py new file mode 100644 index 00000000000..43e27fe1b6d --- /dev/null +++ b/packages/aws-library/src/aws_library/ec2/utils.py @@ -0,0 +1,10 @@ +from textwrap import dedent + + +def compose_user_data(docker_join_bash_command: str) -> str: + return dedent( + f"""\ +#!/bin/bash +{docker_join_bash_command} +""" + ) diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py new file mode 100644 index 00000000000..6ff2d08e7c6 --- /dev/null +++ b/packages/settings-library/src/settings_library/ec2.py @@ -0,0 +1,67 @@ +import datetime + +from pydantic import Field + +from .base import BaseCustomSettings + + +class EC2Settings(BaseCustomSettings): + EC2_ACCESS_KEY_ID: str + EC2_ENDPOINT: str | None = Field( + default=None, description="do not define if using standard AWS" + ) + EC2_REGION_NAME: str = "us-east-1" + EC2_SECRET_ACCESS_KEY: str + + +class EC2InstancesSettings(BaseCustomSettings): + EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( + ..., + min_items=1, + unique_items=True, + description="Defines which EC2 instances are considered as candidates for new EC2 instance", + ) + EC2_INSTANCES_AMI_ID: str = Field( + ..., + min_length=1, + description="Defines the AMI (Amazon Machine Image) ID used to start a new EC2 instance", + ) + EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: list[str] = Field( + default_factory=list, + description="script(s) to run on EC2 instance startup (be careful!), each entry is run one after the other using '&&' operator", + ) + EC2_INSTANCES_KEY_NAME: str = Field( + ..., + min_length=1, + description="SSH key filename (without ext) to access the instance through SSH" + " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html)," + "this is required to start a new EC2 instance", + ) + EC2_INSTANCES_MAX_INSTANCES: int = Field( + default=10, + description="Defines the maximum number of instances the autoscaling app may create", + ) + EC2_INSTANCES_NAME_PREFIX: str = Field( + default="autoscaling", + min_length=1, + description="prefix used to name the EC2 instances created by this instance of autoscaling", + ) + EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( + ..., + min_items=1, + description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic" + " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), " + " this is required to start a new EC2 instance", + ) + EC2_INSTANCES_SUBNET_ID: str = Field( + ..., + min_length=1, + description="A subnet is a range of IP addresses in your VPC " + " (https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html), " + "this is required to start a new EC2 instance", + ) + EC2_INSTANCES_TIME_BEFORE_TERMINATION: datetime.timedelta = Field( + default=datetime.timedelta(minutes=1), + description="Time after which an EC2 instance may be terminated (0<=T<=59 minutes, is automatically capped)" + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", + ) From 7cdc987ed1c000317b21103a0207148035cf2872 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:32:36 +0100 Subject: [PATCH 15/78] rename --- packages/aws-library/src/aws_library/ec2/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py index 1d2af87f045..ce07dec423f 100644 --- a/packages/aws-library/src/aws_library/ec2/client.py +++ b/packages/aws-library/src/aws_library/ec2/client.py @@ -22,13 +22,13 @@ @dataclass(frozen=True) -class AutoscalingEC2: +class SimcoreEC2API: client: EC2Client session: aioboto3.Session exit_stack: contextlib.AsyncExitStack @classmethod - async def create(cls, settings: EC2Settings) -> "AutoscalingEC2": + async def create(cls, settings: EC2Settings) -> "SimcoreEC2API": session = aioboto3.Session() session_client = session.client( "ec2", From db83065914c8822fe2f54a602199ee19aaf06609 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:42:22 +0100 Subject: [PATCH 16/78] testing --- packages/aws-library/tests/test_ec2_models.py | 124 ++++++++++++++++++ packages/aws-library/tests/test_ec2_utils.py | 7 + 2 files changed, 131 insertions(+) create mode 100644 packages/aws-library/tests/test_ec2_models.py create mode 100644 packages/aws-library/tests/test_ec2_utils.py diff --git a/packages/aws-library/tests/test_ec2_models.py b/packages/aws-library/tests/test_ec2_models.py new file mode 100644 index 00000000000..aac4a1d7863 --- /dev/null +++ b/packages/aws-library/tests/test_ec2_models.py @@ -0,0 +1,124 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import pytest +from aws_library.ec2.models import Resources +from pydantic import ByteSize + + +@pytest.mark.parametrize( + "a,b,a_greater_or_equal_than_b", + [ + ( + Resources(cpus=0.2, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.1, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.1, ram=ByteSize(1)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.05, ram=ByteSize(1)), + Resources(cpus=0.1, ram=ByteSize(0)), + False, + ), + ( + Resources(cpus=0.1, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(1)), + False, + ), + ], +) +def test_resources_ge_operator( + a: Resources, b: Resources, a_greater_or_equal_than_b: bool +): + assert (a >= b) is a_greater_or_equal_than_b + + +@pytest.mark.parametrize( + "a,b,a_greater_than_b", + [ + ( + Resources(cpus=0.2, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.1, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(0)), + False, + ), + ( + Resources(cpus=0.1, ram=ByteSize(1)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.05, ram=ByteSize(1)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.1, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(1)), + False, + ), + ], +) +def test_resources_gt_operator(a: Resources, b: Resources, a_greater_than_b: bool): + assert (a > b) is a_greater_than_b + + +@pytest.mark.parametrize( + "a,b,result", + [ + ( + Resources(cpus=0, ram=ByteSize(0)), + Resources(cpus=1, ram=ByteSize(34)), + Resources(cpus=1, ram=ByteSize(34)), + ), + ( + Resources(cpus=0.1, ram=ByteSize(-1)), + Resources(cpus=1, ram=ByteSize(34)), + Resources(cpus=1.1, ram=ByteSize(33)), + ), + ], +) +def test_resources_add(a: Resources, b: Resources, result: Resources): + assert a + b == result + a += b + assert a == result + + +def test_resources_create_as_empty(): + assert Resources.create_as_empty() == Resources(cpus=0, ram=ByteSize(0)) + + +@pytest.mark.parametrize( + "a,b,result", + [ + ( + Resources(cpus=0, ram=ByteSize(0)), + Resources(cpus=1, ram=ByteSize(34)), + Resources.construct(cpus=-1, ram=ByteSize(-34)), + ), + ( + Resources(cpus=0.1, ram=ByteSize(-1)), + Resources(cpus=1, ram=ByteSize(34)), + Resources.construct(cpus=-0.9, ram=ByteSize(-35)), + ), + ], +) +def test_resources_sub(a: Resources, b: Resources, result: Resources): + assert a - b == result + a -= b + assert a == result diff --git a/packages/aws-library/tests/test_ec2_utils.py b/packages/aws-library/tests/test_ec2_utils.py new file mode 100644 index 00000000000..ce03ef066b9 --- /dev/null +++ b/packages/aws-library/tests/test_ec2_utils.py @@ -0,0 +1,7 @@ +from aws_library.ec2.utils import compose_user_data +from faker import Faker + + +def test_compose_user_data(faker: Faker): + assert compose_user_data(faker.pystr()).startswith("#!/bin/bash\n") + assert compose_user_data(faker.pystr()).endswith("\n") From 85c768c7bf07df15dc2c0ba1ef60c0024ae598d1 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:52:34 +0100 Subject: [PATCH 17/78] creating tests --- packages/aws-library/tests/test_ec2_client.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/aws-library/tests/test_ec2_client.py diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py new file mode 100644 index 00000000000..e69de29bb2d From 00d6704676f7eecf6a9b258c65c05336f01198ef Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sat, 11 Nov 2023 22:29:42 +0100 Subject: [PATCH 18/78] add testing dependencies --- packages/aws-library/requirements/_base.txt | 2 +- packages/aws-library/requirements/_test.in | 2 + packages/aws-library/requirements/_test.txt | 206 ++++++++++++++++++- packages/aws-library/requirements/_tools.txt | 1 + 4 files changed, 209 insertions(+), 2 deletions(-) diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index f54fc3ff41b..59679efba0b 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -31,7 +31,7 @@ botocore==1.31.64 # aiobotocore # boto3 # s3transfer -botocore-stubs==1.31.83 +botocore-stubs==1.31.84 # via types-aiobotocore charset-normalizer==3.3.2 # via aiohttp diff --git a/packages/aws-library/requirements/_test.in b/packages/aws-library/requirements/_test.in index 83367d97185..75b32719060 100644 --- a/packages/aws-library/requirements/_test.in +++ b/packages/aws-library/requirements/_test.in @@ -11,6 +11,7 @@ # testing coverage faker +moto[server] pint pytest pytest-aiohttp # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76 @@ -20,4 +21,5 @@ pytest-instafail pytest-mock pytest-runner pytest-sugar +python-dotenv pyyaml diff --git a/packages/aws-library/requirements/_test.txt b/packages/aws-library/requirements/_test.txt index 7284cc793c2..d7be7112cd6 100644 --- a/packages/aws-library/requirements/_test.txt +++ b/packages/aws-library/requirements/_test.txt @@ -21,46 +21,175 @@ attrs==23.1.0 # via # -c requirements/_base.txt # aiohttp + # jschema-to-python + # jsonschema + # referencing + # sarif-om +aws-sam-translator==1.79.0 + # via cfn-lint +aws-xray-sdk==2.12.1 + # via moto +blinker==1.7.0 + # via flask +boto3==1.28.64 + # via + # -c requirements/_base.txt + # aws-sam-translator + # moto +botocore==1.31.64 + # via + # -c requirements/_base.txt + # aws-xray-sdk + # boto3 + # moto + # s3transfer +certifi==2023.7.22 + # via + # -c requirements/../../../requirements/constraints.txt + # requests +cffi==1.16.0 + # via cryptography +cfn-lint==0.83.1 + # via moto charset-normalizer==3.3.2 # via # -c requirements/_base.txt # aiohttp + # requests +click==8.1.7 + # via flask coverage==7.3.2 # via # -r requirements/_test.in # pytest-cov +cryptography==41.0.5 + # via + # -c requirements/../../../requirements/constraints.txt + # moto + # python-jose + # sshpubkeys +docker==6.1.3 + # via moto +ecdsa==0.18.0 + # via + # moto + # python-jose + # sshpubkeys exceptiongroup==1.1.3 # via pytest -faker==19.13.0 +faker==20.0.0 # via -r requirements/_test.in +flask==3.0.0 + # via + # flask-cors + # moto +flask-cors==4.0.0 + # via moto frozenlist==1.4.0 # via # -c requirements/_base.txt # aiohttp # aiosignal +graphql-core==3.2.3 + # via moto icdiff==2.0.7 # via pytest-icdiff idna==3.4 # via # -c requirements/_base.txt + # requests # yarl iniconfig==2.0.0 # via pytest +itsdangerous==2.1.2 + # via flask +jinja2==3.1.2 + # via + # -c requirements/../../../requirements/constraints.txt + # flask + # moto +jmespath==1.0.1 + # via + # -c requirements/_base.txt + # boto3 + # botocore +jschema-to-python==1.2.3 + # via cfn-lint +jsondiff==2.0.0 + # via moto +jsonpatch==1.33 + # via cfn-lint +jsonpickle==3.0.2 + # via jschema-to-python +jsonpointer==2.4 + # via jsonpatch +jsonschema==4.19.2 + # via + # aws-sam-translator + # cfn-lint + # openapi-schema-validator + # openapi-spec-validator +jsonschema-path==0.3.1 + # via openapi-spec-validator +jsonschema-specifications==2023.7.1 + # via + # jsonschema + # openapi-schema-validator +junit-xml==1.9 + # via cfn-lint +lazy-object-proxy==1.9.0 + # via openapi-spec-validator +markupsafe==2.1.3 + # via + # jinja2 + # werkzeug +moto==4.2.8 + # via -r requirements/_test.in +mpmath==1.3.0 + # via sympy multidict==6.0.4 # via # -c requirements/_base.txt # aiohttp # yarl +networkx==3.2.1 + # via cfn-lint +openapi-schema-validator==0.6.2 + # via openapi-spec-validator +openapi-spec-validator==0.7.1 + # via moto packaging==23.2 # via + # docker # pytest # pytest-sugar +pathable==0.4.3 + # via jsonschema-path +pbr==6.0.0 + # via + # jschema-to-python + # sarif-om pint==0.22 # via -r requirements/_test.in pluggy==1.3.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff +py-partiql-parser==0.4.2 + # via moto +pyasn1==0.5.0 + # via + # python-jose + # rsa +pycparser==2.21 + # via cffi +pydantic==1.10.13 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # aws-sam-translator +pyparsing==3.1.1 + # via moto pytest==7.4.3 # via # -r requirements/_test.in @@ -90,15 +219,65 @@ pytest-sugar==0.9.7 python-dateutil==2.8.2 # via # -c requirements/_base.txt + # botocore # faker + # moto +python-dotenv==1.0.0 + # via -r requirements/_test.in +python-jose==3.3.0 + # via + # moto + # python-jose pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in + # cfn-lint + # jsonschema-path + # moto + # responses +referencing==0.30.2 + # via + # jsonschema + # jsonschema-path + # jsonschema-specifications +regex==2023.10.3 + # via cfn-lint +requests==2.31.0 + # via + # docker + # jsonschema-path + # moto + # responses +responses==0.24.0 + # via moto +rfc3339-validator==0.1.4 + # via openapi-schema-validator +rpds-py==0.12.0 + # via + # jsonschema + # referencing +rsa==4.9 + # via + # -c requirements/../../../requirements/constraints.txt + # python-jose +s3transfer==0.7.0 + # via + # -c requirements/_base.txt + # boto3 +sarif-om==1.0.4 + # via cfn-lint six==1.16.0 # via # -c requirements/_base.txt + # ecdsa + # junit-xml # python-dateutil + # rfc3339-validator +sshpubkeys==3.3.1 + # via moto +sympy==1.12 + # via cfn-lint termcolor==2.3.0 # via pytest-sugar tomli==2.0.1 @@ -108,8 +287,33 @@ tomli==2.0.1 typing-extensions==4.8.0 # via # -c requirements/_base.txt + # aws-sam-translator # pint + # pydantic +urllib3==2.0.7 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # botocore + # docker + # requests + # responses +websocket-client==1.6.4 + # via docker +werkzeug==3.0.1 + # via + # flask + # moto +wrapt==1.16.0 + # via + # -c requirements/_base.txt + # aws-xray-sdk +xmltodict==0.13.0 + # via moto yarl==1.9.2 # via # -c requirements/_base.txt # aiohttp + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/aws-library/requirements/_tools.txt b/packages/aws-library/requirements/_tools.txt index f1e127151e7..543d0ade5fd 100644 --- a/packages/aws-library/requirements/_tools.txt +++ b/packages/aws-library/requirements/_tools.txt @@ -16,6 +16,7 @@ cfgv==3.4.0 # via pre-commit click==8.1.7 # via + # -c requirements/_test.txt # black # pip-tools dill==0.3.7 From 0e07f485ec6241924c71e83ac1459b7aa16dfb05 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sat, 11 Nov 2023 22:31:41 +0100 Subject: [PATCH 19/78] test runs on client --- packages/aws-library/tests/conftest.py | 1 + packages/aws-library/tests/test_ec2_client.py | 139 ++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/packages/aws-library/tests/conftest.py b/packages/aws-library/tests/conftest.py index d004b25966f..46bcb419c36 100644 --- a/packages/aws-library/tests/conftest.py +++ b/packages/aws-library/tests/conftest.py @@ -7,6 +7,7 @@ import pytest pytest_plugins = [ + "pytest_simcore.environment_configs", "pytest_simcore.repository_paths", "pytest_simcore.pydantic_models", "pytest_simcore.pytest_global_environs", diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index e69de29bb2d..d5ea20ec5db 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -0,0 +1,139 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + + +import json +from collections.abc import AsyncIterator, Iterator + +import botocore.exceptions +import pytest +import requests +from aiohttp.test_utils import unused_port +from aws_library.ec2.client import SimcoreEC2API +from faker import Faker +from moto.server import ThreadedMotoServer +from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip +from settings_library.ec2 import EC2Settings +from types_aiobotocore_ec2 import EC2Client +from types_aiobotocore_ec2.literals import InstanceTypeType + + +@pytest.fixture(scope="session") +def ec2_instances() -> list[InstanceTypeType]: + # these are some examples + return ["t2.nano", "m5.12xlarge"] + + +@pytest.fixture +def app_environment( + mock_env_devel_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + faker: Faker, + ec2_instances: list[InstanceTypeType], +) -> EnvVarsDict: + # SEE https://faker.readthedocs.io/en/master/providers/faker.providers.internet.html?highlight=internet#faker-providers-internet + envs = setenvs_from_dict( + monkeypatch, + { + "EC2_ACCESS_KEY_ID": faker.pystr(), + "EC2_SECRET_ACCESS_KEY": faker.pystr(), + "EC2_INSTANCES_KEY_NAME": faker.pystr(), + "EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( + faker.pylist(allowed_types=(str,)) + ), + "EC2_INSTANCES_SUBNET_ID": faker.pystr(), + "EC2_INSTANCES_AMI_ID": faker.pystr(), + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps(ec2_instances), + }, + ) + return mock_env_devel_environment | envs + + +@pytest.fixture +def ec2_settings( + app_environment: EnvVarsDict, +) -> EC2Settings: + return EC2Settings.create_from_envs() + + +@pytest.fixture +async def simcore_ec2_api(ec2_settings: EC2Settings) -> AsyncIterator[SimcoreEC2API]: + ec2 = await SimcoreEC2API.create(settings=ec2_settings) + assert ec2 + assert ec2.client + assert ec2.exit_stack + assert ec2.session + yield ec2 + await ec2.close() + + +@pytest.fixture +def ec2_client(simcore_ec2_api: SimcoreEC2API) -> EC2Client: + return simcore_ec2_api.client + + +@pytest.fixture(scope="module") +def mocked_aws_server() -> Iterator[ThreadedMotoServer]: + """creates a moto-server that emulates AWS services in place + NOTE: Never use a bucket with underscores it fails!! + """ + server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) + # pylint: disable=protected-access + print( + f"--> started mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 + ) + print( + f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" # noqa: SLF001 + ) + server.start() + yield server + server.stop() + print( + f"<-- stopped mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 + ) + + +@pytest.fixture +def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[None]: + # NOTE: reset_aws_server_state [http://docs.getmoto.org/en/latest/docs/server_mode.html#reset-api] + yield + # pylint: disable=protected-access + requests.post( + f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 + timeout=10, + ) + + +@pytest.fixture +def mocked_aws_server_envs( + app_environment: EnvVarsDict, + mocked_aws_server: ThreadedMotoServer, + reset_aws_server_state: None, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + changed_envs: EnvVarsDict = { + "EC2_ENDPOINT": f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 + "EC2_ACCESS_KEY_ID": "xxx", + "EC2_SECRET_ACCESS_KEY": "xxx", + } + return app_environment | setenvs_from_dict(monkeypatch, changed_envs) + + +async def test_ec2_client_lifespan(simcore_ec2_api: SimcoreEC2API): + ... + + +async def test_ec2_client_raises_when_no_connection_available(ec2_client: EC2Client): + with pytest.raises( + botocore.exceptions.ClientError, match=r".+ AWS was not able to validate .+" + ): + await ec2_client.describe_account_attributes(DryRun=True) + + +async def test_ec2_client_with_mock_server( + mocked_aws_server_envs: None, ec2_client: EC2Client +): + # passes without exception + await ec2_client.describe_account_attributes(DryRun=True) From 9114070e2de212dc09c839106320042c6807fd49 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sat, 11 Nov 2023 22:32:14 +0100 Subject: [PATCH 20/78] moved get_local_host method to dockerless utils --- .../src/pytest_simcore/aioresponses_mocker.py | 2 +- .../src/pytest_simcore/aws_services.py | 2 +- .../src/pytest_simcore/docker_registry.py | 2 +- .../src/pytest_simcore/docker_swarm.py | 2 +- .../src/pytest_simcore/helpers/utils_docker.py | 14 -------------- .../src/pytest_simcore/helpers/utils_host.py | 14 ++++++++++++++ .../src/pytest_simcore/httpbin_service.py | 2 +- .../src/pytest_simcore/minio_service.py | 3 +-- .../src/pytest_simcore/postgres_service.py | 2 +- .../src/pytest_simcore/rabbit_service.py | 2 +- .../src/pytest_simcore/redis_service.py | 2 +- .../src/pytest_simcore/simcore_services.py | 2 +- .../src/pytest_simcore/simcore_storage_service.py | 2 +- services/api-server/tests/unit/conftest.py | 3 +-- services/autoscaling/tests/unit/conftest.py | 2 +- services/clusters-keeper/tests/unit/conftest.py | 2 +- .../integration/02/test_dynamic_services_routes.py | 2 +- .../test_dynamic_sidecar_nodeports_integration.py | 2 +- ...est_mixed_dynamic_sidecar_and_legacy_project.py | 2 +- services/director-v2/tests/integration/02/utils.py | 2 +- .../tests/integration/conftest.py | 6 +++--- .../tests/integration/test_clusters.py | 2 +- .../tests/integration/test_dask_sidecar.py | 2 +- .../tests/integration/test_gateway.py | 2 +- .../tests/system/test_deploy.py | 2 +- services/storage/tests/conftest.py | 2 +- 26 files changed, 40 insertions(+), 42 deletions(-) create mode 100644 packages/pytest-simcore/src/pytest_simcore/helpers/utils_host.py diff --git a/packages/pytest-simcore/src/pytest_simcore/aioresponses_mocker.py b/packages/pytest-simcore/src/pytest_simcore/aioresponses_mocker.py index 11beaa7e325..c91edc673cb 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aioresponses_mocker.py +++ b/packages/pytest-simcore/src/pytest_simcore/aioresponses_mocker.py @@ -1,7 +1,7 @@ import pytest from aioresponses import aioresponses as AioResponsesMock -from .helpers.utils_docker import get_localhost_ip +from .helpers.utils_host import get_localhost_ip # WARNING: any request done through the client will go through aioresponses. It is # unfortunate but that means any valid request (like calling the test server) prefix must be set as passthrough. diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_services.py b/packages/pytest-simcore/src/pytest_simcore/aws_services.py index b96bbfd6c98..084f331aa4e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aws_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/aws_services.py @@ -10,7 +10,7 @@ from aiobotocore.session import get_session from aiohttp.test_utils import unused_port from moto.server import ThreadedMotoServer -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip @pytest.fixture(scope="module") diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py index e063e23adb2..d1627698946 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -15,7 +15,7 @@ import tenacity from pytest import FixtureRequest -from .helpers.utils_docker import get_localhost_ip +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py index 37b8c0379d8..e7a7fbe7c54 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py @@ -24,7 +24,7 @@ from .helpers.constants import HEADER_STR, MINUTE from .helpers.typing_env import EnvVarsDict from .helpers.utils_dict import copy_from_dict -from .helpers.utils_docker import get_localhost_ip +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py index 3d5502e6bfb..29b81200a7e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py @@ -2,7 +2,6 @@ import logging import os import re -import socket import subprocess from enum import Enum from pathlib import Path @@ -46,19 +45,6 @@ class ContainerStatus(str, Enum): log = logging.getLogger(__name__) -def get_localhost_ip(default="127.0.0.1") -> str: - """Return the IP address for localhost""" - local_ip = default - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # doesn't even have to be reachable - s.connect(("10.255.255.255", 1)) - local_ip = s.getsockname()[0] - finally: - s.close() - return f"{local_ip}" - - @retry( wait=wait_fixed(2), stop=stop_after_attempt(10), diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_host.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_host.py new file mode 100644 index 00000000000..e259053694e --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_host.py @@ -0,0 +1,14 @@ +import socket + + +def get_localhost_ip(default="127.0.0.1") -> str: + """Return the IP address for localhost""" + local_ip = default + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + # doesn't even have to be reachable + s.connect(("10.255.255.255", 1)) + local_ip = s.getsockname()[0] + finally: + s.close() + return f"{local_ip}" diff --git a/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py b/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py index 901c2789519..62baba3623e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py @@ -21,7 +21,7 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from .helpers.utils_docker import get_localhost_ip +from .helpers.utils_host import get_localhost_ip @pytest.fixture(scope="session") diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index c24c3d639dc..cb1101eb04b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -16,7 +16,7 @@ from tenacity.stop import stop_after_attempt from tenacity.wait import wait_fixed -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_host import get_localhost_ip, get_service_published_port log = logging.getLogger(__name__) @@ -69,7 +69,6 @@ def minio_config( @pytest.fixture(scope="module") def minio_service(minio_config: dict[str, str]) -> Iterator[Minio]: - client = Minio(**minio_config["client"]) for attempt in Retrying( diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py index ec250c00876..d058e95b774 100644 --- a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py @@ -15,7 +15,7 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_host import get_localhost_ip, get_service_published_port from .helpers.utils_postgres import PostgresTestConfig, migrated_pg_tables_context _TEMPLATE_DB_TO_RESTORE = "template_simcore_db" diff --git a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py index 3e90d2a7bcd..771407c3531 100644 --- a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py @@ -16,7 +16,7 @@ from tenacity.wait import wait_fixed from .helpers.typing_env import EnvVarsDict -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_host import get_localhost_ip, get_service_published_port _logger = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/redis_service.py b/packages/pytest-simcore/src/pytest_simcore/redis_service.py index 20cabcfe006..4879348578d 100644 --- a/packages/pytest-simcore/src/pytest_simcore/redis_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/redis_service.py @@ -14,7 +14,7 @@ from tenacity.wait import wait_fixed from yarl import URL -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_host import get_localhost_ip, get_service_published_port log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 34081e39670..1928949e265 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -20,7 +20,7 @@ from .helpers.constants import MINUTE from .helpers.typing_env import EnvVarsDict -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_host import get_localhost_ip, get_service_published_port log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index 2e1c6f4dc86..f1dad21210a 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -15,7 +15,7 @@ from servicelib.minio_utils import MinioRetryPolicyUponInitialization from yarl import URL -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_host import get_localhost_ip, get_service_published_port @pytest.fixture(scope="module") diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index b1046bafb27..3aa9a1d6cf1 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -36,8 +36,8 @@ from pydantic import HttpUrl, parse_obj_as from pytest import MonkeyPatch # noqa: PT013 from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from pytest_simcore.simcore_webserver_projects_rest_api import GET_PROJECT from requests.auth import HTTPBasicAuth from respx import MockRouter @@ -269,7 +269,6 @@ def mocked_webserver_service_api_base( assert_all_called=False, assert_all_mocked=True, ) as respx_mock: - # healthcheck_readiness_probe, healthcheck_liveness_probe response_body = { "name": "webserver", diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index 2e32db769fd..8bc11071bc2 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -41,8 +41,8 @@ from moto.server import ThreadedMotoServer from pydantic import ByteSize, PositiveInt, parse_obj_as from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from settings_library.rabbit import RabbitSettings from simcore_service_autoscaling.core.application import create_app from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index 8fcb73ceff0..a6486b52bff 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -26,8 +26,8 @@ from models_library.wallets import WalletID from moto.server import ThreadedMotoServer from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.rabbitmq import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from simcore_service_clusters_keeper.core.application import create_app diff --git a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py index 8b52cbeae8f..32cbcbf5fc5 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py +++ b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py @@ -25,8 +25,8 @@ from models_library.users import UserID from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.common_headers import ( X_DYNAMIC_SIDECAR_REQUEST_DNS, X_DYNAMIC_SIDECAR_REQUEST_SCHEME, diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 66f94472efa..ea71f7fbb44 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -46,8 +46,8 @@ from models_library.projects_state import RunningState from models_library.users import UserID from pydantic import AnyHttpUrl, parse_obj_as -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.fastapi.long_running_tasks.client import ( Client, ProgressMessage, diff --git a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py index fae73a6e62e..ca528632c9d 100644 --- a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py +++ b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py @@ -20,8 +20,8 @@ from models_library.users import UserID from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from utils import ( diff --git a/services/director-v2/tests/integration/02/utils.py b/services/director-v2/tests/integration/02/utils.py index 209d98f3911..1dc81ec3c2f 100644 --- a/services/director-v2/tests/integration/02/utils.py +++ b/services/director-v2/tests/integration/02/utils.py @@ -21,7 +21,7 @@ ) from models_library.users import UserID from pydantic import PositiveInt, parse_obj_as -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.common_headers import ( X_DYNAMIC_SIDECAR_REQUEST_DNS, X_DYNAMIC_SIDECAR_REQUEST_SCHEME, diff --git a/services/osparc-gateway-server/tests/integration/conftest.py b/services/osparc-gateway-server/tests/integration/conftest.py index 1e1161670ad..7ac15707662 100644 --- a/services/osparc-gateway-server/tests/integration/conftest.py +++ b/services/osparc-gateway-server/tests/integration/conftest.py @@ -4,7 +4,7 @@ import asyncio import json -from typing import Any, AsyncIterator, Awaitable, Callable, Union +from typing import Any, AsyncIterator, Awaitable, Callable import aiodocker import dask_gateway @@ -19,7 +19,7 @@ OSPARC_SCHEDULER_API_PORT, OSPARC_SCHEDULER_DASHBOARD_PORT, ) -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from tenacity._asyncio import AsyncRetrying from tenacity.wait import wait_fixed @@ -56,7 +56,7 @@ def gateway_password(faker: Faker) -> str: return faker.password() -def _convert_to_dict(c: Union[traitlets.config.Config, dict]) -> dict[str, Any]: +def _convert_to_dict(c: traitlets.config.Config | dict) -> dict[str, Any]: converted_dict = {} for x, y in c.items(): if isinstance(y, (dict, traitlets.config.Config)): diff --git a/services/osparc-gateway-server/tests/integration/test_clusters.py b/services/osparc-gateway-server/tests/integration/test_clusters.py index 1ba132b9ef9..87f776bd5c3 100644 --- a/services/osparc-gateway-server/tests/integration/test_clusters.py +++ b/services/osparc-gateway-server/tests/integration/test_clusters.py @@ -11,7 +11,7 @@ from aiodocker import Docker from dask_gateway import Gateway from faker import Faker -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed diff --git a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py index 4b46e5e226e..fa7e3554870 100644 --- a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py +++ b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py @@ -7,7 +7,7 @@ import aiodocker import pytest from faker import Faker -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed diff --git a/services/osparc-gateway-server/tests/integration/test_gateway.py b/services/osparc-gateway-server/tests/integration/test_gateway.py index 205772a0cee..44ae68ea3e3 100644 --- a/services/osparc-gateway-server/tests/integration/test_gateway.py +++ b/services/osparc-gateway-server/tests/integration/test_gateway.py @@ -9,7 +9,7 @@ from dask_gateway_server.app import DaskGateway from faker import Faker from osparc_gateway_server.backend.osparc import OsparcBackend -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip @pytest.fixture( diff --git a/services/osparc-gateway-server/tests/system/test_deploy.py b/services/osparc-gateway-server/tests/system/test_deploy.py index 4dd4e114ec3..79cc1221884 100644 --- a/services/osparc-gateway-server/tests/system/test_deploy.py +++ b/services/osparc-gateway-server/tests/system/test_deploy.py @@ -12,7 +12,7 @@ import dask_gateway import pytest from faker import Faker -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 531d4f67927..8864901f427 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -42,7 +42,7 @@ from moto.server import ThreadedMotoServer from pydantic import ByteSize, parse_obj_as from pytest_simcore.helpers.utils_assert import assert_status -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from simcore_postgres_database.storage_models import file_meta_data, projects, users from simcore_service_storage.application import create from simcore_service_storage.dsm import get_dsm_provider From 349409df98be77d78297cfe75823eb3edf74304e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 14:12:07 +0100 Subject: [PATCH 21/78] correct imports + ruff --- .../src/pytest_simcore/minio_service.py | 6 ++++-- .../src/pytest_simcore/postgres_service.py | 3 ++- .../src/pytest_simcore/rabbit_service.py | 3 ++- .../src/pytest_simcore/redis_service.py | 17 ++++++++++------- .../src/pytest_simcore/simcore_services.py | 3 ++- .../pytest_simcore/simcore_storage_service.py | 7 ++++--- 6 files changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index cb1101eb04b..7ff284f9213 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -3,7 +3,8 @@ # pylint: disable=unused-variable import logging -from typing import Any, Iterator +from collections.abc import Iterator +from typing import Any import pytest from minio import Minio @@ -16,7 +17,8 @@ from tenacity.stop import stop_after_attempt from tenacity.wait import wait_fixed -from .helpers.utils_host import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py index d058e95b774..0c05f587652 100644 --- a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py @@ -15,7 +15,8 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from .helpers.utils_host import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip from .helpers.utils_postgres import PostgresTestConfig, migrated_pg_tables_context _TEMPLATE_DB_TO_RESTORE = "template_simcore_db" diff --git a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py index 771407c3531..d190d711ff6 100644 --- a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py @@ -16,7 +16,8 @@ from tenacity.wait import wait_fixed from .helpers.typing_env import EnvVarsDict -from .helpers.utils_host import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip _logger = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/redis_service.py b/packages/pytest-simcore/src/pytest_simcore/redis_service.py index 4879348578d..9651255198b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/redis_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/redis_service.py @@ -3,18 +3,20 @@ # pylint:disable=redefined-outer-name import logging -from typing import AsyncIterator +from collections.abc import AsyncIterator import pytest import tenacity from redis.asyncio import Redis, from_url +from settings_library.basic_types import PortInt from settings_library.redis import RedisDatabase, RedisSettings from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed from yarl import URL -from .helpers.utils_host import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) @@ -33,13 +35,13 @@ async def redis_settings( "simcore_redis", testing_environ_vars["REDIS_PORT"] ) # test runner is running on the host computer - settings = RedisSettings(REDIS_HOST=get_localhost_ip(), REDIS_PORT=int(port)) + settings = RedisSettings(REDIS_HOST=get_localhost_ip(), REDIS_PORT=PortInt(port)) await wait_till_redis_responsive(settings.build_redis_dsn(RedisDatabase.RESOURCES)) return settings -@pytest.fixture(scope="function") +@pytest.fixture() def redis_service( redis_settings: RedisSettings, monkeypatch: pytest.MonkeyPatch, @@ -53,7 +55,7 @@ def redis_service( return redis_settings -@pytest.fixture(scope="function") +@pytest.fixture() async def redis_client( redis_settings: RedisSettings, ) -> AsyncIterator[Redis]: @@ -70,7 +72,7 @@ async def redis_client( await client.close(close_connection_pool=True) -@pytest.fixture(scope="function") +@pytest.fixture() async def redis_locks_client( redis_settings: RedisSettings, ) -> AsyncIterator[Redis]: @@ -98,6 +100,7 @@ async def wait_till_redis_responsive(redis_url: URL | str) -> None: try: if not await client.ping(): - raise ConnectionError(f"{redis_url=} not available") + msg = f"{redis_url=} not available" + raise ConnectionError(msg) finally: await client.close(close_connection_pool=True) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 1928949e265..406e1ac269e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -20,7 +20,8 @@ from .helpers.constants import MINUTE from .helpers.typing_env import EnvVarsDict -from .helpers.utils_host import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index f1dad21210a..9b0032c51fd 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -2,8 +2,8 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name import os +from collections.abc import Callable, Iterable from copy import deepcopy -from typing import Callable, Iterable import aiohttp import pytest @@ -15,7 +15,8 @@ from servicelib.minio_utils import MinioRetryPolicyUponInitialization from yarl import URL -from .helpers.utils_host import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip @pytest.fixture(scope="module") @@ -38,7 +39,7 @@ def storage_endpoint(docker_stack: dict, testing_environ_vars: dict) -> Iterable os.environ = old_environ -@pytest.fixture(scope="function") +@pytest.fixture() async def storage_service( minio_service: Minio, storage_endpoint: URL, docker_stack: dict ) -> URL: From 8b99f81fde556716cfb2669b514839c89168f206 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 16:00:14 +0100 Subject: [PATCH 22/78] add moto server as pytest-simcore fixtures --- packages/pytest-simcore/setup.py | 1 + .../src/pytest_simcore/aws_server.py | 60 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 packages/pytest-simcore/src/pytest_simcore/aws_server.py diff --git a/packages/pytest-simcore/setup.py b/packages/pytest-simcore/setup.py index 61874d47ea8..b9ee764e695 100644 --- a/packages/pytest-simcore/setup.py +++ b/packages/pytest-simcore/setup.py @@ -25,6 +25,7 @@ "aiohttp", "aioredis", "docker", + "moto[server]", "python-socketio", "PyYAML", "sqlalchemy[postgresql_psycopg2binary]", diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_server.py b/packages/pytest-simcore/src/pytest_simcore/aws_server.py new file mode 100644 index 00000000000..b8198c0a778 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/aws_server.py @@ -0,0 +1,60 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import + +from collections.abc import Iterator + +import pytest +import requests +from aiohttp.test_utils import unused_port +from moto.server import ThreadedMotoServer + +from .helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from .helpers.utils_host import get_localhost_ip + + +@pytest.fixture(scope="module") +def mocked_aws_server() -> Iterator[ThreadedMotoServer]: + """creates a moto-server that emulates AWS services in place + NOTE: Never use a bucket with underscores it fails!! + """ + server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) + # pylint: disable=protected-access + print( + f"--> started mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 + ) + print( + f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" # noqa: SLF001 + ) + server.start() + yield server + server.stop() + print( + f"<-- stopped mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 + ) + + +@pytest.fixture +def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[None]: + # NOTE: reset_aws_server_state [http://docs.getmoto.org/en/latest/docs/server_mode.html#reset-api] + yield + # pylint: disable=protected-access + requests.post( + f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 + timeout=10, + ) + + +@pytest.fixture +def mocked_aws_server_envs( + client_environment: EnvVarsDict, + mocked_aws_server: ThreadedMotoServer, + reset_aws_server_state: None, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + changed_envs: EnvVarsDict = { + "EC2_ENDPOINT": f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 + "EC2_ACCESS_KEY_ID": "xxx", + "EC2_SECRET_ACCESS_KEY": "xxx", + } + return client_environment | setenvs_from_dict(monkeypatch, changed_envs) From 40f39d63d547b4d49b107f4dbcb3e998ce6174cc Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 17:13:39 +0100 Subject: [PATCH 23/78] fixed import --- .../src/pytest_simcore/docker_compose.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index db34d158938..424f6bb7da2 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -15,9 +15,10 @@ import shutil import subprocess import sys +from collections.abc import Iterator from copy import deepcopy from pathlib import Path -from typing import Any, Iterator +from typing import Any import pytest import yaml @@ -30,11 +31,8 @@ ) from .helpers.constants import HEADER_STR from .helpers.typing_env import EnvVarsDict -from .helpers.utils_docker import ( - get_localhost_ip, - run_docker_compose_config, - save_docker_infos, -) +from .helpers.utils_docker import run_docker_compose_config, save_docker_infos +from .helpers.utils_host import get_localhost_ip @pytest.fixture(scope="session") @@ -79,9 +77,9 @@ def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict: env_devel["API_SERVER_DEV_FEATURES_ENABLED"] = "1" - if not "DOCKER_REGISTRY" in os.environ: + if "DOCKER_REGISTRY" not in os.environ: env_devel["DOCKER_REGISTRY"] = "local" - if not "DOCKER_IMAGE_TAG" in os.environ: + if "DOCKER_IMAGE_TAG" not in os.environ: env_devel["DOCKER_IMAGE_TAG"] = "production" return {key: value for key, value in env_devel.items() if value is not None} @@ -264,8 +262,7 @@ def core_docker_compose_file( @pytest.fixture(scope="module") def ops_services_selection(request) -> list[str]: """Selection of services from the ops stack""" - ops_services = getattr(request.module, FIXTURE_CONFIG_OPS_SERVICES_SELECTION, []) - return ops_services + return getattr(request.module, FIXTURE_CONFIG_OPS_SERVICES_SELECTION, []) @pytest.fixture(scope="module") @@ -370,7 +367,7 @@ def _filter_services_and_dump( with docker_compose_path.open("wt") as fh: if "TRAVIS" in os.environ: # in travis we do not have access to file - print(f"{str(docker_compose_path):-^100}") + print(f"{docker_compose_path!s:-^100}") yaml.dump(content, sys.stdout, default_flow_style=False) print("-" * 100) else: From 339db72fd2ad7f4c8722eb9ee0cd0b920198a7a3 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 17:14:24 +0100 Subject: [PATCH 24/78] preparing for testing --- packages/aws-library/tests/conftest.py | 1 + packages/aws-library/tests/test_ec2_client.py | 388 ++++++++++++++---- .../src/pytest_simcore/aws_server.py | 144 ++++++- 3 files changed, 453 insertions(+), 80 deletions(-) diff --git a/packages/aws-library/tests/conftest.py b/packages/aws-library/tests/conftest.py index 46bcb419c36..aa9455bc600 100644 --- a/packages/aws-library/tests/conftest.py +++ b/packages/aws-library/tests/conftest.py @@ -7,6 +7,7 @@ import pytest pytest_plugins = [ + "pytest_simcore.aws_server", "pytest_simcore.environment_configs", "pytest_simcore.repository_paths", "pytest_simcore.pydantic_models", diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index d5ea20ec5db..83dd0965b36 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -3,35 +3,39 @@ # pylint:disable=redefined-outer-name +import datetime import json -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator +from typing import cast import botocore.exceptions import pytest -import requests -from aiohttp.test_utils import unused_port from aws_library.ec2.client import SimcoreEC2API +from aws_library.ec2.errors import EC2TooManyInstancesError +from aws_library.ec2.models import EC2InstanceType, EC2Tags from faker import Faker from moto.server import ThreadedMotoServer from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict -from pytest_simcore.helpers.utils_host import get_localhost_ip -from settings_library.ec2 import EC2Settings +from settings_library.ec2 import EC2InstancesSettings, EC2Settings from types_aiobotocore_ec2 import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType +def _ec2_allowed_types() -> list[InstanceTypeType]: + return ["t2.nano", "m5.12xlarge", "g4dn.4xlarge"] + + @pytest.fixture(scope="session") -def ec2_instances() -> list[InstanceTypeType]: - # these are some examples - return ["t2.nano", "m5.12xlarge"] +def ec2_allowed_instances() -> list[InstanceTypeType]: + return _ec2_allowed_types() @pytest.fixture -def app_environment( +def client_environment( mock_env_devel_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker, - ec2_instances: list[InstanceTypeType], + ec2_allowed_instances: list[InstanceTypeType], ) -> EnvVarsDict: # SEE https://faker.readthedocs.io/en/master/providers/faker.providers.internet.html?highlight=internet#faker-providers-internet envs = setenvs_from_dict( @@ -45,22 +49,39 @@ def app_environment( ), "EC2_INSTANCES_SUBNET_ID": faker.pystr(), "EC2_INSTANCES_AMI_ID": faker.pystr(), - "EC2_INSTANCES_ALLOWED_TYPES": json.dumps(ec2_instances), + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps(ec2_allowed_instances), }, ) return mock_env_devel_environment | envs @pytest.fixture -def ec2_settings( - app_environment: EnvVarsDict, -) -> EC2Settings: - return EC2Settings.create_from_envs() +def ec2_instances_settings( + aws_vpc_id: str, + aws_subnet_id: str, + aws_security_group_id: str, + aws_ami_id: str, + ec2_allowed_instances: list[InstanceTypeType], + faker: Faker, +) -> EC2InstancesSettings: + return EC2InstancesSettings( + EC2_INSTANCES_ALLOWED_TYPES=[f"{i}" for i in ec2_allowed_instances], + EC2_INSTANCES_AMI_ID=aws_ami_id, + EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS=faker.pylist(allowed_types=(str,)), + EC2_INSTANCES_KEY_NAME=faker.pystr(), + EC2_INSTANCES_MAX_INSTANCES=10, + EC2_INSTANCES_NAME_PREFIX=faker.pystr(), + EC2_INSTANCES_SECURITY_GROUP_IDS=[aws_security_group_id], + EC2_INSTANCES_SUBNET_ID=aws_subnet_id, + EC2_INSTANCES_TIME_BEFORE_TERMINATION=datetime.timedelta(seconds=10), + ) @pytest.fixture -async def simcore_ec2_api(ec2_settings: EC2Settings) -> AsyncIterator[SimcoreEC2API]: - ec2 = await SimcoreEC2API.create(settings=ec2_settings) +async def simcore_ec2_api( + mocked_ec2_server_settings: EC2Settings, +) -> AsyncIterator[SimcoreEC2API]: + ec2 = await SimcoreEC2API.create(settings=mocked_ec2_server_settings) assert ec2 assert ec2.client assert ec2.exit_stack @@ -69,71 +90,296 @@ async def simcore_ec2_api(ec2_settings: EC2Settings) -> AsyncIterator[SimcoreEC2 await ec2.close() -@pytest.fixture -def ec2_client(simcore_ec2_api: SimcoreEC2API) -> EC2Client: - return simcore_ec2_api.client - - -@pytest.fixture(scope="module") -def mocked_aws_server() -> Iterator[ThreadedMotoServer]: - """creates a moto-server that emulates AWS services in place - NOTE: Never use a bucket with underscores it fails!! - """ - server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) - # pylint: disable=protected-access - print( - f"--> started mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 +async def test_ec2_client_lifespan(simcore_ec2_api: SimcoreEC2API): + ... + + +async def test_aiobotocore_ec2_client_when_ec2_server_goes_up_and_down( + mocked_aws_server: ThreadedMotoServer, + ec2_client: EC2Client, +): + # passes without exception + await ec2_client.describe_account_attributes(DryRun=True) + mocked_aws_server.stop() + with pytest.raises(botocore.exceptions.EndpointConnectionError): + await ec2_client.describe_account_attributes(DryRun=True) + + # restart + mocked_aws_server.start() + # passes without exception + await ec2_client.describe_account_attributes(DryRun=True) + + +async def test_ping( + mocked_aws_server: ThreadedMotoServer, + simcore_ec2_api: SimcoreEC2API, +): + assert await simcore_ec2_api.ping() is True + mocked_aws_server.stop() + assert await simcore_ec2_api.ping() is False + mocked_aws_server.start() + assert await simcore_ec2_api.ping() is True + + +async def test_get_ec2_instance_capabilities( + simcore_ec2_api: SimcoreEC2API, + ec2_instances_settings: EC2InstancesSettings, +): + instance_types: list[ + EC2InstanceType + ] = await simcore_ec2_api.get_ec2_instance_capabilities( + cast( + set[InstanceTypeType], + set(ec2_instances_settings.EC2_INSTANCES_ALLOWED_TYPES), + ) ) - print( - f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" # noqa: SLF001 + assert instance_types + assert len(instance_types) == len( + ec2_instances_settings.EC2_INSTANCES_ALLOWED_TYPES ) - server.start() - yield server - server.stop() - print( - f"<-- stopped mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 + + # all the instance names are found and valid + assert all( + i.name in ec2_instances_settings.EC2_INSTANCES_ALLOWED_TYPES + for i in instance_types ) + for instance_type_name in ec2_instances_settings.EC2_INSTANCES_ALLOWED_TYPES: + assert any(i.name == instance_type_name for i in instance_types) -@pytest.fixture -def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[None]: - # NOTE: reset_aws_server_state [http://docs.getmoto.org/en/latest/docs/server_mode.html#reset-api] - yield - # pylint: disable=protected-access - requests.post( - f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 - timeout=10, - ) +@pytest.fixture(params=_ec2_allowed_types()) +async def fake_ec2_instance_type( + simcore_ec2_api: SimcoreEC2API, + request: pytest.FixtureRequest, +) -> EC2InstanceType: + instance_type_name: InstanceTypeType = request.param + instance_types: list[ + EC2InstanceType + ] = await simcore_ec2_api.get_ec2_instance_capabilities({instance_type_name}) + assert len(instance_types) == 1 + return instance_types[0] -@pytest.fixture -def mocked_aws_server_envs( - app_environment: EnvVarsDict, - mocked_aws_server: ThreadedMotoServer, - reset_aws_server_state: None, - monkeypatch: pytest.MonkeyPatch, -) -> EnvVarsDict: - changed_envs: EnvVarsDict = { - "EC2_ENDPOINT": f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - "EC2_ACCESS_KEY_ID": "xxx", - "EC2_SECRET_ACCESS_KEY": "xxx", - } - return app_environment | setenvs_from_dict(monkeypatch, changed_envs) +async def _assert_no_instances_in_ec2(ec2_client: EC2Client) -> None: + all_instances = await ec2_client.describe_instances() + assert not all_instances["Reservations"] -async def test_ec2_client_lifespan(simcore_ec2_api: SimcoreEC2API): - ... +async def _assert_instances_in_ec2( + ec2_client: EC2Client, + *, + expected_num_reservations: int, + expected_num_instances: int, + expected_instance_type: EC2InstanceType, + expected_tags: EC2Tags, +) -> None: + all_instances = await ec2_client.describe_instances() + assert len(all_instances["Reservations"]) == expected_num_reservations + for reservation in all_instances["Reservations"]: + assert "Instances" in reservation + assert len(reservation["Instances"]) == expected_num_instances + for instance in reservation["Instances"]: + assert "InstanceType" in instance + assert instance["InstanceType"] == expected_instance_type.name + assert "Tags" in instance + assert instance["Tags"] == [ + {"Key": key, "Value": value} for key, value in expected_tags.items() + ] -async def test_ec2_client_raises_when_no_connection_available(ec2_client: EC2Client): - with pytest.raises( - botocore.exceptions.ClientError, match=r".+ AWS was not able to validate .+" - ): - await ec2_client.describe_account_attributes(DryRun=True) + +async def test_start_aws_instance( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + faker: Faker, + fake_ec2_instance_type: EC2InstanceType, + ec2_instances_settings: EC2InstancesSettings, +): + await _assert_no_instances_in_ec2(ec2_client) + + tags = faker.pydict(allowed_types=(str,)) + startup_script = faker.pystr() + number_of_instances = 1 + + # let's create a first reservation and check that it is correctly created in EC2 + await simcore_ec2_api.start_aws_instance( + ec2_instances_settings, + fake_ec2_instance_type, + tags=tags, + startup_script=startup_script, + number_of_instances=number_of_instances, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=number_of_instances, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + ) + + # create a second reservation + await simcore_ec2_api.start_aws_instance( + ec2_instances_settings, + fake_ec2_instance_type, + tags=tags, + startup_script=startup_script, + number_of_instances=number_of_instances, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=2, + expected_num_instances=number_of_instances, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + ) -async def test_ec2_client_with_mock_server( - mocked_aws_server_envs: None, ec2_client: EC2Client +async def test_start_aws_instance_is_limited_in_number_of_instances( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + faker: Faker, + fake_ec2_instance_type: EC2InstanceType, + ec2_instances_settings: EC2InstancesSettings, ): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) + await _assert_no_instances_in_ec2(ec2_client) + + tags = faker.pydict(allowed_types=(str,)) + startup_script = faker.pystr() + + # create many instances in one go shall fail + with pytest.raises(EC2TooManyInstancesError): + await simcore_ec2_api.start_aws_instance( + ec2_instances_settings, + fake_ec2_instance_type, + tags=tags, + startup_script=startup_script, + number_of_instances=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES + 1, + ) + await _assert_no_instances_in_ec2(ec2_client) + + # create instances 1 by 1 + for _ in range(ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES): + await simcore_ec2_api.start_aws_instance( + ec2_instances_settings, + fake_ec2_instance_type, + tags=tags, + startup_script=startup_script, + number_of_instances=1, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES, + expected_num_instances=1, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + ) + + # now creating one more shall fail + with pytest.raises(EC2TooManyInstancesError): + await simcore_ec2_api.start_aws_instance( + ec2_instances_settings, + fake_ec2_instance_type, + tags=tags, + startup_script=startup_script, + number_of_instances=1, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES, + expected_num_instances=1, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + ) + + +# async def test_get_instances( +# mocked_ec2_server_envs: None, +# aws_vpc_id: str, +# aws_subnet_id: str, +# aws_security_group_id: str, +# aws_ami_id: str, +# ec2_client: EC2Client, +# autoscaling_ec2: AutoscalingEC2, +# app_settings: ApplicationSettings, +# faker: Faker, +# fake_ec2_instance_type: EC2InstanceType, +# ): +# assert app_settings.AUTOSCALING_EC2_INSTANCES +# # we have nothing running now in ec2 +# all_instances = await ec2_client.describe_instances() +# assert not all_instances["Reservations"] +# assert ( +# await autoscaling_ec2.get_instances(app_settings.AUTOSCALING_EC2_INSTANCES, {}) +# == [] +# ) + +# # create some instance +# tags = faker.pydict(allowed_types=(str,)) +# startup_script = faker.pystr() +# created_instances = await autoscaling_ec2.start_aws_instance( +# app_settings.AUTOSCALING_EC2_INSTANCES, +# fake_ec2_instance_type, +# tags=tags, +# startup_script=startup_script, +# number_of_instances=1, +# ) +# assert len(created_instances) == 1 + +# instance_received = await autoscaling_ec2.get_instances( +# app_settings.AUTOSCALING_EC2_INSTANCES, +# tags=tags, +# ) +# assert created_instances == instance_received + + +# async def test_terminate_instance( +# mocked_ec2_server_envs: None, +# aws_vpc_id: str, +# aws_subnet_id: str, +# aws_security_group_id: str, +# aws_ami_id: str, +# ec2_client: EC2Client, +# autoscaling_ec2: AutoscalingEC2, +# app_settings: ApplicationSettings, +# faker: Faker, +# fake_ec2_instance_type: EC2InstanceType, +# ): +# assert app_settings.AUTOSCALING_EC2_INSTANCES +# # we have nothing running now in ec2 +# all_instances = await ec2_client.describe_instances() +# assert not all_instances["Reservations"] +# # create some instance +# tags = faker.pydict(allowed_types=(str,)) +# startup_script = faker.pystr() +# created_instances = await autoscaling_ec2.start_aws_instance( +# app_settings.AUTOSCALING_EC2_INSTANCES, +# fake_ec2_instance_type, +# tags=tags, +# startup_script=startup_script, +# number_of_instances=1, +# ) +# assert len(created_instances) == 1 + +# # terminate the instance +# await autoscaling_ec2.terminate_instances(created_instances) +# # calling it several times is ok, the instance stays a while +# await autoscaling_ec2.terminate_instances(created_instances) + + +# async def test_terminate_instance_not_existing_raises( +# mocked_ec2_server_envs: None, +# aws_vpc_id: str, +# aws_subnet_id: str, +# aws_security_group_id: str, +# aws_ami_id: str, +# ec2_client: EC2Client, +# autoscaling_ec2: AutoscalingEC2, +# app_settings: ApplicationSettings, +# fake_ec2_instance_data: Callable[..., EC2InstanceData], +# ): +# assert app_settings.AUTOSCALING_EC2_INSTANCES +# # we have nothing running now in ec2 +# all_instances = await ec2_client.describe_instances() +# assert not all_instances["Reservations"] +# with pytest.raises(Ec2InstanceNotFoundError): +# await autoscaling_ec2.terminate_instances([fake_ec2_instance_data()]) diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_server.py b/packages/pytest-simcore/src/pytest_simcore/aws_server.py index b8198c0a778..667682b8288 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aws_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/aws_server.py @@ -2,12 +2,20 @@ # pylint: disable=unused-argument # pylint: disable=unused-import -from collections.abc import Iterator +import contextlib +import random +from collections.abc import AsyncIterator, Iterator +from typing import cast +import aioboto3 import pytest import requests +from aiobotocore.session import ClientCreatorContext from aiohttp.test_utils import unused_port +from faker import Faker from moto.server import ThreadedMotoServer +from settings_library.ec2 import EC2Settings +from types_aiobotocore_ec2.client import EC2Client from .helpers.utils_envs import EnvVarsDict, setenvs_from_dict from .helpers.utils_host import get_localhost_ip @@ -43,18 +51,136 @@ def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[No f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 timeout=10, ) + print( + f"<-- cleaned mock AWS server on {mocked_aws_server._ip_address}:{mocked_aws_server._port}" # noqa: SLF001 + ) @pytest.fixture -def mocked_aws_server_envs( - client_environment: EnvVarsDict, +def mocked_ec2_server_settings( mocked_aws_server: ThreadedMotoServer, reset_aws_server_state: None, +) -> EC2Settings: + return EC2Settings( + EC2_ACCESS_KEY_ID="xxx", + EC2_ENDPOINT=f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 + EC2_SECRET_ACCESS_KEY="xxx", # noqa: S106 + ) + + +@pytest.fixture +def mocked_ec2_server_envs( + mocked_ec2_server_settings: EC2Settings, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: - changed_envs: EnvVarsDict = { - "EC2_ENDPOINT": f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - "EC2_ACCESS_KEY_ID": "xxx", - "EC2_SECRET_ACCESS_KEY": "xxx", - } - return client_environment | setenvs_from_dict(monkeypatch, changed_envs) + changed_envs: EnvVarsDict = mocked_ec2_server_settings.dict() + return setenvs_from_dict(monkeypatch, changed_envs) + + +@pytest.fixture +async def ec2_client( + mocked_ec2_server_settings: EC2Settings, +) -> AsyncIterator[EC2Client]: + session = aioboto3.Session() + exit_stack = contextlib.AsyncExitStack() + session_client = session.client( + "ec2", + endpoint_url=mocked_ec2_server_settings.EC2_ENDPOINT, + aws_access_key_id=mocked_ec2_server_settings.EC2_ACCESS_KEY_ID, + aws_secret_access_key=mocked_ec2_server_settings.EC2_SECRET_ACCESS_KEY, + region_name=mocked_ec2_server_settings.EC2_REGION_NAME, + ) + assert isinstance(session_client, ClientCreatorContext) + ec2_client = cast(EC2Client, await exit_stack.enter_async_context(session_client)) + + yield ec2_client + + await exit_stack.aclose() + + +@pytest.fixture(scope="session") +def vpc_cidr_block() -> str: + return "10.0.0.0/16" + + +@pytest.fixture +async def aws_vpc_id( + ec2_client: EC2Client, + vpc_cidr_block: str, +) -> AsyncIterator[str]: + vpc = await ec2_client.create_vpc( + CidrBlock=vpc_cidr_block, + ) + vpc_id = vpc["Vpc"]["VpcId"] # type: ignore + print(f"--> Created Vpc in AWS with {vpc_id=}") + yield vpc_id + + await ec2_client.delete_vpc(VpcId=vpc_id) + print(f"<-- Deleted Vpc in AWS with {vpc_id=}") + + +@pytest.fixture(scope="session") +def subnet_cidr_block() -> str: + return "10.0.1.0/24" + + +@pytest.fixture +async def aws_subnet_id( + aws_vpc_id: str, + ec2_client: EC2Client, + subnet_cidr_block: str, +) -> AsyncIterator[str]: + subnet = await ec2_client.create_subnet( + CidrBlock=subnet_cidr_block, VpcId=aws_vpc_id + ) + assert "Subnet" in subnet + assert "SubnetId" in subnet["Subnet"] + subnet_id = subnet["Subnet"]["SubnetId"] + print(f"--> Created Subnet in AWS with {subnet_id=}") + + yield subnet_id + + # all the instances in the subnet must be terminated before that works + instances_in_subnet = await ec2_client.describe_instances( + Filters=[{"Name": "subnet-id", "Values": [subnet_id]}] + ) + if instances_in_subnet["Reservations"]: + print(f"--> terminating {len(instances_in_subnet)} instances in subnet") + await ec2_client.terminate_instances( + InstanceIds=[ + instance["Instances"][0]["InstanceId"] # type: ignore + for instance in instances_in_subnet["Reservations"] + ] + ) + print(f"<-- terminated {len(instances_in_subnet)} instances in subnet") + + await ec2_client.delete_subnet(SubnetId=subnet_id) + subnets = await ec2_client.describe_subnets() + print(f"<-- Deleted Subnet in AWS with {subnet_id=}") + print(f"current {subnets=}") + + +@pytest.fixture +async def aws_security_group_id( + faker: Faker, + aws_vpc_id: str, + ec2_client: EC2Client, +) -> AsyncIterator[str]: + security_group = await ec2_client.create_security_group( + Description=faker.text(), GroupName=faker.pystr(), VpcId=aws_vpc_id + ) + security_group_id = security_group["GroupId"] + print(f"--> Created Security Group in AWS with {security_group_id=}") + yield security_group_id + await ec2_client.delete_security_group(GroupId=security_group_id) + print(f"<-- Deleted Security Group in AWS with {security_group_id=}") + + +@pytest.fixture +async def aws_ami_id( + ec2_client: EC2Client, +) -> str: + images = await ec2_client.describe_images() + image = random.choice(images["Images"]) # noqa: S311 + assert "ImageId" in image + return image["ImageId"] From 149d9684a1aaddce2ecf3900afa974b16b2ae55e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 18:11:31 +0100 Subject: [PATCH 25/78] refactoring --- packages/aws-library/tests/test_ec2_client.py | 257 +++++++++++------- 1 file changed, 163 insertions(+), 94 deletions(-) diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index 83dd0965b36..d05c47bf8c7 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -6,19 +6,19 @@ import datetime import json from collections.abc import AsyncIterator -from typing import cast +from typing import cast, get_args import botocore.exceptions import pytest from aws_library.ec2.client import SimcoreEC2API -from aws_library.ec2.errors import EC2TooManyInstancesError +from aws_library.ec2.errors import EC2InstanceNotFoundError, EC2TooManyInstancesError from aws_library.ec2.models import EC2InstanceType, EC2Tags from faker import Faker from moto.server import ThreadedMotoServer from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict from settings_library.ec2 import EC2InstancesSettings, EC2Settings from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceTypeType +from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType def _ec2_allowed_types() -> list[InstanceTypeType]: @@ -173,6 +173,7 @@ async def _assert_instances_in_ec2( expected_num_instances: int, expected_instance_type: EC2InstanceType, expected_tags: EC2Tags, + expected_state: str, ) -> None: all_instances = await ec2_client.describe_instances() assert len(all_instances["Reservations"]) == expected_num_reservations @@ -186,6 +187,7 @@ async def _assert_instances_in_ec2( assert instance["Tags"] == [ {"Key": key, "Value": value} for key, value in expected_tags.items() ] + assert instance["State"]["Name"] == expected_state async def test_start_aws_instance( @@ -215,6 +217,7 @@ async def test_start_aws_instance( expected_num_instances=number_of_instances, expected_instance_type=fake_ec2_instance_type, expected_tags=tags, + expected_state="running", ) # create a second reservation @@ -231,6 +234,7 @@ async def test_start_aws_instance( expected_num_instances=number_of_instances, expected_instance_type=fake_ec2_instance_type, expected_tags=tags, + expected_state="running", ) @@ -272,6 +276,7 @@ async def test_start_aws_instance_is_limited_in_number_of_instances( expected_num_instances=1, expected_instance_type=fake_ec2_instance_type, expected_tags=tags, + expected_state="running", ) # now creating one more shall fail @@ -289,97 +294,161 @@ async def test_start_aws_instance_is_limited_in_number_of_instances( expected_num_instances=1, expected_instance_type=fake_ec2_instance_type, expected_tags=tags, + expected_state="running", ) -# async def test_get_instances( -# mocked_ec2_server_envs: None, -# aws_vpc_id: str, -# aws_subnet_id: str, -# aws_security_group_id: str, -# aws_ami_id: str, -# ec2_client: EC2Client, -# autoscaling_ec2: AutoscalingEC2, -# app_settings: ApplicationSettings, -# faker: Faker, -# fake_ec2_instance_type: EC2InstanceType, -# ): -# assert app_settings.AUTOSCALING_EC2_INSTANCES -# # we have nothing running now in ec2 -# all_instances = await ec2_client.describe_instances() -# assert not all_instances["Reservations"] -# assert ( -# await autoscaling_ec2.get_instances(app_settings.AUTOSCALING_EC2_INSTANCES, {}) -# == [] -# ) - -# # create some instance -# tags = faker.pydict(allowed_types=(str,)) -# startup_script = faker.pystr() -# created_instances = await autoscaling_ec2.start_aws_instance( -# app_settings.AUTOSCALING_EC2_INSTANCES, -# fake_ec2_instance_type, -# tags=tags, -# startup_script=startup_script, -# number_of_instances=1, -# ) -# assert len(created_instances) == 1 - -# instance_received = await autoscaling_ec2.get_instances( -# app_settings.AUTOSCALING_EC2_INSTANCES, -# tags=tags, -# ) -# assert created_instances == instance_received - - -# async def test_terminate_instance( -# mocked_ec2_server_envs: None, -# aws_vpc_id: str, -# aws_subnet_id: str, -# aws_security_group_id: str, -# aws_ami_id: str, -# ec2_client: EC2Client, -# autoscaling_ec2: AutoscalingEC2, -# app_settings: ApplicationSettings, -# faker: Faker, -# fake_ec2_instance_type: EC2InstanceType, -# ): -# assert app_settings.AUTOSCALING_EC2_INSTANCES -# # we have nothing running now in ec2 -# all_instances = await ec2_client.describe_instances() -# assert not all_instances["Reservations"] -# # create some instance -# tags = faker.pydict(allowed_types=(str,)) -# startup_script = faker.pystr() -# created_instances = await autoscaling_ec2.start_aws_instance( -# app_settings.AUTOSCALING_EC2_INSTANCES, -# fake_ec2_instance_type, -# tags=tags, -# startup_script=startup_script, -# number_of_instances=1, -# ) -# assert len(created_instances) == 1 - -# # terminate the instance -# await autoscaling_ec2.terminate_instances(created_instances) -# # calling it several times is ok, the instance stays a while -# await autoscaling_ec2.terminate_instances(created_instances) - - -# async def test_terminate_instance_not_existing_raises( -# mocked_ec2_server_envs: None, -# aws_vpc_id: str, -# aws_subnet_id: str, -# aws_security_group_id: str, -# aws_ami_id: str, -# ec2_client: EC2Client, -# autoscaling_ec2: AutoscalingEC2, -# app_settings: ApplicationSettings, -# fake_ec2_instance_data: Callable[..., EC2InstanceData], -# ): -# assert app_settings.AUTOSCALING_EC2_INSTANCES -# # we have nothing running now in ec2 -# all_instances = await ec2_client.describe_instances() -# assert not all_instances["Reservations"] -# with pytest.raises(Ec2InstanceNotFoundError): -# await autoscaling_ec2.terminate_instances([fake_ec2_instance_data()]) +async def test_get_instances( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + faker: Faker, + fake_ec2_instance_type: EC2InstanceType, + ec2_instances_settings: EC2InstancesSettings, +): + # we have nothing running now in ec2 + await _assert_no_instances_in_ec2(ec2_client) + assert ( + await simcore_ec2_api.get_instances( + key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], tags={} + ) + == [] + ) + + # create some instance + tags = faker.pydict(allowed_types=(str,)) + startup_script = faker.pystr() + num_instances = faker.pyint( + min_value=1, max_value=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES + ) + created_instances = await simcore_ec2_api.start_aws_instance( + ec2_instances_settings, + fake_ec2_instance_type, + tags=tags, + startup_script=startup_script, + number_of_instances=num_instances, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + expected_state="running", + ) + # this returns all the entries using thes key names + instance_received = await simcore_ec2_api.get_instances( + key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], tags={} + ) + assert created_instances == instance_received + + # passing the tags will return the same + instance_received = await simcore_ec2_api.get_instances( + key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], tags=tags + ) + assert created_instances == instance_received + + # asking for running state will also return the same + instance_received = await simcore_ec2_api.get_instances( + key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], + tags=tags, + state_names=["running"], + ) + assert created_instances == instance_received + + # asking for other states shall return nothing + for state in get_args(InstanceStateNameType): + instance_received = await simcore_ec2_api.get_instances( + key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], + tags=tags, + state_names=[state], + ) + if state == "running": + assert created_instances == instance_received + else: + assert not instance_received + + +async def test_terminate_instance( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + faker: Faker, + fake_ec2_instance_type: EC2InstanceType, + ec2_instances_settings: EC2InstancesSettings, +): + # we have nothing running now in ec2 + await _assert_no_instances_in_ec2(ec2_client) + # create some instance + tags = faker.pydict(allowed_types=(str,)) + startup_script = faker.pystr() + num_instances = faker.pyint( + min_value=1, max_value=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES + ) + created_instances = await simcore_ec2_api.start_aws_instance( + ec2_instances_settings, + fake_ec2_instance_type, + tags=tags, + startup_script=startup_script, + number_of_instances=num_instances, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + expected_state="running", + ) + + # terminate the instance + await simcore_ec2_api.terminate_instances(created_instances) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + expected_state="terminated", + ) + # calling it several times is ok, the instance stays a while + await simcore_ec2_api.terminate_instances(created_instances) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + expected_state="terminated", + ) + + +@pytest.fixture +def fake_ec2_instance_data(faker: Faker) -> Callable[..., EC2InstanceData]: + def _creator(**overrides) -> EC2InstanceData: + return EC2InstanceData( + **( + { + "launch_time": faker.date_time(tzinfo=timezone.utc), + "id": faker.uuid4(), + "aws_private_dns": f"ip-{faker.ipv4().replace('.', '-')}.ec2.internal", + "type": faker.pystr(), + "state": faker.pystr(), + "resources": Resources(cpus=4.0, ram=ByteSize(1024 * 1024)), + } + | overrides + ) + ) + + return _creator + + +async def test_terminate_instance_not_existing_raises( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + faker: Faker, + fake_ec2_instance_type: EC2InstanceType, + ec2_instances_settings: EC2InstancesSettings, +): + # we have nothing running now in ec2 + await _assert_no_instances_in_ec2(ec2_client) + with pytest.raises(EC2InstanceNotFoundError): + await simcore_ec2_api.terminate_instances([fake_ec2_instance_data()]) From 85f72e6ee910e51a51edd83490d1265d9c10c0d4 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 22:04:05 +0100 Subject: [PATCH 26/78] 100% coverage --- .../aws-library/src/aws_library/ec2/client.py | 29 +++++---- packages/aws-library/tests/test_ec2_client.py | 62 +++++++++++++++++-- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py index ce07dec423f..ad82efefccc 100644 --- a/packages/aws-library/src/aws_library/ec2/client.py +++ b/packages/aws-library/src/aws_library/ec2/client.py @@ -230,15 +230,20 @@ async def terminate_instances(self, instance_datas: list[EC2InstanceData]) -> No async def set_instances_tags( self, instances: list[EC2InstanceData], *, tags: EC2Tags ) -> None: - with log_context( - _logger, - logging.DEBUG, - msg=f"setting {tags=} on instances '[{[i.id for i in instances]}]'", - ): - await self.client.create_tags( - Resources=[i.id for i in instances], - Tags=[ - {"Key": tag_key, "Value": tag_value} - for tag_key, tag_value in tags.items() - ], - ) + try: + with log_context( + _logger, + logging.DEBUG, + msg=f"setting {tags=} on instances '[{[i.id for i in instances]}]'", + ): + await self.client.create_tags( + Resources=[i.id for i in instances], + Tags=[ + {"Key": tag_key, "Value": tag_value} + for tag_key, tag_value in tags.items() + ], + ) + except botocore.exceptions.ClientError as exc: + if exc.response.get("Error", {}).get("Code", "") == "InvalidID": + raise EC2InstanceNotFoundError from exc + raise # pragma: no cover diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index d05c47bf8c7..0c009939de5 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -6,15 +6,16 @@ import datetime import json from collections.abc import AsyncIterator -from typing import cast, get_args +from typing import Callable, cast, get_args import botocore.exceptions import pytest from aws_library.ec2.client import SimcoreEC2API from aws_library.ec2.errors import EC2InstanceNotFoundError, EC2TooManyInstancesError -from aws_library.ec2.models import EC2InstanceType, EC2Tags +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, EC2Tags, Resources from faker import Faker from moto.server import ThreadedMotoServer +from pydantic import ByteSize from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict from settings_library.ec2 import EC2InstancesSettings, EC2Settings from types_aiobotocore_ec2 import EC2Client @@ -187,6 +188,8 @@ async def _assert_instances_in_ec2( assert instance["Tags"] == [ {"Key": key, "Value": value} for key, value in expected_tags.items() ] + assert "State" in instance + assert "Name" in instance["State"] assert instance["State"]["Name"] == expected_state @@ -427,7 +430,7 @@ def _creator(**overrides) -> EC2InstanceData: return EC2InstanceData( **( { - "launch_time": faker.date_time(tzinfo=timezone.utc), + "launch_time": faker.date_time(tzinfo=datetime.timezone.utc), "id": faker.uuid4(), "aws_private_dns": f"ip-{faker.ipv4().replace('.', '-')}.ec2.internal", "type": faker.pystr(), @@ -442,13 +445,62 @@ def _creator(**overrides) -> EC2InstanceData: async def test_terminate_instance_not_existing_raises( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + fake_ec2_instance_data: Callable[..., EC2InstanceData], +): + await _assert_no_instances_in_ec2(ec2_client) + with pytest.raises(EC2InstanceNotFoundError): + await simcore_ec2_api.terminate_instances([fake_ec2_instance_data()]) + + +async def test_set_instance_tags( simcore_ec2_api: SimcoreEC2API, ec2_client: EC2Client, faker: Faker, fake_ec2_instance_type: EC2InstanceType, ec2_instances_settings: EC2InstancesSettings, ): - # we have nothing running now in ec2 + await _assert_no_instances_in_ec2(ec2_client) + # create some instance + tags = faker.pydict(allowed_types=(str,)) + startup_script = faker.pystr() + num_instances = faker.pyint( + min_value=1, max_value=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES + ) + created_instances = await simcore_ec2_api.start_aws_instance( + ec2_instances_settings, + fake_ec2_instance_type, + tags=tags, + startup_script=startup_script, + number_of_instances=num_instances, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags, + expected_state="running", + ) + + new_tags = faker.pydict(allowed_types=(str,)) + await simcore_ec2_api.set_instances_tags(created_instances, tags=new_tags) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=fake_ec2_instance_type, + expected_tags=tags | new_tags, + expected_state="running", + ) + + +async def test_set_instance_tags_not_existing_raises( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + fake_ec2_instance_data: Callable[..., EC2InstanceData], +): await _assert_no_instances_in_ec2(ec2_client) with pytest.raises(EC2InstanceNotFoundError): - await simcore_ec2_api.terminate_instances([fake_ec2_instance_data()]) + await simcore_ec2_api.set_instances_tags([fake_ec2_instance_data()], tags={}) From 0781d2a9ed003c14836ed165bc11358ed6dd0113 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 22:04:17 +0100 Subject: [PATCH 27/78] ruff --- packages/aws-library/tests/test_ec2_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index 0c009939de5..cbe30b713bc 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -5,8 +5,8 @@ import datetime import json -from collections.abc import AsyncIterator -from typing import Callable, cast, get_args +from collections.abc import AsyncIterator, Callable +from typing import cast, get_args import botocore.exceptions import pytest From cc4c2d5b389ff92f85f7621d6f91002f316373ba Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 22:54:56 +0100 Subject: [PATCH 28/78] removed ec2instancessettings --- .../aws-library/src/aws_library/ec2/client.py | 47 ++-- .../aws-library/src/aws_library/ec2/models.py | 12 + packages/aws-library/tests/test_ec2_client.py | 256 +++++++----------- .../src/settings_library/ec2.py | 55 ---- 4 files changed, 131 insertions(+), 239 deletions(-) diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py index ad82efefccc..dd0d61164b2 100644 --- a/packages/aws-library/src/aws_library/ec2/client.py +++ b/packages/aws-library/src/aws_library/ec2/client.py @@ -9,13 +9,19 @@ from aiocache import cached from pydantic import ByteSize from servicelib.logging_utils import log_context -from settings_library.ec2 import EC2InstancesSettings, EC2Settings +from settings_library.ec2 import EC2Settings from types_aiobotocore_ec2 import EC2Client from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType from types_aiobotocore_ec2.type_defs import FilterTypeDef from .errors import EC2InstanceNotFoundError, EC2TooManyInstancesError -from .models import EC2InstanceData, EC2InstanceType, EC2Tags, Resources +from .models import ( + EC2InstanceConfig, + EC2InstanceData, + EC2InstanceType, + EC2Tags, + Resources, +) from .utils import compose_user_data _logger = logging.getLogger(__name__) @@ -79,48 +85,41 @@ async def get_ec2_instance_capabilities( async def start_aws_instance( self, - instance_settings: EC2InstancesSettings, - instance_type: EC2InstanceType, - tags: dict[str, str], - startup_script: str, + instance_config: EC2InstanceConfig, number_of_instances: int, + max_number_of_instances: int = 10, ) -> list[EC2InstanceData]: with log_context( _logger, logging.INFO, - msg=f"launching {number_of_instances} AWS instance(s) {instance_type.name} with {tags=}", + msg=f"launching {number_of_instances} AWS instance(s) {instance_config.type.name} with {instance_config.tags=}", ): # first check the max amount is not already reached current_instances = await self.get_instances( - key_names=[instance_settings.EC2_INSTANCES_KEY_NAME], tags=tags + key_names=[instance_config.key_name], tags=instance_config.tags ) - if ( - len(current_instances) + number_of_instances - > instance_settings.EC2_INSTANCES_MAX_INSTANCES - ): - raise EC2TooManyInstancesError( - num_instances=instance_settings.EC2_INSTANCES_MAX_INSTANCES - ) + if len(current_instances) + number_of_instances > max_number_of_instances: + raise EC2TooManyInstancesError(num_instances=max_number_of_instances) instances = await self.client.run_instances( - ImageId=instance_settings.EC2_INSTANCES_AMI_ID, + ImageId=instance_config.ami_id, MinCount=number_of_instances, MaxCount=number_of_instances, - InstanceType=instance_type.name, + InstanceType=instance_config.type.name, InstanceInitiatedShutdownBehavior="terminate", - KeyName=instance_settings.EC2_INSTANCES_KEY_NAME, - SubnetId=instance_settings.EC2_INSTANCES_SUBNET_ID, + KeyName=instance_config.key_name, + SubnetId=instance_config.subnet_id, TagSpecifications=[ { "ResourceType": "instance", "Tags": [ {"Key": tag_key, "Value": tag_value} - for tag_key, tag_value in tags.items() + for tag_key, tag_value in instance_config.tags.items() ], } ], - UserData=compose_user_data(startup_script), - SecurityGroupIds=instance_settings.EC2_INSTANCES_SECURITY_GROUP_IDS, + UserData=compose_user_data(instance_config.startup_script), + SecurityGroupIds=instance_config.security_group_ids, ) instance_ids = [i["InstanceId"] for i in instances["Instances"]] _logger.info( @@ -143,7 +142,9 @@ async def start_aws_instance( aws_private_dns=instance["PrivateDnsName"], type=instance["InstanceType"], state=instance["State"]["Name"], - resources=Resources(cpus=instance_type.cpus, ram=instance_type.ram), + resources=Resources( + cpus=instance_config.type.cpus, ram=instance_config.type.ram + ), ) for instance in instances["Reservations"][0]["Instances"] ] diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py index b48a25cbb0d..27aa127455d 100644 --- a/packages/aws-library/src/aws_library/ec2/models.py +++ b/packages/aws-library/src/aws_library/ec2/models.py @@ -60,3 +60,15 @@ class EC2InstanceData: type: InstanceTypeType # noqa: A003 state: InstanceStateNameType resources: Resources + + +@dataclass(frozen=True) +class EC2InstanceConfig: + type: EC2InstanceType + tags: EC2Tags + startup_script: str + + ami_id: str + key_name: str + security_group_ids: list[str] + subnet_id: str diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index cbe30b713bc..42cc3e49242 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -4,7 +4,6 @@ import datetime -import json from collections.abc import AsyncIterator, Callable from typing import cast, get_args @@ -12,12 +11,17 @@ import pytest from aws_library.ec2.client import SimcoreEC2API from aws_library.ec2.errors import EC2InstanceNotFoundError, EC2TooManyInstancesError -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, EC2Tags, Resources +from aws_library.ec2.models import ( + EC2InstanceConfig, + EC2InstanceData, + EC2InstanceType, + EC2Tags, + Resources, +) from faker import Faker from moto.server import ThreadedMotoServer from pydantic import ByteSize -from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict -from settings_library.ec2 import EC2InstancesSettings, EC2Settings +from settings_library.ec2 import EC2Settings from types_aiobotocore_ec2 import EC2Client from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType @@ -31,53 +35,6 @@ def ec2_allowed_instances() -> list[InstanceTypeType]: return _ec2_allowed_types() -@pytest.fixture -def client_environment( - mock_env_devel_environment: EnvVarsDict, - monkeypatch: pytest.MonkeyPatch, - faker: Faker, - ec2_allowed_instances: list[InstanceTypeType], -) -> EnvVarsDict: - # SEE https://faker.readthedocs.io/en/master/providers/faker.providers.internet.html?highlight=internet#faker-providers-internet - envs = setenvs_from_dict( - monkeypatch, - { - "EC2_ACCESS_KEY_ID": faker.pystr(), - "EC2_SECRET_ACCESS_KEY": faker.pystr(), - "EC2_INSTANCES_KEY_NAME": faker.pystr(), - "EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( - faker.pylist(allowed_types=(str,)) - ), - "EC2_INSTANCES_SUBNET_ID": faker.pystr(), - "EC2_INSTANCES_AMI_ID": faker.pystr(), - "EC2_INSTANCES_ALLOWED_TYPES": json.dumps(ec2_allowed_instances), - }, - ) - return mock_env_devel_environment | envs - - -@pytest.fixture -def ec2_instances_settings( - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_allowed_instances: list[InstanceTypeType], - faker: Faker, -) -> EC2InstancesSettings: - return EC2InstancesSettings( - EC2_INSTANCES_ALLOWED_TYPES=[f"{i}" for i in ec2_allowed_instances], - EC2_INSTANCES_AMI_ID=aws_ami_id, - EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS=faker.pylist(allowed_types=(str,)), - EC2_INSTANCES_KEY_NAME=faker.pystr(), - EC2_INSTANCES_MAX_INSTANCES=10, - EC2_INSTANCES_NAME_PREFIX=faker.pystr(), - EC2_INSTANCES_SECURITY_GROUP_IDS=[aws_security_group_id], - EC2_INSTANCES_SUBNET_ID=aws_subnet_id, - EC2_INSTANCES_TIME_BEFORE_TERMINATION=datetime.timedelta(seconds=10), - ) - - @pytest.fixture async def simcore_ec2_api( mocked_ec2_server_settings: EC2Settings, @@ -122,32 +79,54 @@ async def test_ping( assert await simcore_ec2_api.ping() is True +@pytest.fixture +def ec2_instance_config( + fake_ec2_instance_type: EC2InstanceType, + faker: Faker, + aws_subnet_id: str, + aws_security_group_id: str, + aws_ami_id: str, +) -> EC2InstanceConfig: + return EC2InstanceConfig( + type=fake_ec2_instance_type, + tags=faker.pydict(allowed_types=(str,)), + startup_script=faker.pystr(), + ami_id=aws_ami_id, + key_name=faker.pystr(), + security_group_ids=[aws_security_group_id], + subnet_id=aws_subnet_id, + ) + + async def test_get_ec2_instance_capabilities( simcore_ec2_api: SimcoreEC2API, - ec2_instances_settings: EC2InstancesSettings, + ec2_allowed_instances: list[InstanceTypeType], ): instance_types: list[ EC2InstanceType ] = await simcore_ec2_api.get_ec2_instance_capabilities( cast( set[InstanceTypeType], - set(ec2_instances_settings.EC2_INSTANCES_ALLOWED_TYPES), + set(ec2_allowed_instances), ) ) assert instance_types - assert len(instance_types) == len( - ec2_instances_settings.EC2_INSTANCES_ALLOWED_TYPES - ) + assert len(instance_types) == len(ec2_allowed_instances) # all the instance names are found and valid - assert all( - i.name in ec2_instances_settings.EC2_INSTANCES_ALLOWED_TYPES - for i in instance_types - ) - for instance_type_name in ec2_instances_settings.EC2_INSTANCES_ALLOWED_TYPES: + assert all(i.name in ec2_allowed_instances for i in instance_types) + for instance_type_name in ec2_allowed_instances: assert any(i.name == instance_type_name for i in instance_types) +async def test_get_ec2_instance_capabilities_empty_list_returns_all_options( + simcore_ec2_api: SimcoreEC2API, +): + instance_types = await simcore_ec2_api.get_ec2_instance_capabilities(set()) + assert instance_types + assert len(instance_types) > 50 + + @pytest.fixture(params=_ec2_allowed_types()) async def fake_ec2_instance_type( simcore_ec2_api: SimcoreEC2API, @@ -196,107 +175,86 @@ async def _assert_instances_in_ec2( async def test_start_aws_instance( simcore_ec2_api: SimcoreEC2API, ec2_client: EC2Client, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, - ec2_instances_settings: EC2InstancesSettings, + ec2_instance_config: EC2InstanceConfig, ): await _assert_no_instances_in_ec2(ec2_client) - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() number_of_instances = 1 # let's create a first reservation and check that it is correctly created in EC2 await simcore_ec2_api.start_aws_instance( - ec2_instances_settings, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=number_of_instances, + ec2_instance_config, number_of_instances=number_of_instances ) await _assert_instances_in_ec2( ec2_client, expected_num_reservations=1, expected_num_instances=number_of_instances, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="running", ) # create a second reservation await simcore_ec2_api.start_aws_instance( - ec2_instances_settings, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=number_of_instances, + ec2_instance_config, number_of_instances=number_of_instances ) await _assert_instances_in_ec2( ec2_client, expected_num_reservations=2, expected_num_instances=number_of_instances, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="running", ) +@pytest.mark.parametrize("max_num_instances", [13]) async def test_start_aws_instance_is_limited_in_number_of_instances( simcore_ec2_api: SimcoreEC2API, ec2_client: EC2Client, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, - ec2_instances_settings: EC2InstancesSettings, + ec2_instance_config: EC2InstanceConfig, + max_num_instances: int, ): await _assert_no_instances_in_ec2(ec2_client) - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - # create many instances in one go shall fail with pytest.raises(EC2TooManyInstancesError): await simcore_ec2_api.start_aws_instance( - ec2_instances_settings, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES + 1, + ec2_instance_config, + number_of_instances=max_num_instances + 1, + max_number_of_instances=max_num_instances, ) await _assert_no_instances_in_ec2(ec2_client) # create instances 1 by 1 - for _ in range(ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES): + for _ in range(max_num_instances): await simcore_ec2_api.start_aws_instance( - ec2_instances_settings, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, + ec2_instance_config, number_of_instances=1, + max_number_of_instances=max_num_instances, ) await _assert_instances_in_ec2( ec2_client, - expected_num_reservations=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES, + expected_num_reservations=max_num_instances, expected_num_instances=1, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="running", ) # now creating one more shall fail with pytest.raises(EC2TooManyInstancesError): await simcore_ec2_api.start_aws_instance( - ec2_instances_settings, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, + ec2_instance_config, number_of_instances=1, + max_number_of_instances=max_num_instances, ) await _assert_instances_in_ec2( ec2_client, - expected_num_reservations=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES, + expected_num_reservations=max_num_instances, expected_num_instances=1, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="running", ) @@ -305,55 +263,47 @@ async def test_get_instances( simcore_ec2_api: SimcoreEC2API, ec2_client: EC2Client, faker: Faker, - fake_ec2_instance_type: EC2InstanceType, - ec2_instances_settings: EC2InstancesSettings, + ec2_instance_config: EC2InstanceConfig, ): # we have nothing running now in ec2 await _assert_no_instances_in_ec2(ec2_client) assert ( await simcore_ec2_api.get_instances( - key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], tags={} + key_names=[ec2_instance_config.key_name], tags={} ) == [] ) # create some instance - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - num_instances = faker.pyint( - min_value=1, max_value=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES - ) + _MAX_NUM_INSTANCES = 10 + num_instances = faker.pyint(min_value=1, max_value=_MAX_NUM_INSTANCES) created_instances = await simcore_ec2_api.start_aws_instance( - ec2_instances_settings, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=num_instances, + ec2_instance_config, number_of_instances=num_instances ) await _assert_instances_in_ec2( ec2_client, expected_num_reservations=1, expected_num_instances=num_instances, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="running", ) # this returns all the entries using thes key names instance_received = await simcore_ec2_api.get_instances( - key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], tags={} + key_names=[ec2_instance_config.key_name], tags={} ) assert created_instances == instance_received # passing the tags will return the same instance_received = await simcore_ec2_api.get_instances( - key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], tags=tags + key_names=[ec2_instance_config.key_name], tags=ec2_instance_config.tags ) assert created_instances == instance_received # asking for running state will also return the same instance_received = await simcore_ec2_api.get_instances( - key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], - tags=tags, + key_names=[ec2_instance_config.key_name], + tags=ec2_instance_config.tags, state_names=["running"], ) assert created_instances == instance_received @@ -361,8 +311,8 @@ async def test_get_instances( # asking for other states shall return nothing for state in get_args(InstanceStateNameType): instance_received = await simcore_ec2_api.get_instances( - key_names=[ec2_instances_settings.EC2_INSTANCES_KEY_NAME], - tags=tags, + key_names=[ec2_instance_config.key_name], + tags=ec2_instance_config.tags, state_names=[state], ) if state == "running": @@ -375,30 +325,22 @@ async def test_terminate_instance( simcore_ec2_api: SimcoreEC2API, ec2_client: EC2Client, faker: Faker, - fake_ec2_instance_type: EC2InstanceType, - ec2_instances_settings: EC2InstancesSettings, + ec2_instance_config: EC2InstanceConfig, ): # we have nothing running now in ec2 await _assert_no_instances_in_ec2(ec2_client) # create some instance - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - num_instances = faker.pyint( - min_value=1, max_value=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES - ) + _NUM_INSTANCES = 10 + num_instances = faker.pyint(min_value=1, max_value=_NUM_INSTANCES) created_instances = await simcore_ec2_api.start_aws_instance( - ec2_instances_settings, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=num_instances, + ec2_instance_config, number_of_instances=num_instances ) await _assert_instances_in_ec2( ec2_client, expected_num_reservations=1, expected_num_instances=num_instances, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="running", ) @@ -408,8 +350,8 @@ async def test_terminate_instance( ec2_client, expected_num_reservations=1, expected_num_instances=num_instances, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="terminated", ) # calling it several times is ok, the instance stays a while @@ -418,8 +360,8 @@ async def test_terminate_instance( ec2_client, expected_num_reservations=1, expected_num_instances=num_instances, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="terminated", ) @@ -458,29 +400,21 @@ async def test_set_instance_tags( simcore_ec2_api: SimcoreEC2API, ec2_client: EC2Client, faker: Faker, - fake_ec2_instance_type: EC2InstanceType, - ec2_instances_settings: EC2InstancesSettings, + ec2_instance_config: EC2InstanceConfig, ): await _assert_no_instances_in_ec2(ec2_client) # create some instance - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - num_instances = faker.pyint( - min_value=1, max_value=ec2_instances_settings.EC2_INSTANCES_MAX_INSTANCES - ) + _MAX_NUM_INSTANCES = 10 + num_instances = faker.pyint(min_value=1, max_value=_MAX_NUM_INSTANCES) created_instances = await simcore_ec2_api.start_aws_instance( - ec2_instances_settings, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=num_instances, + ec2_instance_config, number_of_instances=num_instances ) await _assert_instances_in_ec2( ec2_client, expected_num_reservations=1, expected_num_instances=num_instances, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, expected_state="running", ) @@ -490,8 +424,8 @@ async def test_set_instance_tags( ec2_client, expected_num_reservations=1, expected_num_instances=num_instances, - expected_instance_type=fake_ec2_instance_type, - expected_tags=tags | new_tags, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags | new_tags, expected_state="running", ) diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py index 6ff2d08e7c6..111e52adfb0 100644 --- a/packages/settings-library/src/settings_library/ec2.py +++ b/packages/settings-library/src/settings_library/ec2.py @@ -1,5 +1,3 @@ -import datetime - from pydantic import Field from .base import BaseCustomSettings @@ -12,56 +10,3 @@ class EC2Settings(BaseCustomSettings): ) EC2_REGION_NAME: str = "us-east-1" EC2_SECRET_ACCESS_KEY: str - - -class EC2InstancesSettings(BaseCustomSettings): - EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( - ..., - min_items=1, - unique_items=True, - description="Defines which EC2 instances are considered as candidates for new EC2 instance", - ) - EC2_INSTANCES_AMI_ID: str = Field( - ..., - min_length=1, - description="Defines the AMI (Amazon Machine Image) ID used to start a new EC2 instance", - ) - EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: list[str] = Field( - default_factory=list, - description="script(s) to run on EC2 instance startup (be careful!), each entry is run one after the other using '&&' operator", - ) - EC2_INSTANCES_KEY_NAME: str = Field( - ..., - min_length=1, - description="SSH key filename (without ext) to access the instance through SSH" - " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html)," - "this is required to start a new EC2 instance", - ) - EC2_INSTANCES_MAX_INSTANCES: int = Field( - default=10, - description="Defines the maximum number of instances the autoscaling app may create", - ) - EC2_INSTANCES_NAME_PREFIX: str = Field( - default="autoscaling", - min_length=1, - description="prefix used to name the EC2 instances created by this instance of autoscaling", - ) - EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( - ..., - min_items=1, - description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic" - " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), " - " this is required to start a new EC2 instance", - ) - EC2_INSTANCES_SUBNET_ID: str = Field( - ..., - min_length=1, - description="A subnet is a range of IP addresses in your VPC " - " (https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html), " - "this is required to start a new EC2 instance", - ) - EC2_INSTANCES_TIME_BEFORE_TERMINATION: datetime.timedelta = Field( - default=datetime.timedelta(minutes=1), - description="Time after which an EC2 instance may be terminated (0<=T<=59 minutes, is automatically capped)" - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", - ) From e277b25aee866732f952bf4d6cf6b4e7c5e3688a Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 12 Nov 2023 23:15:22 +0100 Subject: [PATCH 29/78] Squashed commit of the following: commit cb161ed01c30195ceea59d62e35c59e6c19476a5 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun Nov 12 23:14:58 2023 +0100 refactoring autoscaling commit 52594c3c06b9bbc4b512a6f9363ac11fef544553 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun Nov 12 22:08:02 2023 +0100 add dependency on aws-lib --- services/autoscaling/requirements/_base.in | 3 +- services/autoscaling/requirements/_base.txt | 19 +- .../core/settings.py | 10 +- .../modules/auto_scaling_core.py | 23 +- .../modules/ec2.py | 236 +----------------- services/autoscaling/tests/unit/conftest.py | 10 +- ...test_modules_auto_scaling_computational.py | 4 +- .../unit/test_modules_auto_scaling_dynamic.py | 7 +- .../tests/unit/test_modules_ec2.py | 25 +- .../tests/unit/test_utils_dynamic_scaling.py | 3 +- 10 files changed, 65 insertions(+), 275 deletions(-) diff --git a/services/autoscaling/requirements/_base.in b/services/autoscaling/requirements/_base.in index 09555b35987..ae362ec2744 100644 --- a/services/autoscaling/requirements/_base.in +++ b/services/autoscaling/requirements/_base.in @@ -9,14 +9,13 @@ # intra-repo required dependencies --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/aws-library/requirements/_base.in # service-library[fastapi] --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in aiocache aiodocker -aioboto3 dask[distributed] fastapi packaging -types-aiobotocore[ec2] diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index 0c81d2664c3..bb189297c0e 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -9,13 +9,15 @@ aio-pika==9.3.0 # -c requirements/../../../packages/service-library/requirements/./_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aioboto3==12.0.0 - # via -r requirements/_base.in + # via -r requirements/../../../packages/aws-library/requirements/_base.in aiobotocore==2.7.0 # via # aioboto3 # aiobotocore aiocache==0.12.2 - # via -r requirements/_base.in + # via + # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/_base.in aiodebug==2.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in @@ -31,6 +33,7 @@ aiofiles==23.2.1 # -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.8.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -79,6 +82,7 @@ botocore-stubs==1.31.77 # via types-aiobotocore certifi==2023.7.22 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -121,6 +125,7 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -148,6 +153,7 @@ httpcore==0.18.0 # via httpx httpx==0.25.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -170,6 +176,7 @@ importlib-metadata==6.8.0 # dask jinja2==3.1.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -236,6 +243,7 @@ psutil==5.9.5 # distributed pydantic==1.10.13 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -246,6 +254,7 @@ pydantic==1.10.13 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -266,6 +275,7 @@ python-dateutil==2.8.2 # botocore pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -282,6 +292,7 @@ pyyaml==6.0.1 # distributed redis==5.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -323,6 +334,7 @@ sortedcontainers==2.4.0 # distributed starlette==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -363,7 +375,7 @@ typer==0.9.0 # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in types-aiobotocore==2.7.0 - # via -r requirements/_base.in + # via -r requirements/../../../packages/aws-library/requirements/_base.in types-aiobotocore-ec2==2.7.0 # via types-aiobotocore types-awscrt==0.19.8 @@ -382,6 +394,7 @@ typing-extensions==4.8.0 # uvicorn urllib3==1.26.16 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py index 057d59aa912..be378286fad 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py @@ -21,6 +21,7 @@ ) from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings +from settings_library.ec2 import EC2Settings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from settings_library.utils_logging import MixinLoggingSettings @@ -29,15 +30,6 @@ from .._meta import API_VERSION, API_VTAG, APP_NAME -class EC2Settings(BaseCustomSettings): - EC2_ACCESS_KEY_ID: str - EC2_ENDPOINT: str | None = Field( - default=None, description="do not define if using standard AWS" - ) - EC2_REGION_NAME: str = "us-east-1" - EC2_SECRET_ACCESS_KEY: str - - class EC2InstancesSettings(BaseCustomSettings): EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( ..., diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py index ef2c5ab5a8b..1b86e643d2d 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py @@ -7,7 +7,7 @@ from typing import cast import arrow -from aws_library.ec2.models import EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceConfig, EC2InstanceType, Resources from fastapi import FastAPI from models_library.generated_models.docker_rest_api import ( Availability, @@ -53,12 +53,13 @@ async def _analyze_current_cluster( # get the EC2 instances we have existing_ec2_instances = await get_ec2_client(app).get_instances( - app_settings.AUTOSCALING_EC2_INSTANCES, auto_scaling_mode.get_ec2_tags(app) + key_names=[app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME], + tags=auto_scaling_mode.get_ec2_tags(app), ) terminated_ec2_instances = await get_ec2_client(app).get_instances( - app_settings.AUTOSCALING_EC2_INSTANCES, - auto_scaling_mode.get_ec2_tags(app), + key_names=[app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME], + tags=auto_scaling_mode.get_ec2_tags(app), state_names=["terminated"], ) @@ -402,11 +403,17 @@ async def _start_instances( results = await asyncio.gather( *[ ec2_client.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - instance_type=instance_type, - tags=instance_tags, - startup_script=instance_startup_script, + EC2InstanceConfig( + type=instance_type, + tags=instance_tags, + startup_script=instance_startup_script, + ami_id=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_AMI_ID, + key_name=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME, + security_group_ids=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SECURITY_GROUP_IDS, + subnet_id=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SUBNET_ID, + ), number_of_instances=instance_num, + max_number_of_instances=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES, ) for instance_type, instance_num in needed_instances.items() ], diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py b/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py index a97baf83b70..420d63e7762 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py @@ -1,233 +1,17 @@ -import contextlib import logging -from dataclasses import dataclass from typing import cast -import aioboto3 -import botocore.exceptions -from aiobotocore.session import ClientCreatorContext -from aiocache import cached -from aws_library.ec2.models import EC2InstanceType, Resources +from aws_library.ec2.client import SimcoreEC2API from fastapi import FastAPI -from pydantic import ByteSize, parse_obj_as -from servicelib.logging_utils import log_context from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_random_exponential -from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType -from types_aiobotocore_ec2.type_defs import FilterTypeDef -from ..core.errors import ( - ConfigurationError, - Ec2InstanceNotFoundError, - Ec2NotConnectedError, - Ec2TooManyInstancesError, -) -from ..core.settings import EC2InstancesSettings, EC2Settings -from ..models import EC2InstanceData -from ..utils.utils_ec2 import compose_user_data +from ..core.errors import ConfigurationError, Ec2NotConnectedError +from ..core.settings import EC2Settings -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class AutoscalingEC2: - client: EC2Client - session: aioboto3.Session - exit_stack: contextlib.AsyncExitStack - - @classmethod - async def create(cls, settings: EC2Settings) -> "AutoscalingEC2": - session = aioboto3.Session() - session_client = session.client( - "ec2", - endpoint_url=settings.EC2_ENDPOINT, - aws_access_key_id=settings.EC2_ACCESS_KEY_ID, - aws_secret_access_key=settings.EC2_SECRET_ACCESS_KEY, - region_name=settings.EC2_REGION_NAME, - ) - assert isinstance(session_client, ClientCreatorContext) # nosec - exit_stack = contextlib.AsyncExitStack() - ec2_client = cast( - EC2Client, await exit_stack.enter_async_context(session_client) - ) - return cls(ec2_client, session, exit_stack) - - async def close(self) -> None: - await self.exit_stack.aclose() - - async def ping(self) -> bool: - try: - await self.client.describe_account_attributes() - return True - except Exception: # pylint: disable=broad-except - return False - - @cached(noself=True) - async def get_ec2_instance_capabilities( - self, - instance_type_names: set[InstanceTypeType], - ) -> list[EC2InstanceType]: - """instance_type_names must be a set of unique values""" - instance_types = await self.client.describe_instance_types( - InstanceTypes=list(instance_type_names) - ) - list_instances: list[EC2InstanceType] = [] - for instance in instance_types.get("InstanceTypes", []): - with contextlib.suppress(KeyError): - list_instances.append( - EC2InstanceType( - name=instance["InstanceType"], - cpus=instance["VCpuInfo"]["DefaultVCpus"], - ram=parse_obj_as( - ByteSize, f"{instance['MemoryInfo']['SizeInMiB']}MiB" - ), - ) - ) - return list_instances - - async def start_aws_instance( - self, - instance_settings: EC2InstancesSettings, - instance_type: EC2InstanceType, - tags: dict[str, str], - startup_script: str, - number_of_instances: int, - ) -> list[EC2InstanceData]: - with log_context( - logger, - logging.INFO, - msg=f"launching {number_of_instances} AWS instance(s) {instance_type.name} with {tags=}", - ): - # first check the max amount is not already reached - current_instances = await self.get_instances(instance_settings, tags) - if ( - len(current_instances) + number_of_instances - > instance_settings.EC2_INSTANCES_MAX_INSTANCES - ): - raise Ec2TooManyInstancesError( - num_instances=instance_settings.EC2_INSTANCES_MAX_INSTANCES - ) - - instances = await self.client.run_instances( - ImageId=instance_settings.EC2_INSTANCES_AMI_ID, - MinCount=number_of_instances, - MaxCount=number_of_instances, - InstanceType=instance_type.name, - InstanceInitiatedShutdownBehavior="terminate", - KeyName=instance_settings.EC2_INSTANCES_KEY_NAME, - SubnetId=instance_settings.EC2_INSTANCES_SUBNET_ID, - TagSpecifications=[ - { - "ResourceType": "instance", - "Tags": [ - {"Key": tag_key, "Value": tag_value} - for tag_key, tag_value in tags.items() - ], - } - ], - UserData=compose_user_data(startup_script), - SecurityGroupIds=instance_settings.EC2_INSTANCES_SECURITY_GROUP_IDS, - ) - instance_ids = [i["InstanceId"] for i in instances["Instances"]] - logger.info( - "New instances launched: %s, waiting for them to start now...", - instance_ids, - ) - - # wait for the instance to be in a pending state - # NOTE: reference to EC2 states https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html - waiter = self.client.get_waiter("instance_exists") - await waiter.wait(InstanceIds=instance_ids) - logger.info("instances %s exists now.", instance_ids) - - # get the private IPs - instances = await self.client.describe_instances(InstanceIds=instance_ids) - instance_datas = [ - EC2InstanceData( - launch_time=instance["LaunchTime"], - id=instance["InstanceId"], - aws_private_dns=instance["PrivateDnsName"], - type=instance["InstanceType"], - state=instance["State"]["Name"], - resources=Resources(cpus=instance_type.cpus, ram=instance_type.ram), - ) - for instance in instances["Reservations"][0]["Instances"] - ] - logger.info( - "%s is available, happy computing!!", - f"{instance_datas=}", - ) - return instance_datas - - async def get_instances( - self, - instance_settings: EC2InstancesSettings, - tags: dict[str, str], - *, - state_names: list[InstanceStateNameType] | None = None, - ) -> list[EC2InstanceData]: - # NOTE: be careful: Name=instance-state-name,Values=["pending", "running"] means pending OR running - # NOTE2: AND is done by repeating Name=instance-state-name,Values=pending Name=instance-state-name,Values=running - if state_names is None: - state_names = ["pending", "running"] - - filters: list[FilterTypeDef] = [ - { - "Name": "key-name", - "Values": [instance_settings.EC2_INSTANCES_KEY_NAME], - }, - {"Name": "instance-state-name", "Values": state_names}, - ] - filters.extend( - [{"Name": f"tag:{key}", "Values": [value]} for key, value in tags.items()] - ) - - instances = await self.client.describe_instances(Filters=filters) - all_instances = [] - for reservation in instances["Reservations"]: - assert "Instances" in reservation # nosec - for instance in reservation["Instances"]: - assert "LaunchTime" in instance # nosec - assert "InstanceId" in instance # nosec - assert "PrivateDnsName" in instance # nosec - assert "InstanceType" in instance # nosec - assert "State" in instance # nosec - assert "Name" in instance["State"] # nosec - ec2_instance_types = await self.get_ec2_instance_capabilities( - {instance["InstanceType"]} - ) - assert len(ec2_instance_types) == 1 # nosec - all_instances.append( - EC2InstanceData( - launch_time=instance["LaunchTime"], - id=instance["InstanceId"], - aws_private_dns=instance["PrivateDnsName"], - type=instance["InstanceType"], - state=instance["State"]["Name"], - resources=Resources( - cpus=ec2_instance_types[0].cpus, - ram=ec2_instance_types[0].ram, - ), - ) - ) - logger.debug("received: %s", f"{all_instances=}") - return all_instances - - async def terminate_instances(self, instance_datas: list[EC2InstanceData]) -> None: - try: - await self.client.terminate_instances( - InstanceIds=[i.id for i in instance_datas] - ) - except botocore.exceptions.ClientError as exc: - if ( - exc.response.get("Error", {}).get("Code", "") - == "InvalidInstanceID.NotFound" - ): - raise Ec2InstanceNotFoundError from exc - raise # pragma: no cover +_logger = logging.getLogger(__name__) def setup(app: FastAPI) -> None: @@ -236,16 +20,16 @@ async def on_startup() -> None: settings: EC2Settings | None = app.state.settings.AUTOSCALING_EC2_ACCESS if not settings: - logger.warning("EC2 client is de-activated in the settings") + _logger.warning("EC2 client is de-activated in the settings") return - app.state.ec2_client = client = await AutoscalingEC2.create(settings) + app.state.ec2_client = client = await SimcoreEC2API.create(settings) async for attempt in AsyncRetrying( reraise=True, stop=stop_after_delay(120), wait=wait_random_exponential(max=30), - before_sleep=before_sleep_log(logger, logging.WARNING), + before_sleep=before_sleep_log(_logger, logging.WARNING), ): with attempt: connected = await client.ping() @@ -254,15 +38,15 @@ async def on_startup() -> None: async def on_shutdown() -> None: if app.state.ec2_client: - await cast(AutoscalingEC2, app.state.ec2_client).close() + await cast(SimcoreEC2API, app.state.ec2_client).close() app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) -def get_ec2_client(app: FastAPI) -> AutoscalingEC2: +def get_ec2_client(app: FastAPI) -> SimcoreEC2API: if not app.state.ec2_client: raise ConfigurationError( msg="EC2 client is not available. Please check the configuration." ) - return cast(AutoscalingEC2, app.state.ec2_client) + return cast(SimcoreEC2API, app.state.ec2_client) diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index 8bc11071bc2..896e7b1831e 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -23,7 +23,8 @@ import simcore_service_autoscaling from aiohttp.test_utils import unused_port from asgi_lifespan import LifespanManager -from aws_library.ec2.models import Resources +from aws_library.ec2.client import SimcoreEC2API +from aws_library.ec2.models import EC2InstanceData, Resources from deepdiff import DeepDiff from faker import Faker from fakeredis.aioredis import FakeRedis @@ -48,7 +49,6 @@ from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings from simcore_service_autoscaling.models import Cluster, DaskTaskResources from simcore_service_autoscaling.modules.docker import AutoscalingDocker -from simcore_service_autoscaling.modules.ec2 import AutoscalingEC2, EC2InstanceData from tenacity import retry from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -637,9 +637,9 @@ async def aws_ami_id( @pytest.fixture async def autoscaling_ec2( app_environment: EnvVarsDict, -) -> AsyncIterator[AutoscalingEC2]: +) -> AsyncIterator[SimcoreEC2API]: settings = EC2Settings.create_from_envs() - ec2 = await AutoscalingEC2.create(settings) + ec2 = await SimcoreEC2API.create(settings) assert ec2 yield ec2 await ec2.close() @@ -647,7 +647,7 @@ async def autoscaling_ec2( @pytest.fixture async def ec2_client( - autoscaling_ec2: AutoscalingEC2, + autoscaling_ec2: SimcoreEC2API, ) -> EC2Client: return autoscaling_ec2.client diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index cea344f7e03..d5393382fde 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -220,7 +220,7 @@ def mock_rabbitmq_post_message(mocker: MockerFixture) -> Iterator[mock.Mock]: @pytest.fixture def mock_terminate_instances(mocker: MockerFixture) -> Iterator[mock.Mock]: return mocker.patch( - "simcore_service_autoscaling.modules.ec2.AutoscalingEC2.terminate_instances", + "simcore_service_autoscaling.modules.ec2.SimcoreEC2API.terminate_instances", autospec=True, ) @@ -232,7 +232,7 @@ def mock_start_aws_instance( fake_ec2_instance_data: Callable[..., EC2InstanceData], ) -> Iterator[mock.Mock]: return mocker.patch( - "simcore_service_autoscaling.modules.ec2.AutoscalingEC2.start_aws_instance", + "simcore_service_autoscaling.modules.ec2.SimcoreEC2API.start_aws_instance", autospec=True, return_value=fake_ec2_instance_data(aws_private_dns=aws_instance_private_dns), ) diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index c7ec52b6237..873d5385bc5 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -16,7 +16,7 @@ import aiodocker import pytest -from aws_library.ec2.models import Resources +from aws_library.ec2.models import EC2InstanceData, Resources from faker import Faker from fastapi import FastAPI from models_library.docker import ( @@ -51,7 +51,6 @@ AutoscalingDocker, get_docker_client, ) -from simcore_service_autoscaling.modules.ec2 import EC2InstanceData from types_aiobotocore_ec2.client import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType @@ -59,7 +58,7 @@ @pytest.fixture def mock_terminate_instances(mocker: MockerFixture) -> Iterator[mock.Mock]: return mocker.patch( - "simcore_service_autoscaling.modules.ec2.AutoscalingEC2.terminate_instances", + "simcore_service_autoscaling.modules.ec2.SimcoreEC2API.terminate_instances", autospec=True, ) @@ -71,7 +70,7 @@ def mock_start_aws_instance( fake_ec2_instance_data: Callable[..., EC2InstanceData], ) -> Iterator[mock.Mock]: return mocker.patch( - "simcore_service_autoscaling.modules.ec2.AutoscalingEC2.start_aws_instance", + "simcore_service_autoscaling.modules.ec2.SimcoreEC2API.start_aws_instance", autospec=True, return_value=fake_ec2_instance_data(aws_private_dns=aws_instance_private_dns), ) diff --git a/services/autoscaling/tests/unit/test_modules_ec2.py b/services/autoscaling/tests/unit/test_modules_ec2.py index 727501adec5..629e20c1e5d 100644 --- a/services/autoscaling/tests/unit/test_modules_ec2.py +++ b/services/autoscaling/tests/unit/test_modules_ec2.py @@ -7,7 +7,8 @@ import botocore.exceptions import pytest -from aws_library.ec2.models import EC2InstanceType +from aws_library.ec2.client import SimcoreEC2API +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType from faker import Faker from fastapi import FastAPI from moto.server import ThreadedMotoServer @@ -19,11 +20,7 @@ Ec2TooManyInstancesError, ) from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings -from simcore_service_autoscaling.modules.ec2 import ( - AutoscalingEC2, - EC2InstanceData, - get_ec2_client, -) +from simcore_service_autoscaling.modules.ec2 import get_ec2_client from types_aiobotocore_ec2 import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType @@ -43,7 +40,7 @@ def app_settings( async def test_ec2_client_lifespan(ec2_settings: EC2Settings): - ec2 = await AutoscalingEC2.create(settings=ec2_settings) + ec2 = await SimcoreEC2API.create(settings=ec2_settings) assert ec2 assert ec2.client assert ec2.exit_stack @@ -100,7 +97,7 @@ async def test_ping( mocked_aws_server_envs: None, aws_allowed_ec2_instance_type_names_env: list[str], app_settings: ApplicationSettings, - autoscaling_ec2: AutoscalingEC2, + autoscaling_ec2: SimcoreEC2API, ): assert await autoscaling_ec2.ping() is True mocked_aws_server.stop() @@ -113,7 +110,7 @@ async def test_get_ec2_instance_capabilities( mocked_aws_server_envs: None, aws_allowed_ec2_instance_type_names_env: list[str], app_settings: ApplicationSettings, - autoscaling_ec2: AutoscalingEC2, + autoscaling_ec2: SimcoreEC2API, ): assert app_settings.AUTOSCALING_EC2_INSTANCES assert app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES @@ -173,7 +170,7 @@ async def test_start_aws_instance( aws_security_group_id: str, aws_ami_id: str, ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, + autoscaling_ec2: SimcoreEC2API, app_settings: ApplicationSettings, faker: Faker, fake_ec2_instance_type: EC2InstanceType, @@ -216,7 +213,7 @@ async def test_start_aws_instance_is_limited_in_number_of_instances( aws_security_group_id: str, aws_ami_id: str, ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, + autoscaling_ec2: SimcoreEC2API, app_settings: ApplicationSettings, faker: Faker, fake_ec2_instance_type: EC2InstanceType, @@ -257,7 +254,7 @@ async def test_get_instances( aws_security_group_id: str, aws_ami_id: str, ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, + autoscaling_ec2: SimcoreEC2API, app_settings: ApplicationSettings, faker: Faker, fake_ec2_instance_type: EC2InstanceType, @@ -297,7 +294,7 @@ async def test_terminate_instance( aws_security_group_id: str, aws_ami_id: str, ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, + autoscaling_ec2: SimcoreEC2API, app_settings: ApplicationSettings, faker: Faker, fake_ec2_instance_type: EC2InstanceType, @@ -331,7 +328,7 @@ async def test_terminate_instance_not_existing_raises( aws_security_group_id: str, aws_ami_id: str, ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, + autoscaling_ec2: SimcoreEC2API, app_settings: ApplicationSettings, fake_ec2_instance_data: Callable[..., EC2InstanceData], ): diff --git a/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py b/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py index 16eee35cd5d..d27bc0dd28f 100644 --- a/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py +++ b/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py @@ -8,12 +8,11 @@ from datetime import timedelta import pytest -from aws_library.ec2.models import EC2InstanceType +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType from faker import Faker from models_library.generated_models.docker_rest_api import Task from pydantic import ByteSize from pytest_mock import MockerFixture -from simcore_service_autoscaling.modules.ec2 import EC2InstanceData from simcore_service_autoscaling.utils.dynamic_scaling import ( try_assigning_task_to_pending_instances, ) From bc268ab5e6a001c4eaf81b1cea3384adae204889 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:47:07 +0100 Subject: [PATCH 30/78] removed duplicated tests --- .../autoscaling/tests/unit/test_models.py | 81 +---- .../tests/unit/test_modules_ec2.py | 322 +----------------- 2 files changed, 2 insertions(+), 401 deletions(-) diff --git a/services/autoscaling/tests/unit/test_models.py b/services/autoscaling/tests/unit/test_models.py index 2525377823b..f859ff591d6 100644 --- a/services/autoscaling/tests/unit/test_models.py +++ b/services/autoscaling/tests/unit/test_models.py @@ -8,88 +8,9 @@ import aiodocker import pytest -from aws_library.ec2.models import Resources from models_library.docker import DockerLabelKey, StandardSimcoreDockerLabels from models_library.generated_models.docker_rest_api import Service, Task -from pydantic import ByteSize, ValidationError, parse_obj_as - - -@pytest.mark.parametrize( - "a,b,a_greater_or_equal_than_b", - [ - ( - Resources(cpus=0.2, ram=ByteSize(0)), - Resources(cpus=0.1, ram=ByteSize(0)), - True, - ), - ( - Resources(cpus=0.1, ram=ByteSize(0)), - Resources(cpus=0.1, ram=ByteSize(0)), - True, - ), - ( - Resources(cpus=0.1, ram=ByteSize(1)), - Resources(cpus=0.1, ram=ByteSize(0)), - True, - ), - ( - Resources(cpus=0.05, ram=ByteSize(1)), - Resources(cpus=0.1, ram=ByteSize(0)), - False, - ), - ( - Resources(cpus=0.1, ram=ByteSize(0)), - Resources(cpus=0.1, ram=ByteSize(1)), - False, - ), - ], -) -def test_resources_ge_operator( - a: Resources, b: Resources, a_greater_or_equal_than_b: bool -): - assert (a >= b) is a_greater_or_equal_than_b - - -@pytest.mark.parametrize( - "a,b,result", - [ - ( - Resources(cpus=0, ram=ByteSize(0)), - Resources(cpus=1, ram=ByteSize(34)), - Resources(cpus=1, ram=ByteSize(34)), - ), - ( - Resources(cpus=0.1, ram=ByteSize(-1)), - Resources(cpus=1, ram=ByteSize(34)), - Resources(cpus=1.1, ram=ByteSize(33)), - ), - ], -) -def test_resources_add(a: Resources, b: Resources, result: Resources): - assert a + b == result - a += b - assert a == result - - -@pytest.mark.parametrize( - "a,b,result", - [ - ( - Resources(cpus=0, ram=ByteSize(0)), - Resources(cpus=1, ram=ByteSize(34)), - Resources.construct(cpus=-1, ram=ByteSize(-34)), - ), - ( - Resources(cpus=0.1, ram=ByteSize(-1)), - Resources(cpus=1, ram=ByteSize(34)), - Resources.construct(cpus=-0.9, ram=ByteSize(-35)), - ), - ], -) -def test_resources_sub(a: Resources, b: Resources, result: Resources): - assert a - b == result - a -= b - assert a == result +from pydantic import ValidationError, parse_obj_as async def test_get_simcore_service_docker_labels_from_task_with_missing_labels_raises( diff --git a/services/autoscaling/tests/unit/test_modules_ec2.py b/services/autoscaling/tests/unit/test_modules_ec2.py index 629e20c1e5d..0be93d22055 100644 --- a/services/autoscaling/tests/unit/test_modules_ec2.py +++ b/services/autoscaling/tests/unit/test_modules_ec2.py @@ -2,65 +2,10 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -from collections.abc import Callable -from typing import cast - -import botocore.exceptions import pytest -from aws_library.ec2.client import SimcoreEC2API -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType -from faker import Faker from fastapi import FastAPI -from moto.server import ThreadedMotoServer -from pydantic import ByteSize, parse_obj_as -from pytest_simcore.helpers.utils_envs import EnvVarsDict -from simcore_service_autoscaling.core.errors import ( - ConfigurationError, - Ec2InstanceNotFoundError, - Ec2TooManyInstancesError, -) -from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings +from simcore_service_autoscaling.core.errors import ConfigurationError from simcore_service_autoscaling.modules.ec2 import get_ec2_client -from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceTypeType - - -@pytest.fixture -def ec2_settings( - app_environment: EnvVarsDict, -) -> EC2Settings: - return EC2Settings.create_from_envs() - - -@pytest.fixture -def app_settings( - app_environment: EnvVarsDict, -) -> ApplicationSettings: - return ApplicationSettings.create_from_envs() - - -async def test_ec2_client_lifespan(ec2_settings: EC2Settings): - ec2 = await SimcoreEC2API.create(settings=ec2_settings) - assert ec2 - assert ec2.client - assert ec2.exit_stack - assert ec2.session - - await ec2.close() - - -async def test_ec2_client_raises_when_no_connection_available(ec2_client: EC2Client): - with pytest.raises( - botocore.exceptions.ClientError, match=r".+ AWS was not able to validate .+" - ): - await ec2_client.describe_account_attributes(DryRun=True) - - -async def test_ec2_client_with_mock_server( - mocked_aws_server_envs: None, ec2_client: EC2Client -): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) async def test_ec2_does_not_initialize_if_deactivated( @@ -73,268 +18,3 @@ async def test_ec2_does_not_initialize_if_deactivated( assert initialized_app.state.ec2_client is None with pytest.raises(ConfigurationError): get_ec2_client(initialized_app) - - -async def test_ec2_client_when_ec2_server_goes_up_and_down( - mocked_aws_server: ThreadedMotoServer, - mocked_aws_server_envs: None, - ec2_client: EC2Client, -): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) - mocked_aws_server.stop() - with pytest.raises(botocore.exceptions.EndpointConnectionError): - await ec2_client.describe_account_attributes(DryRun=True) - - # restart - mocked_aws_server.start() - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) - - -async def test_ping( - mocked_aws_server: ThreadedMotoServer, - mocked_aws_server_envs: None, - aws_allowed_ec2_instance_type_names_env: list[str], - app_settings: ApplicationSettings, - autoscaling_ec2: SimcoreEC2API, -): - assert await autoscaling_ec2.ping() is True - mocked_aws_server.stop() - assert await autoscaling_ec2.ping() is False - mocked_aws_server.start() - assert await autoscaling_ec2.ping() is True - - -async def test_get_ec2_instance_capabilities( - mocked_aws_server_envs: None, - aws_allowed_ec2_instance_type_names_env: list[str], - app_settings: ApplicationSettings, - autoscaling_ec2: SimcoreEC2API, -): - assert app_settings.AUTOSCALING_EC2_INSTANCES - assert app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES - instance_types = await autoscaling_ec2.get_ec2_instance_capabilities( - cast( - set[InstanceTypeType], - set(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES), - ) - ) - assert instance_types - assert len(instance_types) == len( - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES - ) - - # all the instance names are found and valid - assert all( - i.name in app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES - for i in instance_types - ) - for ( - instance_type_name - ) in app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES: - assert any(i.name == instance_type_name for i in instance_types) - - -@pytest.fixture -async def fake_ec2_instance_type( - mocked_aws_server_envs: None, - ec2_client: EC2Client, -) -> EC2InstanceType: - instance_type_name: InstanceTypeType = parse_obj_as(InstanceTypeType, "c3.8xlarge") - instance_types = await ec2_client.describe_instance_types( - InstanceTypes=[instance_type_name] - ) - assert instance_types - assert "InstanceTypes" in instance_types - assert instance_types["InstanceTypes"] - assert "MemoryInfo" in instance_types["InstanceTypes"][0] - assert "SizeInMiB" in instance_types["InstanceTypes"][0]["MemoryInfo"] - assert "VCpuInfo" in instance_types["InstanceTypes"][0] - assert "DefaultVCpus" in instance_types["InstanceTypes"][0]["VCpuInfo"] - - return EC2InstanceType( - name=instance_type_name, - cpus=instance_types["InstanceTypes"][0]["VCpuInfo"]["DefaultVCpus"], - ram=parse_obj_as( - ByteSize, - f"{instance_types['InstanceTypes'][0]['MemoryInfo']['SizeInMiB']}MiB", - ), - ) - - -async def test_start_aws_instance( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: SimcoreEC2API, - app_settings: ApplicationSettings, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, -): - assert app_settings.AUTOSCALING_EC2_ACCESS - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - # check we have that now in ec2 - all_instances = await ec2_client.describe_instances() - assert len(all_instances["Reservations"]) == 1 - running_instance = all_instances["Reservations"][0] - assert "Instances" in running_instance - assert len(running_instance["Instances"]) == 1 - running_instance = running_instance["Instances"][0] - assert "InstanceType" in running_instance - assert running_instance["InstanceType"] == fake_ec2_instance_type.name - assert "Tags" in running_instance - assert running_instance["Tags"] == [ - {"Key": key, "Value": value} for key, value in tags.items() - ] - - -async def test_start_aws_instance_is_limited_in_number_of_instances( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: SimcoreEC2API, - app_settings: ApplicationSettings, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, -): - assert app_settings.AUTOSCALING_EC2_ACCESS - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - - # create as many instances as we can - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - for _ in range(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES): - await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - # now creating one more shall fail - with pytest.raises(Ec2TooManyInstancesError): - await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - -async def test_get_instances( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: SimcoreEC2API, - app_settings: ApplicationSettings, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, -): - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - assert ( - await autoscaling_ec2.get_instances(app_settings.AUTOSCALING_EC2_INSTANCES, {}) - == [] - ) - - # create some instance - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - created_instances = await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - assert len(created_instances) == 1 - - instance_received = await autoscaling_ec2.get_instances( - app_settings.AUTOSCALING_EC2_INSTANCES, - tags=tags, - ) - assert created_instances == instance_received - - -async def test_terminate_instance( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: SimcoreEC2API, - app_settings: ApplicationSettings, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, -): - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - # create some instance - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - created_instances = await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - assert len(created_instances) == 1 - - # terminate the instance - await autoscaling_ec2.terminate_instances(created_instances) - # calling it several times is ok, the instance stays a while - await autoscaling_ec2.terminate_instances(created_instances) - - -async def test_terminate_instance_not_existing_raises( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: SimcoreEC2API, - app_settings: ApplicationSettings, - fake_ec2_instance_data: Callable[..., EC2InstanceData], -): - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - with pytest.raises(Ec2InstanceNotFoundError): - await autoscaling_ec2.terminate_instances([fake_ec2_instance_data()]) From 44fb75c30c5fa57a6273bebfe0c71645e49e5574 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:59:20 +0100 Subject: [PATCH 31/78] improve client --- .../aws-library/src/aws_library/ec2/client.py | 53 +++++++++++++------ .../aws-library/src/aws_library/ec2/errors.py | 4 ++ packages/aws-library/tests/test_ec2_client.py | 9 ++++ 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py index dd0d61164b2..bac58dc473b 100644 --- a/packages/aws-library/src/aws_library/ec2/client.py +++ b/packages/aws-library/src/aws_library/ec2/client.py @@ -14,7 +14,12 @@ from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType from types_aiobotocore_ec2.type_defs import FilterTypeDef -from .errors import EC2InstanceNotFoundError, EC2TooManyInstancesError +from .errors import ( + EC2InstanceNotFoundError, + EC2InstanceTypeInvalidError, + EC2RuntimeError, + EC2TooManyInstancesError, +) from .models import ( EC2InstanceConfig, EC2InstanceData, @@ -65,23 +70,37 @@ async def get_ec2_instance_capabilities( self, instance_type_names: set[InstanceTypeType], ) -> list[EC2InstanceType]: - """instance_type_names must be a set of unique values""" - instance_types = await self.client.describe_instance_types( - InstanceTypes=list(instance_type_names) - ) - list_instances: list[EC2InstanceType] = [] - for instance in instance_types.get("InstanceTypes", []): - with contextlib.suppress(KeyError): - list_instances.append( - EC2InstanceType( - name=instance["InstanceType"], - cpus=instance["VCpuInfo"]["DefaultVCpus"], - ram=ByteSize( - int(instance["MemoryInfo"]["SizeInMiB"]) * 1024 * 1024 - ), + """returns the ec2 instance types from a list of instance type names + NOTE: the order might differ! + Arguments: + instance_type_names -- the types to filter with + + Raises: + Ec2InstanceTypeInvalidError: some invalid types were used as filter + ClustersKeeperRuntimeError: unexpected error communicating with EC2 + + """ + try: + instance_types = await self.client.describe_instance_types( + InstanceTypes=list(instance_type_names) + ) + list_instances: list[EC2InstanceType] = [] + for instance in instance_types.get("InstanceTypes", []): + with contextlib.suppress(KeyError): + list_instances.append( + EC2InstanceType( + name=instance["InstanceType"], + cpus=instance["VCpuInfo"]["DefaultVCpus"], + ram=ByteSize( + int(instance["MemoryInfo"]["SizeInMiB"]) * 1024 * 1024 + ), + ) ) - ) - return list_instances + return list_instances + except botocore.exceptions.ClientError as exc: + if exc.response.get("Error", {}).get("Code", "") == "InvalidInstanceType": + raise EC2InstanceTypeInvalidError from exc + raise EC2RuntimeError from exc # pragma: no cover async def start_aws_instance( self, diff --git a/packages/aws-library/src/aws_library/ec2/errors.py b/packages/aws-library/src/aws_library/ec2/errors.py index 4de33f15020..2253fd17680 100644 --- a/packages/aws-library/src/aws_library/ec2/errors.py +++ b/packages/aws-library/src/aws_library/ec2/errors.py @@ -9,6 +9,10 @@ class EC2InstanceNotFoundError(EC2RuntimeError): msg_template: str = "EC2 instance was not found" +class EC2InstanceTypeInvalidError(EC2RuntimeError): + msg_template: str = "EC2 instance type invalid" + + class EC2TooManyInstancesError(EC2RuntimeError): msg_template: str = ( "The maximum amount of instances {num_instances} is already reached!" diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index 42cc3e49242..b34722b0fa1 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -22,6 +22,7 @@ from moto.server import ThreadedMotoServer from pydantic import ByteSize from settings_library.ec2 import EC2Settings +from simcore_service_autoscaling.core.errors import Ec2InstanceInvalidError from types_aiobotocore_ec2 import EC2Client from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType @@ -127,6 +128,14 @@ async def test_get_ec2_instance_capabilities_empty_list_returns_all_options( assert len(instance_types) > 50 +async def test_get_ec2_instance_capabilities_with_invalid_type_raises( + simcore_ec2_api: SimcoreEC2API, + faker: Faker, +): + with pytest.raises(Ec2InstanceInvalidError): + await simcore_ec2_api.get_ec2_instance_capabilities(set(faker.pystr())) + + @pytest.fixture(params=_ec2_allowed_types()) async def fake_ec2_instance_type( simcore_ec2_api: SimcoreEC2API, From 04d9d712ee9724b32598e4212368d0aa6b1c545d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:06:46 +0100 Subject: [PATCH 32/78] update client and models based on clusters-keeper --- .../aws-library/src/aws_library/ec2/client.py | 20 +++++++++++++++++-- .../aws-library/src/aws_library/ec2/models.py | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py index bac58dc473b..4f5e9e6c752 100644 --- a/packages/aws-library/src/aws_library/ec2/client.py +++ b/packages/aws-library/src/aws_library/ec2/client.py @@ -127,7 +127,6 @@ async def start_aws_instance( InstanceType=instance_config.type.name, InstanceInitiatedShutdownBehavior="terminate", KeyName=instance_config.key_name, - SubnetId=instance_config.subnet_id, TagSpecifications=[ { "ResourceType": "instance", @@ -138,6 +137,14 @@ async def start_aws_instance( } ], UserData=compose_user_data(instance_config.startup_script), + NetworkInterfaces=[ + { + "AssociatePublicIpAddress": True, + "DeviceIndex": 0, + "SubnetId": instance_config.subnet_id, + "Groups": instance_config.security_group_ids, + } + ], SecurityGroupIds=instance_config.security_group_ids, ) instance_ids = [i["InstanceId"] for i in instances["Instances"]] @@ -159,8 +166,12 @@ async def start_aws_instance( launch_time=instance["LaunchTime"], id=instance["InstanceId"], aws_private_dns=instance["PrivateDnsName"], + aws_public_ip=instance["PublicIpAddress"] + if "PublicIpAddress" in instance + else None, type=instance["InstanceType"], state=instance["State"]["Name"], + tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]}, resources=Resources( cpus=instance_config.type.cpus, ram=instance_config.type.ram ), @@ -177,7 +188,7 @@ async def get_instances( self, *, key_names: list[str], - tags: dict[str, str], + tags: EC2Tags, state_names: list[InstanceStateNameType] | None = None, ) -> list[EC2InstanceData]: # NOTE: be careful: Name=instance-state-name,Values=["pending", "running"] means pending OR running @@ -211,17 +222,22 @@ async def get_instances( {instance["InstanceType"]} ) assert len(ec2_instance_types) == 1 # nosec + assert "Tags" in instance # nosec all_instances.append( EC2InstanceData( launch_time=instance["LaunchTime"], id=instance["InstanceId"], aws_private_dns=instance["PrivateDnsName"], + aws_public_ip=instance["PublicIpAddress"] + if "PublicIpAddress" in instance + else None, type=instance["InstanceType"], state=instance["State"]["Name"], resources=Resources( cpus=ec2_instance_types[0].cpus, ram=ec2_instance_types[0].ram, ), + tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]}, ) ) _logger.debug( diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py index 27aa127455d..d124763cebd 100644 --- a/packages/aws-library/src/aws_library/ec2/models.py +++ b/packages/aws-library/src/aws_library/ec2/models.py @@ -57,9 +57,11 @@ class EC2InstanceData: launch_time: datetime.datetime id: str # noqa: A003 aws_private_dns: InstancePrivateDNSName + aws_public_ip: str | None type: InstanceTypeType # noqa: A003 state: InstanceStateNameType resources: Resources + tags: EC2Tags @dataclass(frozen=True) From 7afdda708479481b5baad4121904c7eb2470e6c6 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:08:55 +0100 Subject: [PATCH 33/78] re-use ec2 client from aws-library --- .../modules/ec2.py | 279 +----------------- 1 file changed, 5 insertions(+), 274 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py index 6a66adf6f00..70c52c9692f 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py @@ -1,288 +1,19 @@ -import contextlib import logging -from dataclasses import dataclass from typing import cast -import aioboto3 -import botocore.exceptions -from aiobotocore.session import ClientCreatorContext +from aws_library.ec2.client import SimcoreEC2API from fastapi import FastAPI -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType -from pydantic import ByteSize -from servicelib.logging_utils import log_context from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_random_exponential -from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType -from types_aiobotocore_ec2.type_defs import FilterTypeDef -from ..core.errors import ( - ClustersKeeperRuntimeError, - ConfigurationError, - Ec2InstanceNotFoundError, - Ec2InstanceTypeInvalidError, - Ec2NotConnectedError, - Ec2TooManyInstancesError, -) -from ..core.settings import ( - EC2ClustersKeeperSettings, - PrimaryEC2InstancesSettings, - get_application_settings, -) -from ..models import EC2InstanceData, EC2Tags -from ..utils.ec2 import compose_user_data +from ..core.errors import ConfigurationError, Ec2NotConnectedError +from ..core.settings import EC2ClustersKeeperSettings, get_application_settings logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class ClustersKeeperEC2: - client: EC2Client - session: aioboto3.Session - exit_stack: contextlib.AsyncExitStack - - @classmethod - async def create(cls, settings: EC2ClustersKeeperSettings) -> "ClustersKeeperEC2": - session = aioboto3.Session() - session_client = session.client( - "ec2", - endpoint_url=settings.EC2_CLUSTERS_KEEPER_ENDPOINT, - aws_access_key_id=settings.EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID, - aws_secret_access_key=settings.EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY, - region_name=settings.EC2_CLUSTERS_KEEPER_REGION_NAME, - ) - assert isinstance(session_client, ClientCreatorContext) # nosec - exit_stack = contextlib.AsyncExitStack() - ec2_client = cast( - EC2Client, await exit_stack.enter_async_context(session_client) - ) - return cls(ec2_client, session, exit_stack) - - async def close(self) -> None: - await self.exit_stack.aclose() - - async def ping(self) -> bool: - try: - await self.client.describe_account_attributes() - return True - except Exception: # pylint: disable=broad-except - return False - - async def get_ec2_instance_capabilities( - self, - instance_type_names: set[InstanceTypeType], - ) -> list[EC2InstanceType]: - """returns the ec2 instance types from a list of instance type names - NOTE: the order might differ! - Arguments: - instance_type_names -- the types to filter with - - Raises: - Ec2InstanceTypeInvalidError: some invalid types were used as filter - ClustersKeeperRuntimeError: unexpected error communicating with EC2 - - """ - try: - instance_types = await self.client.describe_instance_types( - InstanceTypes=list(instance_type_names) - ) - list_instances: list[EC2InstanceType] = [] - for instance in instance_types.get("InstanceTypes", []): - with contextlib.suppress(KeyError): - list_instances.append( - EC2InstanceType( - name=instance["InstanceType"], - cpus=instance["VCpuInfo"]["DefaultVCpus"], - ram=ByteSize( - int(instance["MemoryInfo"]["SizeInMiB"]) * 1024 * 1024 - ), - ) - ) - return list_instances - except botocore.exceptions.ClientError as exc: - if exc.response.get("Error", {}).get("Code", "") == "InvalidInstanceType": - raise Ec2InstanceTypeInvalidError from exc - raise ClustersKeeperRuntimeError from exc # pragma: no cover - - async def start_aws_instance( - self, - instance_settings: PrimaryEC2InstancesSettings, - instance_type: InstanceTypeType, - tags: EC2Tags, - startup_script: str, - number_of_instances: int, - ) -> list[EC2InstanceData]: - with log_context( - logger, - logging.INFO, - msg=f"launching {number_of_instances} AWS instance(s) {instance_type} with {tags=}", - ): - # first check the max amount is not already reached - current_instances = await self.get_instances( - key_names=[instance_settings.PRIMARY_EC2_INSTANCES_KEY_NAME], tags=tags - ) - if ( - len(current_instances) + number_of_instances - > instance_settings.PRIMARY_EC2_INSTANCES_MAX_INSTANCES - ): - raise Ec2TooManyInstancesError( - num_instances=instance_settings.PRIMARY_EC2_INSTANCES_MAX_INSTANCES - ) - - instances = await self.client.run_instances( - ImageId=instance_settings.PRIMARY_EC2_INSTANCES_AMI_ID, - MinCount=number_of_instances, - MaxCount=number_of_instances, - InstanceType=instance_type, - InstanceInitiatedShutdownBehavior="terminate", - KeyName=instance_settings.PRIMARY_EC2_INSTANCES_KEY_NAME, - TagSpecifications=[ - { - "ResourceType": "instance", - "Tags": [ - {"Key": tag_key, "Value": tag_value} - for tag_key, tag_value in tags.items() - ], - } - ], - UserData=compose_user_data(startup_script), - NetworkInterfaces=[ - { - "AssociatePublicIpAddress": True, - "DeviceIndex": 0, - "SubnetId": instance_settings.PRIMARY_EC2_INSTANCES_SUBNET_ID, - "Groups": instance_settings.PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS, - } - ], - ) - instance_ids = [i["InstanceId"] for i in instances["Instances"]] - logger.info( - "New instances launched: %s, waiting for them to start now...", - instance_ids, - ) - - # wait for the instance to be in a pending state - # NOTE: reference to EC2 states https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html - waiter = self.client.get_waiter("instance_exists") - await waiter.wait(InstanceIds=instance_ids) - logger.info("instances %s exists now.", instance_ids) - - # get the private IPs - instances = await self.client.describe_instances(InstanceIds=instance_ids) - instance_datas = [ - EC2InstanceData( - launch_time=instance["LaunchTime"], - id=instance["InstanceId"], - aws_private_dns=instance["PrivateDnsName"], - aws_public_ip=instance["PublicIpAddress"] - if "PublicIpAddress" in instance - else None, - type=instance["InstanceType"], - state=instance["State"]["Name"], - tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]}, - ) - for instance in instances["Reservations"][0]["Instances"] - ] - logger.info( - "%s is available, happy computing!!", - f"{instance_datas=}", - ) - return instance_datas - - async def get_instances( - self, - *, - key_names: list[str], - tags: EC2Tags, - state_names: list[InstanceStateNameType] | None = None, - ) -> list[EC2InstanceData]: - # NOTE: be careful: Name=instance-state-name,Values=["pending", "running"] means pending OR running - # NOTE2: AND is done by repeating Name=instance-state-name,Values=pending Name=instance-state-name,Values=running - if state_names is None: - state_names = ["pending", "running"] - - filters: list[FilterTypeDef] = [ - { - "Name": "key-name", - "Values": key_names, - }, - {"Name": "instance-state-name", "Values": state_names}, - ] - filters.extend( - [{"Name": f"tag:{key}", "Values": [value]} for key, value in tags.items()] - ) - - instances = await self.client.describe_instances(Filters=filters) - all_instances = [] - for reservation in instances["Reservations"]: - assert "Instances" in reservation # nosec - for instance in reservation["Instances"]: - assert "LaunchTime" in instance # nosec - assert "InstanceId" in instance # nosec - assert "PrivateDnsName" in instance # nosec - assert "InstanceType" in instance # nosec - assert "State" in instance # nosec - assert "Name" in instance["State"] # nosec - assert "Tags" in instance # nosec - all_instances.append( - EC2InstanceData( - launch_time=instance["LaunchTime"], - id=instance["InstanceId"], - aws_private_dns=instance["PrivateDnsName"], - aws_public_ip=instance["PublicIpAddress"] - if "PublicIpAddress" in instance - else None, - type=instance["InstanceType"], - state=instance["State"]["Name"], - tags={ - tag["Key"]: tag["Value"] - for tag in instance["Tags"] - if all(k in tag for k in ["Key", "Value"]) - }, - ) - ) - logger.debug( - "received: %s instances with %s", f"{len(all_instances)}", f"{state_names=}" - ) - return all_instances - - async def terminate_instances(self, instance_datas: list[EC2InstanceData]) -> None: - try: - with log_context( - logger, - logging.INFO, - msg=f"terminating instances {[i.id for i in instance_datas]}", - ): - await self.client.terminate_instances( - InstanceIds=[i.id for i in instance_datas] - ) - except botocore.exceptions.ClientError as exc: - if ( - exc.response.get("Error", {}).get("Code", "") - == "InvalidInstanceID.NotFound" - ): - raise Ec2InstanceNotFoundError from exc - raise # pragma: no cover - - async def set_instances_tags( - self, instances: list[EC2InstanceData], *, tags: EC2Tags - ) -> None: - with log_context( - logger, - logging.DEBUG, - msg=f"setting {tags=} on instances '[{[i.id for i in instances]}]'", - ): - await self.client.create_tags( - Resources=[i.id for i in instances], - Tags=[ - {"Key": tag_key, "Value": tag_value} - for tag_key, tag_value in tags.items() - ], - ) - - def setup(app: FastAPI) -> None: async def on_startup() -> None: app.state.ec2_client = None @@ -295,7 +26,7 @@ async def on_startup() -> None: logger.warning("EC2 client is de-activated in the settings") return - app.state.ec2_client = client = await ClustersKeeperEC2.create(settings) + app.state.ec2_client = client = await SimcoreEC2API.create(settings) async for attempt in AsyncRetrying( reraise=True, @@ -310,7 +41,7 @@ async def on_startup() -> None: async def on_shutdown() -> None: if app.state.ec2_client: - await cast(ClustersKeeperEC2, app.state.ec2_client).close() + await cast(SimcoreEC2API, app.state.ec2_client).close() app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) From ced2317d218d1626bfcb52b7c25be9509077457b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:10:41 +0100 Subject: [PATCH 34/78] re-use ec2 settings --- .../core/settings.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index 4b69f776900..11f02ae5d4d 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -12,6 +12,7 @@ from pydantic import Field, NonNegativeInt, PositiveInt, parse_obj_as, validator from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings +from settings_library.ec2 import EC2Settings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from settings_library.utils_logging import MixinLoggingSettings @@ -20,15 +21,6 @@ from .._meta import API_VERSION, API_VTAG, APP_NAME -class EC2ClustersKeeperSettings(BaseCustomSettings): - EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID: str - EC2_CLUSTERS_KEEPER_ENDPOINT: str | None = Field( - default=None, description="do not define if using standard AWS" - ) - EC2_CLUSTERS_KEEPER_REGION_NAME: str = "us-east-1" - EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY: str - - class WorkersEC2InstancesSettings(BaseCustomSettings): WORKERS_EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( ..., @@ -181,9 +173,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - CLUSTERS_KEEPER_EC2_ACCESS: EC2ClustersKeeperSettings | None = Field( - auto_default_from_env=True - ) + CLUSTERS_KEEPER_EC2_ACCESS: EC2Settings | None = Field(auto_default_from_env=True) CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES: PrimaryEC2InstancesSettings | None = Field( auto_default_from_env=True From 8c3c4b6506745e7586da6776a9003972927a989e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:12:44 +0100 Subject: [PATCH 35/78] re-use ec2 client --- .../src/simcore_service_clusters_keeper/modules/ec2.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py index 70c52c9692f..bd74dc85f56 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py @@ -3,13 +3,14 @@ from aws_library.ec2.client import SimcoreEC2API from fastapi import FastAPI +from settings_library.ec2 import EC2Settings from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_random_exponential from ..core.errors import ConfigurationError, Ec2NotConnectedError -from ..core.settings import EC2ClustersKeeperSettings, get_application_settings +from ..core.settings import get_application_settings logger = logging.getLogger(__name__) @@ -18,7 +19,7 @@ def setup(app: FastAPI) -> None: async def on_startup() -> None: app.state.ec2_client = None - settings: EC2ClustersKeeperSettings | None = get_application_settings( + settings: EC2Settings | None = get_application_settings( app ).CLUSTERS_KEEPER_EC2_ACCESS @@ -47,9 +48,9 @@ async def on_shutdown() -> None: app.add_event_handler("shutdown", on_shutdown) -def get_ec2_client(app: FastAPI) -> ClustersKeeperEC2: +def get_ec2_client(app: FastAPI) -> SimcoreEC2API: if not app.state.ec2_client: raise ConfigurationError( msg="EC2 client is not available. Please check the configuration." ) - return cast(ClustersKeeperEC2, app.state.ec2_client) + return cast(SimcoreEC2API, app.state.ec2_client) From 47fa781adc39612c419b46aeb0b1b7ab7da1545f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:18:28 +0100 Subject: [PATCH 36/78] add aws-lib dependency --- services/clusters-keeper/requirements/_base.in | 3 +-- services/clusters-keeper/requirements/_base.txt | 17 +++++++++++++++-- services/clusters-keeper/requirements/ci.txt | 1 + services/clusters-keeper/requirements/dev.txt | 1 + services/clusters-keeper/requirements/prod.txt | 1 + services/clusters-keeper/setup.py | 1 + 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/services/clusters-keeper/requirements/_base.in b/services/clusters-keeper/requirements/_base.in index 25eb0daeb8a..dc3b222d6db 100644 --- a/services/clusters-keeper/requirements/_base.in +++ b/services/clusters-keeper/requirements/_base.in @@ -9,14 +9,13 @@ # intra-repo required dependencies --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/aws-library/requirements/_base.in # service-library[fastapi] --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in -aioboto3 dask[distributed] fastapi packaging -types-aiobotocore[ec2] diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 2202b2fea87..26dd9a5edd3 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -9,9 +9,11 @@ aio-pika==9.3.0 # -c requirements/../../../packages/service-library/requirements/./_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aioboto3==12.0.0 - # via -r requirements/_base.in + # via -r requirements/../../../packages/aws-library/requirements/_base.in aiobotocore==2.7.0 # via aioboto3 +aiocache==0.12.2 + # via -r requirements/../../../packages/aws-library/requirements/_base.in aiodebug==2.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in @@ -26,6 +28,7 @@ aiofiles==23.2.1 # -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.8.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -74,6 +77,7 @@ botocore-stubs==1.31.80 # via types-aiobotocore certifi==2023.7.22 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -116,6 +120,7 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -143,6 +148,7 @@ httpcore==1.0.1 # via httpx httpx==0.25.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -165,6 +171,7 @@ importlib-metadata==6.8.0 # dask jinja2==3.1.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -231,6 +238,7 @@ psutil==5.9.5 # distributed pydantic==1.10.13 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -241,6 +249,7 @@ pydantic==1.10.13 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -261,6 +270,7 @@ python-dateutil==2.8.2 # botocore pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -277,6 +287,7 @@ pyyaml==6.0.1 # distributed redis==5.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -317,6 +328,7 @@ sortedcontainers==2.4.0 # distributed starlette==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -357,7 +369,7 @@ typer==0.9.0 # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in types-aiobotocore==2.7.0 - # via -r requirements/_base.in + # via -r requirements/../../../packages/aws-library/requirements/_base.in types-aiobotocore-ec2==2.7.0 # via types-aiobotocore types-awscrt==0.19.8 @@ -376,6 +388,7 @@ typing-extensions==4.8.0 # uvicorn urllib3==1.26.16 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/ci.txt b/services/clusters-keeper/requirements/ci.txt index 1a24f9bf9fe..972b44964e6 100644 --- a/services/clusters-keeper/requirements/ci.txt +++ b/services/clusters-keeper/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +../../packages/aws-library ../../packages/models-library ../../packages/pytest-simcore ../../packages/service-library[fastapi] diff --git a/services/clusters-keeper/requirements/dev.txt b/services/clusters-keeper/requirements/dev.txt index f8a28b5d52d..5324f4c79f7 100644 --- a/services/clusters-keeper/requirements/dev.txt +++ b/services/clusters-keeper/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/aws-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/clusters-keeper/requirements/prod.txt b/services/clusters-keeper/requirements/prod.txt index 8e90729c4fc..f009eafa289 100644 --- a/services/clusters-keeper/requirements/prod.txt +++ b/services/clusters-keeper/requirements/prod.txt @@ -10,6 +10,7 @@ --requirement _base.txt # installs this repo's packages +../../packages/aws-library ../../packages/models-library ../../packages/service-library[fastapi] ../../packages/settings-library diff --git a/services/clusters-keeper/setup.py b/services/clusters-keeper/setup.py index 63b4d8ed7c5..8a67e39e66d 100755 --- a/services/clusters-keeper/setup.py +++ b/services/clusters-keeper/setup.py @@ -30,6 +30,7 @@ def read_reqs(reqs_path: Path) -> set[str]: PROD_REQUIREMENTS = tuple( read_reqs(CURRENT_DIR / "requirements" / "_base.txt") | { + "simcore-aws-library", "simcore-models-library", "simcore-service-library[fastapi]", "simcore-settings-library", From cf50a0701acb9540965b1661de8a5a263cc2a20d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:25:07 +0100 Subject: [PATCH 37/78] all models are already defined in aws-lib --- .../simcore_service_clusters_keeper/models.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 services/clusters-keeper/src/simcore_service_clusters_keeper/models.py diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/models.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/models.py deleted file mode 100644 index 6217940b44d..00000000000 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/models.py +++ /dev/null @@ -1,19 +0,0 @@ -import datetime -from dataclasses import dataclass -from typing import TypeAlias - -from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType - -InstancePrivateDNSName = str -EC2Tags: TypeAlias = dict[str, str] - - -@dataclass(frozen=True) -class EC2InstanceData: - launch_time: datetime.datetime - id: str # noqa: A003 - aws_private_dns: InstancePrivateDNSName - aws_public_ip: str | None - type: InstanceTypeType # noqa: A003 - state: InstanceStateNameType - tags: EC2Tags From 7d2f230fccd8288acc39073d3d534c55a7e0cb30 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:57:17 +0100 Subject: [PATCH 38/78] missing entry --- .vscode/settings.template.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json index 3a4405e9594..8df39917def 100644 --- a/.vscode/settings.template.json +++ b/.vscode/settings.template.json @@ -32,6 +32,7 @@ "python.analysis.autoImportCompletions": true, "python.analysis.typeCheckingMode": "basic", "python.analysis.extraPaths": [ + "./packages/aws-library/src", "./packages/models-library/src", "./packages/postgres-database/src", "./packages/postgres-database/tests", From 1a886d6b94960624f8ce975524b161aecceaee91 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:57:43 +0100 Subject: [PATCH 39/78] typo --- packages/aws-library/tests/test_ec2_client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index b34722b0fa1..33e4800f084 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -10,7 +10,11 @@ import botocore.exceptions import pytest from aws_library.ec2.client import SimcoreEC2API -from aws_library.ec2.errors import EC2InstanceNotFoundError, EC2TooManyInstancesError +from aws_library.ec2.errors import ( + EC2InstanceNotFoundError, + EC2InstanceTypeInvalidError, + EC2TooManyInstancesError, +) from aws_library.ec2.models import ( EC2InstanceConfig, EC2InstanceData, @@ -22,7 +26,6 @@ from moto.server import ThreadedMotoServer from pydantic import ByteSize from settings_library.ec2 import EC2Settings -from simcore_service_autoscaling.core.errors import Ec2InstanceInvalidError from types_aiobotocore_ec2 import EC2Client from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType @@ -132,7 +135,7 @@ async def test_get_ec2_instance_capabilities_with_invalid_type_raises( simcore_ec2_api: SimcoreEC2API, faker: Faker, ): - with pytest.raises(Ec2InstanceInvalidError): + with pytest.raises(EC2InstanceTypeInvalidError): await simcore_ec2_api.get_ec2_instance_capabilities(set(faker.pystr())) From c920c842d7f5590d51eefee6acc5f71c6a3de0d9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:59:37 +0100 Subject: [PATCH 40/78] missing arguments --- packages/aws-library/tests/test_ec2_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index 33e4800f084..9a81519bbb8 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -387,9 +387,11 @@ def _creator(**overrides) -> EC2InstanceData: "launch_time": faker.date_time(tzinfo=datetime.timezone.utc), "id": faker.uuid4(), "aws_private_dns": f"ip-{faker.ipv4().replace('.', '-')}.ec2.internal", + "aws_public_ip": faker.ipv4(), "type": faker.pystr(), "state": faker.pystr(), "resources": Resources(cpus=4.0, ram=ByteSize(1024 * 1024)), + "tags": faker.pydict(allowed_types=(str,)), } | overrides ) From 0693768b6e7da812858e011cdc488d5c6473eb8d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:04:54 +0100 Subject: [PATCH 41/78] mypy --- .../modules/auto_scaling_core.py | 13 +++++++++---- .../modules/auto_scaling_mode_base.py | 4 ++-- .../modules/auto_scaling_mode_computational.py | 4 ++-- .../modules/auto_scaling_mode_dynamic.py | 4 ++-- .../src/simcore_service_autoscaling/modules/dask.py | 10 ++-------- .../src/simcore_service_autoscaling/modules/ec2.py | 2 +- .../utils/auto_scaling_core.py | 7 ++++--- .../utils/computational_scaling.py | 4 ++-- .../utils/dynamic_scaling.py | 4 ++-- .../utils/utils_docker.py | 5 ++--- 10 files changed, 28 insertions(+), 29 deletions(-) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py index 1b86e643d2d..382b7f85c6c 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py @@ -7,7 +7,12 @@ from typing import cast import arrow -from aws_library.ec2.models import EC2InstanceConfig, EC2InstanceType, Resources +from aws_library.ec2.models import ( + EC2InstanceConfig, + EC2InstanceData, + EC2InstanceType, + Resources, +) from fastapi import FastAPI from models_library.generated_models.docker_rest_api import ( Availability, @@ -25,7 +30,7 @@ Ec2TooManyInstancesError, ) from ..core.settings import ApplicationSettings, get_application_settings -from ..models import AssociatedInstance, Cluster, EC2InstanceData +from ..models import AssociatedInstance, Cluster from ..utils import utils_docker, utils_ec2 from ..utils.auto_scaling_core import ( associate_ec2_instances_with_nodes, @@ -108,7 +113,7 @@ def _node_not_ready(node: Node) -> bool: async def _cleanup_disconnected_nodes(app: FastAPI, cluster: Cluster) -> Cluster: if cluster.disconnected_nodes: await utils_docker.remove_nodes( - get_docker_client(app), cluster.disconnected_nodes + get_docker_client(app), nodes=cluster.disconnected_nodes ) return dataclasses.replace(cluster, disconnected_nodes=[]) @@ -592,7 +597,7 @@ async def _try_scale_down_cluster(app: FastAPI, cluster: Cluster) -> Cluster: await utils_docker.remove_nodes( get_docker_client(app), - [i.node for i in terminateable_instances], + nodes=[i.node for i in terminateable_instances], force=True, ) terminated_instance_ids = [i.ec2_instance.id for i in terminateable_instances] diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py index 9a2cb291dd6..08ae67d435f 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py @@ -2,14 +2,14 @@ from collections.abc import Iterable from dataclasses import dataclass -from aws_library.ec2.models import EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import DockerLabelKey from models_library.generated_models.docker_rest_api import Node as DockerNode from servicelib.logging_utils import LogLevelInt from types_aiobotocore_ec2.literals import InstanceTypeType -from ..models import AssociatedInstance, EC2InstanceData +from ..models import AssociatedInstance @dataclass diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py index 8a4f8e7e810..16dc36cb240 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py @@ -2,7 +2,7 @@ import logging from collections.abc import Iterable -from aws_library.ec2.models import EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import ( DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY, @@ -15,7 +15,7 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import AssociatedInstance, DaskTask, EC2InstanceData +from ..models import AssociatedInstance, DaskTask from ..utils import computational_scaling as utils from ..utils import utils_docker, utils_ec2 from . import dask diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py index 3d5c673fbab..b2107f2a043 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py @@ -1,6 +1,6 @@ from collections.abc import Iterable -from aws_library.ec2.models import EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import DockerLabelKey from models_library.generated_models.docker_rest_api import Node, Task @@ -8,7 +8,7 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import AssociatedInstance, EC2InstanceData +from ..models import AssociatedInstance from ..utils import dynamic_scaling as utils from ..utils import utils_docker, utils_ec2 from ..utils.rabbitmq import log_tasks_message, progress_tasks_message diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py b/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py index 07716fbbd26..a40dacbb17e 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py @@ -4,7 +4,7 @@ from typing import Any, Final, TypeAlias import distributed -from aws_library.ec2.models import Resources +from aws_library.ec2.models import EC2InstanceData, Resources from pydantic import AnyUrl, ByteSize, parse_obj_as from ..core.errors import ( @@ -12,13 +12,7 @@ DaskSchedulerNotFoundError, DaskWorkerNotFoundError, ) -from ..models import ( - AssociatedInstance, - DaskTask, - DaskTaskId, - DaskTaskResources, - EC2InstanceData, -) +from ..models import AssociatedInstance, DaskTask, DaskTaskId, DaskTaskResources from ..utils.auto_scaling_core import ( node_host_name_from_ec2_private_dns, node_ip_from_ec2_private_dns, diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py b/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py index 420d63e7762..c3464eb6f0b 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py @@ -3,13 +3,13 @@ from aws_library.ec2.client import SimcoreEC2API from fastapi import FastAPI +from settings_library.ec2 import EC2Settings from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_random_exponential from ..core.errors import ConfigurationError, Ec2NotConnectedError -from ..core.settings import EC2Settings _logger = logging.getLogger(__name__) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py index 0c0dd33cb00..701f9d1ae39 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py @@ -3,13 +3,13 @@ import re from typing import Final -from aws_library.ec2.models import EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from models_library.generated_models.docker_rest_api import Node from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.errors import Ec2InstanceInvalidError, Ec2InvalidDnsNameError from ..core.settings import ApplicationSettings -from ..models import AssociatedInstance, EC2InstanceData +from ..models import AssociatedInstance from ..modules.auto_scaling_mode_base import BaseAutoscaling from . import utils_docker @@ -25,7 +25,8 @@ def node_host_name_from_ec2_private_dns( Ec2InvalidDnsNameError: if the dns name does not follow the expected pattern """ if match := re.match(_EC2_INTERNAL_DNS_RE, ec2_instance_data.aws_private_dns): - return match.group("host_name") + host_name: str = match.group("host_name") + return host_name raise Ec2InvalidDnsNameError(aws_private_dns_name=ec2_instance_data.aws_private_dns) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py b/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py index 7425914875e..8fcefb5c8de 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py @@ -3,14 +3,14 @@ from collections.abc import Iterable from typing import Final -from aws_library.ec2.models import EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from dask_task_models_library.constants import DASK_TASK_EC2_RESOURCE_RESTRICTION_KEY from fastapi import FastAPI from servicelib.utils_formatting import timedelta_as_minute_second from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import AssociatedInstance, DaskTask, EC2InstanceData +from ..models import AssociatedInstance, DaskTask _logger = logging.getLogger(__name__) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py b/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py index 7f82b07d062..93af22f7f4e 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py @@ -2,13 +2,13 @@ import logging from collections.abc import Iterable -from aws_library.ec2.models import EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from fastapi import FastAPI from models_library.generated_models.docker_rest_api import Task from servicelib.utils_formatting import timedelta_as_minute_second from ..core.settings import get_application_settings -from ..models import AssociatedInstance, EC2InstanceData +from ..models import AssociatedInstance from . import utils_docker from .rabbitmq import log_tasks_message, progress_tasks_message diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py index 61e8c64be44..edea9cee264 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py @@ -13,7 +13,7 @@ from typing import Final, cast import yaml -from aws_library.ec2.models import Resources +from aws_library.ec2.models import EC2InstanceData, Resources from models_library.docker import ( DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY, DockerGenericTag, @@ -34,7 +34,6 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import ApplicationSettings -from ..models import EC2InstanceData from ..modules.docker import AutoscalingDocker logger = logging.getLogger(__name__) @@ -78,7 +77,7 @@ async def get_worker_nodes(docker_client: AutoscalingDocker) -> list[Node]: async def remove_nodes( - docker_client: AutoscalingDocker, nodes: list[Node], force: bool = False + docker_client: AutoscalingDocker, *, nodes: list[Node], force: bool = False ) -> list[Node]: """removes docker nodes that are in the down state (unless force is used and they will be forcibly removed)""" From 9941be9fad3f0706cc63b3b42d2ec5b77803af9d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:05:12 +0100 Subject: [PATCH 42/78] ruff --- .../modules/auto_scaling_mode_computational.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py index 16dc36cb240..f627e0398a1 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py @@ -70,6 +70,7 @@ async def try_assigning_task_to_instances( *, notify_progress: bool ) -> bool: + assert type_to_instance_map # nosec return await utils.try_assigning_task_to_instances( app, pending_task, From 6f5b6fc9d78e95d0e941b3aa71a69d4b0152a075 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:06:10 +0100 Subject: [PATCH 43/78] linter --- services/autoscaling/tests/unit/test_utils_docker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/autoscaling/tests/unit/test_utils_docker.py b/services/autoscaling/tests/unit/test_utils_docker.py index 8a0a3a7c3af..e9ee5c21c27 100644 --- a/services/autoscaling/tests/unit/test_utils_docker.py +++ b/services/autoscaling/tests/unit/test_utils_docker.py @@ -145,14 +145,14 @@ async def test_worker_nodes( async def test_remove_monitored_down_nodes_with_empty_list_does_nothing( autoscaling_docker: AutoscalingDocker, ): - assert await remove_nodes(autoscaling_docker, []) == [] + assert await remove_nodes(autoscaling_docker, nodes=[]) == [] async def test_remove_monitored_down_nodes_of_non_down_node_does_nothing( autoscaling_docker: AutoscalingDocker, host_node: Node, ): - assert await remove_nodes(autoscaling_docker, [host_node]) == [] + assert await remove_nodes(autoscaling_docker, nodes=[host_node]) == [] @pytest.fixture @@ -174,7 +174,7 @@ async def test_remove_monitored_down_nodes_of_down_node( assert fake_docker_node.Status fake_docker_node.Status.State = NodeState.down assert fake_docker_node.Status.State == NodeState.down - assert await remove_nodes(autoscaling_docker, [fake_docker_node]) == [ + assert await remove_nodes(autoscaling_docker, nodes=[fake_docker_node]) == [ fake_docker_node ] # NOTE: this is the same as calling with aiodocker.Docker() as docker: docker.nodes.remove() @@ -190,7 +190,7 @@ async def test_remove_monitored_down_node_with_unexpected_state_does_nothing( assert fake_docker_node.Status fake_docker_node.Status = None assert not fake_docker_node.Status - assert await remove_nodes(autoscaling_docker, [fake_docker_node]) == [] + assert await remove_nodes(autoscaling_docker, nodes=[fake_docker_node]) == [] async def test_pending_service_task_with_insufficient_resources_with_no_service( From 8a42bcd961d9067edb67c55d78179463ff4c0260 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:10:53 +0100 Subject: [PATCH 44/78] split fixtures --- .../src/pytest_simcore/aws_ec2_service.py | 124 ++++++++++++++++++ .../src/pytest_simcore/aws_server.py | 118 +---------------- 2 files changed, 125 insertions(+), 117 deletions(-) create mode 100644 packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py b/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py new file mode 100644 index 00000000000..2a4aa4296d2 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py @@ -0,0 +1,124 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import + +import contextlib +import random +from collections.abc import AsyncIterator +from typing import cast + +import aioboto3 +import pytest +from aiobotocore.session import ClientCreatorContext +from faker import Faker +from settings_library.ec2 import EC2Settings +from types_aiobotocore_ec2.client import EC2Client + + +@pytest.fixture +async def ec2_client( + mocked_ec2_server_settings: EC2Settings, +) -> AsyncIterator[EC2Client]: + session = aioboto3.Session() + exit_stack = contextlib.AsyncExitStack() + session_client = session.client( + "ec2", + endpoint_url=mocked_ec2_server_settings.EC2_ENDPOINT, + aws_access_key_id=mocked_ec2_server_settings.EC2_ACCESS_KEY_ID, + aws_secret_access_key=mocked_ec2_server_settings.EC2_SECRET_ACCESS_KEY, + region_name=mocked_ec2_server_settings.EC2_REGION_NAME, + ) + assert isinstance(session_client, ClientCreatorContext) + ec2_client = cast(EC2Client, await exit_stack.enter_async_context(session_client)) + + yield ec2_client + + await exit_stack.aclose() + + +@pytest.fixture(scope="session") +def vpc_cidr_block() -> str: + return "10.0.0.0/16" + + +@pytest.fixture +async def aws_vpc_id( + ec2_client: EC2Client, + vpc_cidr_block: str, +) -> AsyncIterator[str]: + vpc = await ec2_client.create_vpc( + CidrBlock=vpc_cidr_block, + ) + vpc_id = vpc["Vpc"]["VpcId"] # type: ignore + print(f"--> Created Vpc in AWS with {vpc_id=}") + yield vpc_id + + await ec2_client.delete_vpc(VpcId=vpc_id) + print(f"<-- Deleted Vpc in AWS with {vpc_id=}") + + +@pytest.fixture(scope="session") +def subnet_cidr_block() -> str: + return "10.0.1.0/24" + + +@pytest.fixture +async def aws_subnet_id( + aws_vpc_id: str, + ec2_client: EC2Client, + subnet_cidr_block: str, +) -> AsyncIterator[str]: + subnet = await ec2_client.create_subnet( + CidrBlock=subnet_cidr_block, VpcId=aws_vpc_id + ) + assert "Subnet" in subnet + assert "SubnetId" in subnet["Subnet"] + subnet_id = subnet["Subnet"]["SubnetId"] + print(f"--> Created Subnet in AWS with {subnet_id=}") + + yield subnet_id + + # all the instances in the subnet must be terminated before that works + instances_in_subnet = await ec2_client.describe_instances( + Filters=[{"Name": "subnet-id", "Values": [subnet_id]}] + ) + if instances_in_subnet["Reservations"]: + print(f"--> terminating {len(instances_in_subnet)} instances in subnet") + await ec2_client.terminate_instances( + InstanceIds=[ + instance["Instances"][0]["InstanceId"] # type: ignore + for instance in instances_in_subnet["Reservations"] + ] + ) + print(f"<-- terminated {len(instances_in_subnet)} instances in subnet") + + await ec2_client.delete_subnet(SubnetId=subnet_id) + subnets = await ec2_client.describe_subnets() + print(f"<-- Deleted Subnet in AWS with {subnet_id=}") + print(f"current {subnets=}") + + +@pytest.fixture +async def aws_security_group_id( + faker: Faker, + aws_vpc_id: str, + ec2_client: EC2Client, +) -> AsyncIterator[str]: + security_group = await ec2_client.create_security_group( + Description=faker.text(), GroupName=faker.pystr(), VpcId=aws_vpc_id + ) + security_group_id = security_group["GroupId"] + print(f"--> Created Security Group in AWS with {security_group_id=}") + yield security_group_id + await ec2_client.delete_security_group(GroupId=security_group_id) + print(f"<-- Deleted Security Group in AWS with {security_group_id=}") + + +@pytest.fixture +async def aws_ami_id( + ec2_client: EC2Client, +) -> str: + images = await ec2_client.describe_images() + image = random.choice(images["Images"]) # noqa: S311 + assert "ImageId" in image + return image["ImageId"] diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_server.py b/packages/pytest-simcore/src/pytest_simcore/aws_server.py index 667682b8288..aa6a83f95fc 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aws_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/aws_server.py @@ -2,20 +2,13 @@ # pylint: disable=unused-argument # pylint: disable=unused-import -import contextlib -import random -from collections.abc import AsyncIterator, Iterator -from typing import cast +from collections.abc import Iterator -import aioboto3 import pytest import requests -from aiobotocore.session import ClientCreatorContext from aiohttp.test_utils import unused_port -from faker import Faker from moto.server import ThreadedMotoServer from settings_library.ec2 import EC2Settings -from types_aiobotocore_ec2.client import EC2Client from .helpers.utils_envs import EnvVarsDict, setenvs_from_dict from .helpers.utils_host import get_localhost_ip @@ -75,112 +68,3 @@ def mocked_ec2_server_envs( ) -> EnvVarsDict: changed_envs: EnvVarsDict = mocked_ec2_server_settings.dict() return setenvs_from_dict(monkeypatch, changed_envs) - - -@pytest.fixture -async def ec2_client( - mocked_ec2_server_settings: EC2Settings, -) -> AsyncIterator[EC2Client]: - session = aioboto3.Session() - exit_stack = contextlib.AsyncExitStack() - session_client = session.client( - "ec2", - endpoint_url=mocked_ec2_server_settings.EC2_ENDPOINT, - aws_access_key_id=mocked_ec2_server_settings.EC2_ACCESS_KEY_ID, - aws_secret_access_key=mocked_ec2_server_settings.EC2_SECRET_ACCESS_KEY, - region_name=mocked_ec2_server_settings.EC2_REGION_NAME, - ) - assert isinstance(session_client, ClientCreatorContext) - ec2_client = cast(EC2Client, await exit_stack.enter_async_context(session_client)) - - yield ec2_client - - await exit_stack.aclose() - - -@pytest.fixture(scope="session") -def vpc_cidr_block() -> str: - return "10.0.0.0/16" - - -@pytest.fixture -async def aws_vpc_id( - ec2_client: EC2Client, - vpc_cidr_block: str, -) -> AsyncIterator[str]: - vpc = await ec2_client.create_vpc( - CidrBlock=vpc_cidr_block, - ) - vpc_id = vpc["Vpc"]["VpcId"] # type: ignore - print(f"--> Created Vpc in AWS with {vpc_id=}") - yield vpc_id - - await ec2_client.delete_vpc(VpcId=vpc_id) - print(f"<-- Deleted Vpc in AWS with {vpc_id=}") - - -@pytest.fixture(scope="session") -def subnet_cidr_block() -> str: - return "10.0.1.0/24" - - -@pytest.fixture -async def aws_subnet_id( - aws_vpc_id: str, - ec2_client: EC2Client, - subnet_cidr_block: str, -) -> AsyncIterator[str]: - subnet = await ec2_client.create_subnet( - CidrBlock=subnet_cidr_block, VpcId=aws_vpc_id - ) - assert "Subnet" in subnet - assert "SubnetId" in subnet["Subnet"] - subnet_id = subnet["Subnet"]["SubnetId"] - print(f"--> Created Subnet in AWS with {subnet_id=}") - - yield subnet_id - - # all the instances in the subnet must be terminated before that works - instances_in_subnet = await ec2_client.describe_instances( - Filters=[{"Name": "subnet-id", "Values": [subnet_id]}] - ) - if instances_in_subnet["Reservations"]: - print(f"--> terminating {len(instances_in_subnet)} instances in subnet") - await ec2_client.terminate_instances( - InstanceIds=[ - instance["Instances"][0]["InstanceId"] # type: ignore - for instance in instances_in_subnet["Reservations"] - ] - ) - print(f"<-- terminated {len(instances_in_subnet)} instances in subnet") - - await ec2_client.delete_subnet(SubnetId=subnet_id) - subnets = await ec2_client.describe_subnets() - print(f"<-- Deleted Subnet in AWS with {subnet_id=}") - print(f"current {subnets=}") - - -@pytest.fixture -async def aws_security_group_id( - faker: Faker, - aws_vpc_id: str, - ec2_client: EC2Client, -) -> AsyncIterator[str]: - security_group = await ec2_client.create_security_group( - Description=faker.text(), GroupName=faker.pystr(), VpcId=aws_vpc_id - ) - security_group_id = security_group["GroupId"] - print(f"--> Created Security Group in AWS with {security_group_id=}") - yield security_group_id - await ec2_client.delete_security_group(GroupId=security_group_id) - print(f"<-- Deleted Security Group in AWS with {security_group_id=}") - - -@pytest.fixture -async def aws_ami_id( - ec2_client: EC2Client, -) -> str: - images = await ec2_client.describe_images() - image = random.choice(images["Images"]) # noqa: S311 - assert "ImageId" in image - return image["ImageId"] From e0999db3dc504966bb09a1131f7ad7ab3fdab7e7 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:11:26 +0100 Subject: [PATCH 45/78] use ec2 fixtures --- packages/aws-library/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/aws-library/tests/conftest.py b/packages/aws-library/tests/conftest.py index aa9455bc600..9d04f458812 100644 --- a/packages/aws-library/tests/conftest.py +++ b/packages/aws-library/tests/conftest.py @@ -8,6 +8,7 @@ pytest_plugins = [ "pytest_simcore.aws_server", + "pytest_simcore.aws_ec2_service", "pytest_simcore.environment_configs", "pytest_simcore.repository_paths", "pytest_simcore.pydantic_models", From 647ad058c89d806dd4499a8ac7abb0a1812dde90 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:14:28 +0100 Subject: [PATCH 46/78] use pytest-simcore fixtures --- packages/aws-library/tests/test_ec2_client.py | 25 ----------------- .../src/pytest_simcore/aws_ec2_service.py | 27 ++++++++++++++++++- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index 9a81519bbb8..5ba224a73ce 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -3,7 +3,6 @@ # pylint:disable=redefined-outer-name -import datetime from collections.abc import AsyncIterator, Callable from typing import cast, get_args @@ -20,11 +19,9 @@ EC2InstanceData, EC2InstanceType, EC2Tags, - Resources, ) from faker import Faker from moto.server import ThreadedMotoServer -from pydantic import ByteSize from settings_library.ec2 import EC2Settings from types_aiobotocore_ec2 import EC2Client from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType @@ -378,28 +375,6 @@ async def test_terminate_instance( ) -@pytest.fixture -def fake_ec2_instance_data(faker: Faker) -> Callable[..., EC2InstanceData]: - def _creator(**overrides) -> EC2InstanceData: - return EC2InstanceData( - **( - { - "launch_time": faker.date_time(tzinfo=datetime.timezone.utc), - "id": faker.uuid4(), - "aws_private_dns": f"ip-{faker.ipv4().replace('.', '-')}.ec2.internal", - "aws_public_ip": faker.ipv4(), - "type": faker.pystr(), - "state": faker.pystr(), - "resources": Resources(cpus=4.0, ram=ByteSize(1024 * 1024)), - "tags": faker.pydict(allowed_types=(str,)), - } - | overrides - ) - ) - - return _creator - - async def test_terminate_instance_not_existing_raises( simcore_ec2_api: SimcoreEC2API, ec2_client: EC2Client, diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py b/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py index 2a4aa4296d2..641664a5292 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py @@ -3,14 +3,17 @@ # pylint: disable=unused-import import contextlib +import datetime import random -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from typing import cast import aioboto3 import pytest from aiobotocore.session import ClientCreatorContext +from aws_library.ec2.models import EC2InstanceData, Resources from faker import Faker +from pydantic import ByteSize from settings_library.ec2 import EC2Settings from types_aiobotocore_ec2.client import EC2Client @@ -122,3 +125,25 @@ async def aws_ami_id( image = random.choice(images["Images"]) # noqa: S311 assert "ImageId" in image return image["ImageId"] + + +@pytest.fixture +def fake_ec2_instance_data(faker: Faker) -> Callable[..., EC2InstanceData]: + def _creator(**overrides) -> EC2InstanceData: + return EC2InstanceData( + **( + { + "launch_time": faker.date_time(tzinfo=datetime.timezone.utc), + "id": faker.uuid4(), + "aws_private_dns": f"ip-{faker.ipv4().replace('.', '-')}.ec2.internal", + "aws_public_ip": faker.ipv4(), + "type": faker.pystr(), + "state": faker.pystr(), + "resources": Resources(cpus=4.0, ram=ByteSize(1024 * 1024)), + "tags": faker.pydict(allowed_types=(str,)), + } + | overrides + ) + ) + + return _creator From 6c7dcd2a6cd072e66c4286756b2c0856c856584a Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:19:53 +0100 Subject: [PATCH 47/78] ruff --- .../autoscaling/tests/unit/test_core_settings.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/autoscaling/tests/unit/test_core_settings.py b/services/autoscaling/tests/unit/test_core_settings.py index 45376d5963e..8576d5fdb35 100644 --- a/services/autoscaling/tests/unit/test_core_settings.py +++ b/services/autoscaling/tests/unit/test_core_settings.py @@ -56,16 +56,16 @@ def test_invalid_EC2_INSTANCES_TIME_BEFORE_TERMINATION( assert settings.AUTOSCALING_EC2_INSTANCES assert settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION assert ( - settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION - == datetime.timedelta(minutes=59) + datetime.timedelta(minutes=59) + == settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION ) monkeypatch.setenv("EC2_INSTANCES_TIME_BEFORE_TERMINATION", "-1:05:00") settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES assert ( - settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION - == datetime.timedelta(minutes=0) + datetime.timedelta(minutes=0) + == settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION ) @@ -74,7 +74,7 @@ def test_EC2_INSTANCES_PRE_PULL_IMAGES( ): settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES - assert settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES == [] + assert not settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES # passing an invalid image tag name will fail monkeypatch.setenv( @@ -97,9 +97,9 @@ def test_EC2_INSTANCES_PRE_PULL_IMAGES( ) settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES - assert settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES == [ + assert [ "nginx:latest", "itisfoundation/my-very-nice-service:latest", "simcore/services/dynamic/another-nice-one:2.4.5", "asd", - ] + ] == settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES From b269fb5987307a27449debd18838f7dcd18b1022 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:20:02 +0100 Subject: [PATCH 48/78] ruff --- .../tests/unit/test_modules_auto_scaling_computational.py | 4 +++- .../tests/unit/test_modules_auto_scaling_dynamic.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index d5393382fde..6e3397ee727 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -556,7 +556,9 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 - datetime.timedelta(seconds=1) ).isoformat() await auto_scale_cluster(app=initialized_app, auto_scaling_mode=auto_scaling_mode) - mocked_docker_remove_node.assert_called_once_with(mock.ANY, [fake_node], force=True) + mocked_docker_remove_node.assert_called_once_with( + mock.ANY, nodes=[fake_node], force=True + ) await _assert_ec2_instances( ec2_client, num_reservations=1, diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index 873d5385bc5..081f87929c6 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -676,7 +676,9 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 - datetime.timedelta(seconds=1) ).isoformat() await auto_scale_cluster(app=initialized_app, auto_scaling_mode=auto_scaling_mode) - mocked_docker_remove_node.assert_called_once_with(mock.ANY, [fake_node], force=True) + mocked_docker_remove_node.assert_called_once_with( + mock.ANY, nodes=[fake_node], force=True + ) await _assert_ec2_instances( ec2_client, num_reservations=1, From 5c44a8e2d70a7a83732addf29eeaeb766857c8d5 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:20:30 +0100 Subject: [PATCH 49/78] re-use fixtures --- services/autoscaling/tests/unit/conftest.py | 24 ++------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index 896e7b1831e..740ec392c31 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -9,7 +9,6 @@ import random from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from copy import deepcopy -from datetime import timezone from pathlib import Path from typing import Any, Final, cast from unittest import mock @@ -24,7 +23,7 @@ from aiohttp.test_utils import unused_port from asgi_lifespan import LifespanManager from aws_library.ec2.client import SimcoreEC2API -from aws_library.ec2.models import EC2InstanceData, Resources +from aws_library.ec2.models import EC2InstanceData from deepdiff import DeepDiff from faker import Faker from fakeredis.aioredis import FakeRedis @@ -58,6 +57,7 @@ from types_aiobotocore_ec2.literals import InstanceTypeType pytest_plugins = [ + "pytest_simcore.aws_ec2_service", "pytest_simcore.dask_scheduler", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", @@ -680,26 +680,6 @@ def aws_instance_private_dns() -> str: return "ip-10-23-40-12.ec2.internal" -@pytest.fixture -def fake_ec2_instance_data(faker: Faker) -> Callable[..., EC2InstanceData]: - def _creator(**overrides) -> EC2InstanceData: - return EC2InstanceData( - **( - { - "launch_time": faker.date_time(tzinfo=timezone.utc), - "id": faker.uuid4(), - "aws_private_dns": f"ip-{faker.ipv4().replace('.', '-')}.ec2.internal", - "type": faker.pystr(), - "state": faker.pystr(), - "resources": Resources(cpus=4.0, ram=ByteSize(1024 * 1024)), - } - | overrides - ) - ) - - return _creator - - @pytest.fixture def fake_localhost_ec2_instance_data( fake_ec2_instance_data: Callable[..., EC2InstanceData] From 86d8d807475da199cf1605232d94c44f9d352e96 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:30:15 +0100 Subject: [PATCH 50/78] removed aws_services.py --- .../src/pytest_simcore/aws_server.py | 16 ++++ .../src/pytest_simcore/aws_services.py | 83 ------------------- packages/simcore-sdk/tests/conftest.py | 2 +- services/agent/tests/conftest.py | 2 +- services/dask-sidecar/tests/unit/conftest.py | 2 +- 5 files changed, 19 insertions(+), 86 deletions(-) delete mode 100644 packages/pytest-simcore/src/pytest_simcore/aws_services.py diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_server.py b/packages/pytest-simcore/src/pytest_simcore/aws_server.py index aa6a83f95fc..48baff0e4c2 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aws_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/aws_server.py @@ -68,3 +68,19 @@ def mocked_ec2_server_envs( ) -> EnvVarsDict: changed_envs: EnvVarsDict = mocked_ec2_server_settings.dict() return setenvs_from_dict(monkeypatch, changed_envs) + + +@pytest.fixture +async def mocked_s3_server_envs( + mocked_aws_server: ThreadedMotoServer, + reset_aws_server_state: None, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + changed_envs = { + "S3_SECURE": "false", + "S3_ENDPOINT": f"{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 + "S3_ACCESS_KEY": "xxx", + "S3_SECRET_KEY": "xxx", + "S3_BUCKET_NAME": "pytestbucket", + } + return setenvs_from_dict(monkeypatch, changed_envs) diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_services.py b/packages/pytest-simcore/src/pytest_simcore/aws_services.py deleted file mode 100644 index 084f331aa4e..00000000000 --- a/packages/pytest-simcore/src/pytest_simcore/aws_services.py +++ /dev/null @@ -1,83 +0,0 @@ -# pylint: disable=protected-access -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable - -import asyncio -from typing import AsyncIterator, Iterator - -import pytest -from aiobotocore.session import get_session -from aiohttp.test_utils import unused_port -from moto.server import ThreadedMotoServer -from pytest_simcore.helpers.utils_host import get_localhost_ip - - -@pytest.fixture(scope="module") -def mocked_s3_server() -> Iterator[ThreadedMotoServer]: - """creates a moto-server that emulates AWS services in place - NOTE: Never use a bucket with underscores it fails!! - """ - server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) - # pylint: disable=protected-access - print(f"--> started mock S3 server on {server._ip_address}:{server._port}") - print( - f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" - ) - server.start() - yield server - server.stop() - print(f"<-- stopped mock S3 server on {server._ip_address}:{server._port}") - - -async def _clean_bucket_content(aiobotore_s3_client, bucket: str): - response = await aiobotore_s3_client.list_objects_v2(Bucket=bucket) - while response["KeyCount"] > 0: - await aiobotore_s3_client.delete_objects( - Bucket=bucket, - Delete={ - "Objects": [ - {"Key": obj["Key"]} for obj in response["Contents"] if "Key" in obj - ] - }, - ) - response = await aiobotore_s3_client.list_objects_v2(Bucket=bucket) - - -async def _remove_all_buckets(aiobotore_s3_client): - response = await aiobotore_s3_client.list_buckets() - bucket_names = [ - bucket["Name"] for bucket in response["Buckets"] if "Name" in bucket - ] - await asyncio.gather( - *(_clean_bucket_content(aiobotore_s3_client, bucket) for bucket in bucket_names) - ) - await asyncio.gather( - *(aiobotore_s3_client.delete_bucket(Bucket=bucket) for bucket in bucket_names) - ) - - -@pytest.fixture -async def mocked_s3_server_envs( - mocked_s3_server: ThreadedMotoServer, monkeypatch: pytest.MonkeyPatch -) -> AsyncIterator[None]: - monkeypatch.setenv("S3_SECURE", "false") - monkeypatch.setenv( - "S3_ENDPOINT", - f"{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # pylint: disable=protected-access - ) - monkeypatch.setenv("S3_ACCESS_KEY", "xxx") - monkeypatch.setenv("S3_SECRET_KEY", "xxx") - monkeypatch.setenv("S3_BUCKET_NAME", "pytestbucket") - - yield - - # cleanup the buckets - session = get_session() - async with session.create_client( - "s3", - endpoint_url=f"http://{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # pylint: disable=protected-access - aws_secret_access_key="xxx", - aws_access_key_id="xxx", - ) as client: - await _remove_all_buckets(client) diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index f691a844b28..62a11df75ac 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -18,7 +18,7 @@ pytest_plugins = [ - "pytest_simcore.aws_services", + "pytest_simcore.aws_server", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", "pytest_simcore.file_extra", diff --git a/services/agent/tests/conftest.py b/services/agent/tests/conftest.py index 92ec5d411c0..3228359a835 100644 --- a/services/agent/tests/conftest.py +++ b/services/agent/tests/conftest.py @@ -22,7 +22,7 @@ pytestmark = pytest.mark.asyncio pytest_plugins = [ - "pytest_simcore.aws_services", + "pytest_simcore.aws_server", "pytest_simcore.repository_paths", ] diff --git a/services/dask-sidecar/tests/unit/conftest.py b/services/dask-sidecar/tests/unit/conftest.py index 97c916cf5b1..40fa1466bc3 100644 --- a/services/dask-sidecar/tests/unit/conftest.py +++ b/services/dask-sidecar/tests/unit/conftest.py @@ -25,7 +25,7 @@ from yarl import URL pytest_plugins = [ - "pytest_simcore.aws_services", + "pytest_simcore.aws_server", "pytest_simcore.cli_runner", "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", From afdeac6080c12aa0d92d341d41973a0e42c8bf2b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:33:28 +0100 Subject: [PATCH 51/78] migration done? --- services/autoscaling/tests/unit/conftest.py | 187 +++--------------- .../autoscaling/tests/unit/test_api_health.py | 2 +- ...test_modules_auto_scaling_computational.py | 6 +- .../unit/test_modules_auto_scaling_dynamic.py | 6 +- .../unit/test_modules_auto_scaling_task.py | 2 +- 5 files changed, 32 insertions(+), 171 deletions(-) diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index 740ec392c31..fb6dca9f814 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -6,8 +6,7 @@ import dataclasses import datetime import json -import random -from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable from copy import deepcopy from pathlib import Path from typing import Any, Final, cast @@ -18,9 +17,7 @@ import httpx import psutil import pytest -import requests import simcore_service_autoscaling -from aiohttp.test_utils import unused_port from asgi_lifespan import LifespanManager from aws_library.ec2.client import SimcoreEC2API from aws_library.ec2.models import EC2InstanceData @@ -38,7 +35,6 @@ ResourceObject, Service, ) -from moto.server import ThreadedMotoServer from pydantic import ByteSize, PositiveInt, parse_obj_as from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict @@ -53,10 +49,10 @@ from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from types_aiobotocore_ec2.client import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType pytest_plugins = [ + "pytest_simcore.aws_server", "pytest_simcore.aws_ec2_service", "pytest_simcore.dask_scheduler", "pytest_simcore.docker_compose", @@ -115,6 +111,30 @@ def app_environment( return mock_env_devel_environment | envs +@pytest.fixture +def mocked_ec2_instances_envs( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + aws_security_group_id: str, + aws_subnet_id: str, + aws_ami_id: str, + aws_allowed_ec2_instance_type_names: list[InstanceTypeType], +) -> EnvVarsDict: + envs = setenvs_from_dict( + monkeypatch, + { + "EC2_INSTANCES_KEY_NAME": "osparc-pytest", + "EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps([aws_security_group_id]), + "EC2_INSTANCES_SUBNET_ID": aws_subnet_id, + "EC2_INSTANCES_AMI_ID": aws_ami_id, + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + aws_allowed_ec2_instance_type_names + ), + }, + ) + return app_environment | envs + + @pytest.fixture def disable_dynamic_service_background_task(mocker: MockerFixture) -> None: mocker.patch( @@ -461,53 +481,6 @@ async def assert_for_service_state( ) -@pytest.fixture(scope="module") -def mocked_aws_server() -> Iterator[ThreadedMotoServer]: - """creates a moto-server that emulates AWS services in place - NOTE: Never use a bucket with underscores it fails!! - """ - server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) - # pylint: disable=protected-access - print( - f"--> started mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 - ) - print( - f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" # noqa: SLF001 - ) - server.start() - yield server - server.stop() - print( - f"<-- stopped mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 - ) - - -@pytest.fixture -def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[None]: - # NOTE: reset_aws_server_state [http://docs.getmoto.org/en/latest/docs/server_mode.html#reset-api] - yield - # pylint: disable=protected-access - requests.post( - f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 - timeout=10, - ) - - -@pytest.fixture -def mocked_aws_server_envs( - app_environment: EnvVarsDict, - mocked_aws_server: ThreadedMotoServer, - reset_aws_server_state: None, - monkeypatch: pytest.MonkeyPatch, -) -> EnvVarsDict: - changed_envs: EnvVarsDict = { - "EC2_ENDPOINT": f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - "EC2_ACCESS_KEY_ID": "xxx", - "EC2_SECRET_ACCESS_KEY": "xxx", - } - return app_environment | setenvs_from_dict(monkeypatch, changed_envs) - - @pytest.fixture(scope="session") def aws_allowed_ec2_instance_type_names() -> list[InstanceTypeType]: return [ @@ -533,107 +506,6 @@ def aws_allowed_ec2_instance_type_names_env( return app_environment | setenvs_from_dict(monkeypatch, changed_envs) -@pytest.fixture(scope="session") -def vpc_cidr_block() -> str: - return "10.0.0.0/16" - - -@pytest.fixture -async def aws_vpc_id( - mocked_aws_server_envs: None, - app_environment: EnvVarsDict, - monkeypatch: pytest.MonkeyPatch, - ec2_client: EC2Client, - vpc_cidr_block: str, -) -> AsyncIterator[str]: - vpc = await ec2_client.create_vpc( - CidrBlock=vpc_cidr_block, - ) - vpc_id = vpc["Vpc"]["VpcId"] # type: ignore - print(f"--> Created Vpc in AWS with {vpc_id=}") - yield vpc_id - - await ec2_client.delete_vpc(VpcId=vpc_id) - print(f"<-- Deleted Vpc in AWS with {vpc_id=}") - - -@pytest.fixture(scope="session") -def subnet_cidr_block() -> str: - return "10.0.1.0/24" - - -@pytest.fixture -async def aws_subnet_id( - monkeypatch: pytest.MonkeyPatch, - aws_vpc_id: str, - ec2_client: EC2Client, - subnet_cidr_block: str, -) -> AsyncIterator[str]: - subnet = await ec2_client.create_subnet( - CidrBlock=subnet_cidr_block, VpcId=aws_vpc_id - ) - assert "Subnet" in subnet - assert "SubnetId" in subnet["Subnet"] - subnet_id = subnet["Subnet"]["SubnetId"] - print(f"--> Created Subnet in AWS with {subnet_id=}") - - monkeypatch.setenv("EC2_INSTANCES_SUBNET_ID", subnet_id) - yield subnet_id - - # all the instances in the subnet must be terminated before that works - instances_in_subnet = await ec2_client.describe_instances( - Filters=[{"Name": "subnet-id", "Values": [subnet_id]}] - ) - if instances_in_subnet["Reservations"]: - print(f"--> terminating {len(instances_in_subnet)} instances in subnet") - await ec2_client.terminate_instances( - InstanceIds=[ - instance["Instances"][0]["InstanceId"] # type: ignore - for instance in instances_in_subnet["Reservations"] - ] - ) - print(f"<-- terminated {len(instances_in_subnet)} instances in subnet") - - await ec2_client.delete_subnet(SubnetId=subnet_id) - subnets = await ec2_client.describe_subnets() - print(f"<-- Deleted Subnet in AWS with {subnet_id=}") - print(f"current {subnets=}") - - -@pytest.fixture -async def aws_security_group_id( - monkeypatch: pytest.MonkeyPatch, - faker: Faker, - aws_vpc_id: str, - ec2_client: EC2Client, -) -> AsyncIterator[str]: - security_group = await ec2_client.create_security_group( - Description=faker.text(), GroupName=faker.pystr(), VpcId=aws_vpc_id - ) - security_group_id = security_group["GroupId"] - print(f"--> Created Security Group in AWS with {security_group_id=}") - monkeypatch.setenv( - "EC2_INSTANCES_SECURITY_GROUP_IDS", json.dumps([security_group_id]) - ) - yield security_group_id - await ec2_client.delete_security_group(GroupId=security_group_id) - print(f"<-- Deleted Security Group in AWS with {security_group_id=}") - - -@pytest.fixture -async def aws_ami_id( - app_environment: EnvVarsDict, - mocked_aws_server_envs: None, - monkeypatch: pytest.MonkeyPatch, - ec2_client: EC2Client, -) -> str: - images = await ec2_client.describe_images() - image = random.choice(images["Images"]) # noqa: S311 - ami_id = image["ImageId"] # type: ignore - monkeypatch.setenv("EC2_INSTANCES_AMI_ID", ami_id) - return ami_id - - @pytest.fixture async def autoscaling_ec2( app_environment: EnvVarsDict, @@ -645,13 +517,6 @@ async def autoscaling_ec2( await ec2.close() -@pytest.fixture -async def ec2_client( - autoscaling_ec2: SimcoreEC2API, -) -> EC2Client: - return autoscaling_ec2.client - - @pytest.fixture def host_cpu_count() -> int: return psutil.cpu_count() diff --git a/services/autoscaling/tests/unit/test_api_health.py b/services/autoscaling/tests/unit/test_api_health.py index 1ad9bbda13c..82caa442b6b 100644 --- a/services/autoscaling/tests/unit/test_api_health.py +++ b/services/autoscaling/tests/unit/test_api_health.py @@ -20,7 +20,7 @@ def app_environment( app_environment: EnvVarsDict, enabled_rabbitmq: None, - mocked_aws_server_envs: None, + mocked_ec2_server_envs: None, mocked_redis_server: None, ) -> EnvVarsDict: return app_environment diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index 6e3397ee727..9885fca105a 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -66,14 +66,12 @@ def local_dask_scheduler_server_envs( @pytest.fixture def minimal_configuration( docker_swarm: None, + mocked_ec2_server_envs: EnvVarsDict, enabled_computational_mode: EnvVarsDict, local_dask_scheduler_server_envs: EnvVarsDict, + mocked_ec2_instances_envs: EnvVarsDict, disabled_rabbitmq: None, disable_dynamic_service_background_task: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, ) -> None: ... diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index 081f87929c6..b908b9e3d8e 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -197,13 +197,11 @@ async def drained_host_node( @pytest.fixture def minimal_configuration( docker_swarm: None, + mocked_ec2_server_envs: EnvVarsDict, enabled_dynamic_mode: EnvVarsDict, + mocked_ec2_instances_envs: EnvVarsDict, disabled_rabbitmq: None, disable_dynamic_service_background_task: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, ) -> None: ... diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py index ceb7816531b..7f3663ef7dc 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py @@ -20,7 +20,7 @@ def app_environment( app_environment: EnvVarsDict, disabled_rabbitmq: None, - mocked_aws_server_envs: None, + mocked_ec2_server_envs: None, mocked_redis_server: None, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: From de838e6f9b7d900e0769b6183c050b4b3ac14a64 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:41:27 +0100 Subject: [PATCH 52/78] ruff --- services/autoscaling/tests/unit/test_utils_ec2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/autoscaling/tests/unit/test_utils_ec2.py b/services/autoscaling/tests/unit/test_utils_ec2.py index 94ac0e16e9a..8697c8d5f11 100644 --- a/services/autoscaling/tests/unit/test_utils_ec2.py +++ b/services/autoscaling/tests/unit/test_utils_ec2.py @@ -59,10 +59,11 @@ async def test_find_best_fitting_ec2_instance_closest_instance_policy_with_resou [ ( Resources(cpus=n, ram=ByteSize(n)), - EC2InstanceType(name="fake", cpus=n, ram=ByteSize(n)), + EC2InstanceType(name="c5ad.12xlarge", cpus=n, ram=ByteSize(n)), ) for n in range(1, 30) ], + ids=str, ) async def test_find_best_fitting_ec2_instance_closest_instance_policy( needed_resources: Resources, @@ -76,7 +77,7 @@ async def test_find_best_fitting_ec2_instance_closest_instance_policy( ) SKIPPED_KEYS = ["name"] - for k in found_instance.__dict__.keys(): + for k in found_instance.__dict__: if k not in SKIPPED_KEYS: assert getattr(found_instance, k) == getattr(expected_ec2_instance, k) From c7ad24bde8b0a295f725508ca91fb96ec529c006 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:42:48 +0100 Subject: [PATCH 53/78] missing dependency --- services/agent/requirements/_base.in | 1 + services/agent/requirements/_base.txt | 2 ++ services/agent/requirements/_test.txt | 8 ++++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/services/agent/requirements/_base.in b/services/agent/requirements/_base.in index c195fb17f5c..ea2c803f81b 100644 --- a/services/agent/requirements/_base.in +++ b/services/agent/requirements/_base.in @@ -14,4 +14,5 @@ aiodocker fastapi packaging pydantic +python-dotenv uvicorn diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index 2619c197119..c3d0645fe49 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -115,6 +115,8 @@ pyrsistent==0.19.2 # via jsonschema python-dateutil==2.8.2 # via arrow +python-dotenv==1.0.0 + # via -r requirements/_base.in pyyaml==6.0.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt diff --git a/services/agent/requirements/_test.txt b/services/agent/requirements/_test.txt index af900eaf1d6..ac1398a936d 100644 --- a/services/agent/requirements/_test.txt +++ b/services/agent/requirements/_test.txt @@ -9,7 +9,9 @@ aioboto3==9.6.0 # -c requirements/./constraints.txt # -r requirements/_test.in aiobotocore==2.3.0 - # via aioboto3 + # via + # aioboto3 + # aiobotocore aiohttp==3.8.5 # via # -c requirements/../../../requirements/constraints.txt @@ -229,7 +231,9 @@ python-dateutil==2.8.2 # faker # moto python-jose==3.3.0 - # via moto + # via + # moto + # python-jose pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt From 7813d5f9d21b927e838fa24270d7c942127150a8 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:45:39 +0100 Subject: [PATCH 54/78] cleanup --- .../test_modules_auto_scaling_computational.py | 14 -------------- .../unit/test_modules_auto_scaling_dynamic.py | 9 --------- 2 files changed, 23 deletions(-) diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index 9885fca105a..7d4f1755ff3 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -88,11 +88,6 @@ def dask_workers_config() -> dict[str, Any]: } -@pytest.fixture -def empty_cluster(cluster: Callable[..., Cluster]) -> Cluster: - return cluster() - - async def _assert_ec2_instances( ec2_client: EC2Client, *, @@ -190,15 +185,6 @@ def mock_find_node_with_name( ) -@pytest.fixture -def mock_cluster_used_resources(mocker: MockerFixture) -> mock.Mock: - return mocker.patch( - "simcore_service_autoscaling.modules.auto_scaling_core.utils_docker.compute_cluster_used_resources", - autospec=True, - return_value=Resources.create_as_empty(), - ) - - @pytest.fixture def mock_compute_node_used_resources(mocker: MockerFixture) -> mock.Mock: return mocker.patch( diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index b908b9e3d8e..3f4caf35d1d 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -114,15 +114,6 @@ def mock_remove_nodes(mocker: MockerFixture) -> mock.Mock: ) -@pytest.fixture -def mock_cluster_used_resources(mocker: MockerFixture) -> mock.Mock: - return mocker.patch( - "simcore_service_autoscaling.modules.auto_scaling_core.utils_docker.compute_cluster_used_resources", - autospec=True, - return_value=Resources.create_as_empty(), - ) - - @pytest.fixture def mock_compute_node_used_resources(mocker: MockerFixture) -> mock.Mock: return mocker.patch( From c4c94fc56f269ed3914c829cdfccae5c60bc378b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:10:01 +0100 Subject: [PATCH 55/78] ruff --- .../modules/clusters_management_core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py index 81fb41ff967..ce02d45166c 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py @@ -2,12 +2,12 @@ import logging import arrow +from aws_library.ec2.models import EC2InstanceData from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID from ..core.settings import get_application_settings -from ..models import EC2InstanceData from ..modules.clusters import ( delete_clusters, get_all_clusters, @@ -25,7 +25,8 @@ def _get_instance_last_heartbeat(instance: EC2InstanceData) -> datetime.datetime if last_heartbeat := instance.tags.get(HEARTBEAT_TAG_KEY, None): last_heartbeat_time: datetime.datetime = arrow.get(last_heartbeat).datetime return last_heartbeat_time - return instance.launch_time + launch_time: datetime.datetime = instance.launch_time + return launch_time async def _find_terminateable_instances( From b3efa5992071b838b23ce50966c8d47ae62bb164 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:10:14 +0100 Subject: [PATCH 56/78] migrate --- .../modules/clusters.py | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py index 895db73fdc8..95d77879149 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py @@ -2,6 +2,7 @@ import logging from typing import cast +from aws_library.ec2.models import EC2InstanceConfig, EC2InstanceData from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID @@ -10,7 +11,6 @@ from ..core.errors import Ec2InstanceNotFoundError from ..core.settings import get_application_settings -from ..models import EC2InstanceData from ..utils.clusters import create_startup_script from ..utils.ec2 import ( HEARTBEAT_TAG_KEY, @@ -30,16 +30,20 @@ async def create_cluster( ec2_client = get_ec2_client(app) app_settings = get_application_settings(app) assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES # nosec - return await ec2_client.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - instance_type=cast( - InstanceTypeType, - next( - iter( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ) - ), - ), + ec2_instance_type = await ec2_client.get_ec2_instance_capabilities( + instance_type_names={ + cast( + InstanceTypeType, + next( + iter( + app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES + ) + ), + ) + } + ) + instance_config = EC2InstanceConfig( + type=ec2_instance_type, tags=creation_ec2_tags(app_settings, user_id=user_id, wallet_id=wallet_id), startup_script=create_startup_script( app_settings, @@ -47,20 +51,30 @@ async def create_cluster( app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False ), ), + ami_id=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_AMI_ID, + key_name=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME, + security_group_ids=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS, + subnet_id=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_SUBNET_ID, + ) + new_ec2_instance_data: list[EC2InstanceData] = await ec2_client.start_aws_instance( + instance_config, number_of_instances=1, + max_number_of_instances=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_MAX_INSTANCES, ) + return new_ec2_instance_data async def get_all_clusters(app: FastAPI) -> list[EC2InstanceData]: app_settings = get_application_settings(app) assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES # nosec - return await get_ec2_client(app).get_instances( + ec2_instance_data: list[EC2InstanceData] = await get_ec2_client(app).get_instances( key_names=[ app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME ], tags=all_created_ec2_instances_filter(app_settings), state_names=["running"], ) + return ec2_instance_data async def get_cluster( @@ -86,7 +100,7 @@ async def get_cluster_workers( ) -> list[EC2InstanceData]: app_settings = get_application_settings(app) assert app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES # nosec - return await get_ec2_client(app).get_instances( + ec2_instance_data: list[EC2InstanceData] = await get_ec2_client(app).get_instances( key_names=[ app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_KEY_NAME ], @@ -94,6 +108,7 @@ async def get_cluster_workers( "Name": f"{get_cluster_name(app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False)}*" }, ) + return ec2_instance_data async def cluster_heartbeat( From 4059e89fc0995f8f3f81b3351979c1a29606dacb Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:10:21 +0100 Subject: [PATCH 57/78] ruff --- .../src/simcore_service_clusters_keeper/rpc/ec2_instances.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py index 758564cf9dc..fb8b0bb8c2e 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py @@ -11,4 +11,7 @@ async def get_instance_type_details( app: FastAPI, *, instance_type_names: set[str] ) -> list[EC2InstanceType]: - return await get_ec2_client(app).get_ec2_instance_capabilities(instance_type_names) + instance_capabilities: list[EC2InstanceType] = await get_ec2_client( + app + ).get_ec2_instance_capabilities(instance_type_names) + return instance_capabilities From b3efb81a7b67b8453af32621a0d8dff7882caf85 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:10:38 +0100 Subject: [PATCH 58/78] migrating --- .../clusters-keeper/tests/unit/conftest.py | 27 +------------------ .../tests/unit/test_utils_clusters.py | 2 +- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index a6486b52bff..f9247057ca9 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -6,7 +6,6 @@ import json import random from collections.abc import AsyncIterator, Awaitable, Callable, Iterator -from datetime import timezone from pathlib import Path from typing import Any @@ -35,10 +34,7 @@ ApplicationSettings, EC2ClustersKeeperSettings, ) -from simcore_service_clusters_keeper.modules.ec2 import ( - ClustersKeeperEC2, - EC2InstanceData, -) +from simcore_service_clusters_keeper.modules.ec2 import ClustersKeeperEC2 from simcore_service_clusters_keeper.utils.ec2 import get_cluster_name from types_aiobotocore_ec2.client import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType @@ -361,27 +357,6 @@ async def ec2_client( return clusters_keeper_ec2.client -@pytest.fixture -def fake_ec2_instance_data(faker: Faker) -> Callable[..., EC2InstanceData]: - def _creator(**overrides) -> EC2InstanceData: - return EC2InstanceData( - **( - { - "launch_time": faker.date_time(tzinfo=timezone.utc), - "id": faker.uuid4(), - "aws_private_dns": faker.name(), - "aws_public_ip": faker.ipv4_public(), - "type": faker.pystr(), - "state": faker.pystr(), - "tags": faker.pydict(allowed_types=(str,)), - } - | overrides - ) - ) - - return _creator - - @pytest.fixture async def mocked_redis_server(mocker: MockerFixture) -> None: mock_redis = FakeRedis() diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py index 495cdbb3d5f..b796970e468 100644 --- a/services/clusters-keeper/tests/unit/test_utils_clusters.py +++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py @@ -7,11 +7,11 @@ from typing import Any import pytest +from aws_library.ec2.models import EC2InstanceData from faker import Faker from models_library.api_schemas_clusters_keeper.clusters import ClusterState from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_clusters_keeper.core.settings import ApplicationSettings -from simcore_service_clusters_keeper.models import EC2InstanceData from simcore_service_clusters_keeper.utils.clusters import ( create_cluster_from_ec2_instance, create_startup_script, From e577020100c328d37c1851c274badc15830700e0 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 18:21:28 +0100 Subject: [PATCH 59/78] fix fixture name --- services/agent/tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/agent/tests/conftest.py b/services/agent/tests/conftest.py index 3228359a835..97da425621a 100644 --- a/services/agent/tests/conftest.py +++ b/services/agent/tests/conftest.py @@ -188,9 +188,9 @@ def caplog_info_debug(caplog: LogCaptureFixture) -> Iterable[LogCaptureFixture]: @pytest.fixture(scope="module") -def mocked_s3_server_url(mocked_s3_server: ThreadedMotoServer) -> HttpUrl: +def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrl: # pylint: disable=protected-access return parse_obj_as( HttpUrl, - f"http://{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # noqa: SLF001 + f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # noqa: SLF001 ) From 9243b59378fe358eba2f925cb177361a791fb119 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 18:22:46 +0100 Subject: [PATCH 60/78] add flaky --- .../dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py b/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py index 0d5e6f2ef17..d42108bbdc2 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py @@ -337,6 +337,7 @@ async def test_does_not_trigger_on_attribute_change( assert mock_event_filter_upload_trigger.call_count == 1 +@pytest.mark.flaky(max_runs=3) async def test_port_key_sequential_event_generation( mock_long_running_upload_outputs: AsyncMock, mounted_volumes: MountedVolumes, From 5853e0b48c0b1e7267dd9d9e6a08b6f24003cac9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 18:25:19 +0100 Subject: [PATCH 61/78] fixed imports --- .../src/simcore_service_clusters_keeper/rpc/clusters.py | 2 +- .../src/simcore_service_clusters_keeper/utils/clusters.py | 2 +- .../src/simcore_service_clusters_keeper/utils/dask.py | 3 +-- services/clusters-keeper/tests/unit/test_modules_clusters.py | 2 +- .../tests/unit/test_modules_clusters_management_core.py | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/clusters.py index babb123e210..ce6ab0b54c1 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/clusters.py @@ -1,3 +1,4 @@ +from aws_library.ec2.models import EC2InstanceData from fastapi import FastAPI from models_library.api_schemas_clusters_keeper.clusters import OnDemandCluster from models_library.users import UserID @@ -5,7 +6,6 @@ from servicelib.rabbitmq import RPCRouter from ..core.errors import Ec2InstanceNotFoundError -from ..models import EC2InstanceData from ..modules import clusters from ..modules.dask import ping_scheduler from ..modules.redis import get_redis_client diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py index f139a36dc19..444a3ef1a26 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py @@ -3,6 +3,7 @@ import functools from typing import Any, Final +from aws_library.ec2.models import EC2InstanceData from models_library.api_schemas_clusters_keeper.clusters import ( ClusterState, OnDemandCluster, @@ -14,7 +15,6 @@ from .._meta import PACKAGE_DATA_FOLDER from ..core.settings import ApplicationSettings -from ..models import EC2InstanceData from .dask import get_scheduler_url _DOCKER_COMPOSE_FILE_NAME: Final[str] = "docker-compose.yml" diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py index 3db6435c820..ac5bb9400de 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py @@ -1,7 +1,6 @@ +from aws_library.ec2.models import EC2InstanceData from pydantic import AnyUrl, parse_obj_as -from ..models import EC2InstanceData - def get_scheduler_url(ec2_instance: EC2InstanceData) -> AnyUrl: url: AnyUrl = parse_obj_as(AnyUrl, f"tcp://{ec2_instance.aws_public_ip}:8786") diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters.py b/services/clusters-keeper/tests/unit/test_modules_clusters.py index 945bb781fde..d24b9dd4184 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters.py @@ -9,6 +9,7 @@ import arrow import pytest +from aws_library.ec2.models import EC2InstanceData from faker import Faker from fastapi import FastAPI from models_library.users import UserID @@ -20,7 +21,6 @@ ApplicationSettings, get_application_settings, ) -from simcore_service_clusters_keeper.models import EC2InstanceData from simcore_service_clusters_keeper.modules.clusters import ( cluster_heartbeat, create_cluster, diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py b/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py index 4801c0233be..51b68670640 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py @@ -9,6 +9,7 @@ import pytest from attr import dataclass +from aws_library.ec2.models import EC2InstanceData from faker import Faker from fastapi import FastAPI from models_library.users import UserID @@ -16,7 +17,6 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict -from simcore_service_clusters_keeper.models import EC2InstanceData from simcore_service_clusters_keeper.modules.clusters import ( cluster_heartbeat, create_cluster, From 1ec65d59caaf36d1289358e775ad88f12fdc3798 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 18:46:58 +0100 Subject: [PATCH 62/78] fix imports --- .../src/simcore_service_clusters_keeper/utils/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py index 7b99b23f6c3..5774f42e0e4 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py @@ -1,12 +1,12 @@ from textwrap import dedent from typing import Final +from aws_library.ec2.models import EC2Tags from models_library.users import UserID from models_library.wallets import WalletID from .._meta import VERSION from ..core.settings import ApplicationSettings -from ..models import EC2Tags _APPLICATION_TAG_KEY: Final[str] = "io.simcore.clusters-keeper" _APPLICATION_VERSION_TAG: Final[EC2Tags] = { From 85800875cb7a1bb40d4b2572577be34c4eef5808 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 18:47:25 +0100 Subject: [PATCH 63/78] correct fixture name --- .../tests/unit/test_node_ports_common_file_io_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py b/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py index 7c38863c096..cf1f0a59d92 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py @@ -195,7 +195,7 @@ def file_id(faker: Faker) -> str: @pytest.fixture async def create_upload_links( - mocked_s3_server: ThreadedMotoServer, + mocked_aws_server: ThreadedMotoServer, aiobotocore_s3_client: AioBaseClient, faker: Faker, bucket: str, From f00ac46e9f441edd732f872d6e0f268804088f96 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 18:47:43 +0100 Subject: [PATCH 64/78] remove duplicate fixtures --- .../clusters-keeper/tests/unit/conftest.py | 103 ------------------ 1 file changed, 103 deletions(-) diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index f9247057ca9..444eeb747f0 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -4,7 +4,6 @@ import importlib.resources import json -import random from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from pathlib import Path from typing import Any @@ -237,108 +236,6 @@ def aws_allowed_ec2_instance_type_names_env( return app_environment | setenvs_from_dict(monkeypatch, changed_envs) -@pytest.fixture(scope="session") -def vpc_cidr_block() -> str: - return "10.0.0.0/16" - - -@pytest.fixture -async def aws_vpc_id( - mocked_aws_server_envs: None, - app_environment: EnvVarsDict, - monkeypatch: pytest.MonkeyPatch, - ec2_client: EC2Client, - vpc_cidr_block: str, -) -> AsyncIterator[str]: - vpc = await ec2_client.create_vpc( - CidrBlock=vpc_cidr_block, - ) - vpc_id = vpc["Vpc"]["VpcId"] # type: ignore - print(f"--> Created Vpc in AWS with {vpc_id=}") - yield vpc_id - - await ec2_client.delete_vpc(VpcId=vpc_id) - print(f"<-- Deleted Vpc in AWS with {vpc_id=}") - - -@pytest.fixture(scope="session") -def subnet_cidr_block() -> str: - return "10.0.1.0/24" - - -@pytest.fixture -async def aws_subnet_id( - monkeypatch: pytest.MonkeyPatch, - aws_vpc_id: str, - ec2_client: EC2Client, - subnet_cidr_block: str, -) -> AsyncIterator[str]: - subnet = await ec2_client.create_subnet( - CidrBlock=subnet_cidr_block, VpcId=aws_vpc_id - ) - assert "Subnet" in subnet - assert "SubnetId" in subnet["Subnet"] - subnet_id = subnet["Subnet"]["SubnetId"] - print(f"--> Created Subnet in AWS with {subnet_id=}") - - monkeypatch.setenv("PRIMARY_EC2_INSTANCES_SUBNET_ID", subnet_id) - yield subnet_id - - # all the instances in the subnet must be terminated before that works - instances_in_subnet = await ec2_client.describe_instances( - Filters=[{"Name": "subnet-id", "Values": [subnet_id]}] - ) - if instances_in_subnet["Reservations"]: - print(f"--> terminating {len(instances_in_subnet)} instances in subnet") - await ec2_client.terminate_instances( - InstanceIds=[ - instance["Instances"][0]["InstanceId"] # type: ignore - for instance in instances_in_subnet["Reservations"] - ] - ) - print(f"<-- terminated {len(instances_in_subnet)} instances in subnet") - - await ec2_client.delete_subnet(SubnetId=subnet_id) - subnets = await ec2_client.describe_subnets() - print(f"<-- Deleted Subnet in AWS with {subnet_id=}") - print(f"current {subnets=}") - - -@pytest.fixture -async def aws_security_group_id( - monkeypatch: pytest.MonkeyPatch, - faker: Faker, - aws_vpc_id: str, - ec2_client: EC2Client, -) -> AsyncIterator[str]: - security_group = await ec2_client.create_security_group( - Description=faker.text(), GroupName=faker.pystr(), VpcId=aws_vpc_id - ) - security_group_id = security_group["GroupId"] - print(f"--> Created Security Group in AWS with {security_group_id=}") - monkeypatch.setenv( - "PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS", - json.dumps([security_group_id]), - ) - yield security_group_id - await ec2_client.delete_security_group(GroupId=security_group_id) - print(f"<-- Deleted Security Group in AWS with {security_group_id=}") - - -@pytest.fixture -async def aws_ami_id( - app_environment: EnvVarsDict, - mocked_aws_server_envs: None, - monkeypatch: pytest.MonkeyPatch, - ec2_client: EC2Client, -) -> str: - images = await ec2_client.describe_images() - image = random.choice(images["Images"]) # noqa S311 - ami_id = image["ImageId"] # type: ignore - monkeypatch.setenv("PRIMARY_EC2_INSTANCES_AMI_ID", ami_id) - return ami_id - - @pytest.fixture async def clusters_keeper_ec2( app_environment: EnvVarsDict, From cb6adb595abf02a0705ab0597c03184e72010763 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:31:45 +0100 Subject: [PATCH 65/78] added dependencies --- packages/aws-library/requirements/_base.in | 3 + packages/aws-library/requirements/_base.txt | 124 ++++++++++++++++++- packages/aws-library/requirements/_test.txt | 13 +- packages/aws-library/requirements/_tools.txt | 2 + 4 files changed, 134 insertions(+), 8 deletions(-) diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in index efdc1a7b6e2..a0fd39eb41f 100644 --- a/packages/aws-library/requirements/_base.in +++ b/packages/aws-library/requirements/_base.in @@ -2,6 +2,9 @@ # Specifies third-party dependencies for 'aws-library' # --constraint ../../../requirements/constraints.txt +--requirement ../../../packages/models-library/requirements/_base.in +--requirement ../../../packages/service-library/requirements/_base.in +--requirement ../../../packages/settings-library/requirements/_base.in aioboto3 aiocache diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index 59679efba0b..2e125c14ad8 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -4,6 +4,8 @@ # # pip-compile --output-file=requirements/_base.txt --strip-extras requirements/_base.in # +aio-pika==9.3.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in aioboto3==12.0.0 # via -r requirements/_base.in aiobotocore==2.7.0 @@ -12,18 +14,42 @@ aiobotocore==2.7.0 # aiobotocore aiocache==0.12.2 # via -r requirements/_base.in +aiodebug==2.3.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiodocker==0.21.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiofiles==23.2.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.8.6 # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiobotocore + # aiodocker aioitertools==0.11.0 # via aiobotocore +aiormq==6.7.7 + # via aio-pika aiosignal==1.3.1 # via aiohttp +arrow==1.3.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in async-timeout==4.0.3 - # via aiohttp + # via + # aiohttp + # redis attrs==23.1.0 - # via aiohttp + # via + # aiohttp + # jsonschema + # referencing boto3==1.28.64 # via aiobotocore botocore==1.31.64 @@ -31,10 +57,12 @@ botocore==1.31.64 # aiobotocore # boto3 # s3transfer -botocore-stubs==1.31.84 +botocore-stubs==1.31.85 # via types-aiobotocore charset-normalizer==3.3.2 # via aiohttp +click==8.1.7 + # via typer dnspython==2.4.2 # via email-validator email-validator==2.1.0.post1 @@ -51,36 +79,122 @@ jmespath==1.0.1 # via # boto3 # botocore +jsonschema==4.19.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +jsonschema-specifications==2023.7.1 + # via jsonschema +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py multidict==6.0.4 # via # aiohttp # yarl +orjson==3.9.10 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pamqp==3.2.1 + # via aiormq pydantic==1.10.13 # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in +pygments==2.16.1 + # via rich +pyinstrument==4.6.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.8.2 - # via botocore + # via + # arrow + # botocore +pyyaml==6.0.1 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in +redis==5.0.1 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in +referencing==0.29.3 + # via + # -c requirements/../../../packages/service-library/requirements/./constraints.txt + # jsonschema + # jsonschema-specifications +rich==13.6.0 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +rpds-py==0.12.0 + # via + # jsonschema + # referencing s3transfer==0.7.0 # via boto3 six==1.16.0 # via python-dateutil +tenacity==8.2.3 + # via -r requirements/../../../packages/service-library/requirements/_base.in +toolz==0.12.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +tqdm==4.66.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +typer==0.9.0 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in types-aiobotocore==2.7.0 # via -r requirements/_base.in types-aiobotocore-ec2==2.7.0 # via types-aiobotocore types-awscrt==0.19.10 # via botocore-stubs +types-python-dateutil==2.8.19.14 + # via arrow typing-extensions==4.8.0 # via + # aiodebug + # aiodocker # pydantic + # typer # types-aiobotocore # types-aiobotocore-ec2 urllib3==2.0.7 # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # botocore wrapt==1.16.0 # via aiobotocore yarl==1.9.2 - # via aiohttp + # via + # aio-pika + # aiohttp + # aiormq diff --git a/packages/aws-library/requirements/_test.txt b/packages/aws-library/requirements/_test.txt index d7be7112cd6..a588a55a40a 100644 --- a/packages/aws-library/requirements/_test.txt +++ b/packages/aws-library/requirements/_test.txt @@ -49,7 +49,7 @@ certifi==2023.7.22 # requests cffi==1.16.0 # via cryptography -cfn-lint==0.83.1 +cfn-lint==0.83.2 # via moto charset-normalizer==3.3.2 # via @@ -57,7 +57,9 @@ charset-normalizer==3.3.2 # aiohttp # requests click==8.1.7 - # via flask + # via + # -c requirements/_base.txt + # flask coverage==7.3.2 # via # -r requirements/_test.in @@ -125,6 +127,7 @@ jsonpointer==2.4 # via jsonpatch jsonschema==4.19.2 # via + # -c requirements/_base.txt # aws-sam-translator # cfn-lint # openapi-schema-validator @@ -133,6 +136,7 @@ jsonschema-path==0.3.1 # via openapi-spec-validator jsonschema-specifications==2023.7.1 # via + # -c requirements/_base.txt # jsonschema # openapi-schema-validator junit-xml==1.9 @@ -231,13 +235,15 @@ python-jose==3.3.0 pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt # -r requirements/_test.in # cfn-lint # jsonschema-path # moto # responses -referencing==0.30.2 +referencing==0.29.3 # via + # -c requirements/_base.txt # jsonschema # jsonschema-path # jsonschema-specifications @@ -255,6 +261,7 @@ rfc3339-validator==0.1.4 # via openapi-schema-validator rpds-py==0.12.0 # via + # -c requirements/_base.txt # jsonschema # referencing rsa==4.9 diff --git a/packages/aws-library/requirements/_tools.txt b/packages/aws-library/requirements/_tools.txt index 543d0ade5fd..944a95eebf6 100644 --- a/packages/aws-library/requirements/_tools.txt +++ b/packages/aws-library/requirements/_tools.txt @@ -16,6 +16,7 @@ cfgv==3.4.0 # via pre-commit click==8.1.7 # via + # -c requirements/_base.txt # -c requirements/_test.txt # black # pip-tools @@ -60,6 +61,7 @@ pyproject-hooks==1.0.0 pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit ruff==0.1.5 From 5366926895e93b7f90339fc97c6ce7526f6085f8 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:32:06 +0100 Subject: [PATCH 66/78] type --- services/autoscaling/tests/unit/test_api_health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/autoscaling/tests/unit/test_api_health.py b/services/autoscaling/tests/unit/test_api_health.py index 82caa442b6b..9ac0dccac01 100644 --- a/services/autoscaling/tests/unit/test_api_health.py +++ b/services/autoscaling/tests/unit/test_api_health.py @@ -20,7 +20,7 @@ def app_environment( app_environment: EnvVarsDict, enabled_rabbitmq: None, - mocked_ec2_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, mocked_redis_server: None, ) -> EnvVarsDict: return app_environment From dcf380c8bceaf2dfd5e864291e23f2d55a2b7f3a Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:32:13 +0100 Subject: [PATCH 67/78] dependencies --- .../clusters-keeper/requirements/_base.txt | 81 ++++++++++++++++++- .../clusters-keeper/requirements/_test.txt | 4 +- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 26dd9a5edd3..3436e8e8028 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -7,27 +7,38 @@ aio-pika==9.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aioboto3==12.0.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in aiobotocore==2.7.0 - # via aioboto3 + # via + # aioboto3 + # aiobotocore aiocache==0.12.2 # via -r requirements/../../../packages/aws-library/requirements/_base.in aiodebug==2.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiodocker==0.21.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiofiles==23.2.1 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.8.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -53,6 +64,9 @@ anyio==4.0.0 arrow==1.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -77,6 +91,11 @@ botocore-stubs==1.31.80 # via types-aiobotocore certifi==2023.7.22 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -120,6 +139,11 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -148,6 +172,11 @@ httpcore==1.0.1 # via httpx httpx==0.25.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -171,6 +200,11 @@ importlib-metadata==6.8.0 # dask jinja2==3.1.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -189,6 +223,8 @@ jmespath==1.0.1 # botocore jsonschema==4.19.2 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -217,6 +253,8 @@ multidict==6.0.4 # yarl orjson==3.9.10 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -238,6 +276,11 @@ psutil==5.9.5 # distributed pydantic==1.10.13 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -249,6 +292,11 @@ pydantic==1.10.13 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -263,6 +311,7 @@ pygments==2.16.1 pyinstrument==4.6.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.8.2 # via @@ -270,6 +319,11 @@ python-dateutil==2.8.2 # botocore pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -282,11 +336,17 @@ pyyaml==6.0.1 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed redis==5.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -298,15 +358,19 @@ redis==5.0.1 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/./constraints.txt # -c requirements/../../../packages/service-library/requirements/././constraints.txt # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications rich==13.6.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -328,6 +392,11 @@ sortedcontainers==2.4.0 # distributed starlette==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -346,11 +415,13 @@ tblib==2.0.0 tenacity==8.2.3 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in toolz==0.12.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed @@ -362,9 +433,12 @@ tornado==6.3.3 tqdm==4.66.1 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in typer==0.9.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -388,6 +462,11 @@ typing-extensions==4.8.0 # uvicorn urllib3==1.26.16 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index 85dd0cf4dcc..411492a8d69 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -264,7 +264,9 @@ python-dateutil==2.8.2 python-dotenv==1.0.0 # via -r requirements/_test.in python-jose==3.3.0 - # via moto + # via + # moto + # python-jose pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt From 5667173b6a2bcf8ee797aa58800007c110895181 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:32:22 +0100 Subject: [PATCH 68/78] fixed test --- packages/aws-library/tests/test_ec2_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py index 5ba224a73ce..1e85be088d1 100644 --- a/packages/aws-library/tests/test_ec2_client.py +++ b/packages/aws-library/tests/test_ec2_client.py @@ -125,7 +125,8 @@ async def test_get_ec2_instance_capabilities_empty_list_returns_all_options( ): instance_types = await simcore_ec2_api.get_ec2_instance_capabilities(set()) assert instance_types - assert len(instance_types) > 50 + # NOTE: this might need adaptation when moto is updated + assert 700 < len(instance_types) < 800 async def test_get_ec2_instance_capabilities_with_invalid_type_raises( From e2d03540abac9c4abc1c7a05a4a28049657fd7b3 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:32:36 +0100 Subject: [PATCH 69/78] use env_prefix --- .../core/settings.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index 11f02ae5d4d..80a6654b03a 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -1,6 +1,6 @@ import datetime from functools import cached_property -from typing import cast +from typing import Final, cast from fastapi import FastAPI from models_library.basic_types import ( @@ -20,6 +20,13 @@ from .._meta import API_VERSION, API_VTAG, APP_NAME +CLUSTERS_KEEPER_ENV_PREFIX: Final[str] = "CLUSTERS_KEEPER_" + + +class ClustersKeeperEC2Settings(EC2Settings): + class Config: + env_prefix = CLUSTERS_KEEPER_ENV_PREFIX + class WorkersEC2InstancesSettings(BaseCustomSettings): WORKERS_EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( @@ -173,7 +180,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - CLUSTERS_KEEPER_EC2_ACCESS: EC2Settings | None = Field(auto_default_from_env=True) + CLUSTERS_KEEPER_EC2_ACCESS: ClustersKeeperEC2Settings | None = Field( + auto_default_from_env=True + ) CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES: PrimaryEC2InstancesSettings | None = Field( auto_default_from_env=True From 281de5cf7c763a61ca5f3a85a394223bbddbc95f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:32:50 +0100 Subject: [PATCH 70/78] typo --- .../simcore_service_clusters_keeper/modules/clusters.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py index 95d77879149..a3a5a7229aa 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py @@ -2,7 +2,7 @@ import logging from typing import cast -from aws_library.ec2.models import EC2InstanceConfig, EC2InstanceData +from aws_library.ec2.models import EC2InstanceConfig, EC2InstanceData, EC2InstanceType from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID @@ -30,7 +30,9 @@ async def create_cluster( ec2_client = get_ec2_client(app) app_settings = get_application_settings(app) assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES # nosec - ec2_instance_type = await ec2_client.get_ec2_instance_capabilities( + ec2_instance_types: list[ + EC2InstanceType + ] = await ec2_client.get_ec2_instance_capabilities( instance_type_names={ cast( InstanceTypeType, @@ -42,8 +44,9 @@ async def create_cluster( ) } ) + assert len(ec2_instance_types) == 1 # nosec instance_config = EC2InstanceConfig( - type=ec2_instance_type, + type=ec2_instance_types[0], tags=creation_ec2_tags(app_settings, user_id=user_id, wallet_id=wallet_id), startup_script=create_startup_script( app_settings, From f464b17bc639034e2bf8236c85f150aa6c99f2f7 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:32:58 +0100 Subject: [PATCH 71/78] fix environs --- .../src/simcore_service_clusters_keeper/utils/clusters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py index 444a3ef1a26..64cdf4fcb10 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py @@ -40,8 +40,8 @@ def _convert_to_env_list(entries: list[Any]) -> str: environment_variables = [ f"DOCKER_IMAGE_TAG={app_settings.CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG}", - f"EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID}", - f"EC2_CLUSTERS_KEEPER_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_ENDPOINT}", + f"EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ACCESS_KEY_ID}", + f"EC2_CLUSTERS_KEEPER_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ENDPOINT}", f"WORKERS_EC2_INSTANCES_ALLOWED_TYPES={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_ALLOWED_TYPES)}", f"WORKERS_EC2_INSTANCES_AMI_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_AMI_ID}", f"WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS)}", @@ -52,8 +52,8 @@ def _convert_to_env_list(entries: list[Any]) -> str: f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}", f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}", f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}", - f"EC2_CLUSTERS_KEEPER_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_REGION_NAME}", - f"EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY}", + f"EC2_CLUSTERS_KEEPER_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_REGION_NAME}", + f"EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_SECRET_ACCESS_KEY}", f"LOG_LEVEL={app_settings.LOG_LEVEL}", ] From bb47cd8fbe2aac90ed5740de90d6eb29a5979dc7 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:48:37 +0100 Subject: [PATCH 72/78] separate api_schema from internal model --- .../ec2_instances.py | 2 +- .../clusters_keeper/ec2_instances.py | 6 +- .../rpc/ec2_instances.py | 9 +- .../clusters-keeper/tests/unit/conftest.py | 110 +++--- .../tests/unit/test_api_health.py | 2 +- .../tests/unit/test_modules_clusters.py | 9 +- .../test_modules_clusters_management_core.py | 6 +- .../test_modules_clusters_management_task.py | 2 +- .../tests/unit/test_modules_ec2.py | 340 +----------------- .../tests/unit/test_rpc_clusters.py | 7 +- .../tests/unit/test_rpc_ec2_instances.py | 10 +- .../tests/unit/test_utils_clusters.py | 2 +- .../tests/unit/test_utils_ec2.py | 5 +- .../db/repositories/comp_tasks/_utils.py | 6 +- .../with_dbs/test_api_route_computations.py | 11 +- 15 files changed, 86 insertions(+), 441 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_clusters_keeper/ec2_instances.py b/packages/models-library/src/models_library/api_schemas_clusters_keeper/ec2_instances.py index 373a12af823..16f47d2a3fd 100644 --- a/packages/models-library/src/models_library/api_schemas_clusters_keeper/ec2_instances.py +++ b/packages/models-library/src/models_library/api_schemas_clusters_keeper/ec2_instances.py @@ -4,7 +4,7 @@ @dataclass(frozen=True) -class EC2InstanceType: +class EC2InstanceTypeGet: name: str cpus: PositiveInt ram: ByteSize diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py index 2663c82d520..433eee07fa3 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py @@ -1,5 +1,5 @@ from models_library.api_schemas_clusters_keeper import CLUSTERS_KEEPER_RPC_NAMESPACE -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.rabbitmq_basic_types import RPCMethodName from ..._client_rpc import RabbitMQRPCClient @@ -8,14 +8,14 @@ async def get_instance_type_details( client: RabbitMQRPCClient, *, instance_type_names: set[str] -) -> list[EC2InstanceType]: +) -> list[EC2InstanceTypeGet]: """**Remote method** Raises: RPCServerError -- if anything happens remotely """ - instance_types: list[EC2InstanceType] = await client.request( + instance_types: list[EC2InstanceTypeGet] = await client.request( CLUSTERS_KEEPER_RPC_NAMESPACE, RPCMethodName("get_instance_type_details"), timeout_s=RPC_REMOTE_METHOD_TIMEOUT_S, diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py index fb8b0bb8c2e..a68760ad762 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py @@ -1,5 +1,8 @@ +from dataclasses import asdict + +from aws_library.ec2.models import EC2InstanceType from fastapi import FastAPI -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from servicelib.rabbitmq import RPCRouter from ..modules.ec2 import get_ec2_client @@ -10,8 +13,8 @@ @router.expose() async def get_instance_type_details( app: FastAPI, *, instance_type_names: set[str] -) -> list[EC2InstanceType]: +) -> list[EC2InstanceTypeGet]: instance_capabilities: list[EC2InstanceType] = await get_ec2_client( app ).get_ec2_instance_capabilities(instance_type_names) - return instance_capabilities + return [EC2InstanceTypeGet(**asdict(t)) for t in instance_capabilities] diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index 444eeb747f0..26c97e27084 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -11,34 +11,33 @@ import aiodocker import httpx import pytest -import requests import simcore_service_clusters_keeper import simcore_service_clusters_keeper.data import yaml -from aiohttp.test_utils import unused_port from asgi_lifespan import LifespanManager +from aws_library.ec2.client import SimcoreEC2API from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID -from moto.server import ThreadedMotoServer from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict -from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.rabbitmq import RabbitMQRPCClient +from settings_library.ec2 import EC2Settings from settings_library.rabbit import RabbitSettings from simcore_service_clusters_keeper.core.application import create_app from simcore_service_clusters_keeper.core.settings import ( + CLUSTERS_KEEPER_ENV_PREFIX, ApplicationSettings, - EC2ClustersKeeperSettings, ) -from simcore_service_clusters_keeper.modules.ec2 import ClustersKeeperEC2 from simcore_service_clusters_keeper.utils.ec2 import get_cluster_name from types_aiobotocore_ec2.client import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType pytest_plugins = [ + "pytest_simcore.aws_server", + "pytest_simcore.aws_ec2_service", "pytest_simcore.dask_scheduler", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", @@ -71,6 +70,19 @@ def ec2_instances() -> list[InstanceTypeType]: return ["t2.nano", "m5.12xlarge"] +@pytest.fixture +def mocked_ec2_server_envs( + mocked_ec2_server_settings: EC2Settings, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + # NOTE: overrides the EC2Settings with what clusters-keeper expects + changed_envs: EnvVarsDict = { + f"{CLUSTERS_KEEPER_ENV_PREFIX}{k}": v + for k, v in mocked_ec2_server_settings.dict().items() + } + return setenvs_from_dict(monkeypatch, changed_envs) + + @pytest.fixture def app_environment( mock_env_devel_environment: EnvVarsDict, @@ -83,8 +95,8 @@ def app_environment( monkeypatch, { "CLUSTERS_KEEPER_EC2_ACCESS": "{}", - "EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID": faker.pystr(), - "EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY": faker.pystr(), + "CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID": faker.pystr(), + "CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY": faker.pystr(), "CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES": "{}", "PRIMARY_EC2_INSTANCES_KEY_NAME": faker.pystr(), "PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( @@ -106,6 +118,28 @@ def app_environment( return mock_env_devel_environment | envs +@pytest.fixture +def mocked_primary_ec2_instances_envs( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + aws_security_group_id: str, + aws_subnet_id: str, + aws_ami_id: str, +) -> EnvVarsDict: + envs = setenvs_from_dict( + monkeypatch, + { + "PRIMARY_EC2_INSTANCES_KEY_NAME": "osparc-pytest", + "PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( + [aws_security_group_id] + ), + "PRIMARY_EC2_INSTANCES_SUBNET_ID": aws_subnet_id, + "PRIMARY_EC2_INSTANCES_AMI_ID": aws_ami_id, + }, + ) + return app_environment | envs + + @pytest.fixture def disable_clusters_management_background_task( mocker: MockerFixture, @@ -170,53 +204,6 @@ async def async_client(initialized_app: FastAPI) -> AsyncIterator[httpx.AsyncCli yield client -@pytest.fixture(scope="module") -def mocked_aws_server() -> Iterator[ThreadedMotoServer]: - """creates a moto-server that emulates AWS services in place - NOTE: Never use a bucket with underscores it fails!! - """ - server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) - # pylint: disable=protected-access - print( - f"--> started mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 - ) - print( - f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" # noqa: SLF001 - ) - server.start() - yield server - server.stop() - print( - f"<-- stopped mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 - ) - - -@pytest.fixture -def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[None]: - # NOTE: reset_aws_server_state [http://docs.getmoto.org/en/latest/docs/server_mode.html#reset-api] - yield - # pylint: disable=protected-access - requests.post( - f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 - timeout=10, - ) - - -@pytest.fixture -def mocked_aws_server_envs( - app_environment: EnvVarsDict, - mocked_aws_server: ThreadedMotoServer, - reset_aws_server_state: None, - monkeypatch: pytest.MonkeyPatch, -) -> EnvVarsDict: - changed_envs = { - "EC2_CLUSTERS_KEEPER_ENDPOINT": f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - "EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID": "xxx", - "EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY": "xxx", - } - return app_environment | setenvs_from_dict(monkeypatch, changed_envs) - - @pytest.fixture def aws_allowed_ec2_instance_type_names_env( app_environment: EnvVarsDict, @@ -239,21 +226,14 @@ def aws_allowed_ec2_instance_type_names_env( @pytest.fixture async def clusters_keeper_ec2( app_environment: EnvVarsDict, -) -> AsyncIterator[ClustersKeeperEC2]: - settings = EC2ClustersKeeperSettings.create_from_envs() - ec2 = await ClustersKeeperEC2.create(settings) +) -> AsyncIterator[SimcoreEC2API]: + settings = EC2Settings.create_from_envs() + ec2 = await SimcoreEC2API.create(settings) assert ec2 yield ec2 await ec2.close() -@pytest.fixture -async def ec2_client( - clusters_keeper_ec2: ClustersKeeperEC2, -) -> EC2Client: - return clusters_keeper_ec2.client - - @pytest.fixture async def mocked_redis_server(mocker: MockerFixture) -> None: mock_redis = FakeRedis() diff --git a/services/clusters-keeper/tests/unit/test_api_health.py b/services/clusters-keeper/tests/unit/test_api_health.py index 411081de36d..dcd67cb4e98 100644 --- a/services/clusters-keeper/tests/unit/test_api_health.py +++ b/services/clusters-keeper/tests/unit/test_api_health.py @@ -20,7 +20,7 @@ def app_environment( app_environment: EnvVarsDict, enabled_rabbitmq: None, - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, mocked_redis_server: None, ) -> EnvVarsDict: return app_environment diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters.py b/services/clusters-keeper/tests/unit/test_modules_clusters.py index d24b9dd4184..6d0e82cd8da 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters.py @@ -5,7 +5,7 @@ import asyncio import datetime -from typing import Awaitable, Callable +from collections.abc import Awaitable, Callable import arrow import pytest @@ -15,6 +15,7 @@ from models_library.users import UserID from models_library.wallets import WalletID from parse import Result, search +from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_clusters_keeper._meta import VERSION as APP_VERSION from simcore_service_clusters_keeper.core.errors import Ec2InstanceNotFoundError from simcore_service_clusters_keeper.core.settings import ( @@ -50,11 +51,9 @@ def wallet_id(faker: Faker) -> WalletID: def _base_configuration( docker_swarm: None, disabled_rabbitmq: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, + mocked_ec2_server_envs: EnvVarsDict, + mocked_primary_ec2_instances_envs: EnvVarsDict, ) -> None: ... diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py b/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py index 51b68670640..e09cc48c7dd 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py @@ -58,12 +58,10 @@ def app_environment( @pytest.fixture def _base_configuration( - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], disabled_rabbitmq: None, mocked_redis_server: None, + mocked_ec2_server_envs: EnvVarsDict, + mocked_primary_ec2_instances_envs: EnvVarsDict, ) -> None: ... diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters_management_task.py b/services/clusters-keeper/tests/unit/test_modules_clusters_management_task.py index 18d440aa541..d738a5cc05e 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters_management_task.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters_management_task.py @@ -36,7 +36,7 @@ def mock_background_task(mocker: MockerFixture) -> mock.Mock: async def test_clusters_management_task_created_and_deleted( disabled_rabbitmq: None, - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, mocked_redis_server: None, mock_background_task: mock.Mock, initialized_app: FastAPI, diff --git a/services/clusters-keeper/tests/unit/test_modules_ec2.py b/services/clusters-keeper/tests/unit/test_modules_ec2.py index eb253cbe289..0820ada5818 100644 --- a/services/clusters-keeper/tests/unit/test_modules_ec2.py +++ b/services/clusters-keeper/tests/unit/test_modules_ec2.py @@ -2,71 +2,11 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -from collections.abc import Callable -from typing import cast -import botocore.exceptions import pytest -from faker import Faker from fastapi import FastAPI -from moto.server import ThreadedMotoServer -from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.utils_envs import EnvVarsDict -from simcore_service_clusters_keeper.core.errors import ( - ConfigurationError, - Ec2InstanceNotFoundError, - Ec2InstanceTypeInvalidError, - Ec2TooManyInstancesError, -) -from simcore_service_clusters_keeper.core.settings import ( - ApplicationSettings, - EC2ClustersKeeperSettings, -) -from simcore_service_clusters_keeper.modules.ec2 import ( - ClustersKeeperEC2, - EC2InstanceData, - get_ec2_client, -) -from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceTypeType - - -@pytest.fixture -def ec2_settings( - app_environment: EnvVarsDict, -) -> EC2ClustersKeeperSettings: - return EC2ClustersKeeperSettings.create_from_envs() - - -@pytest.fixture -def app_settings( - app_environment: EnvVarsDict, -) -> ApplicationSettings: - return ApplicationSettings.create_from_envs() - - -async def test_ec2_client_lifespan(ec2_settings: EC2ClustersKeeperSettings): - ec2 = await ClustersKeeperEC2.create(settings=ec2_settings) - assert ec2 - assert ec2.client - assert ec2.exit_stack - assert ec2.session - - await ec2.close() - - -async def test_ec2_client_raises_when_no_connection_available(ec2_client: EC2Client): - with pytest.raises( - botocore.exceptions.ClientError, match=r".+ AWS was not able to validate .+" - ): - await ec2_client.describe_account_attributes(DryRun=True) - - -async def test_ec2_client_with_mock_server( - mocked_aws_server_envs: None, ec2_client: EC2Client -): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) +from simcore_service_clusters_keeper.core.errors import ConfigurationError +from simcore_service_clusters_keeper.modules.ec2 import get_ec2_client async def test_ec2_does_not_initialize_if_deactivated( @@ -79,279 +19,3 @@ async def test_ec2_does_not_initialize_if_deactivated( assert initialized_app.state.ec2_client is None with pytest.raises(ConfigurationError): get_ec2_client(initialized_app) - - -async def test_ec2_client_when_ec2_server_goes_up_and_down( - mocked_aws_server: ThreadedMotoServer, - mocked_aws_server_envs: None, - ec2_client: EC2Client, -): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) - mocked_aws_server.stop() - with pytest.raises(botocore.exceptions.EndpointConnectionError): - await ec2_client.describe_account_attributes(DryRun=True) - - # restart - mocked_aws_server.start() - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) - - -async def test_ping( - mocked_aws_server: ThreadedMotoServer, - mocked_aws_server_envs: None, - aws_allowed_ec2_instance_type_names_env: list[str], - app_settings: ApplicationSettings, - clusters_keeper_ec2: ClustersKeeperEC2, -): - assert await clusters_keeper_ec2.ping() is True - mocked_aws_server.stop() - assert await clusters_keeper_ec2.ping() is False - mocked_aws_server.start() - assert await clusters_keeper_ec2.ping() is True - - -async def test_get_ec2_instance_capabilities( - mocked_aws_server_envs: None, - aws_allowed_ec2_instance_type_names_env: list[str], - app_settings: ApplicationSettings, - clusters_keeper_ec2: ClustersKeeperEC2, -): - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - assert ( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ) - instance_types = await clusters_keeper_ec2.get_ec2_instance_capabilities( - cast( - set[InstanceTypeType], - set( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ), - ) - ) - assert instance_types - assert len(instance_types) == len( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ) - - # all the instance names are found and valid - assert all( - i.name - in app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - for i in instance_types - ) - for ( - instance_type_name - ) in ( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ): - assert any(i.name == instance_type_name for i in instance_types) - - -async def test_get_ec2_instance_capabilities_with_empty_set_returns_all_options( - mocked_aws_server_envs: None, - clusters_keeper_ec2: ClustersKeeperEC2, -): - instance_types = await clusters_keeper_ec2.get_ec2_instance_capabilities(set()) - assert instance_types - # NOTE: this might need adaptation when moto is updated - assert 700 < len(instance_types) < 800 - - -async def test_get_ec2_instance_capabilities_with_invalid_names( - mocked_aws_server_envs: None, clusters_keeper_ec2: ClustersKeeperEC2, faker: Faker -): - with pytest.raises(Ec2InstanceTypeInvalidError): - await clusters_keeper_ec2.get_ec2_instance_capabilities( - faker.pyset(allowed_types=(str,)) - ) - - -async def test_start_aws_instance( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - faker: Faker, - mocker: MockerFixture, -): - assert app_settings.CLUSTERS_KEEPER_EC2_ACCESS - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - - instance_type = faker.pystr() - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - # check we have that now in ec2 - all_instances = await ec2_client.describe_instances() - assert len(all_instances["Reservations"]) == 1 - running_instance = all_instances["Reservations"][0] - assert "Instances" in running_instance - assert len(running_instance["Instances"]) == 1 - running_instance = running_instance["Instances"][0] - assert "InstanceType" in running_instance - assert running_instance["InstanceType"] == instance_type - assert "Tags" in running_instance - assert running_instance["Tags"] == [ - {"Key": key, "Value": value} for key, value in tags.items() - ] - - -async def test_start_aws_instance_is_limited_in_number_of_instances( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - faker: Faker, - mocker: MockerFixture, -): - assert app_settings.CLUSTERS_KEEPER_EC2_ACCESS - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - - # create as many instances as we can - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - for _ in range( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_MAX_INSTANCES - ): - await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - faker.pystr(), - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - # now creating one more shall fail - with pytest.raises(Ec2TooManyInstancesError): - await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - faker.pystr(), - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - -async def test_get_instances( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - faker: Faker, - mocker: MockerFixture, -): - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - assert ( - await clusters_keeper_ec2.get_instances( - key_names=[ - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME - ], - tags={}, - ) - == [] - ) - - # create some instance - instance_type = faker.pystr() - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - created_instances = await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - assert len(created_instances) == 1 - - instance_received = await clusters_keeper_ec2.get_instances( - key_names=[ - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME - ], - tags=tags, - ) - assert created_instances == instance_received - - -async def test_terminate_instance( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - faker: Faker, - mocker: MockerFixture, -): - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - # create some instance - instance_type = faker.pystr() - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - created_instances = await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - assert len(created_instances) == 1 - - # terminate the instance - await clusters_keeper_ec2.terminate_instances(created_instances) - # calling it several times is ok, the instance stays a while - await clusters_keeper_ec2.terminate_instances(created_instances) - - -async def test_terminate_instance_not_existing_raises( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - fake_ec2_instance_data: Callable[..., EC2InstanceData], -): - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - with pytest.raises(Ec2InstanceNotFoundError): - await clusters_keeper_ec2.terminate_instances([fake_ec2_instance_data()]) diff --git a/services/clusters-keeper/tests/unit/test_rpc_clusters.py b/services/clusters-keeper/tests/unit/test_rpc_clusters.py index 34d4dad6fec..7d84e6425c8 100644 --- a/services/clusters-keeper/tests/unit/test_rpc_clusters.py +++ b/services/clusters-keeper/tests/unit/test_rpc_clusters.py @@ -15,6 +15,7 @@ from models_library.users import UserID from models_library.wallets import WalletID from pytest_mock.plugin import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.clusters_keeper.clusters import ( get_or_create_cluster, @@ -43,11 +44,9 @@ def wallet_id(faker: Faker) -> WalletID: def _base_configuration( docker_swarm: None, enabled_rabbitmq: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, + mocked_ec2_server_envs: EnvVarsDict, + mocked_primary_ec2_instances_envs: EnvVarsDict, initialized_app: FastAPI, ) -> None: ... diff --git a/services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py b/services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py index ff8a506309f..d03b6b74502 100644 --- a/services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py +++ b/services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py @@ -4,7 +4,8 @@ import pytest from fastapi import FastAPI -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet +from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient, RPCServerError from servicelib.rabbitmq.rpc_interfaces.clusters_keeper.ec2_instances import ( get_instance_type_details, @@ -21,11 +22,8 @@ def _base_configuration( docker_swarm: None, enabled_rabbitmq: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, + mocked_ec2_server_envs: EnvVarsDict, initialized_app: FastAPI, ) -> None: ... @@ -42,7 +40,7 @@ async def test_get_instance_type_details_all_options( ) assert rpc_response assert isinstance(rpc_response, list) - assert isinstance(rpc_response[0], EC2InstanceType) + assert isinstance(rpc_response[0], EC2InstanceTypeGet) async def test_get_instance_type_details_specific_type_names( diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py index b796970e468..3e772896da0 100644 --- a/services/clusters-keeper/tests/unit/test_utils_clusters.py +++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py @@ -26,7 +26,7 @@ def cluster_machines_name_prefix(faker: Faker) -> str: def test_create_startup_script( disabled_rabbitmq: None, - mocked_aws_server_envs: EnvVarsDict, + mocked_ec2_server_envs: EnvVarsDict, mocked_redis_server: None, app_settings: ApplicationSettings, cluster_machines_name_prefix: str, diff --git a/services/clusters-keeper/tests/unit/test_utils_ec2.py b/services/clusters-keeper/tests/unit/test_utils_ec2.py index e278d3dc890..51ccbe9a7da 100644 --- a/services/clusters-keeper/tests/unit/test_utils_ec2.py +++ b/services/clusters-keeper/tests/unit/test_utils_ec2.py @@ -6,6 +6,7 @@ from faker import Faker from models_library.users import UserID from models_library.wallets import WalletID +from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_clusters_keeper.core.settings import ApplicationSettings from simcore_service_clusters_keeper.utils.ec2 import ( _APPLICATION_TAG_KEY, @@ -52,7 +53,7 @@ def test_get_cluster_name( def test_creation_ec2_tags( - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, disabled_rabbitmq: None, mocked_redis_server: None, app_settings: ApplicationSettings, @@ -79,7 +80,7 @@ def test_creation_ec2_tags( def test_all_created_ec2_instances_filter( - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, disabled_rabbitmq: None, mocked_redis_server: None, app_settings: ApplicationSettings, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py index 57ede61ebd3..214b46d2c8c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py @@ -5,7 +5,7 @@ import aiopg.sa import arrow from dask_task_models_library.container_tasks.protocol import ContainerEnvsDict -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.api_schemas_directorv2.services import ( NodeRequirements, ServiceExtras, @@ -252,7 +252,7 @@ async def _update_project_node_resources_from_hardware_info( return try: unordered_list_ec2_instance_types: list[ - EC2InstanceType + EC2InstanceTypeGet ] = await get_instance_type_details( rabbitmq_rpc_client, instance_type_names=set(hardware_info.aws_ec2_instances), @@ -261,7 +261,7 @@ async def _update_project_node_resources_from_hardware_info( assert unordered_list_ec2_instance_types # nosec # NOTE: with the current implementation, there is no use to get the instance past the first one - def _by_type_name(ec2: EC2InstanceType) -> bool: + def _by_type_name(ec2: EC2InstanceTypeGet) -> bool: return bool(ec2.name == hardware_info.aws_ec2_instances[0]) selected_ec2_instance_type = next( diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index f3a83d487db..c13d6852b88 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -19,9 +19,9 @@ import httpx import pytest import respx +from aws_library.ec2.models import EC2InstanceType from faker import Faker from fastapi import FastAPI -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType from models_library.api_schemas_directorv2.comp_tasks import ( ComputationCreate, ComputationGet, @@ -62,6 +62,7 @@ ) from simcore_service_director_v2.utils.computations import to_node_class from starlette import status +from types_aiobotocore_ec2.literals import InstanceTypeType pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = [ @@ -264,11 +265,13 @@ def default_pricing_plan(request: pytest.FixtureRequest) -> ServicePricingPlanGe @pytest.fixture def default_pricing_plan_aws_ec2_type( default_pricing_plan: ServicePricingPlanGet, -) -> str | None: +) -> InstanceTypeType | None: for p in default_pricing_plan.pricing_units: if p.default: if p.specific_info.aws_ec2_instances: - return p.specific_info.aws_ec2_instances[0] + return parse_obj_as( + InstanceTypeType, p.specific_info.aws_ec2_instances[0] + ) return None pytest.fail("no default pricing plan defined!") msg = "make pylint happy by raising here" @@ -400,7 +403,7 @@ def fake_ec2_ram() -> ByteSize: @pytest.fixture def mocked_clusters_keeper_service_get_instance_type_details( mocker: MockerFixture, - default_pricing_plan_aws_ec2_type: str, + default_pricing_plan_aws_ec2_type: InstanceTypeType, fake_ec2_cpus: PositiveInt, fake_ec2_ram: ByteSize, ) -> mock.Mock: From 7e7a4e4b2d3602b9abd5f66953d59aaa854c833f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 23:53:16 +0100 Subject: [PATCH 73/78] fix test --- .../tests/unit/with_dbs/test_api_route_computations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index c13d6852b88..e8cf5dbdfe3 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -19,9 +19,9 @@ import httpx import pytest import respx -from aws_library.ec2.models import EC2InstanceType from faker import Faker from fastapi import FastAPI +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.api_schemas_directorv2.comp_tasks import ( ComputationCreate, ComputationGet, @@ -410,7 +410,7 @@ def mocked_clusters_keeper_service_get_instance_type_details( return mocker.patch( "simcore_service_director_v2.modules.db.repositories.comp_tasks._utils.get_instance_type_details", return_value=[ - EC2InstanceType( + EC2InstanceTypeGet( name=default_pricing_plan_aws_ec2_type, cpus=fake_ec2_cpus, ram=fake_ec2_ram, @@ -429,7 +429,7 @@ def mocked_clusters_keeper_service_get_instance_type_details_with_invalid_name( return mocker.patch( "simcore_service_director_v2.modules.db.repositories.comp_tasks._utils.get_instance_type_details", return_value=[ - EC2InstanceType( + EC2InstanceTypeGet( name=faker.pystr(), cpus=fake_ec2_cpus, ram=fake_ec2_ram, From af9a79b5d45f2b9e958dffaca25f8031d40a18bf Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 23:54:18 +0100 Subject: [PATCH 74/78] fix test --- .../tests/unit/test_node_ports_common_file_io_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py b/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py index cf1f0a59d92..12353b6cd6a 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py @@ -147,12 +147,12 @@ async def test_upload_file_to_presigned_links_raises_aws_s3_400_request_time_out @pytest.fixture async def aiobotocore_s3_client( - mocked_s3_server: ThreadedMotoServer, + mocked_aws_server: ThreadedMotoServer, ) -> AsyncIterator[AioBaseClient]: session = get_session() async with session.create_client( "s3", - endpoint_url=f"http://{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # pylint: disable=protected-access + endpoint_url=f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access aws_secret_access_key="xxx", aws_access_key_id="xxx", ) as client: From c6a31e12b65b8c36f4d94e08170f4fd70896927d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:17:19 +0100 Subject: [PATCH 75/78] @pcrespov review: add examples --- .../settings-library/src/settings_library/ec2.py | 14 ++++++++++++++ .../core/settings.py | 15 +++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py index 111e52adfb0..5af16e2bc36 100644 --- a/packages/settings-library/src/settings_library/ec2.py +++ b/packages/settings-library/src/settings_library/ec2.py @@ -1,3 +1,5 @@ +from typing import Any, ClassVar + from pydantic import Field from .base import BaseCustomSettings @@ -10,3 +12,15 @@ class EC2Settings(BaseCustomSettings): ) EC2_REGION_NAME: str = "us-east-1" EC2_SECRET_ACCESS_KEY: str + + class Config(BaseCustomSettings.Config): + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + "EC2_ACCESS_KEY_ID": "my_access_key_id", + "EC2_ENDPOINT": "http://my_ec2_endpoint.com", + "EC2_REGION_NAME": "us-east-1", + "EC2_SECRET_ACCESS_KEY": "my_secret_access_key", + } + ], + } diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index 80a6654b03a..491cdeec854 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -1,6 +1,6 @@ import datetime from functools import cached_property -from typing import Final, cast +from typing import Any, ClassVar, Final, cast from fastapi import FastAPI from models_library.basic_types import ( @@ -24,9 +24,20 @@ class ClustersKeeperEC2Settings(EC2Settings): - class Config: + class Config(EC2Settings.Config): env_prefix = CLUSTERS_KEEPER_ENV_PREFIX + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ACCESS_KEY_ID": "my_access_key_id", + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ENDPOINT": "http://my_ec2_endpoint.com", + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_REGION_NAME": "us-east-1", + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_SECRET_ACCESS_KEY": "my_secret_access_key", + } + ], + } + class WorkersEC2InstancesSettings(BaseCustomSettings): WORKERS_EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( From 26a9a4ca6dc5f80fdf9dacd32c04a917932cf3be Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:24:40 +0100 Subject: [PATCH 76/78] remove dependency to types_aiobotocore --- .../tests/unit/with_dbs/test_api_route_computations.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index e8cf5dbdfe3..2bd3363eb26 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -62,7 +62,6 @@ ) from simcore_service_director_v2.utils.computations import to_node_class from starlette import status -from types_aiobotocore_ec2.literals import InstanceTypeType pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = [ @@ -265,13 +264,11 @@ def default_pricing_plan(request: pytest.FixtureRequest) -> ServicePricingPlanGe @pytest.fixture def default_pricing_plan_aws_ec2_type( default_pricing_plan: ServicePricingPlanGet, -) -> InstanceTypeType | None: +) -> str | None: for p in default_pricing_plan.pricing_units: if p.default: if p.specific_info.aws_ec2_instances: - return parse_obj_as( - InstanceTypeType, p.specific_info.aws_ec2_instances[0] - ) + return p.specific_info.aws_ec2_instances[0] return None pytest.fail("no default pricing plan defined!") msg = "make pylint happy by raising here" From 9872f05451c959292aac6484fd53b860d9e177a0 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:45:25 +0100 Subject: [PATCH 77/78] mypy --- packages/settings-library/src/settings_library/ec2.py | 2 +- .../tests/unit/with_dbs/test_api_route_computations.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py index 5af16e2bc36..710db9c6e4f 100644 --- a/packages/settings-library/src/settings_library/ec2.py +++ b/packages/settings-library/src/settings_library/ec2.py @@ -14,7 +14,7 @@ class EC2Settings(BaseCustomSettings): EC2_SECRET_ACCESS_KEY: str class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { + schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] "examples": [ { "EC2_ACCESS_KEY_ID": "my_access_key_id", diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index 2bd3363eb26..1cc63734366 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -400,7 +400,7 @@ def fake_ec2_ram() -> ByteSize: @pytest.fixture def mocked_clusters_keeper_service_get_instance_type_details( mocker: MockerFixture, - default_pricing_plan_aws_ec2_type: InstanceTypeType, + default_pricing_plan_aws_ec2_type: str, fake_ec2_cpus: PositiveInt, fake_ec2_ram: ByteSize, ) -> mock.Mock: From 89e5ba51ca9c1ae836db1a41fbd8786eb16dc81e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:07:45 +0100 Subject: [PATCH 78/78] fix call that fails in real AWS --- packages/aws-library/src/aws_library/ec2/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py index 4f5e9e6c752..a88cf93cbd7 100644 --- a/packages/aws-library/src/aws_library/ec2/client.py +++ b/packages/aws-library/src/aws_library/ec2/client.py @@ -145,7 +145,6 @@ async def start_aws_instance( "Groups": instance_config.security_group_ids, } ], - SecurityGroupIds=instance_config.security_group_ids, ) instance_ids = [i["InstanceId"] for i in instances["Instances"]] _logger.info(