run goth in a separate virtual environment (#1106)

* run `goth` in a separate virtual environment
* bump Python, typing extensions
* replace the poetry installation action
* bump goth to the rc release (0.14.1)
* update the github workflow
* use old runners
* fix running `mid_agreeent_payments` test + add sensible logging
* update the nightly
* remove py 3.7 from tests matrix

.github/workflows/goth-nightly.yml

@@ -38,30 +38,29 @@ jobs:
       fail-fast: false
     name: Run integration tests (nightly) on ${{ matrix.branch }}
     steps:
+      - name: install docker-compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.28.4/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose
+          sudo chmod a+x /usr/local/bin/docker-compose
+
+      - name: install ffi
+        run: sudo apt-get install libffi-dev
+
       - name: Checkout
         uses: actions/checkout@v3
-        with:
-          ref: ${{ matrix.branch }}
 
       - name: Configure python
         uses: actions/setup-python@v4
         with:
           python-version: '3.8.0'
 
-      - name: Configure poetry
-        uses: Gr1N/setup-poetry@v8
-        with:
-          poetry-version: 1.2.2
+      - name: Install and configure Poetry
+        run: python -m pip install -U pip setuptools poetry==1.3.2
 
       - name: Install dependencies
         run: |
           poetry env use python3.8
-          poetry install -E integration-tests
-
-      - name: Install websocat
-        run: |
-          sudo wget https://github.com/vi/websocat/releases/download/v1.9.0/websocat_linux64 -O /usr/local/bin/websocat
-          sudo chmod +x /usr/local/bin/websocat
+          poetry install
 
       - name: Disconnect Docker containers from default network
         continue-on-error: true
@@ -82,19 +81,20 @@ jobs:
       - name: Log in to GitHub Docker repository
         run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin
 
+      - name: Initialize the test suite
+        run: poetry run poe tests_integration_init
+
       - name: Run test suite
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          poetry run poe tests_integration_assets
-          poetry run poe tests_integration
+        run: poetry run poe tests_integration
 
       - name: Upload test logs
         uses: actions/upload-artifact@v2
         if: always()
         with:
-          name: goth-logs
-          path: /tmp/goth-tests
+           name: goth-logs
+           path: /tmp/goth-tests
 
       # Only relevant for self-hosted runners
       - name: Remove test logs

.github/workflows/goth.yml

@@ -16,6 +16,14 @@ jobs:
     name: Run integration tests
     runs-on: [goth, ubuntu-18.04]
     steps:
+      - name: install docker-compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.28.4/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose
+          sudo chmod a+x /usr/local/bin/docker-compose
+
+      - name: install ffi
+        run: sudo apt-get install libffi-dev
+
       - name: Checkout
         uses: actions/checkout@v3
 
@@ -24,15 +32,13 @@ jobs:
         with:
           python-version: '3.8.0'
 
-      - name: Configure poetry
-        uses: Gr1N/setup-poetry@v8
-        with:
-          poetry-version: 1.2.2
+      - name: Install and configure Poetry
+        run: python -m pip install -U pip setuptools poetry==1.3.2
 
       - name: Install dependencies
         run: |
           poetry env use python3.8
-          poetry install -E integration-tests
+          poetry install
 
       - name: Disconnect Docker containers from default network
         continue-on-error: true
@@ -53,12 +59,13 @@ jobs:
       - name: Log in to GitHub Docker repository
         run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin
 
+      - name: Initialize the test suite
+        run: poetry run poe tests_integration_init
+
       - name: Run test suite
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          poetry run poe tests_integration_assets
-          poetry run poe tests_integration
+        run: poetry run poe tests_integration
 
       - name: Upload test logs
         uses: actions/upload-artifact@v2

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10"]
         os:
           - ubuntu-latest
           - macos-latest

docs/sphinx/conf.py

@@ -59,6 +59,7 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path: List[str] = []
 
+
 #   This removes the `yapapi.log` docstrings.
 #   There are two reasons:
 #   *   there are some sections/subsections declared there and it messes up the docs

examples/custom-usage-counter/custom_usage_counter.py

@@ -55,7 +55,6 @@ async def shutdown(self):
 
 
 async def main(running_time_sec, subnet_tag, driver=None, network=None):
-
     # necessary to set maximum price for one unit of custom counter,
     # default strategy does not take custom counter prices into account
     strategy = LeastExpensiveLinearPayuMS(

examples/custom_runtime/custom_runtime.py

@@ -38,7 +38,6 @@ async def get_payload():
 
 
 async def main(subnet_tag, driver=None, network=None):
-
     async with Golem(
         budget=10.0,
         subnet_tag=subnet_tag,

pyproject.toml

@@ -19,18 +19,17 @@ requires = ["poetry_core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.dependencies]
-python = "^3.6.2"
+python = "^3.8"
 
 setuptools = [
-  { version = "*", python = ">=3.7" },
-  { version = "^59.0.0", python = ">=3.6.2, <3.7" },
+  { version = "*", python = ">=3.8" },
 ]
 aiohttp = "^3.6"
 aiohttp-sse-client = "^0.1.7"
 dataclasses = { version = "^0.8", python = ">=3.6, <3.7" }
 more-itertools = "^8.6.0"
 urllib3 = "^1.25.9"
-typing_extensions = "^3.10.0"
+typing-extensions = "^4.5.0"
 
 async_exit_stack = "^1.0.1"
 jsonrpc-base = "^1.0.3"
@@ -42,36 +41,23 @@ colorama = "^0.4.4"
 semantic-version = "^2.8"
 attrs = ">=19.3"
 
-# Adding `goth` to dependencies causes > 40 additional packages to be installed. Given
-# that dependency resolution in `poetry` is rather slow, we'd like to avoid installing
-# `goth` for example in a CI pipeline that only runs linters/unit tests, not integration
-# tests. Therefore we specify `goth` as an "extra" dependency, with `optional = "true"`.
-# It will be then installable with `poetry install -E integration-tests`.
-# Note that putting `goth` in `poetry.dev-dependencies` instead of `poetry.dependencies`
-# would not work: see https://github.com/python-poetry/poetry/issues/129.
-goth = { version = "^0.14", optional = true, python = "^3.8.0" }
-# goth = { git = "https://github.com/golemfactory/goth.git", branch = "master", optional = true, python = "^3.8.0", develop = true }
 Deprecated = "^1.2.12"
 python-statemachine = "^0.8.0"
 
-# goth tests
-pexpect = { version = "^4.8.0", optional = true }
-
 # Docs
 sphinx = { version = "^4.0.1", optional = true }
 sphinx-autodoc-typehints = { version = "^1.12.0", optional = true }
 sphinx-rtd-theme = { version = "^1.0.0", optional = true }
 
 [tool.poetry.extras]
-integration-tests = ['goth', 'pytest', 'pytest-asyncio', 'pexpect']
 docs = ['sphinx', 'sphinx-autodoc-typehints', 'sphinx-rtd-theme']
 
 [tool.poetry.group.dev.dependencies]
 pip = [
   { version = "*", python = ">=3.7" },
   { version = "^21.0.0", python = ">=3.6.2, <3.7" },
 ]
-black = "^21.7b0"
+black = "^23.1.0"
 factory-boy = "^3.2.0"
 isort = "^5.10.1"
 liccheck = "^0.4.7"
@@ -97,14 +83,19 @@ checks_typing  = {cmd = "mypy .", help = "Run only code typing checks" }
 checks_license = {sequence = ["_checks_license_export", "_checks_license_verify"], help = "Run only license compatibility checks"}
 _checks_license_export = "poetry export -f requirements.txt -o .requirements.txt"
 _checks_license_verify = "liccheck -r .requirements.txt"
+
 format = {sequence = ["_format_autoflake", "_format_isort", "_format_black"], help = "Run code auto formatting"}
 _format_autoflake = "pautoflake ."
 _format_isort = "isort ."
 _format_black = "black ."
-tests = {sequence = ["tests_unit", "tests_integration"], help = "Run all available tests"}
+
 tests_unit = {cmd = "pytest --cov=yapapi --cov-report html --cov-report term -sv --ignore tests/goth_tests", help = "Run only unit tests"}
-tests_integration = {cmd = "pytest -svx tests/goth_tests  --config-override docker-compose.build-environment.use-prerelease=true --config-path tests/goth_tests/assets/goth-config-testing.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError", help = "Run only integration tests"}
-tests_integration_assets = {cmd = "python -m goth create-assets tests/goth_tests/assets", help = "Generate assets files for integration tests"}
+tests_integration_init = { sequence = ["_gothv_env", "_gothv_requirements", "_gothv_assets"], help="Initialize the integration test environment"}
+tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests  --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError", help = "Run the integration tests"}
+_gothv_env = "python -m venv .envs/yapapi-goth"
+_gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.14.1 pip pytest pytest-asyncio pytest-rerunfailures pexpect"
+_gothv_assets = ".envs/yapapi-goth/bin/python -m goth create-assets tests/goth_tests/assets"
+
 clean = {cmd = "rm -rf .coverage .requirements.txt dist md handbook build", help = "Clean all development related files" }
 clean_examples = {sequence = ["clean_examples_blender", "clean_examples_hello", "clean_examples_http_proxy", "clean_examples_simple_service", "clean_examples_ssh", "clean_examples_webapp", "clean_examples_yacat"], help = "Clean all example related files"}
 clean_examples_blender = {cmd = "rm -f examples/blender/*.png examples/blender/*.log", help = "Clean \"blender\" example related files"}
@@ -114,6 +105,7 @@ clean_examples_simple_service = {cmd = "rm -f examples/simple-service-poc/*.png
 clean_examples_ssh = {cmd = "rm -f examples/ssh/*.log", help = "Clean \"ssh\" example related files"}
 clean_examples_webapp = {cmd = "rm -f examples/webapp/*.log", help = "Clean \"webapp\" example related files"}
 clean_examples_yacat = {cmd = "rm -f examples/yacat/*.log", help = "Clean \"yacat\" example related files"}
+
 sphinx = {cmd = "sphinx-build docs/sphinx/ build -E", help = "Build Sphinx docs"}
 
 [tool.liccheck.authorized_packages]

tests/conftest.py

@@ -4,7 +4,6 @@
 
 
 def pytest_addoption(parser):
-
     parser.addoption("--ya-api-key", type=str, help="instance api key", dest="yaApiKey")
 
 

tests/drone/drone.py

@@ -59,7 +59,6 @@ async def worker(ctx: WorkContext, tasks):
         payment_driver=payment_driver,
         payment_network=payment_network,
     ) as golem:
-
         print(
             f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n"
             f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, "

tests/executor/test_smartq.py

@@ -16,7 +16,6 @@ async def async_iter(iterable):
 @pytest.mark.asyncio
 @pytest.mark.parametrize("length", [0, 1, 100])
 async def test_smart_queue(length: int):
-
     q = SmartQueue(async_iter(range(length)))
 
     async def worker(i, queue):
@@ -54,7 +53,6 @@ async def stats():
 
 @pytest.mark.asyncio
 async def test_smart_queue_empty():
-
     q: SmartQueue = SmartQueue(async_iter([]))
     with q.new_consumer() as c:
         async for _item in c:

tests/goth_tests/test_resubscription.py → tests/goth_tests/___test_resubscription.py

@@ -138,7 +138,6 @@ async def test_demand_resubscription(
     runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config)
 
     async with runner(goth_config.containers):
-
         requestor = runner.get_probes(probe_type=RequestorProbe)[0]
         env = dict(os.environ)
         env.update(requestor.get_agent_env_vars())
@@ -166,7 +165,6 @@ async def worker(work_ctx, tasks):
             budget=10.0,
             event_consumer=monitor.add_event_sync,
         ) as golem:
-
             task: Task  # mypy needs this for some reason
             async for task in golem.execute_tasks(
                 worker,

tests/goth_tests/conftest.py

@@ -7,8 +7,6 @@
 
 from goth.configuration import Override
 
-from yapapi.payload import vm
-
 
 #  `pytest-rerunfailures` and `pytest-asyncio` don't work together
 #   (https://github.com/pytest-dev/pytest-rerunfailures/issues/154)
@@ -112,15 +110,3 @@ def log_dir() -> Path:
 @pytest.fixture(scope="session")
 def goth_config_path(request) -> Path:
     return request.config.option.config_path
-
-
-@pytest.fixture()
-def blender_vm_package():
-    async def coro():
-        return await vm.repo(
-            image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
-            min_mem_gib=0.5,
-            min_storage_gib=2.0,
-        )
-
-    return coro()

tests/goth_tests/test_agreement_termination/requestor.py

@@ -10,7 +10,6 @@
 
 
 async def main():
-
     package = await vm.repo(
         image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
         min_mem_gib=0.5,
@@ -36,7 +35,6 @@ async def worker(ctx: WorkContext, tasks):
         first_worker = False
 
         async for task in tasks:
-
             script = ctx.new_script()
 
             if should_fail:
@@ -54,7 +52,6 @@ async def worker(ctx: WorkContext, tasks):
         budget=10.0,
         subnet_tag="goth",
     ) as golem:
-
         tasks = [Task(data=n) for n in range(6)]
         async for task in golem.execute_tasks(
             worker,
@@ -69,7 +66,6 @@ async def worker(ctx: WorkContext, tasks):
 
 
 if __name__ == "__main__":
-
     enable_default_logger(log_file="test.log")
 
     console_handler = logging.StreamHandler()

tests/goth_tests/test_agreement_termination/test_agreement_termination.py

@@ -64,7 +64,6 @@ async def test_agreement_termination(
     goth_config_path: Path,
     config_overrides: List[Override],
 ) -> None:
-
     # This is the default configuration with 2 wasm/VM providers
     goth_config = load_yaml(goth_config_path, config_overrides)
     test_script_path = str(Path(__file__).parent / "requestor.py")
@@ -77,15 +76,13 @@ async def test_agreement_termination(
     )
 
     async with runner(goth_config.containers):
-
         requestor = runner.get_probes(probe_type=RequestorProbe)[0]
 
         async with requestor.run_command_on_host(test_script_path, env=os.environ) as (
             _cmd_task,
             cmd_monitor,
             _process_monitor,
         ):
-
             cmd_monitor.add_assertion(assert_all_tasks_computed)
 
             # Wait for worker failure due to command error

tests/goth_tests/test_async_task_generation/requestor.py

@@ -11,7 +11,6 @@
 
 
 async def main():
-
     vm_package = await vm.repo(
         image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
         min_mem_gib=0.5,
@@ -31,7 +30,6 @@ async def worker(work_ctx, tasks):
         subnet_tag="goth",
         event_consumer=log_event_repr,
     ) as golem:
-
         # We use an async task generator that yields tasks removed from
         # an async queue. Each computed task will potentially spawn
         # new tasks -- this is made possible thanks to using async task

tests/goth_tests/test_async_task_generation/test_async_task_generation.py

@@ -32,7 +32,6 @@ async def test_async_task_generation(
     runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config)
 
     async with runner(goth_config.containers):
-
         requestor = runner.get_probes(probe_type=RequestorProbe)[0]
 
         async with requestor.run_command_on_host(