✅ Maintenance: adds session-scoped fixture for httpbin service

packages/pytest-simcore/src/pytest_simcore/httpbin_service.py

@@ -0,0 +1,64 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+# pylint: disable=too-many-arguments
+
+
+import logging
+from contextlib import suppress
+from typing import Iterable
+
+import aiohttp.test_utils
+import docker
+import pytest
+import requests
+import requests.exceptions
+from docker.errors import APIError
+from pydantic import HttpUrl, parse_obj_as
+from tenacity import retry
+from tenacity.after import after_log
+from tenacity.retry import retry_if_exception_type
+from tenacity.stop import stop_after_delay
+from tenacity.wait import wait_fixed
+
+from .helpers.utils_docker import get_localhost_ip
+
+
+@pytest.fixture(scope="session")
+def httpbin_base_url() -> Iterable[HttpUrl]:
+    """
+    Implemented as a fixture since it cannot rely on full availability of https://httpbin.org/ during testing
+    """
+    ip_address = get_localhost_ip()
+    port = aiohttp.test_utils.unused_port()
+    base_url = f"http://{ip_address}:{port}"
+
+    client = docker.from_env()
+    container_name = "httpbin-fixture"
+    try:
+        container = client.containers.run(
+            image="kennethreitz/httpbin",
+            ports={80: port},
+            name=container_name,
+            detach=True,
+        )
+        print(container)
+
+        @retry(
+            wait=wait_fixed(1),
+            retry=retry_if_exception_type(requests.exceptions.RequestException),
+            stop=stop_after_delay(15),
+            after=after_log(logging.getLogger(), logging.DEBUG),
+        )
+        def _wait_until_httpbin_is_responsive():
+            r = requests.get(f"{base_url}/get", timeout=2)
+            r.raise_for_status()
+
+        _wait_until_httpbin_is_responsive()
+
+        yield parse_obj_as(HttpUrl, base_url)
+
+    finally:
+        with suppress(APIError):
+            container = client.containers.get(container_name)
+            container.remove(force=True)

packages/simcore-sdk/setup.cfg

@@ -15,3 +15,8 @@ test = pytest
 
 [tool:pytest]
 asyncio_mode = auto
+markers =
+	slow: marks tests as slow (deselect with '-m "not slow"')
+	acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows."
+	testit: "marks test to run during development"
+	heavy_load: "mark tests that require large amount of data"

services/api-server/tests/conftest.py

@@ -14,6 +14,7 @@
 
 pytest_plugins = [
     "pytest_simcore.cli_runner",
+    "pytest_simcore.httpbin_service",
     "pytest_simcore.pydantic_models",
     "pytest_simcore.pytest_global_environs",
     "pytest_simcore.repository_paths",

services/api-server/tests/unit/test_utils_http_calls_capture.py

@@ -5,65 +5,18 @@
 
 
 import asyncio
-import logging
 import re
-from contextlib import suppress
-from typing import Iterable
 
-import docker
 import httpx
-import pytest
 import respx
-from docker.errors import APIError
 from faker import Faker
 from models_library.basic_regex import UUID_RE_BASE
+from pydantic import HttpUrl
 from simcore_service_api_server.utils.http_calls_capture import HttpApiCallCaptureModel
-from tenacity import retry
-from tenacity.after import after_log
-from tenacity.retry import retry_if_exception_type
-from tenacity.stop import stop_after_delay
-from tenacity.wait import wait_fixed
-
-
-@pytest.fixture(scope="module")
-def httpbin_base_url() -> Iterable[str]:
-    # yield "https://httpbin.org/" # sometimes is not available
-
-    port = 80
-    base_url = f"http://127.0.0.1:{port}"
-
-    client = docker.from_env()
-    container_name = "httpbin-fixture"
-    try:
-        client.containers.run(
-            "kennethreitz/httpbin",
-            ports={port: 80},
-            name=container_name,
-            detach=True,
-        )
-
-        @retry(
-            wait=wait_fixed(1),
-            retry=retry_if_exception_type(httpx.HTTPError),
-            stop=stop_after_delay(10),
-            after=after_log(logging.getLogger(), logging.DEBUG),
-        )
-        def _wait_until_httpbin_is_responsive():
-            r = httpx.get(f"{base_url}/get")
-            r.raise_for_status()
-
-        _wait_until_httpbin_is_responsive()
-
-        yield base_url
-
-    finally:
-        with suppress(APIError):
-            container = client.containers.get(container_name)
-            container.remove(force=True)
 
 
 async def test_capture_http_call(
-    event_loop: asyncio.AbstractEventLoop, httpbin_base_url
+    event_loop: asyncio.AbstractEventLoop, httpbin_base_url: HttpUrl
 ):
     # CAPTURE
     async with httpx.AsyncClient() as client:

services/storage/setup.cfg

@@ -13,3 +13,8 @@ commit_args = --no-verify
 
 [tool:pytest]
 asyncio_mode = auto
+markers =
+	slow: marks tests as slow (deselect with '-m "not slow"')
+	acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows."
+	testit: "marks test to run during development"
+	heavy_load: "mark tests that require large amount of data"

services/storage/tests/conftest.py

@@ -12,7 +12,7 @@
 import uuid
 from pathlib import Path
 from time import perf_counter
-from typing import AsyncIterator, Awaitable, Callable, Iterator, Optional, cast
+from typing import AsyncIterator, Awaitable, Callable, Iterator, cast
 
 import dotenv
 import pytest
@@ -59,18 +59,19 @@
 from yarl import URL
 
 pytest_plugins = [
+    "pytest_simcore.aioresponses_mocker",
     "pytest_simcore.cli_runner",
+    "pytest_simcore.docker_compose",
+    "pytest_simcore.docker_swarm",
+    "pytest_simcore.file_extra",
+    "pytest_simcore.httpbin_service",
+    "pytest_simcore.monkeypatch_extra",
+    "pytest_simcore.postgres_service",
+    "pytest_simcore.pytest_global_environs",
     "pytest_simcore.repository_paths",
+    "pytest_simcore.tmp_path_extra",
     "tests.fixtures.data_models",
     "tests.fixtures.datcore_adapter",
-    "pytest_simcore.pytest_global_environs",
-    "pytest_simcore.postgres_service",
-    "pytest_simcore.docker_swarm",
-    "pytest_simcore.docker_compose",
-    "pytest_simcore.tmp_path_extra",
-    "pytest_simcore.monkeypatch_extra",
-    "pytest_simcore.file_extra",
-    "pytest_simcore.aioresponses_mocker",
 ]
 
 CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
@@ -449,15 +450,15 @@ def upload_file(
     project_id: ProjectID,
     node_id: NodeID,
     create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]],
-    create_file_of_size: Callable[[ByteSize, Optional[str]], Path],
+    create_file_of_size: Callable[[ByteSize, str | None], Path],
     create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID],
 ) -> Callable[
-    [ByteSize, str, Optional[SimcoreS3FileID]], Awaitable[tuple[Path, SimcoreS3FileID]]
+    [ByteSize, str, SimcoreS3FileID | None], Awaitable[tuple[Path, SimcoreS3FileID]]
 ]:
     async def _uploader(
         file_size: ByteSize,
         file_name: str,
-        file_id: Optional[SimcoreS3FileID] = None,
+        file_id: SimcoreS3FileID | None = None,
         wait_for_completion: bool = True,
     ) -> tuple[Path, SimcoreS3FileID]:
         assert client.app
@@ -545,12 +546,12 @@ async def _uploader(
 @pytest.fixture
 def create_simcore_file_id(
     faker: Faker,
-) -> Callable[[ProjectID, NodeID, str, Optional[Path]], SimcoreS3FileID]:
+) -> Callable[[ProjectID, NodeID, str, Path | None], SimcoreS3FileID]:
     def _creator(
         project_id: ProjectID,
         node_id: NodeID,
         file_name: str,
-        file_base_path: Optional[Path] = None,
+        file_base_path: Path | None = None,
     ) -> SimcoreS3FileID:
         s3_file_name = file_name
         if file_base_path:

services/storage/tests/unit/test_utils.py

@@ -1,15 +1,22 @@
+# pylint: disable=protected-access
+# pylint: disable=redefined-outer-name
+# pylint: disable=too-many-arguments
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+
 import datetime
 import random
 from pathlib import Path
-from typing import Callable, Optional
+from typing import Callable
 from uuid import uuid4
 
 import pytest
 from aiohttp import ClientSession
 from faker import Faker
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID, SimcoreS3FileID
-from pydantic import ByteSize, parse_obj_as
+from pydantic import ByteSize, HttpUrl, parse_obj_as
 from simcore_service_storage.constants import S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID
 from simcore_service_storage.models import ETag, FileMetaData, S3BucketName, UploadID
 from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager
@@ -21,14 +28,14 @@
 )
 
 
-async def test_download_files(tmpdir):
+async def test_download_files(tmp_path: Path, httpbin_base_url: HttpUrl):
 
-    destination = Path(tmpdir) / "data"
+    destination = tmp_path / "data"
     expected_size = MAX_CHUNK_SIZE * 3 + 1000
 
     async with ClientSession() as session:
         total_size = await download_to_file_or_raise(
-            session, f"https://httpbin.org/bytes/{expected_size}", destination
+            session, f"{httpbin_base_url}/bytes/{expected_size}", destination
         )
         assert destination.exists()
         assert expected_size == total_size
@@ -62,9 +69,9 @@ async def test_download_files(tmpdir):
 )
 def test_file_entry_valid(
     file_size: ByteSize,
-    entity_tag: Optional[ETag],
-    upload_id: Optional[UploadID],
-    upload_expires_at: Optional[datetime.datetime],
+    entity_tag: ETag | None,
+    upload_id: UploadID | None,
+    upload_expires_at: datetime.datetime | None,
     expected_validity: bool,
     create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID],
     faker: Faker,

services/web/server/setup.cfg

@@ -18,3 +18,4 @@ markers =
 	slow: marks tests as slow (deselect with '-m "not slow"')
 	acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows."
 	testit: "marks test to run during development"
+	heavy_load: "mark tests that require large amount of data"