✨ Migrate autoscaling (⚠️ devops)

.github/workflows/ci-testing-deploy.yml

@@ -544,7 +544,10 @@ jobs:
         run: ./ci/helpers/show_system_versions.bash
       - name: install
         run: ./ci/github/unit-testing/autoscaling.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/autoscaling.bash typecheck
       - name: test
+        if: always()
         run: ./ci/github/unit-testing/autoscaling.bash test
       - uses: codecov/[email protected]
         with:
@@ -1567,7 +1570,7 @@ jobs:
       - name: a previous integration-test job failed
         if: ${{ contains(join(needs.*.result, ','), 'failure') }}
         run: |
-          echo "::error title=ERROR::one of the unit-tests failed!"
+          echo "::error title=ERROR::one of the integration-tests failed!"
           exit 1
       - name: all the previous integration-tests were run successfully or skipped
         if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
@@ -1821,7 +1824,7 @@ jobs:
       - name: a previous system-test job failed
         if: ${{ contains(join(needs.*.result, ','), 'failure') }}
         run: |
-          echo "::error title=ERROR::one of the unit-tests failed!"
+          echo "::error title=ERROR::one of the system-tests failed!"
           exit 1
       - name: all the previous system-tests were run successfully or skipped
         if: ${{ !contains(join(needs.*.result, ','), 'failure') }}

.vscode/settings.template.json

@@ -65,8 +65,10 @@
   "shellcheck.run": "onSave",
   "shellcheck.enableQuickFix": true,
   "python.formatting.provider": "black",
-  "python.sortImports.path": "${workspaceFolder}/.venv/bin/isort",
-  "python.sortImports.args": [
+  "isort.path": [
+    "${workspaceFolder}/.venv/bin/isort"
+  ],
+  "isort.args": [
     "--settings-path=${workspaceFolder}/.isort.cfg"
   ]
 }

packages/models-library/src/models_library/docker.py

@@ -1,7 +1,18 @@
-from pydantic import constr
+import re
+from typing import Optional
+
+from pydantic import ConstrainedStr, constr
 
 DOCKER_IMAGE_KEY_RE = r"[\w/-]+"
 DOCKER_IMAGE_VERSION_RE = r"[\w/.]+"
 
 DockerImageKey = constr(regex=DOCKER_IMAGE_KEY_RE)
 DockerImageVersion = constr(regex=DOCKER_IMAGE_VERSION_RE)
+
+
+class DockerLabelKey(ConstrainedStr):
+    # NOTE: https://docs.docker.com/config/labels-custom-metadata/#key-format-recommendations
+    # good practice: use reverse DNS notation
+    regex: Optional[re.Pattern[str]] = re.compile(
+        r"^(?!(\.|\-|com.docker\.|io.docker\.|org.dockerproject\.))(?!.*(--|\.\.))[a-z0-9\.-]+$"
+    )

packages/models-library/src/models_library/generated_models/docker_rest_api.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  https://docs.docker.com/engine/api/v1.41.yaml
-#   timestamp: 2022-05-12T20:15:58+00:00
+#   timestamp: 2022-11-19T13:23:51+00:00
 
 from __future__ import annotations
 
@@ -577,7 +577,14 @@ class Isolation(Enum):
 
 class ContainerConfig(BaseModel):
     """
-    Configuration for a container that is portable between hosts.
+        Configuration for a container that is portable between hosts.
+
+    When used as `ContainerConfig` field in an image, `ContainerConfig` is an
+    optional field containing the configuration of the container that was last
+    committed when creating the image.
+
+    Previous versions of Docker builder used this field to store build cache,
+    and it is not in active use anymore.
 
     """
 
@@ -783,7 +790,7 @@ class ImageInspect(BaseModel):
 
     Id: Optional[str] = Field(
         None,
-        description="ID is the content-addressable ID of an image.\n\nThis identified is a content-addressable digest calculated from the\nimage's configuration (which includes the digests of layers used by\nthe image).\n\nNote that this digest differs from the `RepoDigests` below, which\nholds digests of image manifests that reference the image.\n",
+        description="ID is the content-addressable ID of an image.\n\nThis identifier is a content-addressable digest calculated from the\nimage's configuration (which includes the digests of layers used by\nthe image).\n\nNote that this digest differs from the `RepoDigests` below, which\nholds digests of image manifests that reference the image.\n",
         example="sha256:ec3f0931a6e6b6855d76b2d7b0be30e81860baccd891b2e243280bf1cd8ad710",
     )
     RepoTags: Optional[list[str]] = Field(
@@ -878,16 +885,67 @@ class ImageInspect(BaseModel):
 
 
 class ImageSummary(BaseModel):
-    Id: str
-    ParentId: str
-    RepoTags: list[str]
-    RepoDigests: list[str]
-    Created: int
-    Size: int
-    SharedSize: int
-    VirtualSize: int
-    Labels: dict[str, str]
-    Containers: int
+    Id: str = Field(
+        ...,
+        description="ID is the content-addressable ID of an image.\n\nThis identifier is a content-addressable digest calculated from the\nimage's configuration (which includes the digests of layers used by\nthe image).\n\nNote that this digest differs from the `RepoDigests` below, which\nholds digests of image manifests that reference the image.\n",
+        example="sha256:ec3f0931a6e6b6855d76b2d7b0be30e81860baccd891b2e243280bf1cd8ad710",
+    )
+    ParentId: str = Field(
+        ...,
+        description="ID of the parent image.\n\nDepending on how the image was created, this field may be empty and\nis only set for images that were built/created locally. This field\nis empty if the image was pulled from an image registry.\n",
+        example="",
+    )
+    RepoTags: list[str] = Field(
+        ...,
+        description='List of image names/tags in the local image cache that reference this\nimage.\n\nMultiple image tags can refer to the same imagem and this list may be\nempty if no tags reference the image, in which case the image is\n"untagged", in which case it can still be referenced by its ID.\n',
+        example=[
+            "example:1.0",
+            "example:latest",
+            "example:stable",
+            "internal.registry.example.com:5000/example:1.0",
+        ],
+    )
+    RepoDigests: list[str] = Field(
+        ...,
+        description="List of content-addressable digests of locally available image manifests\nthat the image is referenced from. Multiple manifests can refer to the\nsame image.\n\nThese digests are usually only available if the image was either pulled\nfrom a registry, or if the image was pushed to a registry, which is when\nthe manifest is generated and its digest calculated.\n",
+        example=[
+            "example@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb",
+            "internal.registry.example.com:5000/example@sha256:b69959407d21e8a062e0416bf13405bb2b71ed7a84dde4158ebafacfa06f5578",
+        ],
+    )
+    Created: int = Field(
+        ...,
+        description="Date and time at which the image was created as a Unix timestamp\n(number of seconds sinds EPOCH).\n",
+        example="1644009612",
+    )
+    Size: int = Field(
+        ...,
+        description="Total size of the image including all layers it is composed of.\n",
+        example=172064416,
+    )
+    SharedSize: int = Field(
+        ...,
+        description="Total size of image layers that are shared between this image and other\nimages.\n\nThis size is not calculated by default. `-1` indicates that the value\nhas not been set / calculated.\n",
+        example=1239828,
+    )
+    VirtualSize: int = Field(
+        ...,
+        description="Total size of the image including all layers it is composed of.\n\nIn versions of Docker before v1.10, this field was calculated from\nthe image itself and all of its parent images. Docker v1.10 and up\nstore images self-contained, and no longer use a parent-chain, making\nthis field an equivalent of the Size field.\n\nThis field is kept for backward compatibility, but may be removed in\na future version of the API.\n",
+        example=172064416,
+    )
+    Labels: dict[str, str] = Field(
+        ...,
+        description="User-defined key/value metadata.",
+        example={
+            "com.example.some-label": "some-value",
+            "com.example.some-other-label": "some-other-value",
+        },
+    )
+    Containers: int = Field(
+        ...,
+        description="Number of containers using this image. Includes both stopped and running\ncontainers.\n\nThis size is not calculated by default, and depends on which API endpoint\nis used. `-1` indicates that the value has not been set / calculated.\n",
+        example=2,
+    )
 
 
 class AuthConfig(BaseModel):
@@ -1022,15 +1080,54 @@ class NetworkContainer(BaseModel):
     IPv6Address: Optional[str] = None
 
 
+class Type4(Enum):
+    """
+    Cache record type.
+
+    """
+
+    internal = "internal"
+    frontend = "frontend"
+    source_local = "source.local"
+    source_git_checkout = "source.git.checkout"
+    exec_cachemount = "exec.cachemount"
+    regular = "regular"
+
+
 class BuildCache(BaseModel):
-    ID: Optional[str] = None
-    Parent: Optional[str] = None
-    Type: Optional[str] = None
-    Description: Optional[str] = None
-    InUse: Optional[bool] = None
-    Shared: Optional[bool] = None
+    """
+    BuildCache contains information about a build cache record.
+
+    """
+
+    ID: Optional[str] = Field(
+        None,
+        description="Unique ID of the build cache record.\n",
+        example="ndlpt0hhvkqcdfkputsk4cq9c",
+    )
+    Parent: Optional[str] = Field(
+        None,
+        description="ID of the parent build cache record.\n",
+        example="hw53o5aio51xtltp5xjp8v7fx",
+    )
+    Type: Optional[Type4] = Field(
+        None, description="Cache record type.\n", example="regular"
+    )
+    Description: Optional[str] = Field(
+        None,
+        description="Description of the build-step that produced the build cache.\n",
+        example="mount / from exec /bin/sh -c echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/keep-cache",
+    )
+    InUse: Optional[bool] = Field(
+        None, description="Indicates if the build cache is in use.\n", example=False
+    )
+    Shared: Optional[bool] = Field(
+        None, description="Indicates if the build cache is shared.\n", example=True
+    )
     Size: Optional[int] = Field(
-        None, description="Amount of disk space used by the build cache (in bytes).\n"
+        None,
+        description="Amount of disk space used by the build cache (in bytes).\n",
+        example=51,
     )
     CreatedAt: Optional[str] = Field(
         None,
@@ -1042,7 +1139,7 @@ class BuildCache(BaseModel):
         description="Date and time at which the build cache was last used in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n",
         example="2017-08-09T07:09:37.632105588Z",
     )
-    UsageCount: Optional[int] = None
+    UsageCount: Optional[int] = Field(None, example=26)
 
 
 class ImageID(BaseModel):
@@ -2741,7 +2838,7 @@ class EventActor(BaseModel):
     )
 
 
-class Type4(Enum):
+class Type5(Enum):
     """
     The type of object emitting the event
     """
@@ -2776,7 +2873,7 @@ class SystemEventsResponse(BaseModel):
 
     """
 
-    Type: Optional[Type4] = Field(
+    Type: Optional[Type5] = Field(
         None, description="The type of object emitting the event", example="container"
     )
     Action: Optional[str] = Field(
@@ -3187,7 +3284,7 @@ class Resources1(BaseModel):
     """
 
     Limits: Optional[Limit] = Field(None, description="Define resources limits.")
-    Reservation: Optional[ResourceObject] = Field(
+    Reservations: Optional[ResourceObject] = Field(
         None, description="Define resources reservation."
     )
 
@@ -3409,7 +3506,7 @@ class ContainerWaitResponse(BaseModel):
     """
 
     StatusCode: int = Field(..., description="Exit code of the container")
-    Error: ContainerWaitExitError
+    Error: Optional[ContainerWaitExitError] = None
 
 
 class RegistryServiceConfig(BaseModel):
@@ -3491,7 +3588,7 @@ class SwarmInfo(BaseModel):
         description="IP address at which this node can be reached by other nodes in the\nswarm.\n",
         example="10.0.0.46",
     )
-    LocalNodeState: Optional[LocalNodeState] = None
+    LocalNodeState: Optional[LocalNodeState] = ""
     ControlAvailable: Optional[bool] = Field(False, example=True)
     Error: Optional[str] = ""
     RemoteManagers: Optional[list[PeerNode]] = Field(
@@ -3929,3 +4026,6 @@ class SystemInfo(BaseModel):
             "WARNING: bridge-nf-call-ip6tables is disabled",
         ],
     )
+
+
+# nopycln: file

packages/models-library/tests/test_docker.py

@@ -0,0 +1,37 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+import pytest
+from models_library.docker import DockerLabelKey
+from pydantic import ValidationError, parse_obj_as
+
+
+@pytest.mark.parametrize(
+    "label_key, valid",
+    (
+        (".com.docker.hello", False),
+        ("_com.docker.hello", False),
+        ("com.docker_man.hello", False),
+        ("io.docker.hello", False),
+        ("org.dockerproject.hello", False),
+        ("com.hey-hey.hello", True),
+        ("com.hey--hey.hello", False),
+        ("com.hey..hey.hello", False),
+        ("com.hey_hey.hello", False),
+        ("node.labels.standard_worker", False),
+        ("Node.labels.standard_worker", False),
+        ("Node.labels.standard_worker", False),
+        ("Node.labels.standardworker", False),
+        ("node.labels.standardworker", True),
+        ("io.osparc.auto-scaler", True),
+    ),
+)
+def test_docker_label_key(label_key: str, valid: bool):
+    # NOTE: https://docs.docker.com/config/labels-custom-metadata/#key-format-recommendations
+    if valid:
+        instance = parse_obj_as(DockerLabelKey, label_key)
+        assert instance
+    else:
+        with pytest.raises(ValidationError):
+            parse_obj_as(DockerLabelKey, label_key)

packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py

@@ -17,7 +17,7 @@
 #
 # monkeypatch using dict
 #
-def setenvs_from_dict(monkeypatch: MonkeyPatch, envs: EnvVarsDict):
+def setenvs_from_dict(monkeypatch: MonkeyPatch, envs: EnvVarsDict) -> EnvVarsDict:
     for key, value in envs.items():
         assert value is not None  # None keys cannot be is defined w/o value
         monkeypatch.setenv(key, str(value))