From 5c82a0ffa70e583fa3e48d9cb6dd0d5572d87a8b Mon Sep 17 00:00:00 2001 From: approxit Date: Thu, 16 Feb 2023 18:00:30 +0100 Subject: [PATCH 1/7] Manifest DTO implementation --- tests/payload/test_manifest.py | 305 +++++++++++++++++++++++++++++++++ tests/test_utils.py | 47 +++++ yapapi/payload/manifest.py | 291 +++++++++++++++++++++++++++++++ yapapi/utils.py | 106 +++++++++++- 4 files changed, 747 insertions(+), 2 deletions(-) create mode 100644 tests/payload/test_manifest.py create mode 100644 tests/test_utils.py create mode 100644 yapapi/payload/manifest.py diff --git a/tests/payload/test_manifest.py b/tests/payload/test_manifest.py new file mode 100644 index 000000000..6d63e1c66 --- /dev/null +++ b/tests/payload/test_manifest.py @@ -0,0 +1,305 @@ +from datetime import datetime +from dateutil.tz import UTC +import json +import pytest +from unittest import mock + +from yapapi.payload.manifest import ( + CompManifest, + CompManifestNet, + CompManifestNetInet, + CompManifestNetInetOut, + CompManifestScript, + Manifest, + ManifestMetadata, + ManifestPayload, + ManifestPayloadPlatform, +) + + +@pytest.fixture +def manifest_obj(): + return Manifest( + version="0.1.0", + created_at=datetime(2022, 12, 1, 0, 0, 0, tzinfo=UTC), + expires_at=datetime(2100, 1, 1, 0, 0, 0, tzinfo=UTC), + metadata=ManifestMetadata( + name="Service1", + description="Description of Service1", + version="0.1.1", + authors=[ + "mf ", + "ng ", + ], + homepage="https://github.com/golemfactory/s1", + ), + payload=[ + ManifestPayload( + platform=ManifestPayloadPlatform( + arch="x86_64", + os="linux", + os_version="6.1.7601", + ), + urls=[ + "http://girepo.dev.golem.network:8000/" + "docker-gas_scanner_backend_image-latest-91c471517a.gvmi", + ], + hash="sha3:05270a8a938ff5f5e30b0e61bc983a8c3e286c5cd414a32e1a077657", + ), + ], + comp_manifest=CompManifest( + version="0.1.0", + script=CompManifestScript( + commands=[ + "run .*", + ], + match="regex", + ), + net=CompManifestNet( + inet=CompManifestNetInet( + out=CompManifestNetInetOut( + protocols=[ + "http", + ], + urls=[ + "http://bor.golem.network", + ], + ), + ), + ), + ), + ) + + +@pytest.fixture +def manifest_obj_with_no_optional_values(manifest_obj): + manifest_obj.metadata = None + manifest_obj.payload[0].platform.os_version = None + + return manifest_obj + + +@pytest.fixture +def manifest_dict(): + return { + "version": "0.1.0", + "created_at": "2022-12-01T00:00:00+00:00", + "expires_at": "2100-01-01T00:00:00+00:00", + "metadata": { + "name": "Service1", + "description": "Description of Service1", + "version": "0.1.1", + "authors": [ + "mf ", + "ng ", + ], + "homepage": "https://github.com/golemfactory/s1", + }, + "payload": [ + { + "platform": { + "arch": "x86_64", + "os": "linux", + "os_version": "6.1.7601", + }, + "urls": [ + "http://girepo.dev.golem.network:8000/" + "docker-gas_scanner_backend_image-latest-91c471517a.gvmi", + ], + "hash": "sha3:05270a8a938ff5f5e30b0e61bc983a8c3e286c5cd414a32e1a077657", + } + ], + "comp_manifest": { + "version": "0.1.0", + "script": { + "commands": [ + "run .*", + ], + "match": "regex", + }, + "net": { + "inet": { + "out": { + "protocols": [ + "http", + ], + "urls": [ + "http://bor.golem.network", + ], + }, + }, + }, + }, + } + + +@pytest.fixture +def manifest_dict_with_alias(manifest_dict): + manifest_dict["createdAt"] = manifest_dict.pop("created_at") + manifest_dict["expiresAt"] = manifest_dict.pop("expires_at") + manifest_dict["compManifest"] = manifest_dict.pop("comp_manifest") + manifest_dict["payload"][0]["platform"]["osVersion"] = manifest_dict["payload"][0][ + "platform" + ].pop("os_version") + + return manifest_dict + + +@pytest.fixture +def manifest_dict_with_no_optional_values(manifest_dict): + manifest_dict.pop("metadata") + manifest_dict["payload"][0]["platform"].pop("os_version") + + return manifest_dict + + +def test_manifest_to_dict_no_alias(manifest_obj, manifest_dict): + assert manifest_obj.dict() == manifest_dict + + +def test_manifest_to_dict_with_alias(manifest_obj, manifest_dict_with_alias): + assert manifest_obj.dict(by_alias=True) == manifest_dict_with_alias + + +def test_manifest_to_dict_missing_values( + manifest_obj_with_no_optional_values, manifest_dict_with_no_optional_values +): + assert manifest_obj_with_no_optional_values.dict() == manifest_dict_with_no_optional_values + + +def test_manifest_to_json(manifest_obj, manifest_dict): + manifest_obj_json = json.dumps(manifest_obj.dict()) + manifest_dict_json = json.dumps(manifest_obj.dict()) + + assert json.loads(manifest_obj_json) == json.loads(manifest_dict_json) + + +def test_manifest_parse_obj(manifest_obj, manifest_dict): + assert Manifest.parse_obj(manifest_dict) == manifest_obj + + +@mock.patch( + "yapapi.payload.manifest.datetime", **{"utcnow.return_value": datetime(2020, 1, 1, tzinfo=UTC)} +) +def test_manifest_with_minimal_data(mocked_datetime): + payload_hash = "asd" + + minimal_manifest_dict = Manifest( + payload=[ + ManifestPayload( + hash=payload_hash, + ), + ], + ).dict() + + assert minimal_manifest_dict == { + "created_at": "2020-01-01T00:00:00+00:00", + "expires_at": "2100-01-01T00:00:00+00:00", + "payload": [ + { + "hash": payload_hash, + "urls": [], + "platform": { + "arch": "x86_64", + "os": "linux", + }, + }, + ], + "version": "", + } + + +@mock.patch( + "yapapi.payload.manifest.datetime", **{"utcnow.return_value": datetime(2020, 1, 1, tzinfo=UTC)} +) +def test_manifest_generate(mocked_datetime): + payload_urls = [ + "some url", + "some other url", + "yet another url", + ] + payload_hash = "some hash" + metadata_name = "example" + metadata_version = "0.0.1" + comp_manifest_script_match = "strict" + + assert Manifest.parse_imploded_obj( + { + "metadata.name": metadata_name, + "metadata.version": metadata_version, + "payload.0.hash": payload_hash, + "payload.0.urls": [ + payload_urls[0], + payload_urls[1], + ], + "payload.0.urls.2": payload_urls[2], + "comp_manifest.script.match": comp_manifest_script_match, + } + ).dict() == { + "metadata": { + "name": metadata_name, + "version": metadata_version, + "authors": [], + "homepage": None, + "description": None, + }, + "comp_manifest": { + "script": { + "commands": [ + "run .*", + ], + "match": comp_manifest_script_match, + }, + "version": "", + }, + "created_at": "2020-01-01T00:00:00+00:00", + "expires_at": "2100-01-01T00:00:00+00:00", + "payload": [ + { + "hash": payload_hash, + "urls": payload_urls, + "platform": { + "arch": "x86_64", + "os": "linux", + }, + }, + ], + "version": "", + } + + +@pytest.mark.asyncio +async def test_manifest_payload_resolve_urls_from_hash(): + payload = ManifestPayload( + hash="05270a8a938ff5f5e30b0e61bc983a8c3e286c5cd414a32e1a077657", + ) + + assert payload.urls == [] + + await payload.resolve_urls_from_hash() + + assert payload.urls == [ + "http://girepo.dev.golem.network:8000/" + "docker-gas_scanner_backend_image-latest-91c471517a.gvmi", + ] + + +@pytest.mark.asyncio +async def test_manifest_payload_resolve_urls_from_hash_is_not_duplicating_or_overriding_values(): + urls = [ + "some url", + "http://girepo.dev.golem.network:8000/" + "docker-gas_scanner_backend_image-latest-91c471517a.gvmi", + "other url", + ] + + payload = ManifestPayload( + hash="05270a8a938ff5f5e30b0e61bc983a8c3e286c5cd414a32e1a077657", + urls=urls, + ) + + assert payload.urls == urls + + await payload.resolve_urls_from_hash() + + assert payload.urls == urls diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..ba6d30ade --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,47 @@ +from yapapi.utils import explode_dict + + +def test_explode_dict(): + assert explode_dict( + { + "root_field": 1, + "nested.field": 2, + "nested.obj": { + "works": "fine", + }, + "nested.obj.with_array": [ + "okay!", + ], + "even.more.nested.field": 3, + "arrays.0.are": { + "supported": "too", + }, + "arrays.1": "works fine", + } + ) == { + "root_field": 1, + "nested": { + "field": 2, + "obj": { + "works": "fine", + "with_array": [ + "okay!", + ], + }, + }, + "even": { + "more": { + "nested": { + "field": 3, + }, + }, + }, + "arrays": [ + { + "are": { + "supported": "too", + }, + }, + "works fine", + ], + } diff --git a/yapapi/payload/manifest.py b/yapapi/payload/manifest.py new file mode 100644 index 000000000..c1f8a5e25 --- /dev/null +++ b/yapapi/payload/manifest.py @@ -0,0 +1,291 @@ +from datetime import datetime +from dateutil.tz import UTC +import sys +from typing import Dict, List, Optional + +from yapapi.payload import vm +from yapapi.utils import explode_dict + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +from dataclasses import dataclass, field + + +@dataclass +class ManifestMetadata: + name: str + version: str + authors: List[str] = field(default_factory=list) + homepage: Optional[str] = None + description: Optional[str] = None + + def dict(self, *, by_alias=False): + return { + "name": self.name, + "description": self.description, + "version": self.version, + "authors": self.authors.copy(), + "homepage": self.homepage, + } + + @classmethod + def parse_obj(cls, obj: Dict) -> "ManifestMetadata": + obj_copy = obj.copy() + + if obj_copy.get("authors") is not None: + obj_copy["authors"] = obj_copy["authors"].copy() + + return cls(**obj_copy) + + +@dataclass +class ManifestPayloadPlatform: + arch: str = "x86_64" + os: str = "linux" + os_version: Optional[str] = None + + def dict(self, *, by_alias=False): + obj = { + "arch": self.arch, + "os": self.os, + } + + if self.os_version is not None: + obj["osVersion" if by_alias else "os_version"] = self.os_version + + return obj + + @classmethod + def parse_obj(cls, obj: Dict) -> "ManifestPayloadPlatform": + obj_copy = obj.copy() + + return cls(**obj_copy) + + +@dataclass +class ManifestPayload: + hash: str + urls: List[str] = field(default_factory=list) + platform: Optional[ManifestPayloadPlatform] = field(default_factory=ManifestPayloadPlatform) + + def dict(self, *, by_alias=False): + obj: Dict = { + "hash": self.hash, + } + + if self.platform is not None: + obj["platform"] = self.platform.dict(by_alias=by_alias) + + if self.urls is not None: + obj["urls"] = self.urls.copy() + + return obj + + @classmethod + def parse_obj(cls, obj: Dict) -> "ManifestPayload": + obj_copy = obj.copy() + + if obj_copy.get("platform") is not None: + obj_copy["platform"] = ManifestPayloadPlatform.parse_obj(obj_copy["platform"]) + + if obj_copy.get("urls") is not None: + obj_copy["urls"] = obj_copy["urls"].copy() + + return cls(**obj_copy) + + async def resolve_urls_from_hash(self) -> None: + package = await vm.repo(image_hash=self.hash) + + full_url = await package.resolve_url() + + image_url = full_url.split(":", 3)[3] + + if image_url not in self.urls: + self.urls.append(image_url) + + +@dataclass +class CompManifestScript: + commands: List[str] = field(default_factory=lambda: ["run .*"]) + match: Literal["strict", "regex"] = "regex" + + def dict(self, *, by_alias=False): + return { + "commands": self.commands.copy(), + "match": self.match, + } + + @classmethod + def parse_obj(cls, obj: Dict) -> "CompManifestScript": + obj_copy = obj.copy() + + if obj_copy.get("commands") is not None: + obj_copy["commands"] = obj_copy["commands"].copy() + + return cls(**obj_copy) + + +@dataclass +class CompManifestNetInetOut: + protocols: List[Literal["http", "https", "ws", "wss"]] = field( + default_factory=lambda: ["http", "https", "ws", "wss"] + ) + urls: Optional[List[str]] = None + + def dict(self, *, by_alias=False): + obj: Dict = { + "protocols": self.protocols.copy(), + } + + if self.urls is not None: + obj["urls"] = self.urls.copy() + + return obj + + @classmethod + def parse_obj(cls, obj: Dict) -> "CompManifestNetInetOut": + obj_copy = obj.copy() + + if obj_copy.get("protocols") is not None: + obj_copy["protocols"] = obj_copy["protocols"].copy() + + if obj_copy.get("urls") is not None: + obj_copy["urls"] = obj_copy["urls"].copy() + + return cls(**obj_copy) + + +@dataclass +class CompManifestNetInet: + out: Optional[CompManifestNetInetOut] = None + + def dict(self, *, by_alias=False): + obj = {} + + if self.out is not None: + obj["out"] = self.out.dict(by_alias=by_alias) + + return obj + + @classmethod + def parse_obj(cls, obj: Dict) -> "CompManifestNetInet": + obj_copy = obj.copy() + + if obj_copy.get("out") is not None: + obj_copy["out"] = CompManifestNetInetOut.parse_obj(obj_copy["out"]) + + return cls(**obj_copy) + + +@dataclass +class CompManifestNet: + inet: Optional[CompManifestNetInet] = None + + def dict(self, *, by_alias=False): + obj = {} + + if self.inet is not None: + obj["inet"] = self.inet.dict(by_alias=by_alias) + + return obj + + @classmethod + def parse_obj(cls, obj: Dict) -> "CompManifestNet": + obj_copy = obj.copy() + + if obj_copy.get("inet") is not None: + obj_copy["inet"] = CompManifestNetInet.parse_obj(obj_copy["inet"]) + + return cls(**obj_copy) + + +@dataclass +class CompManifest: + version: str = "" + script: Optional[CompManifestScript] = None + net: Optional[CompManifestNet] = None + + def dict(self, *, by_alias=False): + obj = { + "version": self.version, + } + + if self.script is not None: + obj["script"] = self.script.dict(by_alias=by_alias) + + if self.net is not None: + obj["net"] = self.net.dict(by_alias=by_alias) + + return obj + + @classmethod + def parse_obj(cls, obj: Dict) -> "CompManifest": + obj_copy = obj.copy() + + if obj_copy.get("script") is not None: + obj_copy["script"] = CompManifestScript.parse_obj(obj_copy["script"]) + + if obj_copy.get("net") is not None: + obj_copy["net"] = CompManifestNet.parse_obj(obj_copy["net"]) + + return cls(**obj_copy) + + +@dataclass +class Manifest: + payload: List[ManifestPayload] + version: str = "" + comp_manifest: Optional[CompManifest] = None + # Using lambda helps with mocking in tests + created_at: datetime = field(default_factory=lambda: datetime.utcnow()) + expires_at: datetime = datetime(2100, 1, 1, tzinfo=UTC) + metadata: Optional[ManifestMetadata] = None + + def dict(self, *, by_alias=False): + obj = { + "version": self.version, + "createdAt" if by_alias else "created_at": self.created_at.isoformat(), + "expiresAt" if by_alias else "expires_at": self.expires_at.isoformat(), + "payload": [payload.dict(by_alias=by_alias) for payload in self.payload], + } + + if self.comp_manifest is not None: + comp_manifest_field_name = "compManifest" if by_alias else "comp_manifest" + obj[comp_manifest_field_name] = self.comp_manifest.dict(by_alias=by_alias) + + if self.metadata is not None: + obj["metadata"] = self.metadata.dict(by_alias=by_alias) + + return obj + + @classmethod + def parse_obj(cls, obj: Dict) -> "Manifest": + obj_copy = obj.copy() + + if isinstance(obj_copy.get("created_at"), str): + obj_copy["created_at"] = datetime.strptime( + obj_copy["created_at"], "%Y-%m-%dT%H:%M:%S%z" + ) + + if isinstance(obj_copy.get("expires_at"), str): + obj_copy["expires_at"] = datetime.strptime( + obj_copy["expires_at"], "%Y-%m-%dT%H:%M:%S%z" + ) + + if obj_copy.get("payload") is not None: + obj_copy["payload"] = [ManifestPayload.parse_obj(p) for p in obj_copy["payload"]] + + if obj_copy.get("comp_manifest") is not None: + obj_copy["comp_manifest"] = CompManifest.parse_obj(obj_copy["comp_manifest"]) + + if obj_copy.get("metadata") is not None: + obj_copy["metadata"] = ManifestMetadata.parse_obj(obj_copy["metadata"]) + + return cls(**obj_copy) + + @classmethod + def parse_imploded_obj(cls, obj: Dict) -> "Manifest": + return cls.parse_obj(explode_dict(obj)) diff --git a/yapapi/utils.py b/yapapi/utils.py index 0db9a7b1c..dfa352b2b 100644 --- a/yapapi/utils.py +++ b/yapapi/utils.py @@ -1,11 +1,11 @@ """Utility functions and classes used within yapapi.""" import asyncio +from datetime import datetime, timezone, tzinfo import enum import functools import logging +from typing import AsyncContextManager, Callable, Dict, List, Optional, Union import warnings -from datetime import datetime, timezone, tzinfo -from typing import AsyncContextManager, Callable, Optional logger = logging.getLogger(__name__) @@ -188,3 +188,105 @@ def strtobool(val): def utc_now() -> datetime: """Get a timezone-aware datetime for _now_.""" return datetime.now(tz=timezone.utc) + + +def explode_dict(imploded_dict: Dict, separator=".") -> Dict: + """Return an exploded dictionary based on path found in keys. + + Example usage: + + assert explode_dict( + { + "root_field": 1, + "nested.field": 2, + "nested.obj": { + "works": "fine", + }, + "nested.obj.with_array": [ + "okay!", + ], + "even.more.nested.field": 3, + "arrays.0.are": { + "supported": "too", + }, + "arrays.1": "works fine", + } + ) == { + "root_field": 1, + "nested": { + "field": 2, + "obj": { + "works": "fine", + "with_array": [ + "okay!", + ], + }, + }, + "even": { + "more": { + "nested": { + "field": 3, + }, + }, + }, + "arrays": [ + { + "are": { + "supported": "too", + }, + }, + "works fine", + ], + } + + """ + obj: Dict = {} + + for path, value in imploded_dict.items(): + parts = path.split(separator) + + nested_obj: Union[List, Dict] = obj + for part, next_part in zip(parts, parts[1:]): + try: + int(next_part) + except ValueError: + is_next_part_number = False + else: + is_next_part_number = True + + try: + part_index = int(part) + except ValueError: + assert isinstance(nested_obj, Dict) + + if is_next_part_number: + nested_obj = nested_obj.setdefault(part, []) + else: + nested_obj = nested_obj.setdefault(part, {}) + else: + + try: + nested_obj = nested_obj[part_index] + except IndexError: + assert isinstance(nested_obj, List) + + new_nested_obj: Union[List, Dict] = [] if is_next_part_number else {} + nested_obj.insert(part_index, new_nested_obj) + nested_obj = new_nested_obj + + try: + last_part = int(parts[-1]) + except ValueError: + assert isinstance(nested_obj, Dict) + last_part = parts[-1] + else: + assert isinstance(nested_obj, List) + + try: + nested_obj[last_part] + except IndexError: + nested_obj.insert(last_part, None) + + nested_obj[last_part] = value + + return obj From a58719fbb508145a7cb7853dd192f94433af60dd Mon Sep 17 00:00:00 2001 From: approxit Date: Tue, 7 Mar 2023 16:03:09 +0100 Subject: [PATCH 2/7] formatting after rebase --- tests/payload/test_manifest.py | 7 ++++--- yapapi/payload/manifest.py | 11 ++++++----- yapapi/utils.py | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/payload/test_manifest.py b/tests/payload/test_manifest.py index 6d63e1c66..0a85e2ddb 100644 --- a/tests/payload/test_manifest.py +++ b/tests/payload/test_manifest.py @@ -1,9 +1,10 @@ -from datetime import datetime -from dateutil.tz import UTC import json -import pytest +from datetime import datetime from unittest import mock +import pytest +from dateutil.tz import UTC + from yapapi.payload.manifest import ( CompManifest, CompManifestNet, diff --git a/yapapi/payload/manifest.py b/yapapi/payload/manifest.py index c1f8a5e25..91afc671c 100644 --- a/yapapi/payload/manifest.py +++ b/yapapi/payload/manifest.py @@ -1,17 +1,18 @@ -from datetime import datetime -from dateutil.tz import UTC import sys +from datetime import datetime from typing import Dict, List, Optional -from yapapi.payload import vm -from yapapi.utils import explode_dict +from dataclasses import dataclass, field if sys.version_info >= (3, 8): from typing import Literal else: from typing_extensions import Literal -from dataclasses import dataclass, field +from dateutil.tz import UTC + +from yapapi.payload import vm +from yapapi.utils import explode_dict @dataclass diff --git a/yapapi/utils.py b/yapapi/utils.py index dfa352b2b..afee57a40 100644 --- a/yapapi/utils.py +++ b/yapapi/utils.py @@ -1,11 +1,11 @@ """Utility functions and classes used within yapapi.""" import asyncio -from datetime import datetime, timezone, tzinfo import enum import functools import logging -from typing import AsyncContextManager, Callable, Dict, List, Optional, Union import warnings +from datetime import datetime, timezone, tzinfo +from typing import AsyncContextManager, Callable, Dict, List, Optional, Union logger = logging.getLogger(__name__) From 295297e6b2090e20cba7b22d8a27ff098517a72d Mon Sep 17 00:00:00 2001 From: approxit Date: Fri, 10 Mar 2023 14:56:42 +0100 Subject: [PATCH 3/7] tmp --- tests/payload/test_manifest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/payload/test_manifest.py b/tests/payload/test_manifest.py index 0a85e2ddb..382e8fdee 100644 --- a/tests/payload/test_manifest.py +++ b/tests/payload/test_manifest.py @@ -170,7 +170,7 @@ def test_manifest_to_dict_missing_values( def test_manifest_to_json(manifest_obj, manifest_dict): manifest_obj_json = json.dumps(manifest_obj.dict()) - manifest_dict_json = json.dumps(manifest_obj.dict()) + manifest_dict_json = json.dumps(manifest_dict) assert json.loads(manifest_obj_json) == json.loads(manifest_dict_json) @@ -285,6 +285,7 @@ async def test_manifest_payload_resolve_urls_from_hash(): ] +@mock.patch("yapapi.payload.vm.repo", mock.AsyncMock(**{'return_value.resolve_url': mock.AsyncMock(return_value='dupa')})) @pytest.mark.asyncio async def test_manifest_payload_resolve_urls_from_hash_is_not_duplicating_or_overriding_values(): urls = [ From df0e2f3cc31f32e5dba91cf916456c34f7735b11 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 13 Mar 2023 16:22:35 +0100 Subject: [PATCH 4/7] mocked out image url resolvers --- tests/payload/test_manifest.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/payload/test_manifest.py b/tests/payload/test_manifest.py index 382e8fdee..f408644c1 100644 --- a/tests/payload/test_manifest.py +++ b/tests/payload/test_manifest.py @@ -269,6 +269,12 @@ def test_manifest_generate(mocked_datetime): } +@mock.patch( + "yapapi.payload.vm.repo", + mock.AsyncMock( + **{"return_value.resolve_url.return_value": "hash:sha3:some_image_hash:some_image_url"} + ), +) @pytest.mark.asyncio async def test_manifest_payload_resolve_urls_from_hash(): payload = ManifestPayload( @@ -280,24 +286,27 @@ async def test_manifest_payload_resolve_urls_from_hash(): await payload.resolve_urls_from_hash() assert payload.urls == [ - "http://girepo.dev.golem.network:8000/" - "docker-gas_scanner_backend_image-latest-91c471517a.gvmi", + "some_image_url", ] -@mock.patch("yapapi.payload.vm.repo", mock.AsyncMock(**{'return_value.resolve_url': mock.AsyncMock(return_value='dupa')})) +@mock.patch( + "yapapi.payload.vm.repo", + mock.AsyncMock( + **{"return_value.resolve_url.return_value": "hash:sha3:some_image_hash:some_image_url"} + ), +) @pytest.mark.asyncio async def test_manifest_payload_resolve_urls_from_hash_is_not_duplicating_or_overriding_values(): urls = [ - "some url", - "http://girepo.dev.golem.network:8000/" - "docker-gas_scanner_backend_image-latest-91c471517a.gvmi", - "other url", + "some_url", + "some_image_url", + "other_url", ] payload = ManifestPayload( hash="05270a8a938ff5f5e30b0e61bc983a8c3e286c5cd414a32e1a077657", - urls=urls, + urls=urls.copy(), ) assert payload.urls == urls From 38fb5f9dad5ef427176ab60be876d2ce85da3b71 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 22 Mar 2023 15:31:31 +0100 Subject: [PATCH 5/7] generate method instead of parse_imploded_dict --- tests/payload/test_manifest.py | 18 ++++++++++++++---- yapapi/payload/manifest.py | 12 ++++++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/payload/test_manifest.py b/tests/payload/test_manifest.py index f408644c1..787759e60 100644 --- a/tests/payload/test_manifest.py +++ b/tests/payload/test_manifest.py @@ -223,19 +223,21 @@ def test_manifest_generate(mocked_datetime): metadata_name = "example" metadata_version = "0.0.1" comp_manifest_script_match = "strict" + outbound_urls = ["url1", "url2"] - assert Manifest.parse_imploded_obj( - { + assert Manifest.generate( + image_hash=payload_hash, + outbound_urls=outbound_urls, + **{ "metadata.name": metadata_name, "metadata.version": metadata_version, - "payload.0.hash": payload_hash, "payload.0.urls": [ payload_urls[0], payload_urls[1], ], "payload.0.urls.2": payload_urls[2], "comp_manifest.script.match": comp_manifest_script_match, - } + }, ).dict() == { "metadata": { "name": metadata_name, @@ -251,6 +253,14 @@ def test_manifest_generate(mocked_datetime): ], "match": comp_manifest_script_match, }, + "net": { + "inet": { + "out": { + "protocols": ["http", "https", "ws", "wss"], + "urls": outbound_urls, + }, + }, + }, "version": "", }, "created_at": "2020-01-01T00:00:00+00:00", diff --git a/yapapi/payload/manifest.py b/yapapi/payload/manifest.py index 91afc671c..b7355c15e 100644 --- a/yapapi/payload/manifest.py +++ b/yapapi/payload/manifest.py @@ -288,5 +288,13 @@ def parse_obj(cls, obj: Dict) -> "Manifest": return cls(**obj_copy) @classmethod - def parse_imploded_obj(cls, obj: Dict) -> "Manifest": - return cls.parse_obj(explode_dict(obj)) + def generate( + cls, image_hash: str = None, outbound_urls: List[str] = None, **kwargs + ) -> "Manifest": + if image_hash is not None: + kwargs["payload.0.hash"] = image_hash + + if outbound_urls is not None: + kwargs["comp_manifest.net.inet.out.urls"] = outbound_urls + + return cls.parse_obj(explode_dict(kwargs)) From 0ca1571e45ead2a543995d008fd5795e0951760a Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 22 Mar 2023 15:35:40 +0100 Subject: [PATCH 6/7] formatting --- yapapi/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yapapi/utils.py b/yapapi/utils.py index afee57a40..828b6b751 100644 --- a/yapapi/utils.py +++ b/yapapi/utils.py @@ -264,7 +264,6 @@ def explode_dict(imploded_dict: Dict, separator=".") -> Dict: else: nested_obj = nested_obj.setdefault(part, {}) else: - try: nested_obj = nested_obj[part_index] except IndexError: From 539830622a16e6cb823453d4e73f35b81dc6ad0b Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 22 Mar 2023 16:06:38 +0100 Subject: [PATCH 7/7] resolving urls in generate --- tests/payload/test_manifest.py | 22 ++++++++++++++++++---- yapapi/payload/manifest.py | 9 +++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/tests/payload/test_manifest.py b/tests/payload/test_manifest.py index 787759e60..ffee97693 100644 --- a/tests/payload/test_manifest.py +++ b/tests/payload/test_manifest.py @@ -210,10 +210,17 @@ def test_manifest_with_minimal_data(mocked_datetime): } +@pytest.mark.asyncio @mock.patch( "yapapi.payload.manifest.datetime", **{"utcnow.return_value": datetime(2020, 1, 1, tzinfo=UTC)} ) -def test_manifest_generate(mocked_datetime): +@mock.patch( + "yapapi.payload.vm.repo", + mock.AsyncMock( + **{"return_value.resolve_url.return_value": "hash:sha3:some_image_hash:some_image_url"} + ), +) +async def test_manifest_generate(mocked_datetime): payload_urls = [ "some url", "some other url", @@ -225,7 +232,7 @@ def test_manifest_generate(mocked_datetime): comp_manifest_script_match = "strict" outbound_urls = ["url1", "url2"] - assert Manifest.generate( + manifest = await Manifest.generate( image_hash=payload_hash, outbound_urls=outbound_urls, **{ @@ -238,7 +245,9 @@ def test_manifest_generate(mocked_datetime): "payload.0.urls.2": payload_urls[2], "comp_manifest.script.match": comp_manifest_script_match, }, - ).dict() == { + ) + + assert manifest.dict() == { "metadata": { "name": metadata_name, "version": metadata_version, @@ -268,7 +277,12 @@ def test_manifest_generate(mocked_datetime): "payload": [ { "hash": payload_hash, - "urls": payload_urls, + "urls": [ + payload_urls[0], + payload_urls[1], + payload_urls[2], + "some_image_url", + ], "platform": { "arch": "x86_64", "os": "linux", diff --git a/yapapi/payload/manifest.py b/yapapi/payload/manifest.py index b7355c15e..426770c3c 100644 --- a/yapapi/payload/manifest.py +++ b/yapapi/payload/manifest.py @@ -1,3 +1,4 @@ +import asyncio import sys from datetime import datetime from typing import Dict, List, Optional @@ -288,7 +289,7 @@ def parse_obj(cls, obj: Dict) -> "Manifest": return cls(**obj_copy) @classmethod - def generate( + async def generate( cls, image_hash: str = None, outbound_urls: List[str] = None, **kwargs ) -> "Manifest": if image_hash is not None: @@ -297,4 +298,8 @@ def generate( if outbound_urls is not None: kwargs["comp_manifest.net.inet.out.urls"] = outbound_urls - return cls.parse_obj(explode_dict(kwargs)) + manifest = cls.parse_obj(explode_dict(kwargs)) + + await asyncio.gather(*[payload.resolve_urls_from_hash() for payload in manifest.payload]) + + return manifest