From 2c17812e29e1a19ec9023731f8cba22d562ac3d2 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 1 Jun 2023 16:12:26 +0200 Subject: [PATCH 01/25] Added image tag selection for blender --- examples/blender/blender.py | 9 +++++++-- examples/utils/__init__.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index b445f9c52..eb93220d3 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -16,6 +16,7 @@ TEXT_COLOR_MAGENTA, TEXT_COLOR_RED, build_parser, + resolve_image_hash_and_url, format_usage, print_env_info, run_golem_example, @@ -23,10 +24,12 @@ async def main( - subnet_tag, min_cpu_threads, payment_driver=None, payment_network=None, show_usage=False + subnet_tag, min_cpu_threads, image_tag, payment_driver=None, payment_network=None, show_usage=False ): + image_hash, image_url = resolve_image_hash_and_url(image_tag) package = await vm.repo( - image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + image_url=image_url, + image_hash=image_hash, # only run on provider nodes that have more than 0.5gb of RAM available min_mem_gib=0.5, # only run on provider nodes that have more than 2gb of storage space available @@ -156,10 +159,12 @@ async def worker(ctx: WorkContext, tasks): parser.set_defaults(log_file=f"blender-yapapi-{now}.log") args = parser.parse_args() + image_tag = args.image_tag or "yapapi/blender:latest" run_golem_example( main( subnet_tag=args.subnet_tag, min_cpu_threads=args.min_cpu_threads, + image_tag=image_tag, payment_driver=args.payment_driver, payment_network=args.payment_network, show_usage=args.show_usage, diff --git a/examples/utils/__init__.py b/examples/utils/__init__.py index 3864dbd20..f98c1a936 100644 --- a/examples/utils/__init__.py +++ b/examples/utils/__init__.py @@ -1,9 +1,11 @@ """Utilities for yapapi example scripts.""" import argparse import asyncio +import json import tempfile from datetime import datetime, timezone from pathlib import Path +from urllib.request import urlopen import colorama # type: ignore @@ -25,6 +27,28 @@ colorama.init() +def sizeof_fmt(num, suffix="B"): + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + if abs(num) < 1024.0: + return f"{num:3.2f}{unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" + + +def resolve_image_hash_and_url(image_tag: str) -> tuple[str, str]: + print(f"Checking golem registry for tag: {image_tag}...") + url = f"https://registry.dev.golem.network/v1/image/info?tag={image_tag}" + resp = urlopen(url) + image_info = json.loads(resp.read()) + image_hash = image_info["sha3"] + image_size = image_info["size"] + image_size_human = sizeof_fmt(image_size) + image_arch = image_info["arch"] + image_url = f"http://registry.dev.golem.network:8010/download/{image_hash}" + print(f"Image found: \n --tag: {image_tag}\n --sha224: {image_hash}\n --arch {image_arch}\n --size: {image_size} ({image_size_human})\n --url: {image_url}") + return image_hash, image_url + + def build_parser(description: str) -> argparse.ArgumentParser: current_time_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z") default_log_path = Path(tempfile.gettempdir()) / f"yapapi_{current_time_str}.log" @@ -36,6 +60,9 @@ def build_parser(description: str) -> argparse.ArgumentParser: parser.add_argument( "--payment-network", "--network", help="Payment network name, for example `rinkeby`" ) + parser.add_argument( + "--image-tag", help="Image tag from golem registry" + ) parser.add_argument("--subnet-tag", help="Subnet name, for example `public`") parser.add_argument( "--log-file", From 74b9d8b58f60c13d446748619912e7c1bf40fa8b Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 14:20:56 +0200 Subject: [PATCH 02/25] Working on registry integration --- examples/blender/blender.py | 4 +- yapapi/config.py | 4 ++ yapapi/payload/package.py | 39 +++++++++++++++--- yapapi/payload/vm.py | 79 +++++++++++++------------------------ 4 files changed, 67 insertions(+), 59 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index eb93220d3..7f30dc9fe 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -26,10 +26,8 @@ async def main( subnet_tag, min_cpu_threads, image_tag, payment_driver=None, payment_network=None, show_usage=False ): - image_hash, image_url = resolve_image_hash_and_url(image_tag) package = await vm.repo( - image_url=image_url, - image_hash=image_hash, + image_tag=image_tag, # only run on provider nodes that have more than 0.5gb of RAM available min_mem_gib=0.5, # only run on provider nodes that have more than 2gb of storage space available diff --git a/yapapi/config.py b/yapapi/config.py index 7c8a7953c..c1dad0edf 100644 --- a/yapapi/config.py +++ b/yapapi/config.py @@ -48,6 +48,10 @@ class ApiConfig: net_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_NET_URL")) activity_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_ACTIVITY_URL")) + repository_url: str = field( + default_factory=partial(os.getenv, "YAPAPI_REPOSITORY_URL", "https://registry.golem.network") + ) + def __post_init__(self): if self.app_key is None: raise MissingConfiguration(key="YAGNA_APPKEY", description="API authentication token") diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 039b35aba..48c3a84c0 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -1,10 +1,12 @@ import abc +import os +from typing import Optional import aiohttp from dataclasses import dataclass from yapapi.payload import Payload - +from yapapi.config import ApiConfig class PackageException(Exception): """Exception raised on any problems related to the package repository.""" @@ -19,7 +21,7 @@ async def resolve_url(self) -> str: """Return package URL.""" -async def resolve_package_url(image_url: str, image_hash: str) -> str: +async def check_package_url(image_url: str, image_hash: str) -> str: async with aiohttp.ClientSession() as client: resp = await client.head(image_url, allow_redirects=True) if resp.status != 200: @@ -28,11 +30,38 @@ async def resolve_package_url(image_url: str, image_hash: str) -> str: return f"hash:sha3:{image_hash}:{image_url}" -async def resolve_package_repo_url(repo_url: str, image_hash: str) -> str: +async def resolve_package_url(repo_url: str, image_tag: Optional[str] = None, image_hash: Optional[str] = None) -> str: async with aiohttp.ClientSession() as client: - resp = await client.get(f"{repo_url}/image.{image_hash}.link") + is_dev = os.getenv("GOLEM_DEV_MODE", False) + is_https = os.getenv("YAPAPI_RESOLVE_USING_HTTPS", False) + + if image_tag is None and image_hash is None: + raise PackageException("Neither image tag nor image hash specified") + + if image_tag and image_hash: + raise PackageException("Both image tag and image hash specified") + + if image_tag: + url_params = f"tag={image_tag}" + else: + url_params = f"hash={image_tag}" + + if is_dev: + # if dev, skip usage statistics, pass dev option for statistics + url_params += "&dev=true" + else: + # resolved by yapapi, so count as used tag (for usage statistics) + url_params += "&count=true" + + resp = await client.get(f"{repo_url}/v1/image/info?{url_params}") if resp.status != 200: resp.raise_for_status() + json_resp = await resp.json() + if is_https: + image_url = json_resp["https"] + else: + image_url = json_resp["http"] + image_hash = json_resp["sha3"] - image_url = await resp.text() return f"hash:sha3:{image_hash}:{image_url}" + diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 97f50dd30..99f1a4a94 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -1,4 +1,5 @@ import logging +import os import sys from enum import Enum from typing import List, Optional @@ -7,6 +8,8 @@ from dns.exception import DNSException from typing_extensions import Final +from yapapi.config import ApiConfig + if sys.version_info > (3, 8): from typing import Literal else: @@ -14,21 +17,12 @@ from srvresolver.srv_resolver import SRVRecord, SRVResolver -from yapapi.payload.package import ( - Package, - PackageException, - resolve_package_repo_url, - resolve_package_url, -) +from yapapi.payload.package import Package, resolve_package_url, check_package_url from yapapi.props import base as prop_base from yapapi.props import inf from yapapi.props.builder import DemandBuilder, Model from yapapi.props.inf import INF_CORES, RUNTIME_VM, ExeUnitManifestRequest, ExeUnitRequest, InfBase -_DEFAULT_REPO_SRV: Final[str] = "_girepo._tcp.dev.golem.network" -_FALLBACK_REPO_URL: Final[str] = "http://girepo.dev.golem.network:8000" -_DEFAULT_TIMEOUT_SECONDS: Final[int] = 10 - logger = logging.getLogger(__name__) VM_CAPS_VPN: str = "vpn" @@ -158,25 +152,22 @@ async def manifest( @dataclass class _VmPackage(Package): - repo_url: str - image_hash: str - image_url: Optional[str] + image_url: str constraints: _VmConstraints async def resolve_url(self) -> str: - if not self.image_url: - return await resolve_package_repo_url(self.repo_url, self.image_hash) - return await resolve_package_url(self.image_url, self.image_hash) + # trivial implementation - already resolved before creation of _VmPackage + return self.image_url async def decorate_demand(self, demand: DemandBuilder): - image_url = await self.resolve_url() demand.ensure(str(self.constraints)) - demand.add(VmRequest(package_url=image_url, package_format=VmPackageFormat.GVMKIT_SQUASH)) + demand.add(VmRequest(package_url=self.image_url, package_format=VmPackageFormat.GVMKIT_SQUASH)) async def repo( *, - image_hash: str, + image_hash: Optional[str] = None, + image_tag: Optional[str] = None, image_url: Optional[str] = None, min_mem_gib: float = 0.5, min_storage_gib: float = 2.0, @@ -187,6 +178,7 @@ async def repo( Build a reference to application package. :param image_hash: hash of the package's image + :param image_tag: Tag of the package to resolve from Golem Registry :param image_url: URL of the package's image :param min_mem_gib: minimal memory required to execute application code :param min_storage_gib: minimal disk storage to execute tasks @@ -230,40 +222,25 @@ async def repo( ) """ + + repo_url = ApiConfig().repository_url + print("USING repo_url", repo_url) + if not image_tag and not image_hash: + raise ValueError("Either image_tag or image_hash must be provided") + elif image_tag and image_url: + raise ValueError("You cannot override image_url when using image_tag, use image_hash instead") + elif not image_url and image_hash: + resolved_image_url = await resolve_package_url(repo_url, image_hash=image_hash) + elif not image_url and image_tag: + resolved_image_url = await resolve_package_url(repo_url, image_tag=image_tag) + elif image_hash and image_url: + resolved_image_url = await check_package_url(image_url, image_hash) + else: + raise ValueError("Invalid combination of arguments") + capabilities = capabilities or list() return _VmPackage( - repo_url=resolve_repo_srv(_DEFAULT_REPO_SRV), - image_hash=image_hash, - image_url=image_url, + image_url=resolved_image_url, constraints=_VmConstraints(min_mem_gib, min_storage_gib, min_cpu_threads, capabilities), ) - -def resolve_repo_srv( - repo_srv: str, fallback_url=_FALLBACK_REPO_URL, timeout=_DEFAULT_TIMEOUT_SECONDS -) -> str: - """ - Get the url of the package repository based on its SRV record address. - - :param repo_srv: the SRV domain name - :param fallback_url: temporary hardcoded fallback url in case there's a problem resolving SRV - :param timeout: socket connection timeout in seconds - :return: the url of the package repository containing the port - :raises: PackageException if no valid service could be reached - """ - try: - try: - srv: Optional[SRVRecord] = SRVResolver.resolve_random(repo_srv, timeout=timeout) - except DNSException as e: - raise PackageException(f"Could not resolve Golem package repository address [{e}].") - - if not srv: - raise PackageException("Golem package repository is currently unavailable.") - except Exception as e: - # this is a temporary fallback for a problem resolving the SRV record - logger.warning( - "Problem resolving %s, falling back to %s, exception: %s", repo_srv, fallback_url, e - ) - return fallback_url - - return f"http://{srv.host}:{srv.port}" From 8174017ffd6b8c84cac0763a4d31b35db0ac5678 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 14:55:17 +0200 Subject: [PATCH 03/25] Working on resolving images using golem registry --- examples/blender/blender.py | 6 ++++-- examples/utils/__init__.py | 8 ++++++-- yapapi/payload/package.py | 7 +++++-- yapapi/payload/vm.py | 12 +++++++++--- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 7f30dc9fe..0f76ae90d 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -24,10 +24,11 @@ async def main( - subnet_tag, min_cpu_threads, image_tag, payment_driver=None, payment_network=None, show_usage=False + subnet_tag, min_cpu_threads, image_tag, image_use_https, payment_driver=None, payment_network=None, show_usage=False ): package = await vm.repo( image_tag=image_tag, + image_use_https=image_use_https, # only run on provider nodes that have more than 0.5gb of RAM available min_mem_gib=0.5, # only run on provider nodes that have more than 2gb of storage space available @@ -157,12 +158,13 @@ async def worker(ctx: WorkContext, tasks): parser.set_defaults(log_file=f"blender-yapapi-{now}.log") args = parser.parse_args() - image_tag = args.image_tag or "yapapi/blender:latest" + image_tag = args.image_tag or "golem/blender:latest" run_golem_example( main( subnet_tag=args.subnet_tag, min_cpu_threads=args.min_cpu_threads, image_tag=image_tag, + image_use_https=args.image_use_https, payment_driver=args.payment_driver, payment_network=args.payment_network, show_usage=args.show_usage, diff --git a/examples/utils/__init__.py b/examples/utils/__init__.py index f98c1a936..ab99b308b 100644 --- a/examples/utils/__init__.py +++ b/examples/utils/__init__.py @@ -58,10 +58,14 @@ def build_parser(description: str) -> argparse.ArgumentParser: "--payment-driver", "--driver", help="Payment driver name, for example `erc20`" ) parser.add_argument( - "--payment-network", "--network", help="Payment network name, for example `rinkeby`" + "--payment-network", "--network", help="Payment network name, for example `goerli`" ) parser.add_argument( - "--image-tag", help="Image tag from golem registry" + "--image-tag", help="Image tag to use when resolving image url from Golem Registry" + ) + parser.add_argument( + "--image-use-https", help="Whether to use https when resolving image url from Golem Registry", + action="store_true" ) parser.add_argument("--subnet-tag", help="Subnet name, for example `public`") parser.add_argument( diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 48c3a84c0..08f43d12e 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -30,7 +30,10 @@ async def check_package_url(image_url: str, image_hash: str) -> str: return f"hash:sha3:{image_hash}:{image_url}" -async def resolve_package_url(repo_url: str, image_tag: Optional[str] = None, image_hash: Optional[str] = None) -> str: +async def resolve_package_url(repo_url: str, + image_tag: Optional[str] = None, + image_hash: Optional[str] = None, + image_use_https: bool = False) -> str: async with aiohttp.ClientSession() as client: is_dev = os.getenv("GOLEM_DEV_MODE", False) is_https = os.getenv("YAPAPI_RESOLVE_USING_HTTPS", False) @@ -57,7 +60,7 @@ async def resolve_package_url(repo_url: str, image_tag: Optional[str] = None, im if resp.status != 200: resp.raise_for_status() json_resp = await resp.json() - if is_https: + if image_use_https: image_url = json_resp["https"] else: image_url = json_resp["http"] diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 99f1a4a94..73bfb9b09 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -169,6 +169,7 @@ async def repo( image_hash: Optional[str] = None, image_tag: Optional[str] = None, image_url: Optional[str] = None, + image_use_https: bool = False, min_mem_gib: float = 0.5, min_storage_gib: float = 2.0, min_cpu_threads: int = 1, @@ -180,6 +181,7 @@ async def repo( :param image_hash: hash of the package's image :param image_tag: Tag of the package to resolve from Golem Registry :param image_url: URL of the package's image + :param image_use_https: whether to resolve to HTTPS or HTTP link (some providers do not support https) :param min_mem_gib: minimal memory required to execute application code :param min_storage_gib: minimal disk storage to execute tasks :param min_cpu_threads: minimal available logical CPU cores @@ -224,20 +226,24 @@ async def repo( """ repo_url = ApiConfig().repository_url - print("USING repo_url", repo_url) if not image_tag and not image_hash: raise ValueError("Either image_tag or image_hash must be provided") elif image_tag and image_url: raise ValueError("You cannot override image_url when using image_tag, use image_hash instead") elif not image_url and image_hash: - resolved_image_url = await resolve_package_url(repo_url, image_hash=image_hash) + logger.info(f"Resolving using {repo_url} by image hash {image_hash}") + resolved_image_url = await resolve_package_url(repo_url, image_hash=image_hash, image_use_https=image_use_https) elif not image_url and image_tag: - resolved_image_url = await resolve_package_url(repo_url, image_tag=image_tag) + logger.info(f"Resolving using {repo_url} by image tag {image_tag}") + resolved_image_url = await resolve_package_url(repo_url, image_tag=image_tag, image_use_https=image_use_https) elif image_hash and image_url: + logger.info(f"Checking if image url is correct for {image_url} and {image_hash}") resolved_image_url = await check_package_url(image_url, image_hash) else: raise ValueError("Invalid combination of arguments") + logger.info(f"Resolved image full link: {resolved_image_url}") + capabilities = capabilities or list() return _VmPackage( image_url=resolved_image_url, From 0ae907e99df332e84d16acf711e9972d4a4383b0 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 19:39:58 +0200 Subject: [PATCH 04/25] Working on resolving packages --- examples/blender/blender.py | 64 ++++++++++++++++++++++--------------- examples/utils/__init__.py | 6 ++++ yapapi/payload/package.py | 14 ++++++-- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 0f76ae90d..87fc63d05 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -23,21 +23,7 @@ ) -async def main( - subnet_tag, min_cpu_threads, image_tag, image_use_https, payment_driver=None, payment_network=None, show_usage=False -): - package = await vm.repo( - image_tag=image_tag, - image_use_https=image_use_https, - # only run on provider nodes that have more than 0.5gb of RAM available - min_mem_gib=0.5, - # only run on provider nodes that have more than 2gb of storage space available - min_storage_gib=2.0, - # only run on provider nodes which a certain number of CPU threads (logical CPU cores) - # available - min_cpu_threads=min_cpu_threads, - ) - +async def start(subnet_tag, package, payment_driver=None, payment_network=None, show_usage=False): async def worker(ctx: WorkContext, tasks): script_dir = pathlib.Path(__file__).resolve().parent scene_path = str(script_dir / "cubes.blend") @@ -145,6 +131,43 @@ async def worker(ctx: WorkContext, tasks): ) +async def main(args): + # Use golem/blender:latest image tag, you can overwrite this option with --image-tag or --image-hash + default_image_tag = "golem/blender:latest" + if args.image_url and args.image_tag: + raise ValueError("Only one of --image-url and --image-tag can be specified") + if args.image_url and not args.image_hash: + raise ValueError("--image-url requires --image-hash to be specified") + if args.image_hash and args.image_tag: + raise ValueError("Only one of --image-hash and --image-tag can be specified") + elif args.image_hash: + image_tag = None + else: + image_tag = args.image_tag or default_image_tag + + # resolve image by tag, hash or direct link + package = await vm.repo( + image_tag=image_tag, + image_hash=args.image_hash, + image_url=args.image_url, + image_use_https=args.image_use_https, + # only run on provider nodes that have more than 0.5gb of RAM available + min_mem_gib=0.5, + # only run on provider nodes that have more than 2gb of storage space available + min_storage_gib=2.0, + # only run on provider nodes which a certain number of CPU threads (logical CPU cores) + # available + min_cpu_threads=args.min_cpu_threads, + ) + + await start( + subnet_tag=args.subnet_tag, + package=package, + payment_driver=args.payment_driver, + payment_network=args.payment_network, + show_usage=args.show_usage, + ) + if __name__ == "__main__": parser = build_parser("Render a Blender scene") parser.add_argument("--show-usage", action="store_true", help="show activity usage and cost") @@ -158,16 +181,7 @@ async def worker(ctx: WorkContext, tasks): parser.set_defaults(log_file=f"blender-yapapi-{now}.log") args = parser.parse_args() - image_tag = args.image_tag or "golem/blender:latest" run_golem_example( - main( - subnet_tag=args.subnet_tag, - min_cpu_threads=args.min_cpu_threads, - image_tag=image_tag, - image_use_https=args.image_use_https, - payment_driver=args.payment_driver, - payment_network=args.payment_network, - show_usage=args.show_usage, - ), + main(args=args), log_file=args.log_file, ) diff --git a/examples/utils/__init__.py b/examples/utils/__init__.py index ab99b308b..ee659345d 100644 --- a/examples/utils/__init__.py +++ b/examples/utils/__init__.py @@ -63,6 +63,12 @@ def build_parser(description: str) -> argparse.ArgumentParser: parser.add_argument( "--image-tag", help="Image tag to use when resolving image url from Golem Registry" ) + parser.add_argument( + "--image-hash", help="Image hash to use when resolving image url from Golem Registry" + ) + parser.add_argument( + "--image-url", help="Direct image url to use instead of resolving from Golem Registry" + ) parser.add_argument( "--image-use-https", help="Whether to use https when resolving image url from Golem Registry", action="store_true" diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 08f43d12e..cf4251f19 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -1,3 +1,4 @@ +import logging import abc import os from typing import Optional @@ -8,6 +9,8 @@ from yapapi.payload import Payload from yapapi.config import ApiConfig +logger = logging.getLogger(__name__) + class PackageException(Exception): """Exception raised on any problems related to the package repository.""" @@ -47,7 +50,7 @@ async def resolve_package_url(repo_url: str, if image_tag: url_params = f"tag={image_tag}" else: - url_params = f"hash={image_tag}" + url_params = f"hash={image_hash}" if is_dev: # if dev, skip usage statistics, pass dev option for statistics @@ -58,7 +61,14 @@ async def resolve_package_url(repo_url: str, resp = await client.get(f"{repo_url}/v1/image/info?{url_params}") if resp.status != 200: - resp.raise_for_status() + try: + text = await resp.text() + except Exception as ex: + logger.error(f"Failed to get body of response: {ex}") + text = "N/A" + + logger.error(f"Failed to resolve image URL: {resp.status} {text}") + raise Exception(f"Failed to resolve image URL: {resp.status} {text}") json_resp = await resp.json() if image_use_https: image_url = json_resp["https"] From a2ebe7e2696ce7940660987c8bc5fc8f2bedeaa7 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 19:52:55 +0200 Subject: [PATCH 05/25] Modified blender example only --- examples/blender/blender.py | 27 ++++++++++++++++++++++----- examples/utils/__init__.py | 13 ------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 87fc63d05..c89bf5ed0 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -131,9 +131,8 @@ async def worker(ctx: WorkContext, tasks): ) -async def main(args): +async def create_package(args, default_image_tag): # Use golem/blender:latest image tag, you can overwrite this option with --image-tag or --image-hash - default_image_tag = "golem/blender:latest" if args.image_url and args.image_tag: raise ValueError("Only one of --image-url and --image-tag can be specified") if args.image_url and not args.image_hash: @@ -160,6 +159,11 @@ async def main(args): min_cpu_threads=args.min_cpu_threads, ) + +async def main(args): + # Create a package using options specified in the command line + package = await create_package(args, default_image_tag="golem/blender:latest") + await start( subnet_tag=args.subnet_tag, package=package, @@ -177,11 +181,24 @@ async def main(args): default=1, help="require the provider nodes to have at least this number of available CPU threads", ) + parser.add_argument( + "--image-tag", help="Image tag to use when resolving image url from Golem Registry" + ) + parser.add_argument( + "--image-hash", help="Image hash to use when resolving image url from Golem Registry" + ) + parser.add_argument( + "--image-url", help="Direct image url to use instead of resolving from Golem Registry" + ) + parser.add_argument( + "--image-use-https", help="Whether to use https when resolving image url from Golem Registry", + action="store_true" + ) now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") parser.set_defaults(log_file=f"blender-yapapi-{now}.log") - args = parser.parse_args() + cmd_args = parser.parse_args() run_golem_example( - main(args=args), - log_file=args.log_file, + main(args=cmd_args), + log_file=cmd_args.log_file, ) diff --git a/examples/utils/__init__.py b/examples/utils/__init__.py index ee659345d..23bce8ed8 100644 --- a/examples/utils/__init__.py +++ b/examples/utils/__init__.py @@ -60,19 +60,6 @@ def build_parser(description: str) -> argparse.ArgumentParser: parser.add_argument( "--payment-network", "--network", help="Payment network name, for example `goerli`" ) - parser.add_argument( - "--image-tag", help="Image tag to use when resolving image url from Golem Registry" - ) - parser.add_argument( - "--image-hash", help="Image hash to use when resolving image url from Golem Registry" - ) - parser.add_argument( - "--image-url", help="Direct image url to use instead of resolving from Golem Registry" - ) - parser.add_argument( - "--image-use-https", help="Whether to use https when resolving image url from Golem Registry", - action="store_true" - ) parser.add_argument("--subnet-tag", help="Subnet name, for example `public`") parser.add_argument( "--log-file", From 0d42e764f0a661714acf9ef112a6b8f74c8f60d4 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 20:29:24 +0200 Subject: [PATCH 06/25] Working on blender example --- examples/blender/blender.py | 2 +- examples/utils/__init__.py | 21 --------------------- yapapi/payload/package.py | 18 +++++++++++++++++- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index c89bf5ed0..d1a7b3acf 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -16,7 +16,6 @@ TEXT_COLOR_MAGENTA, TEXT_COLOR_RED, build_parser, - resolve_image_hash_and_url, format_usage, print_env_info, run_golem_example, @@ -158,6 +157,7 @@ async def create_package(args, default_image_tag): # available min_cpu_threads=args.min_cpu_threads, ) + return package async def main(args): diff --git a/examples/utils/__init__.py b/examples/utils/__init__.py index 23bce8ed8..4a5bb8e67 100644 --- a/examples/utils/__init__.py +++ b/examples/utils/__init__.py @@ -27,27 +27,6 @@ colorama.init() -def sizeof_fmt(num, suffix="B"): - for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: - if abs(num) < 1024.0: - return f"{num:3.2f}{unit}{suffix}" - num /= 1024.0 - return f"{num:.1f}Yi{suffix}" - - -def resolve_image_hash_and_url(image_tag: str) -> tuple[str, str]: - print(f"Checking golem registry for tag: {image_tag}...") - url = f"https://registry.dev.golem.network/v1/image/info?tag={image_tag}" - resp = urlopen(url) - image_info = json.loads(resp.read()) - image_hash = image_info["sha3"] - image_size = image_info["size"] - image_size_human = sizeof_fmt(image_size) - image_arch = image_info["arch"] - image_url = f"http://registry.dev.golem.network:8010/download/{image_hash}" - print(f"Image found: \n --tag: {image_tag}\n --sha224: {image_hash}\n --arch {image_arch}\n --size: {image_size} ({image_size_human})\n --url: {image_url}") - return image_hash, image_url - def build_parser(description: str) -> argparse.ArgumentParser: current_time_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z") diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index cf4251f19..8dc7fb612 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -33,6 +33,14 @@ async def check_package_url(image_url: str, image_hash: str) -> str: return f"hash:sha3:{image_hash}:{image_url}" +def sizeof_fmt(num, suffix="B"): + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + if abs(num) < 1024.0: + return f"{num:3.2f}{unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" + + async def resolve_package_url(repo_url: str, image_tag: Optional[str] = None, image_hash: Optional[str] = None, @@ -59,7 +67,9 @@ async def resolve_package_url(repo_url: str, # resolved by yapapi, so count as used tag (for usage statistics) url_params += "&count=true" - resp = await client.get(f"{repo_url}/v1/image/info?{url_params}") + query_url = f"{repo_url}/v1/image/info?{url_params}" + logger.debug(f"Querying registry portal: {query_url}") + resp = await client.get(query_url) if resp.status != 200: try: text = await resp.text() @@ -75,6 +85,12 @@ async def resolve_package_url(repo_url: str, else: image_url = json_resp["http"] image_hash = json_resp["sha3"] + image_size = json_resp["size"] + logger.debug(f"Resolved image size: {sizeof_fmt(image_size)}") + logger.debug(f"Resolved image hash: {image_hash}") + logger.debug(f"Resolved image url: {image_url}") + # TODO: check if image_arch is ok + # image_arch = image_info["arch"] return f"hash:sha3:{image_hash}:{image_url}" From b822d2c63f790fce59e7bcf26e504268aa407c1e Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 20:32:14 +0200 Subject: [PATCH 07/25] Revert init.py --- examples/utils/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/utils/__init__.py b/examples/utils/__init__.py index 4a5bb8e67..2cfd7755d 100644 --- a/examples/utils/__init__.py +++ b/examples/utils/__init__.py @@ -1,11 +1,9 @@ """Utilities for yapapi example scripts.""" import argparse import asyncio -import json import tempfile from datetime import datetime, timezone from pathlib import Path -from urllib.request import urlopen import colorama # type: ignore @@ -27,7 +25,6 @@ colorama.init() - def build_parser(description: str) -> argparse.ArgumentParser: current_time_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z") default_log_path = Path(tempfile.gettempdir()) / f"yapapi_{current_time_str}.log" From 2c9b50a0d36907554955dbec644cca43ebdf6e1a Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 20:35:52 +0200 Subject: [PATCH 08/25] Removed srv test (no longer needed) --- tests/payload/test_vm.py | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 tests/payload/test_vm.py diff --git a/tests/payload/test_vm.py b/tests/payload/test_vm.py deleted file mode 100644 index 000b01441..000000000 --- a/tests/payload/test_vm.py +++ /dev/null @@ -1,28 +0,0 @@ -from unittest import mock - -from dns.exception import DNSException -from srvresolver.srv_resolver import SRVRecord # type: ignore - -from yapapi.payload.vm import _FALLBACK_REPO_URL, resolve_repo_srv - -_MOCK_HOST = "non.existent.domain" -_MOCK_PORT = 9999 - - -@mock.patch( - "yapapi.payload.vm.SRVResolver.resolve_random", - mock.Mock(return_value=SRVRecord(host=_MOCK_HOST, port=_MOCK_PORT, weight=1, priority=1)), -) -def test_resolve_srv(): - assert resolve_repo_srv("") == f"http://{_MOCK_HOST}:{_MOCK_PORT}" - - -@mock.patch("yapapi.payload.vm.SRVResolver.resolve_random", mock.Mock(side_effect=DNSException())) -def test_resolve_srv_exception(): - # should be: - - # with pytest.raises(DNSException): - # assert resolve_repo_srv("") - - # temporary work-around: - assert resolve_repo_srv("") == _FALLBACK_REPO_URL From a0de9e55ba63c3fe233a35f9a15be39510d856b6 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 20:52:18 +0200 Subject: [PATCH 09/25] Trying to fix styling --- examples/blender/blender.py | 6 ++++-- yapapi/config.py | 3 ++- yapapi/payload/package.py | 12 ++++-------- yapapi/payload/vm.py | 29 +++++++++++++++-------------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index d1a7b3acf..1b4ef526d 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -131,7 +131,8 @@ async def worker(ctx: WorkContext, tasks): async def create_package(args, default_image_tag): - # Use golem/blender:latest image tag, you can overwrite this option with --image-tag or --image-hash + # Use golem/blender:latest image tag, + # you can overwrite this option with --image-tag or --image-hash if args.image_url and args.image_tag: raise ValueError("Only one of --image-url and --image-tag can be specified") if args.image_url and not args.image_hash: @@ -191,7 +192,8 @@ async def main(args): "--image-url", help="Direct image url to use instead of resolving from Golem Registry" ) parser.add_argument( - "--image-use-https", help="Whether to use https when resolving image url from Golem Registry", + "--image-use-https", + help="Whether to use https when resolving image url from Golem Registry", action="store_true" ) now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") diff --git a/yapapi/config.py b/yapapi/config.py index c1dad0edf..a276df708 100644 --- a/yapapi/config.py +++ b/yapapi/config.py @@ -49,7 +49,8 @@ class ApiConfig: activity_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_ACTIVITY_URL")) repository_url: str = field( - default_factory=partial(os.getenv, "YAPAPI_REPOSITORY_URL", "https://registry.golem.network") + default_factory=partial(os.getenv, + "YAPAPI_REPOSITORY_URL", "https://registry.golem.network") ) def __post_init__(self): diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 8dc7fb612..611e49b26 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -1,16 +1,14 @@ -import logging import abc -import os -from typing import Optional - import aiohttp from dataclasses import dataclass - +import logging +import os +from typing import Optional from yapapi.payload import Payload -from yapapi.config import ApiConfig logger = logging.getLogger(__name__) + class PackageException(Exception): """Exception raised on any problems related to the package repository.""" @@ -47,7 +45,6 @@ async def resolve_package_url(repo_url: str, image_use_https: bool = False) -> str: async with aiohttp.ClientSession() as client: is_dev = os.getenv("GOLEM_DEV_MODE", False) - is_https = os.getenv("YAPAPI_RESOLVE_USING_HTTPS", False) if image_tag is None and image_hash is None: raise PackageException("Neither image tag nor image hash specified") @@ -93,4 +90,3 @@ async def resolve_package_url(repo_url: str, # image_arch = image_info["arch"] return f"hash:sha3:{image_hash}:{image_url}" - diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 73bfb9b09..4e0dbe274 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -1,23 +1,19 @@ import logging -import os import sys + from enum import Enum + from typing import List, Optional from dataclasses import dataclass -from dns.exception import DNSException -from typing_extensions import Final - -from yapapi.config import ApiConfig if sys.version_info > (3, 8): from typing import Literal else: from typing_extensions import Literal -from srvresolver.srv_resolver import SRVRecord, SRVResolver - -from yapapi.payload.package import Package, resolve_package_url, check_package_url +from yapapi.config import ApiConfig +from yapapi.payload.package import Package, check_package_url, resolve_package_url from yapapi.props import base as prop_base from yapapi.props import inf from yapapi.props.builder import DemandBuilder, Model @@ -161,7 +157,8 @@ async def resolve_url(self) -> str: async def decorate_demand(self, demand: DemandBuilder): demand.ensure(str(self.constraints)) - demand.add(VmRequest(package_url=self.image_url, package_format=VmPackageFormat.GVMKIT_SQUASH)) + demand.add(VmRequest(package_url=self.image_url, + package_format=VmPackageFormat.GVMKIT_SQUASH)) async def repo( @@ -181,7 +178,7 @@ async def repo( :param image_hash: hash of the package's image :param image_tag: Tag of the package to resolve from Golem Registry :param image_url: URL of the package's image - :param image_use_https: whether to resolve to HTTPS or HTTP link (some providers do not support https) + :param image_use_https: whether to resolve to HTTPS or HTTP when using Golem Registry :param min_mem_gib: minimal memory required to execute application code :param min_storage_gib: minimal disk storage to execute tasks :param min_cpu_threads: minimal available logical CPU cores @@ -229,13 +226,18 @@ async def repo( if not image_tag and not image_hash: raise ValueError("Either image_tag or image_hash must be provided") elif image_tag and image_url: - raise ValueError("You cannot override image_url when using image_tag, use image_hash instead") + raise ValueError( + "You cannot override image_url when using image_tag, use image_hash instead") elif not image_url and image_hash: logger.info(f"Resolving using {repo_url} by image hash {image_hash}") - resolved_image_url = await resolve_package_url(repo_url, image_hash=image_hash, image_use_https=image_use_https) + resolved_image_url = await resolve_package_url(repo_url, + image_hash=image_hash, + image_use_https=image_use_https) elif not image_url and image_tag: logger.info(f"Resolving using {repo_url} by image tag {image_tag}") - resolved_image_url = await resolve_package_url(repo_url, image_tag=image_tag, image_use_https=image_use_https) + resolved_image_url = await resolve_package_url(repo_url, + image_tag=image_tag, + image_use_https=image_use_https) elif image_hash and image_url: logger.info(f"Checking if image url is correct for {image_url} and {image_hash}") resolved_image_url = await check_package_url(image_url, image_hash) @@ -249,4 +251,3 @@ async def repo( image_url=resolved_image_url, constraints=_VmConstraints(min_mem_gib, min_storage_gib, min_cpu_threads, capabilities), ) - From 26b67952878c0047da5dd87cbe3b5899551fd4f3 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 21 Jun 2023 21:02:59 +0200 Subject: [PATCH 10/25] format --- examples/blender/blender.py | 3 ++- yapapi/config.py | 5 +++-- yapapi/payload/package.py | 16 ++++++++++------ yapapi/payload/vm.py | 22 +++++++++++----------- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 1b4ef526d..6ebb9a6aa 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -173,6 +173,7 @@ async def main(args): show_usage=args.show_usage, ) + if __name__ == "__main__": parser = build_parser("Render a Blender scene") parser.add_argument("--show-usage", action="store_true", help="show activity usage and cost") @@ -194,7 +195,7 @@ async def main(args): parser.add_argument( "--image-use-https", help="Whether to use https when resolving image url from Golem Registry", - action="store_true" + action="store_true", ) now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") parser.set_defaults(log_file=f"blender-yapapi-{now}.log") diff --git a/yapapi/config.py b/yapapi/config.py index a276df708..a0c5ed304 100644 --- a/yapapi/config.py +++ b/yapapi/config.py @@ -49,8 +49,9 @@ class ApiConfig: activity_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_ACTIVITY_URL")) repository_url: str = field( - default_factory=partial(os.getenv, - "YAPAPI_REPOSITORY_URL", "https://registry.golem.network") + default_factory=partial( + os.getenv, "YAPAPI_REPOSITORY_URL", "https://registry.golem.network" + ) ) def __post_init__(self): diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 611e49b26..ff1309340 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -1,9 +1,11 @@ import abc -import aiohttp -from dataclasses import dataclass import logging import os from typing import Optional + +import aiohttp +from dataclasses import dataclass + from yapapi.payload import Payload logger = logging.getLogger(__name__) @@ -39,10 +41,12 @@ def sizeof_fmt(num, suffix="B"): return f"{num:.1f}Yi{suffix}" -async def resolve_package_url(repo_url: str, - image_tag: Optional[str] = None, - image_hash: Optional[str] = None, - image_use_https: bool = False) -> str: +async def resolve_package_url( + repo_url: str, + image_tag: Optional[str] = None, + image_hash: Optional[str] = None, + image_use_https: bool = False, +) -> str: async with aiohttp.ClientSession() as client: is_dev = os.getenv("GOLEM_DEV_MODE", False) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 4e0dbe274..38fd52cc6 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -1,8 +1,6 @@ import logging import sys - from enum import Enum - from typing import List, Optional from dataclasses import dataclass @@ -157,8 +155,9 @@ async def resolve_url(self) -> str: async def decorate_demand(self, demand: DemandBuilder): demand.ensure(str(self.constraints)) - demand.add(VmRequest(package_url=self.image_url, - package_format=VmPackageFormat.GVMKIT_SQUASH)) + demand.add( + VmRequest(package_url=self.image_url, package_format=VmPackageFormat.GVMKIT_SQUASH) + ) async def repo( @@ -227,17 +226,18 @@ async def repo( raise ValueError("Either image_tag or image_hash must be provided") elif image_tag and image_url: raise ValueError( - "You cannot override image_url when using image_tag, use image_hash instead") + "You cannot override image_url when using image_tag, use image_hash instead" + ) elif not image_url and image_hash: logger.info(f"Resolving using {repo_url} by image hash {image_hash}") - resolved_image_url = await resolve_package_url(repo_url, - image_hash=image_hash, - image_use_https=image_use_https) + resolved_image_url = await resolve_package_url( + repo_url, image_hash=image_hash, image_use_https=image_use_https + ) elif not image_url and image_tag: logger.info(f"Resolving using {repo_url} by image tag {image_tag}") - resolved_image_url = await resolve_package_url(repo_url, - image_tag=image_tag, - image_use_https=image_use_https) + resolved_image_url = await resolve_package_url( + repo_url, image_tag=image_tag, image_use_https=image_use_https + ) elif image_hash and image_url: logger.info(f"Checking if image url is correct for {image_url} and {image_hash}") resolved_image_url = await check_package_url(image_url, image_hash) From cb0943d14ed666b0cb4dafccf54ace414e7f6043 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Sat, 1 Jul 2023 13:22:48 +0200 Subject: [PATCH 11/25] Fix type checkings --- yapapi/config.py | 2 +- yapapi/payload/vm.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/yapapi/config.py b/yapapi/config.py index a0c5ed304..dfa089738 100644 --- a/yapapi/config.py +++ b/yapapi/config.py @@ -48,7 +48,7 @@ class ApiConfig: net_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_NET_URL")) activity_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_ACTIVITY_URL")) - repository_url: str = field( + repository_url: Optional[str] = field( default_factory=partial( os.getenv, "YAPAPI_REPOSITORY_URL", "https://registry.golem.network" ) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 38fd52cc6..963379c4d 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -229,11 +229,15 @@ async def repo( "You cannot override image_url when using image_tag, use image_hash instead" ) elif not image_url and image_hash: + if not repo_url: + raise ValueError("Repo url is empty or not set") logger.info(f"Resolving using {repo_url} by image hash {image_hash}") resolved_image_url = await resolve_package_url( repo_url, image_hash=image_hash, image_use_https=image_use_https ) elif not image_url and image_tag: + if not repo_url: + raise ValueError("Repo url is empty or not set") logger.info(f"Resolving using {repo_url} by image tag {image_tag}") resolved_image_url = await resolve_package_url( repo_url, image_tag=image_tag, image_use_https=image_use_https From d3b760aa55dfb2232b41db66dfa78c411f27e83e Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 20 Jul 2023 15:42:49 +0200 Subject: [PATCH 12/25] Revert accidental commit --- examples/ssh/ssh.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/ssh/ssh.py b/examples/ssh/ssh.py index 0f163c680..0a035c8ea 100755 --- a/examples/ssh/ssh.py +++ b/examples/ssh/ssh.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import asyncio import pathlib +import random +import string import sys from datetime import datetime, timedelta @@ -50,7 +52,7 @@ async def start(self): async for script in super().start(): yield script - password = "dupa" + password = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) script = self._ctx.new_script(timeout=timedelta(seconds=10)) script.run("/bin/bash", "-c", "syslogd") @@ -74,10 +76,10 @@ async def main(subnet_tag, payment_driver=None, payment_network=None, num_instan # See the documentation of the `yapapi.log` module on how to set # the level of detail and format of the logged information. async with Golem( - budget=1.0, - subnet_tag=subnet_tag, - payment_driver=payment_driver, - payment_network=payment_network, + budget=1.0, + subnet_tag=subnet_tag, + payment_driver=payment_driver, + payment_network=payment_network, ) as golem: print_env_info(golem) From eb33acae2ab510c148a59ece0a959a2275da4d38 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 20 Jul 2023 15:44:11 +0200 Subject: [PATCH 13/25] revert accidental commit --- examples/ssh/ssh.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/ssh/ssh.py b/examples/ssh/ssh.py index 0a035c8ea..4562b15fe 100755 --- a/examples/ssh/ssh.py +++ b/examples/ssh/ssh.py @@ -76,10 +76,10 @@ async def main(subnet_tag, payment_driver=None, payment_network=None, num_instan # See the documentation of the `yapapi.log` module on how to set # the level of detail and format of the logged information. async with Golem( - budget=1.0, - subnet_tag=subnet_tag, - payment_driver=payment_driver, - payment_network=payment_network, + budget=1.0, + subnet_tag=subnet_tag, + payment_driver=payment_driver, + payment_network=payment_network, ) as golem: print_env_info(golem) From 95d98df13fd55bc8e9ab494587e45753e85ab40d Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 20 Jul 2023 15:45:38 +0200 Subject: [PATCH 14/25] Update --- examples/hello-world/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/hello-world/Dockerfile b/examples/hello-world/Dockerfile index 1cc8c0638..fa8c75c18 100644 --- a/examples/hello-world/Dockerfile +++ b/examples/hello-world/Dockerfile @@ -1,3 +1,3 @@ FROM alpine:latest -VOLUME /golem/input /golem/output -WORKDIR /golem/work +VOLUME /golem/input +WORKDIR /golem/work \ No newline at end of file From b68048c65fab3715e7d34e989af465fdbbd26965 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 20 Jul 2023 15:48:59 +0200 Subject: [PATCH 15/25] fix --- examples/hello-world/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/hello-world/Dockerfile b/examples/hello-world/Dockerfile index fa8c75c18..7a79183ff 100644 --- a/examples/hello-world/Dockerfile +++ b/examples/hello-world/Dockerfile @@ -1,3 +1,3 @@ FROM alpine:latest VOLUME /golem/input -WORKDIR /golem/work \ No newline at end of file +WORKDIR /golem/work From eaa3a3d248afb1f57d78151c10508d4a1bed8b36 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 20 Jul 2023 15:55:40 +0200 Subject: [PATCH 16/25] formatting --- yapapi/payload/vm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index a7652e5df..114f8a171 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -1,8 +1,9 @@ import logging from enum import Enum -from typing import Final, List, Literal, Optional +from typing import List, Literal, Optional from dataclasses import dataclass + from yapapi.config import ApiConfig from yapapi.payload.package import Package, check_package_url, resolve_package_url from yapapi.props import base as prop_base From e39174cbbe852343ba3d2f6f1dba15509b88b70e Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 14 Aug 2023 08:23:56 +0200 Subject: [PATCH 17/25] revert the blender example, add registry usage as a separate example --- examples/blender/blender.py | 85 +++------ examples/blender/blender_registry_usage.py | 207 +++++++++++++++++++++ 2 files changed, 230 insertions(+), 62 deletions(-) create mode 100755 examples/blender/blender_registry_usage.py diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 6ebb9a6aa..b445f9c52 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -22,7 +22,20 @@ ) -async def start(subnet_tag, package, payment_driver=None, payment_network=None, show_usage=False): +async def main( + subnet_tag, min_cpu_threads, payment_driver=None, payment_network=None, show_usage=False +): + package = await vm.repo( + image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + # only run on provider nodes that have more than 0.5gb of RAM available + min_mem_gib=0.5, + # only run on provider nodes that have more than 2gb of storage space available + min_storage_gib=2.0, + # only run on provider nodes which a certain number of CPU threads (logical CPU cores) + # available + min_cpu_threads=min_cpu_threads, + ) + async def worker(ctx: WorkContext, tasks): script_dir = pathlib.Path(__file__).resolve().parent scene_path = str(script_dir / "cubes.blend") @@ -130,50 +143,6 @@ async def worker(ctx: WorkContext, tasks): ) -async def create_package(args, default_image_tag): - # Use golem/blender:latest image tag, - # you can overwrite this option with --image-tag or --image-hash - if args.image_url and args.image_tag: - raise ValueError("Only one of --image-url and --image-tag can be specified") - if args.image_url and not args.image_hash: - raise ValueError("--image-url requires --image-hash to be specified") - if args.image_hash and args.image_tag: - raise ValueError("Only one of --image-hash and --image-tag can be specified") - elif args.image_hash: - image_tag = None - else: - image_tag = args.image_tag or default_image_tag - - # resolve image by tag, hash or direct link - package = await vm.repo( - image_tag=image_tag, - image_hash=args.image_hash, - image_url=args.image_url, - image_use_https=args.image_use_https, - # only run on provider nodes that have more than 0.5gb of RAM available - min_mem_gib=0.5, - # only run on provider nodes that have more than 2gb of storage space available - min_storage_gib=2.0, - # only run on provider nodes which a certain number of CPU threads (logical CPU cores) - # available - min_cpu_threads=args.min_cpu_threads, - ) - return package - - -async def main(args): - # Create a package using options specified in the command line - package = await create_package(args, default_image_tag="golem/blender:latest") - - await start( - subnet_tag=args.subnet_tag, - package=package, - payment_driver=args.payment_driver, - payment_network=args.payment_network, - show_usage=args.show_usage, - ) - - if __name__ == "__main__": parser = build_parser("Render a Blender scene") parser.add_argument("--show-usage", action="store_true", help="show activity usage and cost") @@ -183,25 +152,17 @@ async def main(args): default=1, help="require the provider nodes to have at least this number of available CPU threads", ) - parser.add_argument( - "--image-tag", help="Image tag to use when resolving image url from Golem Registry" - ) - parser.add_argument( - "--image-hash", help="Image hash to use when resolving image url from Golem Registry" - ) - parser.add_argument( - "--image-url", help="Direct image url to use instead of resolving from Golem Registry" - ) - parser.add_argument( - "--image-use-https", - help="Whether to use https when resolving image url from Golem Registry", - action="store_true", - ) now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") parser.set_defaults(log_file=f"blender-yapapi-{now}.log") - cmd_args = parser.parse_args() + args = parser.parse_args() run_golem_example( - main(args=cmd_args), - log_file=cmd_args.log_file, + main( + subnet_tag=args.subnet_tag, + min_cpu_threads=args.min_cpu_threads, + payment_driver=args.payment_driver, + payment_network=args.payment_network, + show_usage=args.show_usage, + ), + log_file=args.log_file, ) diff --git a/examples/blender/blender_registry_usage.py b/examples/blender/blender_registry_usage.py new file mode 100755 index 000000000..6ebb9a6aa --- /dev/null +++ b/examples/blender/blender_registry_usage.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +import pathlib +import sys +from datetime import datetime, timedelta + +from yapapi import Golem, Task, WorkContext +from yapapi.payload import vm +from yapapi.rest.activity import BatchTimeoutError + +examples_dir = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(str(examples_dir)) + +from utils import ( + TEXT_COLOR_CYAN, + TEXT_COLOR_DEFAULT, + TEXT_COLOR_MAGENTA, + TEXT_COLOR_RED, + build_parser, + format_usage, + print_env_info, + run_golem_example, +) + + +async def start(subnet_tag, package, payment_driver=None, payment_network=None, show_usage=False): + async def worker(ctx: WorkContext, tasks): + script_dir = pathlib.Path(__file__).resolve().parent + scene_path = str(script_dir / "cubes.blend") + + # Set timeout for the first script executed on the provider. Usually, 30 seconds + # should be more than enough for computing a single frame of the provided scene, + # however a provider may require more time for the first task if it needs to download + # the VM image first. Once downloaded, the VM image will be cached and other tasks that use + # that image will be computed faster. + script = ctx.new_script(timeout=timedelta(minutes=10)) + script.upload_file(scene_path, "/golem/resource/scene.blend") + + async for task in tasks: + frame = task.data + crops = [{"outfilebasename": "out", "borders_x": [0.0, 1.0], "borders_y": [0.0, 1.0]}] + script.upload_json( + { + "scene_file": "/golem/resource/scene.blend", + "resolution": (400, 300), + "use_compositing": False, + "crops": crops, + "samples": 100, + "frames": [frame], + "output_format": "PNG", + "RESOURCES_DIR": "/golem/resources", + "WORK_DIR": "/golem/work", + "OUTPUT_DIR": "/golem/output", + }, + "/golem/work/params.json", + ) + + script.run("/golem/entrypoints/run-blender.sh") + output_file = f"output_{frame}.png" + script.download_file(f"/golem/output/out{frame:04d}.png", output_file) + try: + yield script + # TODO: Check if job results are valid + # and reject by: task.reject_task(reason = 'invalid file') + task.accept_result(result=output_file) + except BatchTimeoutError: + print( + f"{TEXT_COLOR_RED}" + f"Task {task} timed out on {ctx.provider_name}, time: {task.running_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + raise + + # reinitialize the script which we send to the engine to compute subsequent frames + script = ctx.new_script(timeout=timedelta(minutes=1)) + + if show_usage: + raw_state = await ctx.get_raw_state() + usage = format_usage(await ctx.get_usage()) + cost = await ctx.get_cost() + print( + f"{TEXT_COLOR_MAGENTA}" + f" --- {ctx.provider_name} STATE: {raw_state}\n" + f" --- {ctx.provider_name} USAGE: {usage}\n" + f" --- {ctx.provider_name} COST: {cost}" + f"{TEXT_COLOR_DEFAULT}" + ) + + # Iterator over the frame indices that we want to render + frames: range = range(0, 60, 10) + # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) + # TODO: make this dynamic, e.g. depending on the size of files to transfer + init_overhead = 3 + # Providers will not accept work if the timeout is outside of the [5 min, 30min] range. + # We increase the lower bound to 6 min to account for the time needed for our demand to + # reach the providers. + min_timeout, max_timeout = 6, 30 + + timeout = timedelta(minutes=max(min(init_overhead + len(frames) * 2, max_timeout), min_timeout)) + + async with Golem( + budget=10.0, + subnet_tag=subnet_tag, + payment_driver=payment_driver, + payment_network=payment_network, + ) as golem: + print_env_info(golem) + + num_tasks = 0 + start_time = datetime.now() + + completed_tasks = golem.execute_tasks( + worker, + [Task(data=frame) for frame in frames], + payload=package, + max_workers=3, + timeout=timeout, + ) + async for task in completed_tasks: + num_tasks += 1 + print( + f"{TEXT_COLOR_CYAN}" + f"Task computed: {task}, result: {task.result}, time: {task.running_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + + print( + f"{TEXT_COLOR_CYAN}" + f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + + +async def create_package(args, default_image_tag): + # Use golem/blender:latest image tag, + # you can overwrite this option with --image-tag or --image-hash + if args.image_url and args.image_tag: + raise ValueError("Only one of --image-url and --image-tag can be specified") + if args.image_url and not args.image_hash: + raise ValueError("--image-url requires --image-hash to be specified") + if args.image_hash and args.image_tag: + raise ValueError("Only one of --image-hash and --image-tag can be specified") + elif args.image_hash: + image_tag = None + else: + image_tag = args.image_tag or default_image_tag + + # resolve image by tag, hash or direct link + package = await vm.repo( + image_tag=image_tag, + image_hash=args.image_hash, + image_url=args.image_url, + image_use_https=args.image_use_https, + # only run on provider nodes that have more than 0.5gb of RAM available + min_mem_gib=0.5, + # only run on provider nodes that have more than 2gb of storage space available + min_storage_gib=2.0, + # only run on provider nodes which a certain number of CPU threads (logical CPU cores) + # available + min_cpu_threads=args.min_cpu_threads, + ) + return package + + +async def main(args): + # Create a package using options specified in the command line + package = await create_package(args, default_image_tag="golem/blender:latest") + + await start( + subnet_tag=args.subnet_tag, + package=package, + payment_driver=args.payment_driver, + payment_network=args.payment_network, + show_usage=args.show_usage, + ) + + +if __name__ == "__main__": + parser = build_parser("Render a Blender scene") + parser.add_argument("--show-usage", action="store_true", help="show activity usage and cost") + parser.add_argument( + "--min-cpu-threads", + type=int, + default=1, + help="require the provider nodes to have at least this number of available CPU threads", + ) + parser.add_argument( + "--image-tag", help="Image tag to use when resolving image url from Golem Registry" + ) + parser.add_argument( + "--image-hash", help="Image hash to use when resolving image url from Golem Registry" + ) + parser.add_argument( + "--image-url", help="Direct image url to use instead of resolving from Golem Registry" + ) + parser.add_argument( + "--image-use-https", + help="Whether to use https when resolving image url from Golem Registry", + action="store_true", + ) + now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") + parser.set_defaults(log_file=f"blender-yapapi-{now}.log") + cmd_args = parser.parse_args() + + run_golem_example( + main(args=cmd_args), + log_file=cmd_args.log_file, + ) From de43e7e66dec513b577846ff4f465c1b73aa0e25 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 14 Aug 2023 20:58:32 +0200 Subject: [PATCH 18/25] pull request review --- yapapi/config.py | 6 ---- yapapi/payload/package.py | 69 +++++++++++++++++++++------------------ yapapi/payload/vm.py | 64 ++++++++++++++++++++---------------- 3 files changed, 72 insertions(+), 67 deletions(-) diff --git a/yapapi/config.py b/yapapi/config.py index dfa089738..7c8a7953c 100644 --- a/yapapi/config.py +++ b/yapapi/config.py @@ -48,12 +48,6 @@ class ApiConfig: net_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_NET_URL")) activity_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_ACTIVITY_URL")) - repository_url: Optional[str] = field( - default_factory=partial( - os.getenv, "YAPAPI_REPOSITORY_URL", "https://registry.golem.network" - ) - ) - def __post_init__(self): if self.app_key is None: raise MissingConfiguration(key="YAGNA_APPKEY", description="API authentication token") diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index ff1309340..80af2367b 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -33,7 +33,7 @@ async def check_package_url(image_url: str, image_hash: str) -> str: return f"hash:sha3:{image_hash}:{image_url}" -def sizeof_fmt(num, suffix="B"): +def _sizeof_fmt(num, suffix="B"): for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: if abs(num) < 1024.0: return f"{num:3.2f}{unit}{suffix}" @@ -47,30 +47,30 @@ async def resolve_package_url( image_hash: Optional[str] = None, image_use_https: bool = False, ) -> str: - async with aiohttp.ClientSession() as client: - is_dev = os.getenv("GOLEM_DEV_MODE", False) + params = {} + + if image_tag: + params["tag"] = image_tag - if image_tag is None and image_hash is None: - raise PackageException("Neither image tag nor image hash specified") + if image_hash: + params["hash"] = image_hash - if image_tag and image_hash: - raise PackageException("Both image tag and image hash specified") + if not params: + raise PackageException("Neither image tag nor image hash specified") - if image_tag: - url_params = f"tag={image_tag}" - else: - url_params = f"hash={image_hash}" + if "tag" in params and "hash" in params: + raise PackageException("Both image tag and image hash specified") - if is_dev: - # if dev, skip usage statistics, pass dev option for statistics - url_params += "&dev=true" - else: - # resolved by yapapi, so count as used tag (for usage statistics) - url_params += "&count=true" + if os.getenv("GOLEM_DEV_MODE", False): + # if dev, skip usage statistics, pass dev option for statistics + params["dev"] = "true" + else: + params["count"] = "true" - query_url = f"{repo_url}/v1/image/info?{url_params}" - logger.debug(f"Querying registry portal: {query_url}") - resp = await client.get(query_url) + async with aiohttp.ClientSession() as client: + url = f"{repo_url}/v1/image/info" + logger.info(f"Querying registry portal: url={url}, params={params}") + resp = await client.get(url, params=params) if resp.status != 200: try: text = await resp.text() @@ -80,17 +80,22 @@ async def resolve_package_url( logger.error(f"Failed to resolve image URL: {resp.status} {text}") raise Exception(f"Failed to resolve image URL: {resp.status} {text}") + json_resp = await resp.json() - if image_use_https: - image_url = json_resp["https"] - else: - image_url = json_resp["http"] - image_hash = json_resp["sha3"] - image_size = json_resp["size"] - logger.debug(f"Resolved image size: {sizeof_fmt(image_size)}") - logger.debug(f"Resolved image hash: {image_hash}") - logger.debug(f"Resolved image url: {image_url}") - # TODO: check if image_arch is ok - # image_arch = image_info["arch"] - return f"hash:sha3:{image_hash}:{image_url}" + if image_use_https: + image_url = json_resp["https"] + else: + image_url = json_resp["http"] + image_hash = json_resp["sha3"] + image_size = json_resp["size"] + logger.debug( + f"Resolved image: " + f"url={image_url}, " + f"size={_sizeof_fmt(image_size)}, " + f"hash={image_hash}" + ) + # TODO: check if image_arch is ok + # image_arch = image_info["arch"] + + return f"hash:sha3:{image_hash}:{image_url}" diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 114f8a171..49b332297 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -1,10 +1,9 @@ import logging from enum import Enum -from typing import List, Literal, Optional +from typing import Final, List, Literal, Optional from dataclasses import dataclass -from yapapi.config import ApiConfig from yapapi.payload.package import Package, check_package_url, resolve_package_url from yapapi.props import base as prop_base from yapapi.props import inf @@ -13,6 +12,8 @@ logger = logging.getLogger(__name__) +DEFAULT_REPOSITORY_URL: Final[str] = "https://registry.golem.network" + VM_CAPS_VPN: str = "vpn" VM_CAPS_MANIFEST_SUPPORT: str = "manifest-support" @@ -144,7 +145,6 @@ class _VmPackage(Package): constraints: _VmConstraints async def resolve_url(self) -> str: - # trivial implementation - already resolved before creation of _VmPackage return self.image_url async def decorate_demand(self, demand: DemandBuilder): @@ -160,6 +160,7 @@ async def repo( image_tag: Optional[str] = None, image_url: Optional[str] = None, image_use_https: bool = False, + repository_url: str = DEFAULT_REPOSITORY_URL, min_mem_gib: float = 0.5, min_storage_gib: float = 2.0, min_cpu_threads: int = 1, @@ -209,40 +210,45 @@ async def repo( min_mem_gib=0.5, # only run on provider nodes that have more than 2gb of storage space available min_storage_gib=2.0, - # only run on provider nodes which a certain number of CPU threads available + # only run on provider nodes with a certain number of CPU threads available min_cpu_threads=min_cpu_threads, ) """ - repo_url = ApiConfig().repository_url - if not image_tag and not image_hash: - raise ValueError("Either image_tag or image_hash must be provided") - elif image_tag and image_url: - raise ValueError( - "You cannot override image_url when using image_tag, use image_hash instead" - ) - elif not image_url and image_hash: - if not repo_url: - raise ValueError("Repo url is empty or not set") - logger.info(f"Resolving using {repo_url} by image hash {image_hash}") - resolved_image_url = await resolve_package_url( - repo_url, image_hash=image_hash, image_use_https=image_use_https - ) - elif not image_url and image_tag: - if not repo_url: - raise ValueError("Repo url is empty or not set") - logger.info(f"Resolving using {repo_url} by image tag {image_tag}") - resolved_image_url = await resolve_package_url( - repo_url, image_tag=image_tag, image_use_https=image_use_https - ) - elif image_hash and image_url: - logger.info(f"Checking if image url is correct for {image_url} and {image_hash}") + if image_url: + if image_tag: + raise ValueError( + "An image_tag can only be used when resolving from Golem Registry, " + "not with a direct image_url." + ) + if not image_hash: + raise ValueError("An image_hash is required when using a direct image_url.") + logger.debug(f"Verifying if {image_url} exists.") resolved_image_url = await check_package_url(image_url, image_hash) else: - raise ValueError("Invalid combination of arguments") + if image_hash and image_tag: + raise ValueError( + "Golem Registry images can be resolved by " + "either an image_hash or by an image_tag but not both." + ) + if image_hash: + logger.debug(f"Resolving using image hash: {image_hash} on {repository_url}.") + resolved_image_url = await resolve_package_url( + repository_url, image_hash=image_hash, image_use_https=image_use_https + ) + elif image_tag: + logger.debug(f"Resolving using image tag: {image_tag} on {repository_url}.") + resolved_image_url = await resolve_package_url( + repository_url, image_tag=image_tag, image_use_https=image_use_https + ) + else: + raise ValueError( + "Either an image_hash or an image_tag is required " + "to resolve an image URL from the Golem Registry." + ) - logger.info(f"Resolved image full link: {resolved_image_url}") + logger.debug(f"Resolved image: {resolved_image_url}") capabilities = capabilities or list() return _VmPackage( From d8c0e8b2a419712a1bcd819fe8eda5035ae52982 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 14 Aug 2023 22:54:41 +0200 Subject: [PATCH 19/25] add tests for vm.repo --- tests/payload/test_repo.py | 103 +++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tests/payload/test_repo.py diff --git a/tests/payload/test_repo.py b/tests/payload/test_repo.py new file mode 100644 index 000000000..5ecba21c9 --- /dev/null +++ b/tests/payload/test_repo.py @@ -0,0 +1,103 @@ +from aiohttp import ClientResponse +import pytest +from unittest.mock import AsyncMock + +from yapapi.payload import vm + +_MOCK_HTTP_ADDR = "http://test.address/" +_MOCK_HTTPS_ADDR = "https://test.address/" +_MOCK_SHA3 = "abcdef124356789" +_MOCK_SIZE = 2**24 + + +async def _mock_response(*args, **kwargs): + mock = AsyncMock() + mock.status = 200 + mock.json.return_value = { + "http": _MOCK_HTTP_ADDR, + "https": _MOCK_HTTPS_ADDR, + "sha3": _MOCK_SHA3, + "size": _MOCK_SIZE, + } + return mock + + +@pytest.mark.parametrize( + "image_hash, image_tag, image_url, image_use_https, " + "expected_url, expected_error, expected_error_msg", + ( + ("testhash", None, None, False, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTP_ADDR}", None, ""), + (None, "testtag", None, False, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTP_ADDR}", None, ""), + ("testhash", None, None, True, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTPS_ADDR}", None, ""), + (None, "testtag", None, True, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTPS_ADDR}", None, ""), + ("testhash", None, "http://image", False, f"hash:sha3:testhash:http://image", None, ""), + ( + None, + None, + None, + False, + None, + ValueError, + "Either an image_hash or an image_tag is required " + "to resolve an image URL from the Golem Registry", + ), + ( + None, + None, + "http://image", + False, + None, + ValueError, + "An image_hash is required when using a direct image_url", + ), + ( + None, + "testtag", + "http://image", + False, + None, + ValueError, + "An image_tag can only be used when resolving " + "from Golem Registry, not with a direct image_url", + ), + ( + "testhash", + "testtag", + None, + False, + None, + ValueError, + "Golem Registry images can be resolved by either " + "an image_hash or by an image_tag but not both", + ), + ), +) +@pytest.mark.asyncio +async def test_repo( + monkeypatch, + image_hash, + image_tag, + image_url, + image_use_https, + expected_url, + expected_error, + expected_error_msg, +): + monkeypatch.setattr("aiohttp.ClientSession.get", _mock_response) + monkeypatch.setattr("aiohttp.ClientSession.head", _mock_response) + + package_awaitable = vm.repo( + image_hash=image_hash, + image_tag=image_tag, + image_url=image_url, + image_use_https=image_use_https, + ) + + if expected_error: + with pytest.raises(expected_error) as e: + _ = await package_awaitable + assert expected_error_msg in str(e) + else: + package = await package_awaitable + url = await package.resolve_url() + assert url == expected_url From bda598223f3a4c13a337013deb0d1b4a033802d1 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 16 Aug 2023 20:28:54 +0200 Subject: [PATCH 20/25] remove overeager checks --- tests/payload/test_repo.py | 11 ++++++----- yapapi/payload/package.py | 12 +++++++++--- yapapi/payload/vm.py | 30 ++++++++++-------------------- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/tests/payload/test_repo.py b/tests/payload/test_repo.py index 5ecba21c9..9aa591b91 100644 --- a/tests/payload/test_repo.py +++ b/tests/payload/test_repo.py @@ -1,8 +1,9 @@ -from aiohttp import ClientResponse -import pytest from unittest.mock import AsyncMock +import pytest + from yapapi.payload import vm +from yapapi.payload.package import PackageException _MOCK_HTTP_ADDR = "http://test.address/" _MOCK_HTTPS_ADDR = "https://test.address/" @@ -30,14 +31,14 @@ async def _mock_response(*args, **kwargs): (None, "testtag", None, False, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTP_ADDR}", None, ""), ("testhash", None, None, True, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTPS_ADDR}", None, ""), (None, "testtag", None, True, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTPS_ADDR}", None, ""), - ("testhash", None, "http://image", False, f"hash:sha3:testhash:http://image", None, ""), + ("testhash", None, "http://image", False, "hash:sha3:testhash:http://image", None, ""), ( None, None, None, False, None, - ValueError, + PackageException, "Either an image_hash or an image_tag is required " "to resolve an image URL from the Golem Registry", ), @@ -66,7 +67,7 @@ async def _mock_response(*args, **kwargs): None, False, None, - ValueError, + PackageException, "Golem Registry images can be resolved by either " "an image_hash or by an image_tag but not both", ), diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 80af2367b..b22a52beb 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -56,10 +56,16 @@ async def resolve_package_url( params["hash"] = image_hash if not params: - raise PackageException("Neither image tag nor image hash specified") + raise PackageException( + "Either an image_hash or an image_tag is required " + "to resolve an image URL from the Golem Registry." + ) if "tag" in params and "hash" in params: - raise PackageException("Both image tag and image hash specified") + raise PackageException( + "Golem Registry images can be resolved by " + "either an image_hash or by an image_tag but not both." + ) if os.getenv("GOLEM_DEV_MODE", False): # if dev, skip usage statistics, pass dev option for statistics @@ -69,7 +75,7 @@ async def resolve_package_url( async with aiohttp.ClientSession() as client: url = f"{repo_url}/v1/image/info" - logger.info(f"Querying registry portal: url={url}, params={params}") + logger.debug(f"Querying registry portal: url={url}, params={params}") resp = await client.get(url, params=params) if resp.status != 200: try: diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 49b332297..a17ca8e35 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -227,26 +227,16 @@ async def repo( logger.debug(f"Verifying if {image_url} exists.") resolved_image_url = await check_package_url(image_url, image_hash) else: - if image_hash and image_tag: - raise ValueError( - "Golem Registry images can be resolved by " - "either an image_hash or by an image_tag but not both." - ) - if image_hash: - logger.debug(f"Resolving using image hash: {image_hash} on {repository_url}.") - resolved_image_url = await resolve_package_url( - repository_url, image_hash=image_hash, image_use_https=image_use_https - ) - elif image_tag: - logger.debug(f"Resolving using image tag: {image_tag} on {repository_url}.") - resolved_image_url = await resolve_package_url( - repository_url, image_tag=image_tag, image_use_https=image_use_https - ) - else: - raise ValueError( - "Either an image_hash or an image_tag is required " - "to resolve an image URL from the Golem Registry." - ) + logger.debug( + f"Resolving image on {repository_url}: " + f"image_hash={image_hash}, image_tag={image_tag}, image_use_https={image_use_https}." + ) + resolved_image_url = await resolve_package_url( + repository_url, + image_hash=image_hash, + image_tag=image_tag, + image_use_https=image_use_https, + ) logger.debug(f"Resolved image: {resolved_image_url}") From 5505c9474e29ffd96d9c6f1b518ec2f622a49fb7 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 16 Aug 2023 20:38:00 +0200 Subject: [PATCH 21/25] throw PackageException instead of the generic ValueError when resolving an image --- tests/payload/test_repo.py | 4 ++-- yapapi/payload/vm.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/payload/test_repo.py b/tests/payload/test_repo.py index 9aa591b91..6a32268e2 100644 --- a/tests/payload/test_repo.py +++ b/tests/payload/test_repo.py @@ -48,7 +48,7 @@ async def _mock_response(*args, **kwargs): "http://image", False, None, - ValueError, + PackageException, "An image_hash is required when using a direct image_url", ), ( @@ -57,7 +57,7 @@ async def _mock_response(*args, **kwargs): "http://image", False, None, - ValueError, + PackageException, "An image_tag can only be used when resolving " "from Golem Registry, not with a direct image_url", ), diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index a17ca8e35..c8c8cd08e 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from yapapi.payload.package import Package, check_package_url, resolve_package_url +from yapapi.payload.package import Package, PackageException, check_package_url, resolve_package_url from yapapi.props import base as prop_base from yapapi.props import inf from yapapi.props.builder import DemandBuilder, Model @@ -218,12 +218,12 @@ async def repo( if image_url: if image_tag: - raise ValueError( + raise PackageException( "An image_tag can only be used when resolving from Golem Registry, " "not with a direct image_url." ) if not image_hash: - raise ValueError("An image_hash is required when using a direct image_url.") + raise PackageException("An image_hash is required when using a direct image_url.") logger.debug(f"Verifying if {image_url} exists.") resolved_image_url = await check_package_url(image_url, image_hash) else: From 77ae9139ac99ad3efa8b81c049b86f04f631b7f8 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 16 Aug 2023 20:45:23 +0200 Subject: [PATCH 22/25] use `dev_mode` argument to `vm.repo` to mark resolve queries as "dev mode" instead of a hidden env var --- yapapi/payload/package.py | 4 ++-- yapapi/payload/vm.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index b22a52beb..f2cdef465 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -1,6 +1,5 @@ import abc import logging -import os from typing import Optional import aiohttp @@ -46,6 +45,7 @@ async def resolve_package_url( image_tag: Optional[str] = None, image_hash: Optional[str] = None, image_use_https: bool = False, + dev_mode: bool = False, ) -> str: params = {} @@ -67,7 +67,7 @@ async def resolve_package_url( "either an image_hash or by an image_tag but not both." ) - if os.getenv("GOLEM_DEV_MODE", False): + if dev_mode: # if dev, skip usage statistics, pass dev option for statistics params["dev"] = "true" else: diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index c8c8cd08e..6cda613dc 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -161,6 +161,7 @@ async def repo( image_url: Optional[str] = None, image_use_https: bool = False, repository_url: str = DEFAULT_REPOSITORY_URL, + dev_mode: bool = False, # noqa min_mem_gib: float = 0.5, min_storage_gib: float = 2.0, min_cpu_threads: int = 1, @@ -173,6 +174,7 @@ async def repo( :param image_tag: Tag of the package to resolve from Golem Registry :param image_url: URL of the package's image :param image_use_https: whether to resolve to HTTPS or HTTP when using Golem Registry + :param repository_url: override the package repository location :param min_mem_gib: minimal memory required to execute application code :param min_storage_gib: minimal disk storage to execute tasks :param min_cpu_threads: minimal available logical CPU cores @@ -236,6 +238,7 @@ async def repo( image_hash=image_hash, image_tag=image_tag, image_use_https=image_use_https, + dev_mode=dev_mode, ) logger.debug(f"Resolved image: {resolved_image_url}") From 373df2630d98648f9a70c9430d5fd64c3b1bf882 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 17 Aug 2023 14:30:22 +0200 Subject: [PATCH 23/25] remove `srvresolver` from requirements, add re-run for the connection error... --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8bdcc4e2b..132e91b3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ jsonrpc-base = "^1.0.3" ya-aioclient = "^0.6.4" toml = "^0.10.1" -srvresolver = "^0.3.5" colorama = "^0.4.4" semantic-version = "^2.8" attrs = ">=19.3" @@ -82,7 +81,7 @@ _format_black = "black ." tests_unit = {cmd = "pytest --cov=yapapi --cov-report html --cov-report term -sv --ignore tests/goth_tests", help = "Run only unit tests"} tests_integration_init = { sequence = ["_gothv_env", "_gothv_requirements", "_gothv_assets"], help="Initialize the integration test environment"} -tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError", help = "Run the integration tests"} +tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError --only-rerun requests.exceptions.ConnectionError", help = "Run the integration tests"} _gothv_env = "python -m venv .envs/yapapi-goth" _gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.14.1 pip pytest pytest-asyncio pytest-rerunfailures pexpect" _gothv_assets = ".envs/yapapi-goth/bin/python -m goth create-assets tests/goth_tests/assets" From 242149eb5ce72337aa619c05d26ec03ed68a7a4c Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 17 Aug 2023 15:10:43 +0200 Subject: [PATCH 24/25] remove irrelevant `todo` ;) --- yapapi/payload/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index f2cdef465..348e352d9 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -101,7 +101,5 @@ async def resolve_package_url( f"size={_sizeof_fmt(image_size)}, " f"hash={image_hash}" ) - # TODO: check if image_arch is ok - # image_arch = image_info["arch"] return f"hash:sha3:{image_hash}:{image_url}" From 9a062faf6457b8e4dddfbd135947f38043ad0e91 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 17 Aug 2023 16:31:36 +0200 Subject: [PATCH 25/25] add one more error to retry list --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 132e91b3d..43c8a5059 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ _format_black = "black ." tests_unit = {cmd = "pytest --cov=yapapi --cov-report html --cov-report term -sv --ignore tests/goth_tests", help = "Run only unit tests"} tests_integration_init = { sequence = ["_gothv_env", "_gothv_requirements", "_gothv_assets"], help="Initialize the integration test environment"} -tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError --only-rerun requests.exceptions.ConnectionError", help = "Run the integration tests"} +tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError --only-rerun requests.exceptions.ConnectionError --only-rerun OSError", help = "Run the integration tests"} _gothv_env = "python -m venv .envs/yapapi-goth" _gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.14.1 pip pytest pytest-asyncio pytest-rerunfailures pexpect" _gothv_assets = ".envs/yapapi-goth/bin/python -m goth create-assets tests/goth_tests/assets"