From 1beb0c76857d5e56b9d638c1c0052e7c3efe3934 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 15 Sep 2021 15:51:38 +0200 Subject: [PATCH 1/5] add vpn capability properties + support for list constraints --- examples/ssh/ssh.py | 2 ++ tests/props/test_base.py | 51 ++++++++++++++++++++++++++++++---------- yapapi/payload/vm.py | 19 ++++++++++++--- yapapi/props/base.py | 10 ++++++-- 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/examples/ssh/ssh.py b/examples/ssh/ssh.py index 7c9628413..93a77ecd5 100755 --- a/examples/ssh/ssh.py +++ b/examples/ssh/ssh.py @@ -15,6 +15,7 @@ ) from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa from yapapi.payload import vm +from yapapi.payload.vm import VM_CAPS_VPN from yapapi.services import Service examples_dir = pathlib.Path(__file__).resolve().parent.parent @@ -36,6 +37,7 @@ async def get_payload(): image_hash="ea233c6774b1621207a48e10b46e3e1f944d881911f499f5cbac546a", min_mem_gib=0.5, min_storage_gib=2.0, + capabilities=[VM_CAPS_VPN], ) async def run(self): diff --git a/tests/props/test_base.py b/tests/props/test_base.py index 2509d1587..9e664c2a2 100644 --- a/tests/props/test_base.py +++ b/tests/props/test_base.py @@ -9,6 +9,7 @@ class Foo(props_base.Model): bar: str = props_base.prop("bar", "cafebiba") max_baz: int = props_base.constraint("baz", "<=", 100) min_baz: int = props_base.constraint("baz", ">=", 1) + lst: list = props_base.constraint("lst", "=", default_factory=list) @dataclass @@ -23,7 +24,7 @@ class FooZero(props_base.Model): def test_constraint_fields(): fields = Foo.constraint_fields() - assert len(fields) == 2 + assert len(fields) == 3 assert any(f.name == "max_baz" for f in fields) assert all(f.name != "bar" for f in fields) @@ -41,71 +42,95 @@ def test_constraint_to_str(): assert props_base.constraint_to_str(foo.max_baz, max_baz) == "(baz<=42)" +@pytest.mark.parametrize( + "value, constraint_str", [ + (["one"], "(lst=one)"), + (["one", "two"], "(&(lst=one)\n\t(lst=two))"), + ] +) +def test_constraint_to_str_list(value, constraint_str): + foo = Foo(lst=value) + lst = [f for f in foo.constraint_fields() if f.name == "lst"][0] + assert props_base.constraint_to_str(foo.lst, lst) == constraint_str + + def test_constraint_model_serialize(): foo = Foo() constraints = props_base.constraint_model_serialize(foo) - assert constraints == ["(baz<=100)", "(baz>=1)"] + assert constraints == ["(baz<=100)", "(baz>=1)", ""] @pytest.mark.parametrize( "model, operator, result, error", [ ( - Foo, + Foo(), None, "(&(baz<=100)\n\t(baz>=1))", False, ), ( - Foo, + Foo(), "&", "(&(baz<=100)\n\t(baz>=1))", False, ), ( - Foo, + Foo(lst=["one"]), + None, + "(&(baz<=100)\n\t(baz>=1)\n\t(lst=one))", + False, + ), + ( + Foo(lst=["one", "other"]), + None, + "(&(baz<=100)\n\t(baz>=1)\n\t(&(lst=one)\n\t(lst=other)))", + False, + ), + ( + Foo(), "|", "(|(baz<=100)\n\t(baz>=1))", False, ), ( - Foo, + Foo(), "!", None, True, ), ( - FooToo, + FooToo(), "!", "(!(baz=21))", False, ), ( - FooToo, + FooToo(), "&", "(baz=21)", False, ), ( - FooToo, + FooToo(), "|", "(baz=21)", False, ), ( - FooZero, + FooZero(), None, "(&)", False, ), ( - FooZero, + FooZero(), "&", "(&)", False, ), ( - FooZero, + FooZero(), "|", "(|)", False, @@ -113,7 +138,7 @@ def test_constraint_model_serialize(): ], ) def test_join_str_constraints(model, operator, result, error): - args = [props_base.constraint_model_serialize(model())] + args = [props_base.constraint_model_serialize(model)] if operator: args.append(operator) try: diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 1f5fb8ccf..34231e580 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -2,8 +2,14 @@ from dataclasses import dataclass, field from enum import Enum import logging -from typing import Optional +import sys +from typing import Optional, List from typing_extensions import Final +if sys.version_info > (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + from srvresolver.srv_resolver import SRVResolver, SRVRecord # type: ignore from yapapi.payload.package import ( @@ -22,6 +28,9 @@ logger = logging.getLogger(__name__) +VM_CAPS_VPN: str = "vpn" + +VmCaps = Literal[VM_CAPS_VPN,] @dataclass class InfVm(InfBase): @@ -47,7 +56,8 @@ class _VmConstraints: min_mem_gib: float = prop_base.constraint(inf.INF_MEM, operator=">=") min_storage_gib: float = prop_base.constraint(inf.INF_STORAGE, operator=">=") min_cpu_threads: int = prop_base.constraint(inf.INF_THREADS, operator=">=") - # cores: int = prop_base.constraint(inf.INF_CORES, operator=">=") + + capabilities: List[VmCaps] = prop_base.constraint("golem.runtime.capabilities", operator="=", default_factory=list) runtime: str = prop_base.constraint(inf.INF_RUNTIME_NAME, operator="=", default=RUNTIME_VM) @@ -80,6 +90,7 @@ async def repo( min_mem_gib: float = 0.5, min_storage_gib: float = 2.0, min_cpu_threads: int = 1, + capabilities: Optional[List[VmCaps]] = None, ) -> Package: """ Build a reference to application package. @@ -89,13 +100,15 @@ async def repo( :param min_mem_gib: minimal memory required to execute application code :param min_storage_gib: minimal disk storage to execute tasks :param min_cpu_threads: minimal available logical CPU cores + :param capabilities: an optional list of required vm capabilities :return: the payload definition for the given VM image """ + capabilities = capabilities or list() return _VmPackage( repo_url=resolve_repo_srv(_DEFAULT_REPO_SRV), image_hash=image_hash, image_url=image_url, - constraints=_VmConstraints(min_mem_gib, min_storage_gib, min_cpu_threads), + constraints=_VmConstraints(min_mem_gib, min_storage_gib, min_cpu_threads, capabilities), ) diff --git a/yapapi/props/base.py b/yapapi/props/base.py index f9d837296..b0749eaa4 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -205,7 +205,7 @@ class ModelFieldType(enum.Enum): property = "property" -def constraint(key: str, operator: ConstraintOperator = "=", default=MISSING): +def constraint(key: str, operator: ConstraintOperator = "=", default=MISSING, default_factory=MISSING): """ Return a constraint-type dataclass field for a Model. @@ -229,6 +229,7 @@ def constraint(key: str, operator: ConstraintOperator = "=", default=MISSING): """ return field( default=default, + default_factory=default_factory, metadata={ PROP_KEY: key, PROP_OPERATOR: operator, @@ -271,7 +272,10 @@ def constraint_to_str(value, f: Field) -> str: :param value: the value of the the constraint field :param f: the dataclass field for this constraint """ - return f"({f.metadata[PROP_KEY]}{f.metadata[PROP_OPERATOR]}{value})" + if type(value) == list: + return join_str_constraints([constraint_to_str(v, f) for v in value]) if value else "" + else: + return f"({f.metadata[PROP_KEY]}{f.metadata[PROP_OPERATOR]}{value})" def constraint_model_serialize(m: Model) -> List[str]: @@ -316,6 +320,8 @@ def join_str_constraints(constraints: List[str], operator: ConstraintGroupOperat (bar<=128)) ``` """ + constraints = list(filter(bool, constraints)) + if operator == "!": if len(constraints) == 1: return f"({operator}{constraints[0]})" From d4c5a481b54acd14349adaf4402bc2e012db8b09 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 16 Sep 2021 11:00:35 +0200 Subject: [PATCH 2/5] Update yapapi/props/base.py Co-authored-by: johny-b <33967107+johny-b@users.noreply.github.com> --- yapapi/props/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/props/base.py b/yapapi/props/base.py index b0749eaa4..45249c0ff 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -320,7 +320,7 @@ def join_str_constraints(constraints: List[str], operator: ConstraintGroupOperat (bar<=128)) ``` """ - constraints = list(filter(bool, constraints)) + constraints = [c for c in constraints if c] if operator == "!": if len(constraints) == 1: From 7d03d2028e80e6e5a34ab1c092256c842319204c Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 16 Sep 2021 11:41:28 +0200 Subject: [PATCH 3/5] mypy... --- examples/ssh/ssh.py | 3 +-- yapapi/payload/vm.py | 2 +- yapapi/props/base.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/ssh/ssh.py b/examples/ssh/ssh.py index 93a77ecd5..ef8a2007f 100755 --- a/examples/ssh/ssh.py +++ b/examples/ssh/ssh.py @@ -15,7 +15,6 @@ ) from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa from yapapi.payload import vm -from yapapi.payload.vm import VM_CAPS_VPN from yapapi.services import Service examples_dir = pathlib.Path(__file__).resolve().parent.parent @@ -37,7 +36,7 @@ async def get_payload(): image_hash="ea233c6774b1621207a48e10b46e3e1f944d881911f499f5cbac546a", min_mem_gib=0.5, min_storage_gib=2.0, - capabilities=[VM_CAPS_VPN], + capabilities=[vm.VM_CAPS_VPN], ) async def run(self): diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 34231e580..2ebe03a71 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -30,7 +30,7 @@ VM_CAPS_VPN: str = "vpn" -VmCaps = Literal[VM_CAPS_VPN,] +VmCaps = Literal["vpn"] @dataclass class InfVm(InfBase): diff --git a/yapapi/props/base.py b/yapapi/props/base.py index b0749eaa4..df378e243 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -235,7 +235,7 @@ def constraint(key: str, operator: ConstraintOperator = "=", default=MISSING, de PROP_OPERATOR: operator, PROP_MODEL_FIELD_TYPE: ModelFieldType.constraint, }, - ) + ) # type: ignore # the default / default_factory exception is resolved by the `field` function def prop(key: str, default=MISSING): From 344b3e80acd076bbc42c9170a9b1d586c5a8b756 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 16 Sep 2021 11:42:52 +0200 Subject: [PATCH 4/5] black --- tests/props/test_base.py | 5 +++-- yapapi/payload/vm.py | 6 +++++- yapapi/props/base.py | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/props/test_base.py b/tests/props/test_base.py index 9e664c2a2..86ff6a589 100644 --- a/tests/props/test_base.py +++ b/tests/props/test_base.py @@ -43,10 +43,11 @@ def test_constraint_to_str(): @pytest.mark.parametrize( - "value, constraint_str", [ + "value, constraint_str", + [ (["one"], "(lst=one)"), (["one", "two"], "(&(lst=one)\n\t(lst=two))"), - ] + ], ) def test_constraint_to_str_list(value, constraint_str): foo = Foo(lst=value) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 2ebe03a71..31173a5e7 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -5,6 +5,7 @@ import sys from typing import Optional, List from typing_extensions import Final + if sys.version_info > (3, 8): from typing import Literal else: @@ -32,6 +33,7 @@ VmCaps = Literal["vpn"] + @dataclass class InfVm(InfBase): runtime = RUNTIME_VM @@ -57,7 +59,9 @@ class _VmConstraints: min_storage_gib: float = prop_base.constraint(inf.INF_STORAGE, operator=">=") min_cpu_threads: int = prop_base.constraint(inf.INF_THREADS, operator=">=") - capabilities: List[VmCaps] = prop_base.constraint("golem.runtime.capabilities", operator="=", default_factory=list) + capabilities: List[VmCaps] = prop_base.constraint( + "golem.runtime.capabilities", operator="=", default_factory=list + ) runtime: str = prop_base.constraint(inf.INF_RUNTIME_NAME, operator="=", default=RUNTIME_VM) diff --git a/yapapi/props/base.py b/yapapi/props/base.py index df378e243..ec836fe5a 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -205,7 +205,9 @@ class ModelFieldType(enum.Enum): property = "property" -def constraint(key: str, operator: ConstraintOperator = "=", default=MISSING, default_factory=MISSING): +def constraint( + key: str, operator: ConstraintOperator = "=", default=MISSING, default_factory=MISSING +): """ Return a constraint-type dataclass field for a Model. From d84c91a8a9b644cc7c1c39d8e75e65ed42f1b8a3 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 16 Sep 2021 11:51:05 +0200 Subject: [PATCH 5/5] mypy < 3.8 ? ... --- yapapi/props/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 0d96a4f54..856966866 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -229,7 +229,7 @@ def constraint( ['(baz<=100)'] ``` """ - return field( + return field( # type: ignore # the default / default_factory exception is resolved by the `field` function default=default, default_factory=default_factory, metadata={ @@ -237,7 +237,7 @@ def constraint( PROP_OPERATOR: operator, PROP_MODEL_FIELD_TYPE: ModelFieldType.constraint, }, - ) # type: ignore # the default / default_factory exception is resolved by the `field` function + ) def prop(key: str, default=MISSING):