From 20c989a53ab0cd4e886c839a5eb0346f28442f3e Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 23 Mar 2021 10:26:46 +0100 Subject: [PATCH 01/85] + http server example --- examples/http_server/http_server.Dockerfile | 5 + examples/http_server/http_server.py | 147 ++++++++++++++++++++ examples/http_server/http_test.py | 5 + examples/http_server/index.html | 1 + examples/http_server/run-http-server.sh | 1 + 5 files changed, 159 insertions(+) create mode 100644 examples/http_server/http_server.Dockerfile create mode 100644 examples/http_server/http_server.py create mode 100644 examples/http_server/http_test.py create mode 100644 examples/http_server/index.html create mode 100644 examples/http_server/run-http-server.sh diff --git a/examples/http_server/http_server.Dockerfile b/examples/http_server/http_server.Dockerfile new file mode 100644 index 000000000..dfa42dfdb --- /dev/null +++ b/examples/http_server/http_server.Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.8-slim +VOLUME /golem/html /golem/out /golem/test +COPY run-http-server.sh /golem/run +EXPOSE 80 +ENTRYPOINT ["sh", "/golem/run/run-http-server.sh"] diff --git a/examples/http_server/http_server.py b/examples/http_server/http_server.py new file mode 100644 index 000000000..dacd203e3 --- /dev/null +++ b/examples/http_server/http_server.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +import asyncio +from datetime import datetime, timedelta +import pathlib +import sys + +from yapapi import ( + Executor, + NoPaymentAccountError, + Task, + __version__ as yapapi_version, + WorkContext, + windows_event_loop_fix, +) +from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa +from yapapi.package import vm +from yapapi.rest.activity import BatchTimeoutError + +examples_dir = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(str(examples_dir)) + +from utils import ( + build_parser, + TEXT_COLOR_CYAN, + TEXT_COLOR_DEFAULT, + TEXT_COLOR_RED, + TEXT_COLOR_YELLOW, +) + + +async def main(subnet_tag, driver=None, network=None): + package = await vm.repo( + image_hash="54169fddccc723285789278e28899edab2bb3e73514aeae20f9fc3a2", + min_mem_gib=0.5, + min_storage_gib=2.0, + ) + + async def worker(ctx: WorkContext, tasks): # + script_dir = pathlib.Path(__file__).resolve().parent + ctx.send_file(script_dir / "index.html", "/golem/html/index.html") + ctx.send_file(script_dir / "http_test.py", "/golem/test/http_test.py") + + async for task in tasks: + + ctx.run("python", "/golem/out/http_test.py") + ctx.download_file("/golem/out/test", "test") + yield ctx.commit(timeout=timedelta(seconds=120)) + task.accept_result(result="test") + + # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) + # TODO: make this dynamic, e.g. depending on the size of files to transfer + init_overhead = 3 + # Providers will not accept work if the timeout is outside of the [5 min, 30min] range. + # We increase the lower bound to 6 min to account for the time needed for our demand to + # reach the providers. + min_timeout, max_timeout = 6, 30 + + timeout = timedelta(minutes=min_timeout) + + # By passing `event_consumer=log_summary()` we enable summary logging. + # See the documentation of the `yapapi.log` module on how to set + # the level of detail and format of the logged information. + async with Executor( + package=package, + max_workers=3, + budget=10.0, + timeout=timeout, + subnet_tag=subnet_tag, + driver=driver, + network=network, + event_consumer=log_summary(log_event_repr), + ) as executor: + + print( + f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" + f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " + f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" + ) + + num_tasks = 0 + start_time = datetime.now() + + async for task in executor.submit(worker, [Task(data=None)]): + num_tasks += 1 + print( + f"{TEXT_COLOR_CYAN}" + f"Task computed: {task}, result: {task.result}, time: {task.running_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + + print( + f"{TEXT_COLOR_CYAN}" + f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + + +if __name__ == "__main__": + parser = build_parser("Test http") + parser.set_defaults(log_file="http-yapapi.log") + args = parser.parse_args() + + # This is only required when running on Windows with Python prior to 3.8: + windows_event_loop_fix() + + enable_default_logger( + log_file=args.log_file, + debug_activity_api=True, + debug_market_api=True, + debug_payment_api=True, + ) + + loop = asyncio.get_event_loop() + task = loop.create_task( + main(subnet_tag=args.subnet_tag, driver=args.driver, network=args.network) + ) + + try: + loop.run_until_complete(task) + except NoPaymentAccountError as e: + handbook_url = ( + "https://handbook.golem.network/requestor-tutorials/" + "flash-tutorial-of-requestor-development" + ) + print( + f"{TEXT_COLOR_RED}" + f"No payment account initialized for driver `{e.required_driver}` " + f"and network `{e.required_network}`.\n\n" + f"See {handbook_url} on how to initialize payment accounts for a requestor node." + f"{TEXT_COLOR_DEFAULT}" + ) + except KeyboardInterrupt: + print( + f"{TEXT_COLOR_YELLOW}" + "Shutting down gracefully, please wait a short while " + "or press Ctrl+C to exit immediately..." + f"{TEXT_COLOR_DEFAULT}" + ) + task.cancel() + try: + loop.run_until_complete(task) + print( + f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" + ) + except (asyncio.CancelledError, KeyboardInterrupt): + pass diff --git a/examples/http_server/http_test.py b/examples/http_server/http_test.py new file mode 100644 index 000000000..0cd2d4244 --- /dev/null +++ b/examples/http_server/http_test.py @@ -0,0 +1,5 @@ +from http.client import HTTPConnection +h = HTTPConnection('127.0.0.1') +h.request('GET', '/') +with open("/golem/out/test", "w") as f: + f.write(h.getresponse().read()) diff --git a/examples/http_server/index.html b/examples/http_server/index.html new file mode 100644 index 000000000..47ff8f6dd --- /dev/null +++ b/examples/http_server/index.html @@ -0,0 +1 @@ +Hello from a Golem Service. diff --git a/examples/http_server/run-http-server.sh b/examples/http_server/run-http-server.sh new file mode 100644 index 000000000..d9e5e68d0 --- /dev/null +++ b/examples/http_server/run-http-server.sh @@ -0,0 +1 @@ +python3 -m http.serve --directory /golem/html 80 From 948fd65578a53b8b88636885f0373936a2bfcfb3 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 24 Mar 2021 23:06:49 +0100 Subject: [PATCH 02/85] add another service example --- .../http_server.Dockerfile | 0 .../{http_server => service}/http_server.py | 0 .../{http_server => service}/http_test.py | 0 examples/{http_server => service}/index.html | 0 .../run-http-server.sh | 0 .../simple_service/simple_service.Dockerfile | 8 ++ .../service/simple_service/simple_service.py | 117 ++++++++++++++++++ .../simple_service/simulate_observations.py | 14 +++ .../simulate_observations_ctl.py | 29 +++++ 9 files changed, 168 insertions(+) rename examples/{http_server => service}/http_server.Dockerfile (100%) rename examples/{http_server => service}/http_server.py (100%) rename examples/{http_server => service}/http_test.py (100%) rename examples/{http_server => service}/index.html (100%) rename examples/{http_server => service}/run-http-server.sh (100%) create mode 100644 examples/service/simple_service/simple_service.Dockerfile create mode 100644 examples/service/simple_service/simple_service.py create mode 100644 examples/service/simple_service/simulate_observations.py create mode 100644 examples/service/simple_service/simulate_observations_ctl.py diff --git a/examples/http_server/http_server.Dockerfile b/examples/service/http_server.Dockerfile similarity index 100% rename from examples/http_server/http_server.Dockerfile rename to examples/service/http_server.Dockerfile diff --git a/examples/http_server/http_server.py b/examples/service/http_server.py similarity index 100% rename from examples/http_server/http_server.py rename to examples/service/http_server.py diff --git a/examples/http_server/http_test.py b/examples/service/http_test.py similarity index 100% rename from examples/http_server/http_test.py rename to examples/service/http_test.py diff --git a/examples/http_server/index.html b/examples/service/index.html similarity index 100% rename from examples/http_server/index.html rename to examples/service/index.html diff --git a/examples/http_server/run-http-server.sh b/examples/service/run-http-server.sh similarity index 100% rename from examples/http_server/run-http-server.sh rename to examples/service/run-http-server.sh diff --git a/examples/service/simple_service/simple_service.Dockerfile b/examples/service/simple_service/simple_service.Dockerfile new file mode 100644 index 000000000..4d16b3648 --- /dev/null +++ b/examples/service/simple_service/simple_service.Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.8-slim +VOLUME /golem/in /golem/out +COPY simple_service.py /golem/run/simple_service.py +COPY simulate_observations.py /golem/run/simulate_observations.py +COPY simulate_observations_ctl.py /golem/run/simulate_observations_ctl.py +RUN pip install numpy matplotlib +RUN python /golem/run/simple_service.py --init +ENTRYPOINT ["sh"] diff --git a/examples/service/simple_service/simple_service.py b/examples/service/simple_service/simple_service.py new file mode 100644 index 000000000..de5243991 --- /dev/null +++ b/examples/service/simple_service/simple_service.py @@ -0,0 +1,117 @@ +import argparse +from datetime import datetime +import enum +import contextlib +import json +import matplotlib.pyplot as plt +import numpy +import random +import sqlite3 +import string +from pathlib import Path + +DB_PATH = Path(__file__).absolute().parent / "service.db" +PLOT_PATH = Path("/golem/out").absolute() + + +class PlotType(enum.Enum): + time = "time" + dist = "dist" + + +@contextlib.contextmanager +def _connect_db(): + db = sqlite3.connect(DB_PATH) + db.row_factory = sqlite3.Row + try: + yield db.cursor() + db.commit() + finally: + db.close() + + +def init(): + with _connect_db() as db: + db.execute("create table observations(" + "id integer primary key autoincrement not null, " + "val float not null," + "time_added timestamp default current_timestamp not null" + ")") + + +def add(val): + with _connect_db() as db: + db.execute("insert into observations (val) values (?)", [val]) + + +def plot(plot_type): + data = _get_data() + y = [r["val"] for r in data] + + if plot_type == PlotType.dist.value: + plt.hist(y) + elif plot_type == PlotType.time.value: + x = [datetime.strptime(r["time_added"], "%Y-%m-%d %H:%M:%S") for r in data] + plt.plot(x, y) + + plot_filename = PLOT_PATH / ( + "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" + ) + plt.savefig(plot_filename) + print(json.dumps(str(plot_filename))) + + +def dump(): + print(json.dumps(_get_data())) + + +def _get_data(): + with _connect_db() as db: + db.execute( + "select val, time_added from observations order by time_added asc" + ) + return list(map(dict, db.fetchall())) + + +def _get_stats(data=None): + data = data or [r["val"] for r in _get_data()] + return { + "min": min(data), + "max": max(data), + "median": numpy.median(data), + "mean": numpy.mean(data), + "variance": numpy.var(data), + "std dev": numpy.std(data), + "size": len(data), + } + + +def stats(): + print(json.dumps(_get_stats())) + + +def get_arg_parser(): + parser = argparse.ArgumentParser(description="simple service") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--add", type=float) + group.add_argument("--init", action="store_true") + group.add_argument("--plot", choices=[pt.value for pt in list(PlotType)]) + group.add_argument("--dump", action="store_true") + group.add_argument("--stats", action="store_true") + return parser + + +if __name__ == "__main__": + arg_parser = get_arg_parser() + args = arg_parser.parse_args() + + if args.init: + init() + elif args.add: + add(args.add) + elif args.plot: + plot(args.plot) + elif args.dump: + dump() + elif args.stats: + stats() diff --git a/examples/service/simple_service/simulate_observations.py b/examples/service/simple_service/simulate_observations.py new file mode 100644 index 000000000..ae4e28d0e --- /dev/null +++ b/examples/service/simple_service/simulate_observations.py @@ -0,0 +1,14 @@ +import os +from pathlib import Path +import random +import time + +MU = 14 +SIGMA = 3 + +SERVICE_PATH = Path(__file__).absolute().parent / "simple_service.py" + +while True: + v = random.normalvariate(MU, SIGMA) + os.system(f"python {SERVICE_PATH} --add {v}") + time.sleep(1) diff --git a/examples/service/simple_service/simulate_observations_ctl.py b/examples/service/simple_service/simulate_observations_ctl.py new file mode 100644 index 000000000..c59ab4996 --- /dev/null +++ b/examples/service/simple_service/simulate_observations_ctl.py @@ -0,0 +1,29 @@ +import argparse +import os +import subprocess +import signal + +PIDFILE = "/var/run/simulate_observations.pid" +SCRIPT_FILE = "/golem/run/simulate_observations.py" + +parser = argparse.ArgumentParser("start/stop simulation") +group = parser.add_mutually_exclusive_group(required=True) +group.add_argument("--start", action="store_true") +group.add_argument("--stop", action="store_true") + +args = parser.parse_args() + +if args.start: + if os.path.exists(PIDFILE): + raise Exception(f"Cannot start process, {PIDFILE} exists.") + p = subprocess.Popen(["python", SCRIPT_FILE]) + with open(PIDFILE, "w") as pidfile: + pidfile.write(str(p.pid)) +elif args.stop: + if not os.path.exists(PIDFILE): + raise Exception(f"Could not find pidfile: {PIDFILE}.") + with open(PIDFILE, "r") as pidfile: + pid = int(pidfile.read()) + + os.kill(pid, signal.SIGKILL) + os.remove(PIDFILE) From 9880630e167c1fe3b431257ce4cbc19c34983448 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 29 Mar 2021 17:37:48 +0200 Subject: [PATCH 03/85] simple service debug session etc ... --- examples/service/http_server.py | 15 +- .../{ => http_server}/http_server.Dockerfile | 0 .../{ => http_server}/run-http-server.sh | 0 examples/service/simple_service.py | 190 ++++++++++++++++++ .../simple_service/simple_service.Dockerfile | 3 +- .../service/simple_service/simple_service.py | 18 +- .../simple_service/simulate_observations.py | 3 +- .../simulate_observations_ctl.py | 3 +- 8 files changed, 216 insertions(+), 16 deletions(-) rename examples/service/{ => http_server}/http_server.Dockerfile (100%) rename examples/service/{ => http_server}/run-http-server.sh (100%) create mode 100644 examples/service/simple_service.py diff --git a/examples/service/http_server.py b/examples/service/http_server.py index dacd203e3..1e331223f 100644 --- a/examples/service/http_server.py +++ b/examples/service/http_server.py @@ -35,17 +35,18 @@ async def main(subnet_tag, driver=None, network=None): min_storage_gib=2.0, ) - async def worker(ctx: WorkContext, tasks): # + async def service(ctx: WorkContext, tasks): script_dir = pathlib.Path(__file__).resolve().parent ctx.send_file(script_dir / "index.html", "/golem/html/index.html") ctx.send_file(script_dir / "http_test.py", "/golem/test/http_test.py") - async for task in tasks: + ctx.run("sh", "/golem/run/run-http-server.sh") + ctx.run("python", "/golem/out/http_test.py") + ctx.download_file("/golem/out/test", "test") + yield ctx.commit() - ctx.run("python", "/golem/out/http_test.py") - ctx.download_file("/golem/out/test", "test") - yield ctx.commit(timeout=timedelta(seconds=120)) - task.accept_result(result="test") + async for task in tasks: + task.accept_result() # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) # TODO: make this dynamic, e.g. depending on the size of files to transfer @@ -81,7 +82,7 @@ async def worker(ctx: WorkContext, tasks): # num_tasks = 0 start_time = datetime.now() - async for task in executor.submit(worker, [Task(data=None)]): + async for task in executor.submit(service, [Task(data=None)]): num_tasks += 1 print( f"{TEXT_COLOR_CYAN}" diff --git a/examples/service/http_server.Dockerfile b/examples/service/http_server/http_server.Dockerfile similarity index 100% rename from examples/service/http_server.Dockerfile rename to examples/service/http_server/http_server.Dockerfile diff --git a/examples/service/run-http-server.sh b/examples/service/http_server/run-http-server.sh similarity index 100% rename from examples/service/run-http-server.sh rename to examples/service/http_server/run-http-server.sh diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py new file mode 100644 index 000000000..fbff0c668 --- /dev/null +++ b/examples/service/simple_service.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +import asyncio +from datetime import datetime, timedelta +import pathlib +import random +import string +import sys +import tempfile +import time + + +from yapapi import ( + Executor, + NoPaymentAccountError, + Task, + __version__ as yapapi_version, + WorkContext, + windows_event_loop_fix, +) +from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa +from yapapi.package import vm +from yapapi.rest.activity import BatchTimeoutError + +examples_dir = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(str(examples_dir)) + +from utils import ( + build_parser, + TEXT_COLOR_CYAN, + TEXT_COLOR_DEFAULT, + TEXT_COLOR_RED, + TEXT_COLOR_YELLOW, +) + + +async def main(subnet_tag, driver=None, network=None): + package = await vm.repo( + image_hash="8b11df59f84358d47fc6776d0bb7290b0054c15ded2d6f54cf634488", + min_mem_gib=0.5, + min_storage_gib=2.0, + ) + + async def service(ctx: WorkContext, tasks): # + print("-------------------------- 1") + ctx.run("/bin/sh", "-c", "echo aaa") + # ctx.run("/golem/run/simulate_observations_ctl.py", "--start") + # ctx.send_bytes("/golem/in/get_stats.sh", b"/golem/run/simple_service.py --stats > /golem/out/test") + # print("-------------------------- 2") + yield ctx.commit() + + await asyncio.sleep(10) + + ctx.run("/bin/sh", "-c", "echo bbb") + ctx.run("/bin/sh", "-c", "echo ccc") + yield ctx.commit() + + await asyncio.sleep(10) + + ctx.run("/bin/sh", "-c", "echo ddd") + yield ctx.commit() + + # print("-------------------------- 3") + # await asyncio.sleep(10) + # print("-------------------------- 4") + # + # ctx.run("/bin/sh", "/golem/in/get_stats.sh") + # test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".stats" + # ctx.download_file("/golem/out/test", pathlib.Path(__file__).resolve().parent / test_filename) + # ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") + # print("-------------------------- 5") + # yield ctx.commit() + # print("-------------------------- 6") + # + async for task in tasks: + print(f"-------------------------- 7 {task}") + task.accept_result() + + # while True: + # try: + # await asyncio.sleep(10) + # ctx.run("sh", "/golem/run/get_stats.sh") + # test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".stats" + # + # ctx.download_file("/golem/out/test", pathlib.Path(__file__).resolve().parent / test_filename) + # yield ctx.commit() + # finally: + # async for task in tasks: + # print("!!!!!!!!!!!!!!!!!!!!!!!!!!! task accept") + # task.accept_result() + + + + # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) + # TODO: make this dynamic, e.g. depending on the size of files to transfer + init_overhead = 3 + # Providers will not accept work if the timeout is outside of the [5 min, 30min] range. + # We increase the lower bound to 6 min to account for the time needed for our demand to + # reach the providers. + min_timeout, max_timeout = 6, 30 + + timeout = timedelta(minutes=min_timeout) + + # By passing `event_consumer=log_summary()` we enable summary logging. + # See the documentation of the `yapapi.log` module on how to set + # the level of detail and format of the logged information. + async with Executor( + package=package, + max_workers=3, + budget=10.0, + timeout=timeout, + subnet_tag=subnet_tag, + driver=driver, + network=network, + event_consumer=log_summary(log_event_repr), + ) as executor: + + print( + f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" + f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " + f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" + ) + + num_tasks = 0 + start_time = datetime.now() + + async for task in executor.submit(service, [Task(data=None)]): + num_tasks += 1 + print( + f"{TEXT_COLOR_CYAN}" + f"Task computed: {task}, result: {task.result}, time: {task.running_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + + print( + f"{TEXT_COLOR_CYAN}" + f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + + +if __name__ == "__main__": + parser = build_parser("Test http") + parser.set_defaults(log_file="service-yapapi.log") + args = parser.parse_args() + + # This is only required when running on Windows with Python prior to 3.8: + windows_event_loop_fix() + + enable_default_logger( + log_file=args.log_file, + debug_activity_api=True, + debug_market_api=True, + debug_payment_api=True, + ) + + loop = asyncio.get_event_loop() + task = loop.create_task( + main(subnet_tag=args.subnet_tag, driver=args.driver, network=args.network) + ) + + try: + loop.run_until_complete(task) + except NoPaymentAccountError as e: + handbook_url = ( + "https://handbook.golem.network/requestor-tutorials/" + "flash-tutorial-of-requestor-development" + ) + print( + f"{TEXT_COLOR_RED}" + f"No payment account initialized for driver `{e.required_driver}` " + f"and network `{e.required_network}`.\n\n" + f"See {handbook_url} on how to initialize payment accounts for a requestor node." + f"{TEXT_COLOR_DEFAULT}" + ) + except KeyboardInterrupt: + print( + f"{TEXT_COLOR_YELLOW}" + "Shutting down gracefully, please wait a short while " + "or press Ctrl+C to exit immediately..." + f"{TEXT_COLOR_DEFAULT}" + ) + task.cancel() + try: + loop.run_until_complete(task) + print( + f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" + ) + except (asyncio.CancelledError, KeyboardInterrupt): + pass diff --git a/examples/service/simple_service/simple_service.Dockerfile b/examples/service/simple_service/simple_service.Dockerfile index 4d16b3648..8626fd87e 100644 --- a/examples/service/simple_service/simple_service.Dockerfile +++ b/examples/service/simple_service/simple_service.Dockerfile @@ -4,5 +4,6 @@ COPY simple_service.py /golem/run/simple_service.py COPY simulate_observations.py /golem/run/simulate_observations.py COPY simulate_observations_ctl.py /golem/run/simulate_observations_ctl.py RUN pip install numpy matplotlib -RUN python /golem/run/simple_service.py --init +RUN chmod +x /golem/run/* +RUN /golem/run/simple_service.py --init ENTRYPOINT ["sh"] diff --git a/examples/service/simple_service/simple_service.py b/examples/service/simple_service/simple_service.py index de5243991..a17171b80 100644 --- a/examples/service/simple_service/simple_service.py +++ b/examples/service/simple_service/simple_service.py @@ -1,3 +1,4 @@ +#!/usr/local/bin/python import argparse from datetime import datetime import enum @@ -46,6 +47,11 @@ def add(val): def plot(plot_type): data = _get_data() + + if not data: + print(json.dumps("")) + return + y = [r["val"] for r in data] if plot_type == PlotType.dist.value: @@ -76,12 +82,12 @@ def _get_data(): def _get_stats(data=None): data = data or [r["val"] for r in _get_data()] return { - "min": min(data), - "max": max(data), - "median": numpy.median(data), - "mean": numpy.mean(data), - "variance": numpy.var(data), - "std dev": numpy.std(data), + "min": min(data) if data else None, + "max": max(data) if data else None, + "median": numpy.median(data) if data else None, + "mean": numpy.mean(data) if data else None, + "variance": numpy.var(data) if data else None, + "std dev": numpy.std(data) if data else None, "size": len(data), } diff --git a/examples/service/simple_service/simulate_observations.py b/examples/service/simple_service/simulate_observations.py index ae4e28d0e..b8c598abb 100644 --- a/examples/service/simple_service/simulate_observations.py +++ b/examples/service/simple_service/simulate_observations.py @@ -1,3 +1,4 @@ +#!/usr/local/bin/python import os from pathlib import Path import random @@ -10,5 +11,5 @@ while True: v = random.normalvariate(MU, SIGMA) - os.system(f"python {SERVICE_PATH} --add {v}") + os.system(f"{SERVICE_PATH} --add {v}") time.sleep(1) diff --git a/examples/service/simple_service/simulate_observations_ctl.py b/examples/service/simple_service/simulate_observations_ctl.py index c59ab4996..6a083dfe2 100644 --- a/examples/service/simple_service/simulate_observations_ctl.py +++ b/examples/service/simple_service/simulate_observations_ctl.py @@ -1,3 +1,4 @@ +#!/usr/local/bin/python import argparse import os import subprocess @@ -16,7 +17,7 @@ if args.start: if os.path.exists(PIDFILE): raise Exception(f"Cannot start process, {PIDFILE} exists.") - p = subprocess.Popen(["python", SCRIPT_FILE]) + p = subprocess.Popen([SCRIPT_FILE]) with open(PIDFILE, "w") as pidfile: pidfile.write(str(p.pid)) elif args.stop: From 982d7dfed08014b66a1c3c50f0fa130ff6b8b9f3 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 29 Mar 2021 18:58:12 +0200 Subject: [PATCH 04/85] add `WorkContext.send_bytes` command --- tests/executor/test_ctx.py | 57 ++++++++++++++++++++++++++++++- tests/factories/props/__init__.py | 11 ++++++ yapapi/executor/ctx.py | 25 ++++++++++---- 3 files changed, 86 insertions(+), 7 deletions(-) create mode 100644 tests/factories/props/__init__.py diff --git a/tests/executor/test_ctx.py b/tests/executor/test_ctx.py index be0407064..71c899242 100644 --- a/tests/executor/test_ctx.py +++ b/tests/executor/test_ctx.py @@ -1,5 +1,12 @@ -from yapapi.executor.ctx import CommandContainer +import factory import json +import pytest +import sys +from unittest import mock + +from yapapi.executor.ctx import CommandContainer, WorkContext + +from tests.factories.props import NodeInfoFactory def test_command_container(): @@ -29,3 +36,51 @@ def test_command_container(): ] """ assert json.loads(expected_commands) == c.commands() + + +class TestWorkContext: + @staticmethod + def _get_work_context(storage): + return WorkContext( + factory.Faker("pystr"), + node_info=NodeInfoFactory(), + storage=storage + ) + + @staticmethod + def _assert_dst_path(steps, dst_path): + c = CommandContainer() + steps.register(c) + assert c.commands().pop()['transfer']['to'] == f"container:{dst_path}" + + @pytest.mark.asyncio + @pytest.mark.skipif( + sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+" + ) + async def test_send_json(self): + storage = mock.AsyncMock() + dst_path = "/test/path" + data = { + 'param': 'value', + } + ctx = self._get_work_context(storage) + ctx.send_json(dst_path, data) + steps = ctx.commit() + await steps.prepare() + storage.upload_bytes.assert_called_with(json.dumps(data).encode("utf-8")) + self._assert_dst_path(steps, dst_path) + + @pytest.mark.asyncio + @pytest.mark.skipif( + sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+" + ) + async def test_send_bytes(self): + storage = mock.AsyncMock() + dst_path = "/test/path" + data = b"some byte string" + ctx = self._get_work_context(storage) + ctx.send_bytes(dst_path, data) + steps = ctx.commit() + await steps.prepare() + storage.upload_bytes.assert_called_with(data) + self._assert_dst_path(steps, dst_path) diff --git a/tests/factories/props/__init__.py b/tests/factories/props/__init__.py new file mode 100644 index 000000000..0b7c3362b --- /dev/null +++ b/tests/factories/props/__init__.py @@ -0,0 +1,11 @@ +import factory + +from yapapi.props import NodeInfo + + +class NodeInfoFactory(factory.Factory): + class Meta: + model = NodeInfo + + name = factory.Faker("pystr") + subnet_tag = "devnet-beta.1" diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index 3779abab0..59fe7bddd 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -76,20 +76,23 @@ def register(self, commands: CommandContainer): ) -class _SendJson(_SendWork): - def __init__(self, storage: StorageProvider, data: dict, dst_path: str): +class _SendBytes(_SendWork): + def __init__(self, storage: StorageProvider, data: bytes, dst_path: str): super().__init__(storage, dst_path) - self._cnt = 0 - self._data: Optional[bytes] = json.dumps(data).encode(encoding="utf-8") + self._data: Optional[bytes] = data async def do_upload(self, storage: StorageProvider) -> Source: - self._cnt += 1 - assert self._data is not None, f"json buffer unintialized {self._cnt}" + assert self._data is not None, f"buffer unintialized" src = await storage.upload_bytes(self._data) self._data = None return src +class _SendJson(_SendBytes): + def __init__(self, storage: StorageProvider, data: dict, dst_path: str): + super().__init__(storage, json.dumps(data).encode(encoding="utf-8"), dst_path) + + class _SendFile(_SendWork): def __init__(self, storage: StorageProvider, src_path: str, dst_path: str): super(_SendFile, self).__init__(storage, dst_path) @@ -230,6 +233,16 @@ def send_json(self, json_path: str, data: dict): self.__prepare() self._pending_steps.append(_SendJson(self._storage, data, json_path)) + def send_bytes(self, dst_path: str, data: bytes): + """Schedule sending bytes data to the provider. + + :param dst_path: remote (provider) path + :param data: dictionary representing JSON data + :return: None + """ + self.__prepare() + self._pending_steps.append(_SendBytes(self._storage, data, dst_path)) + def send_file(self, src_path: str, dst_path: str): """Schedule sending file to the provider. From 066586545ef354e91713621541f226b092f4a438 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 29 Mar 2021 19:12:54 +0200 Subject: [PATCH 05/85] fix the PollingBatch with the current version of yagna --- yapapi/rest/activity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index 1052f03a5..4a02feebc 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -167,7 +167,7 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: raise BatchTimeoutError() try: results: List[yaa.ExeScriptCommandResult] = await self._api.get_exec_batch_results( - self._activity_id, self._batch_id, timeout=timeout + self._activity_id, self._batch_id #, timeout=timeout ) except ApiException as err: if err.status == 408: From 3e4b0a8a5bb2ba5a5e85fdd36732e6f0e005165f Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 29 Mar 2021 19:33:20 +0200 Subject: [PATCH 06/85] black, additional comment --- yapapi/rest/activity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index 4a02feebc..68475b125 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -167,7 +167,9 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: raise BatchTimeoutError() try: results: List[yaa.ExeScriptCommandResult] = await self._api.get_exec_batch_results( - self._activity_id, self._batch_id #, timeout=timeout + self._activity_id, + self._batch_id, + # , timeout=timeout # timeout argument is currently unsupported ) except ApiException as err: if err.status == 408: From 3524e5bf52eadd68e01d837422f7a45d344a874d Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 29 Mar 2021 19:34:36 +0200 Subject: [PATCH 07/85] - "," --- yapapi/rest/activity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index 68475b125..537413014 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -169,7 +169,7 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: results: List[yaa.ExeScriptCommandResult] = await self._api.get_exec_batch_results( self._activity_id, self._batch_id, - # , timeout=timeout # timeout argument is currently unsupported + # timeout=timeout # timeout argument is currently unsupported ) except ApiException as err: if err.status == 408: From cab9f94f25946d62a242fc83a22aa7c8ac4999bc Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 30 Mar 2021 13:36:53 +0200 Subject: [PATCH 08/85] use aiohttp's general `_request_timeout` --- yapapi/rest/activity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index 537413014..f8db818d1 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -169,7 +169,7 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: results: List[yaa.ExeScriptCommandResult] = await self._api.get_exec_batch_results( self._activity_id, self._batch_id, - # timeout=timeout # timeout argument is currently unsupported + _request_timeout = timeout ) except ApiException as err: if err.status == 408: From f8511240b585744fb4ed3c449cb9ddd631a93962 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 30 Mar 2021 14:02:03 +0200 Subject: [PATCH 09/85] black... --- yapapi/rest/activity.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index f8db818d1..ac463dfc1 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -167,9 +167,7 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: raise BatchTimeoutError() try: results: List[yaa.ExeScriptCommandResult] = await self._api.get_exec_batch_results( - self._activity_id, - self._batch_id, - _request_timeout = timeout + self._activity_id, self._batch_id, _request_timeout=timeout ) except ApiException as err: if err.status == 408: From 32ae650540711ac4319c4fdd2bdadad5631277a3 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 30 Mar 2021 15:56:14 +0200 Subject: [PATCH 10/85] black... --- examples/service/http_test.py | 5 +++-- .../service/simple_service/simple_service.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/examples/service/http_test.py b/examples/service/http_test.py index 0cd2d4244..13fe92659 100644 --- a/examples/service/http_test.py +++ b/examples/service/http_test.py @@ -1,5 +1,6 @@ from http.client import HTTPConnection -h = HTTPConnection('127.0.0.1') -h.request('GET', '/') + +h = HTTPConnection("127.0.0.1") +h.request("GET", "/") with open("/golem/out/test", "w") as f: f.write(h.getresponse().read()) diff --git a/examples/service/simple_service/simple_service.py b/examples/service/simple_service/simple_service.py index a17171b80..c6c40dc04 100644 --- a/examples/service/simple_service/simple_service.py +++ b/examples/service/simple_service/simple_service.py @@ -33,11 +33,13 @@ def _connect_db(): def init(): with _connect_db() as db: - db.execute("create table observations(" - "id integer primary key autoincrement not null, " - "val float not null," - "time_added timestamp default current_timestamp not null" - ")") + db.execute( + "create table observations(" + "id integer primary key autoincrement not null, " + "val float not null," + "time_added timestamp default current_timestamp not null" + ")" + ) def add(val): @@ -61,7 +63,7 @@ def plot(plot_type): plt.plot(x, y) plot_filename = PLOT_PATH / ( - "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" + "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" ) plt.savefig(plot_filename) print(json.dumps(str(plot_filename))) @@ -73,9 +75,7 @@ def dump(): def _get_data(): with _connect_db() as db: - db.execute( - "select val, time_added from observations order by time_added asc" - ) + db.execute("select val, time_added from observations order by time_added asc") return list(map(dict, db.fetchall())) From 4a2081d9b884a54e83bf73d0afe731a3ffe1cc01 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 30 Mar 2021 16:20:18 +0200 Subject: [PATCH 11/85] bring back the "proper" version of the service --- examples/service/simple_service.py | 59 +++++++----------------------- 1 file changed, 13 insertions(+), 46 deletions(-) diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py index fbff0c668..ed2da1638 100644 --- a/examples/service/simple_service.py +++ b/examples/service/simple_service.py @@ -41,54 +41,21 @@ async def main(subnet_tag, driver=None, network=None): ) async def service(ctx: WorkContext, tasks): # - print("-------------------------- 1") - ctx.run("/bin/sh", "-c", "echo aaa") - # ctx.run("/golem/run/simulate_observations_ctl.py", "--start") - # ctx.send_bytes("/golem/in/get_stats.sh", b"/golem/run/simple_service.py --stats > /golem/out/test") - # print("-------------------------- 2") + ctx.run("/golem/run/simulate_observations_ctl.py", "--start") + ctx.send_bytes("/golem/in/get_stats.sh", b"/golem/run/simple_service.py --stats > /golem/out/test") yield ctx.commit() - await asyncio.sleep(10) - - ctx.run("/bin/sh", "-c", "echo bbb") - ctx.run("/bin/sh", "-c", "echo ccc") - yield ctx.commit() - - await asyncio.sleep(10) - - ctx.run("/bin/sh", "-c", "echo ddd") - yield ctx.commit() - - # print("-------------------------- 3") - # await asyncio.sleep(10) - # print("-------------------------- 4") - # - # ctx.run("/bin/sh", "/golem/in/get_stats.sh") - # test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".stats" - # ctx.download_file("/golem/out/test", pathlib.Path(__file__).resolve().parent / test_filename) - # ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") - # print("-------------------------- 5") - # yield ctx.commit() - # print("-------------------------- 6") - # - async for task in tasks: - print(f"-------------------------- 7 {task}") - task.accept_result() - - # while True: - # try: - # await asyncio.sleep(10) - # ctx.run("sh", "/golem/run/get_stats.sh") - # test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".stats" - # - # ctx.download_file("/golem/out/test", pathlib.Path(__file__).resolve().parent / test_filename) - # yield ctx.commit() - # finally: - # async for task in tasks: - # print("!!!!!!!!!!!!!!!!!!!!!!!!!!! task accept") - # task.accept_result() - - + while True: + try: + await asyncio.sleep(10) + ctx.run("/bin/sh", "/golem/in/get_stats.sh") + test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".stats" + ctx.download_file("/golem/out/test", pathlib.Path(__file__).resolve().parent / test_filename) + ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") + yield ctx.commit() + finally: + async for task in tasks: + task.accept_result() # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) # TODO: make this dynamic, e.g. depending on the size of files to transfer From ccb307e2a5a4106b1dc62089c826955d482036c3 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 30 Mar 2021 19:44:42 +0200 Subject: [PATCH 12/85] . --- examples/service/simple_service.py | 17 +++++++++++------ yapapi/rest/activity.py | 4 +++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py index ed2da1638..1e1be1949 100644 --- a/examples/service/simple_service.py +++ b/examples/service/simple_service.py @@ -45,17 +45,22 @@ async def service(ctx: WorkContext, tasks): # ctx.send_bytes("/golem/in/get_stats.sh", b"/golem/run/simple_service.py --stats > /golem/out/test") yield ctx.commit() - while True: - try: + try: + while True: await asyncio.sleep(10) + ctx.run("/bin/sh", "/golem/in/get_stats.sh") test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".stats" ctx.download_file("/golem/out/test", pathlib.Path(__file__).resolve().parent / test_filename) - ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") yield ctx.commit() - finally: - async for task in tasks: - task.accept_result() + + except KeyboardInterrupt: + ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") + yield ctx.commit() + + finally: + async for task in tasks: + task.accept_result() # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) # TODO: make this dynamic, e.g. depending on the size of files to transfer diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index f8db818d1..0377323c3 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -216,8 +216,10 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: batch_id = self._batch_id last_idx = self._size - 1 + evt_src_endpoint = f"{host}/activity/{activity_id}/exec/{batch_id}" + async with sse_client.EventSource( - f"{host}/activity/{activity_id}/exec/{batch_id}", + evt_src_endpoint, headers=headers, timeout=self.seconds_left(), ) as event_source: From eec051066d02d5ecf7508ac61e78129d045497e4 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 31 Mar 2021 12:46:00 +0200 Subject: [PATCH 13/85] a more fleshed-out example --- examples/service/simple_service.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py index 1e1be1949..89a4ffb59 100644 --- a/examples/service/simple_service.py +++ b/examples/service/simple_service.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import asyncio from datetime import datetime, timedelta +import json import pathlib import random import string @@ -40,18 +41,37 @@ async def main(subnet_tag, driver=None, network=None): min_storage_gib=2.0, ) - async def service(ctx: WorkContext, tasks): # + async def service(ctx: WorkContext, tasks): + STATS_PATH = "/golem/out/stats" + PLOT_INFO_PATH = "/golem/out/plot" + SIMPLE_SERVICE = "/golem/run/simple_service.py" + ctx.run("/golem/run/simulate_observations_ctl.py", "--start") - ctx.send_bytes("/golem/in/get_stats.sh", b"/golem/run/simple_service.py --stats > /golem/out/test") + ctx.send_bytes("/golem/in/get_stats.sh", f"{SIMPLE_SERVICE} --stats > {STATS_PATH}".encode()) + ctx.send_bytes("/golem/in/get_plot.sh", f"{SIMPLE_SERVICE} --plot dist > {PLOT_INFO_PATH}".encode()) + yield ctx.commit() + plots_to_download = [] + + def on_plot(out: bytes): + fname = json.loads(out.strip()) + print(f"{TEXT_COLOR_CYAN}plot: {fname}{TEXT_COLOR_DEFAULT}") + plots_to_download.append(fname) + try: while True: await asyncio.sleep(10) ctx.run("/bin/sh", "/golem/in/get_stats.sh") - test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".stats" - ctx.download_file("/golem/out/test", pathlib.Path(__file__).resolve().parent / test_filename) + ctx.download_bytes(STATS_PATH, lambda out: print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}")) + ctx.run("/bin/sh", "/golem/in/get_plot.sh") + ctx.download_bytes(PLOT_INFO_PATH, on_plot) + yield ctx.commit() + + for plot in plots_to_download: + test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" + ctx.download_file(plot, pathlib.Path(__file__).resolve().parent / test_filename) yield ctx.commit() except KeyboardInterrupt: From ccedf057ebe915293743fe363302f9cf0019e3dd Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 31 Mar 2021 12:47:56 +0200 Subject: [PATCH 14/85] `WorkContext.download_bytes` and `WorkContext.download_json` methods --- yapapi/executor/ctx.py | 102 +++++++++++++++++++++++++++++++++---- yapapi/storage/__init__.py | 15 ++++++ 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index 59fe7bddd..e675c3c7d 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -3,12 +3,13 @@ import json from dataclasses import dataclass from datetime import timedelta +from functools import partial from pathlib import Path -from typing import Callable, Iterable, Optional, Dict, List, Tuple, Union +from typing import Callable, Iterable, Optional, Dict, List, Tuple, Union, Any from .events import DownloadStarted, DownloadFinished from ..props import NodeInfo -from ..storage import StorageProvider, Source, Destination +from ..storage import StorageProvider, Source, Destination, DOWNLOAD_BYTES_LIMIT_DEFAULT class CommandContainer: @@ -130,40 +131,99 @@ def register(self, commands: CommandContainer): StorageEvent = Union[DownloadStarted, DownloadFinished] -class _RecvFile(Work): +class _ReceiveContent(Work): def __init__( self, storage: StorageProvider, src_path: str, - dst_path: str, emitter: Optional[Callable[[StorageEvent], None]] = None, ): self._storage = storage - self._dst_path = Path(dst_path) self._src_path: str = src_path self._idx: Optional[int] = None self._dst_slot: Optional[Destination] = None self._emitter: Optional[Callable[[StorageEvent], None]] = emitter + self._dst_path: Optional[str] = None async def prepare(self): self._dst_slot = await self._storage.new_destination(destination_file=self._dst_path) def register(self, commands: CommandContainer): - assert self._dst_slot, "_RecvFile command creation without prepare" + assert self._dst_slot, "_ReceiveFile command creation without prepare" self._idx = commands.transfer( _from=f"container:{self._src_path}", to=self._dst_slot.upload_url ) - async def post(self) -> None: - assert self._dst_slot, "_RecvFile post without prepare" + def _emit_download_start(self): + assert self._dst_slot, f"{self.__class__} post without prepare" if self._emitter: self._emitter(DownloadStarted(path=self._src_path)) - await self._dst_slot.download_file(self._dst_path) + + def _emit_download_end(self): if self._emitter: self._emitter(DownloadFinished(path=str(self._dst_path))) +class _ReceiveFile(_ReceiveContent): + def __init__( + self, + storage: StorageProvider, + src_path: str, + dst_path: str, + emitter: Optional[Callable[[StorageEvent], None]] = None, + ): + super().__init__(storage, src_path, emitter) + self._dst_path = Path(dst_path) + + async def post(self) -> None: + self._emit_download_start() + await self._dst_slot.download_file(self._dst_path) + self._emit_download_end() + + +class _ReceiveBytes(_ReceiveContent): + def __init__( + self, + storage: StorageProvider, + src_path: str, + on_download: Callable[[bytes], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + emitter: Optional[Callable[[StorageEvent], None]] = None, + ): + super().__init__(storage, src_path, emitter) + self._on_download = on_download + self._limit = limit + + async def post(self) -> None: + self._emit_download_start() + output = await self._dst_slot.download_bytes(limit=self._limit) + self._emit_download_end() + self._on_download(output) + + +class _ReceiveJson(_ReceiveBytes): + def __init__( + self, + storage: StorageProvider, + src_path: str, + on_download: Callable[[Any], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + emitter: Optional[Callable[[StorageEvent], None]] = None, + ): + super().__init__( + storage, + src_path, + partial(self.__on_json_download, on_download), + limit, + emitter + ) + + @staticmethod + def __on_json_download(on_download: Callable[[bytes], None], content: bytes): + on_download(json.loads(content)) + + class _Steps(Work): def __init__(self, *steps: Work, timeout: Optional[timedelta] = None): self._steps: Tuple[Work, ...] = steps @@ -280,7 +340,29 @@ def download_file(self, src_path: str, dst_path: str): :return: None """ self.__prepare() - self._pending_steps.append(_RecvFile(self._storage, src_path, dst_path, self._emitter)) + self._pending_steps.append(_ReceiveFile(self._storage, src_path, dst_path, self._emitter)) + + def download_bytes( + self, + src_path: str, + on_download: Callable[[bytes], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + ): + self.__prepare() + self._pending_steps.append( + _ReceiveBytes(self._storage, src_path, on_download, limit, self._emitter) + ) + + def download_json( + self, + src_path: str, + on_download: Callable[[Any], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + ): + self.__prepare() + self._pending_steps.append( + _ReceiveJson(self._storage, src_path, on_download, limit, self._emitter) + ) def commit(self, timeout: Optional[timedelta] = None) -> Work: """Creates sequence of commands to be sent to provider. diff --git a/yapapi/storage/__init__.py b/yapapi/storage/__init__.py index 8b03a6499..1205cb756 100644 --- a/yapapi/storage/__init__.py +++ b/yapapi/storage/__init__.py @@ -12,6 +12,7 @@ import aiohttp _BUF_SIZE = 40960 +DOWNLOAD_BYTES_LIMIT_DEFAULT = 1*1024*1024 AsyncReader = Union[asyncio.streams.StreamReader, aiohttp.streams.StreamReader] @@ -50,6 +51,20 @@ def upload_url(self) -> str: async def download_stream(self) -> Content: raise NotImplementedError + async def download_bytes(self, limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT): + output = b"" + content = await self.download_stream() + + async for chunk in content.stream: + limit_remaining = limit - len(output) + if limit_remaining > len(chunk): + output += chunk + else: + output += chunk[:limit_remaining] + break + + return output + async def download_file(self, destination_file: PathLike): content = await self.download_stream() with open(destination_file, "wb") as f: From f8eeedae493b4db996d32bde5236d0383cf41574 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 31 Mar 2021 12:47:56 +0200 Subject: [PATCH 15/85] `WorkContext.download_bytes` and `WorkContext.download_json` methods --- yapapi/executor/ctx.py | 102 +++++++++++++++++++++++++++++++++---- yapapi/storage/__init__.py | 15 ++++++ 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index 29b378c2c..42bf0fe91 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -3,12 +3,13 @@ import json from dataclasses import dataclass from datetime import timedelta +from functools import partial from pathlib import Path -from typing import Callable, Iterable, Optional, Dict, List, Tuple, Union +from typing import Callable, Iterable, Optional, Dict, List, Tuple, Union, Any from .events import DownloadStarted, DownloadFinished from ..props import NodeInfo -from ..storage import StorageProvider, Source, Destination +from ..storage import StorageProvider, Source, Destination, DOWNLOAD_BYTES_LIMIT_DEFAULT class CommandContainer: @@ -130,40 +131,99 @@ def register(self, commands: CommandContainer): StorageEvent = Union[DownloadStarted, DownloadFinished] -class _RecvFile(Work): +class _ReceiveContent(Work): def __init__( self, storage: StorageProvider, src_path: str, - dst_path: str, emitter: Optional[Callable[[StorageEvent], None]] = None, ): self._storage = storage - self._dst_path = Path(dst_path) self._src_path: str = src_path self._idx: Optional[int] = None self._dst_slot: Optional[Destination] = None self._emitter: Optional[Callable[[StorageEvent], None]] = emitter + self._dst_path: Optional[str] = None async def prepare(self): self._dst_slot = await self._storage.new_destination(destination_file=self._dst_path) def register(self, commands: CommandContainer): - assert self._dst_slot, "_RecvFile command creation without prepare" + assert self._dst_slot, "_ReceiveFile command creation without prepare" self._idx = commands.transfer( _from=f"container:{self._src_path}", to=self._dst_slot.upload_url ) - async def post(self) -> None: - assert self._dst_slot, "_RecvFile post without prepare" + def _emit_download_start(self): + assert self._dst_slot, f"{self.__class__} post without prepare" if self._emitter: self._emitter(DownloadStarted(path=self._src_path)) - await self._dst_slot.download_file(self._dst_path) + + def _emit_download_end(self): if self._emitter: self._emitter(DownloadFinished(path=str(self._dst_path))) +class _ReceiveFile(_ReceiveContent): + def __init__( + self, + storage: StorageProvider, + src_path: str, + dst_path: str, + emitter: Optional[Callable[[StorageEvent], None]] = None, + ): + super().__init__(storage, src_path, emitter) + self._dst_path = Path(dst_path) + + async def post(self) -> None: + self._emit_download_start() + await self._dst_slot.download_file(self._dst_path) + self._emit_download_end() + + +class _ReceiveBytes(_ReceiveContent): + def __init__( + self, + storage: StorageProvider, + src_path: str, + on_download: Callable[[bytes], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + emitter: Optional[Callable[[StorageEvent], None]] = None, + ): + super().__init__(storage, src_path, emitter) + self._on_download = on_download + self._limit = limit + + async def post(self) -> None: + self._emit_download_start() + output = await self._dst_slot.download_bytes(limit=self._limit) + self._emit_download_end() + self._on_download(output) + + +class _ReceiveJson(_ReceiveBytes): + def __init__( + self, + storage: StorageProvider, + src_path: str, + on_download: Callable[[Any], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + emitter: Optional[Callable[[StorageEvent], None]] = None, + ): + super().__init__( + storage, + src_path, + partial(self.__on_json_download, on_download), + limit, + emitter + ) + + @staticmethod + def __on_json_download(on_download: Callable[[bytes], None], content: bytes): + on_download(json.loads(content)) + + class _Steps(Work): def __init__(self, *steps: Work, timeout: Optional[timedelta] = None): self._steps: Tuple[Work, ...] = steps @@ -280,7 +340,29 @@ def download_file(self, src_path: str, dst_path: str): :return: None """ self.__prepare() - self._pending_steps.append(_RecvFile(self._storage, src_path, dst_path, self._emitter)) + self._pending_steps.append(_ReceiveFile(self._storage, src_path, dst_path, self._emitter)) + + def download_bytes( + self, + src_path: str, + on_download: Callable[[bytes], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + ): + self.__prepare() + self._pending_steps.append( + _ReceiveBytes(self._storage, src_path, on_download, limit, self._emitter) + ) + + def download_json( + self, + src_path: str, + on_download: Callable[[Any], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + ): + self.__prepare() + self._pending_steps.append( + _ReceiveJson(self._storage, src_path, on_download, limit, self._emitter) + ) def commit(self, timeout: Optional[timedelta] = None) -> Work: """Creates sequence of commands to be sent to provider. diff --git a/yapapi/storage/__init__.py b/yapapi/storage/__init__.py index 8b03a6499..1205cb756 100644 --- a/yapapi/storage/__init__.py +++ b/yapapi/storage/__init__.py @@ -12,6 +12,7 @@ import aiohttp _BUF_SIZE = 40960 +DOWNLOAD_BYTES_LIMIT_DEFAULT = 1*1024*1024 AsyncReader = Union[asyncio.streams.StreamReader, aiohttp.streams.StreamReader] @@ -50,6 +51,20 @@ def upload_url(self) -> str: async def download_stream(self) -> Content: raise NotImplementedError + async def download_bytes(self, limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT): + output = b"" + content = await self.download_stream() + + async for chunk in content.stream: + limit_remaining = limit - len(output) + if limit_remaining > len(chunk): + output += chunk + else: + output += chunk[:limit_remaining] + break + + return output + async def download_file(self, destination_file: PathLike): content = await self.download_stream() with open(destination_file, "wb") as f: From 9854fc4429062f3eeede92df25a9cdcae23438c0 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Fri, 2 Apr 2021 09:19:44 +0200 Subject: [PATCH 16/85] refine the simple_service.py --- examples/service/simple_service.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py index 89a4ffb59..2f3a99066 100644 --- a/examples/service/simple_service.py +++ b/examples/service/simple_service.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 import asyncio from datetime import datetime, timedelta +import itertools import json import pathlib import random import string import sys -import tempfile -import time from yapapi import ( @@ -54,13 +53,16 @@ async def service(ctx: WorkContext, tasks): plots_to_download = [] + # have a look at asyncio docs and figure out whether to leave the callback or replace it with something + # more asyncio-ic + def on_plot(out: bytes): fname = json.loads(out.strip()) print(f"{TEXT_COLOR_CYAN}plot: {fname}{TEXT_COLOR_DEFAULT}") plots_to_download.append(fname) try: - while True: + async for task in tasks: await asyncio.sleep(10) ctx.run("/bin/sh", "/golem/in/get_stats.sh") @@ -74,13 +76,12 @@ def on_plot(out: bytes): ctx.download_file(plot, pathlib.Path(__file__).resolve().parent / test_filename) yield ctx.commit() + task.accept_result() + except KeyboardInterrupt: ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") yield ctx.commit() - - finally: - async for task in tasks: - task.accept_result() + raise # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) # TODO: make this dynamic, e.g. depending on the size of files to transfer @@ -90,7 +91,7 @@ def on_plot(out: bytes): # reach the providers. min_timeout, max_timeout = 6, 30 - timeout = timedelta(minutes=min_timeout) + timeout = timedelta(minutes=29) # By passing `event_consumer=log_summary()` we enable summary logging. # See the documentation of the `yapapi.log` module on how to set @@ -98,7 +99,7 @@ def on_plot(out: bytes): async with Executor( package=package, max_workers=3, - budget=10.0, + budget=1.0, timeout=timeout, subnet_tag=subnet_tag, driver=driver, @@ -113,20 +114,18 @@ def on_plot(out: bytes): f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" ) - num_tasks = 0 start_time = datetime.now() - async for task in executor.submit(service, [Task(data=None)]): - num_tasks += 1 + async for task in executor.submit(service, (Task(data=n) for n in itertools.count(1))): print( f"{TEXT_COLOR_CYAN}" - f"Task computed: {task}, result: {task.result}, time: {task.running_time}" + f"Script executed: {task}, result: {task.result}, time: {task.running_time}" f"{TEXT_COLOR_DEFAULT}" ) print( f"{TEXT_COLOR_CYAN}" - f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}" + f"Service finished, total time: {datetime.now() - start_time}" f"{TEXT_COLOR_DEFAULT}" ) From 8f1065e81688527d8cdd8469b7844d03dce47148 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 6 Apr 2021 15:02:40 +0200 Subject: [PATCH 17/85] + tests, comments --- tests/executor/test_ctx.py | 42 +++++++++++++++++++++++++++++++++++ tests/storage/test_storage.py | 31 ++++++++++++++++++++++++++ yapapi/executor/ctx.py | 22 +++++++++++++----- 3 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 tests/storage/test_storage.py diff --git a/tests/executor/test_ctx.py b/tests/executor/test_ctx.py index 4485715f2..2bf3aae44 100644 --- a/tests/executor/test_ctx.py +++ b/tests/executor/test_ctx.py @@ -49,6 +49,12 @@ def _assert_dst_path(steps, dst_path): steps.register(c) assert c.commands().pop()["transfer"]["to"] == f"container:{dst_path}" + @staticmethod + def _assert_src_path(steps, src_path): + c = CommandContainer() + steps.register(c) + assert c.commands().pop()["transfer"]["from"] == f"container:{src_path}" + @pytest.mark.asyncio @pytest.mark.skipif(sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+") async def test_send_json(self): @@ -76,3 +82,39 @@ async def test_send_bytes(self): await steps.prepare() storage.upload_bytes.assert_called_with(data) self._assert_dst_path(steps, dst_path) + + @pytest.mark.asyncio + @pytest.mark.skipif(sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+") + async def test_download_bytes(self): + expected = b'some byte string' + + def on_download(data: bytes): + assert data == expected + + storage = mock.AsyncMock() + storage.new_destination.return_value.download_bytes.return_value = expected + src_path = "/test/path" + ctx = self._get_work_context(storage) + ctx.download_bytes(src_path, on_download) + steps = ctx.commit() + await steps.prepare() + await steps.post() + self._assert_src_path(steps, src_path) + + @pytest.mark.asyncio + @pytest.mark.skipif(sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+") + async def test_download_json(self): + expected = {'key': 'val'} + + def on_download(data: bytes): + assert data == expected + + storage = mock.AsyncMock() + storage.new_destination.return_value.download_bytes.return_value = json.dumps(expected).encode("utf-8") + src_path = "/test/path" + ctx = self._get_work_context(storage) + ctx.download_json(src_path, on_download) + steps = ctx.commit() + await steps.prepare() + await steps.post() + self._assert_src_path(steps, src_path) diff --git a/tests/storage/test_storage.py b/tests/storage/test_storage.py new file mode 100644 index 000000000..33329c88c --- /dev/null +++ b/tests/storage/test_storage.py @@ -0,0 +1,31 @@ +import pytest + +from yapapi.storage import Destination, Content + + +class _TestDestination(Destination): + def __init__(self, test_data: bytes): + self._test_data = test_data + + def upload_url(self): + return "" + + async def download_stream(self) -> Content: + async def data(): + for c in [self._test_data]: + yield self._test_data + + return Content(len(self._test_data), data()) + + +@pytest.mark.asyncio +async def test_download_bytes(): + expected = b"some test data" + destination = _TestDestination(expected) + assert await destination.download_bytes() == expected + + +@pytest.mark.asyncio +async def test_download_bytes_with_limit(): + destination = _TestDestination(b"some test data") + assert await destination.download_bytes(limit=4) == b"some" diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index 42bf0fe91..505a5d440 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -224,7 +224,7 @@ def __on_json_download(on_download: Callable[[bytes], None], content: bytes): on_download(json.loads(content)) -class _Steps(Work): +class Steps(Work): def __init__(self, *steps: Work, timeout: Optional[timedelta] = None): self._steps: Tuple[Work, ...] = steps self._timeout: Optional[timedelta] = timeout @@ -348,6 +348,12 @@ def download_bytes( on_download: Callable[[bytes], None], limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, ): + """Schedule downloading a remote file as bytes + :param src_path: remote (provider) path + :param on_download: the callable to run on the received data + :param limit: the maximum length of the expected byte string + :return None + """ self.__prepare() self._pending_steps.append( _ReceiveBytes(self._storage, src_path, on_download, limit, self._emitter) @@ -359,20 +365,26 @@ def download_json( on_download: Callable[[Any], None], limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, ): + """Schedule downloading a remote file as JSON + :param src_path: remote (provider) path + :param on_download: the callable to run on the received JSON data + :param limit: the maximum length of the expected remote file + :return None + """ self.__prepare() self._pending_steps.append( _ReceiveJson(self._storage, src_path, on_download, limit, self._emitter) ) def commit(self, timeout: Optional[timedelta] = None) -> Work: - """Creates sequence of commands to be sent to provider. + """Creates a sequence of commands to be sent to provider. - :return: Work object (the latter contains - sequence commands added before calling this method) + :return: Work object containing the sequence of commands + scheduled within this work context before calling this method) """ steps = self._pending_steps self._pending_steps = [] - return _Steps(*steps, timeout=timeout) + return Steps(*steps, timeout=timeout) class CaptureMode(enum.Enum): From a55f667dcd50c4f00eb70c1d070999c03074f15b Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 6 Apr 2021 15:05:57 +0200 Subject: [PATCH 18/85] black --- tests/executor/test_ctx.py | 8 +++++--- yapapi/executor/ctx.py | 22 +++++++++------------- yapapi/storage/__init__.py | 2 +- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tests/executor/test_ctx.py b/tests/executor/test_ctx.py index 2bf3aae44..ba8760d2f 100644 --- a/tests/executor/test_ctx.py +++ b/tests/executor/test_ctx.py @@ -86,7 +86,7 @@ async def test_send_bytes(self): @pytest.mark.asyncio @pytest.mark.skipif(sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+") async def test_download_bytes(self): - expected = b'some byte string' + expected = b"some byte string" def on_download(data: bytes): assert data == expected @@ -104,13 +104,15 @@ def on_download(data: bytes): @pytest.mark.asyncio @pytest.mark.skipif(sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+") async def test_download_json(self): - expected = {'key': 'val'} + expected = {"key": "val"} def on_download(data: bytes): assert data == expected storage = mock.AsyncMock() - storage.new_destination.return_value.download_bytes.return_value = json.dumps(expected).encode("utf-8") + storage.new_destination.return_value.download_bytes.return_value = json.dumps( + expected + ).encode("utf-8") src_path = "/test/path" ctx = self._get_work_context(storage) ctx.download_json(src_path, on_download) diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index 505a5d440..af0513940 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -212,11 +212,7 @@ def __init__( emitter: Optional[Callable[[StorageEvent], None]] = None, ): super().__init__( - storage, - src_path, - partial(self.__on_json_download, on_download), - limit, - emitter + storage, src_path, partial(self.__on_json_download, on_download), limit, emitter ) @staticmethod @@ -343,10 +339,10 @@ def download_file(self, src_path: str, dst_path: str): self._pending_steps.append(_ReceiveFile(self._storage, src_path, dst_path, self._emitter)) def download_bytes( - self, - src_path: str, - on_download: Callable[[bytes], None], - limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + self, + src_path: str, + on_download: Callable[[bytes], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, ): """Schedule downloading a remote file as bytes :param src_path: remote (provider) path @@ -360,10 +356,10 @@ def download_bytes( ) def download_json( - self, - src_path: str, - on_download: Callable[[Any], None], - limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, + self, + src_path: str, + on_download: Callable[[Any], None], + limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, ): """Schedule downloading a remote file as JSON :param src_path: remote (provider) path diff --git a/yapapi/storage/__init__.py b/yapapi/storage/__init__.py index 1205cb756..1335dd39b 100644 --- a/yapapi/storage/__init__.py +++ b/yapapi/storage/__init__.py @@ -12,7 +12,7 @@ import aiohttp _BUF_SIZE = 40960 -DOWNLOAD_BYTES_LIMIT_DEFAULT = 1*1024*1024 +DOWNLOAD_BYTES_LIMIT_DEFAULT = 1 * 1024 * 1024 AsyncReader = Union[asyncio.streams.StreamReader, aiohttp.streams.StreamReader] From 338ecebd354f13aa13587c5db37d272e7e147e92 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 7 Apr 2021 11:23:18 +0200 Subject: [PATCH 19/85] make the `on_download` callback async --- tests/executor/test_ctx.py | 22 ++++++++++++++-------- yapapi/executor/ctx.py | 12 ++++++------ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/executor/test_ctx.py b/tests/executor/test_ctx.py index ba8760d2f..ef2d08a08 100644 --- a/tests/executor/test_ctx.py +++ b/tests/executor/test_ctx.py @@ -1,4 +1,5 @@ import factory +from functools import partial import json import pytest import sys @@ -39,6 +40,10 @@ def test_command_container(): class TestWorkContext: + @pytest.fixture(autouse=True) + def setUp(self): + self._on_download_executed = False + @staticmethod def _get_work_context(storage): return WorkContext(factory.Faker("pystr"), node_info=NodeInfoFactory(), storage=storage) @@ -55,6 +60,10 @@ def _assert_src_path(steps, src_path): steps.register(c) assert c.commands().pop()["transfer"]["from"] == f"container:{src_path}" + async def _on_download(self, expected, data: bytes): + assert data == expected + self._on_download_executed = True + @pytest.mark.asyncio @pytest.mark.skipif(sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+") async def test_send_json(self): @@ -88,35 +97,32 @@ async def test_send_bytes(self): async def test_download_bytes(self): expected = b"some byte string" - def on_download(data: bytes): - assert data == expected - storage = mock.AsyncMock() storage.new_destination.return_value.download_bytes.return_value = expected + src_path = "/test/path" ctx = self._get_work_context(storage) - ctx.download_bytes(src_path, on_download) + ctx.download_bytes(src_path, partial(self._on_download, expected)) steps = ctx.commit() await steps.prepare() await steps.post() self._assert_src_path(steps, src_path) + assert self._on_download_executed @pytest.mark.asyncio @pytest.mark.skipif(sys.version_info < (3, 8), reason="AsyncMock requires python 3.8+") async def test_download_json(self): expected = {"key": "val"} - def on_download(data: bytes): - assert data == expected - storage = mock.AsyncMock() storage.new_destination.return_value.download_bytes.return_value = json.dumps( expected ).encode("utf-8") src_path = "/test/path" ctx = self._get_work_context(storage) - ctx.download_json(src_path, on_download) + ctx.download_json(src_path, partial(self._on_download, expected)) steps = ctx.commit() await steps.prepare() await steps.post() self._assert_src_path(steps, src_path) + assert self._on_download_executed diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index af0513940..b8e2365cc 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -5,7 +5,7 @@ from datetime import timedelta from functools import partial from pathlib import Path -from typing import Callable, Iterable, Optional, Dict, List, Tuple, Union, Any +from typing import Callable, Iterable, Optional, Dict, List, Tuple, Union, Any, Awaitable from .events import DownloadStarted, DownloadFinished from ..props import NodeInfo @@ -187,7 +187,7 @@ def __init__( self, storage: StorageProvider, src_path: str, - on_download: Callable[[bytes], None], + on_download: Callable[[bytes], Awaitable], limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, emitter: Optional[Callable[[StorageEvent], None]] = None, ): @@ -199,7 +199,7 @@ async def post(self) -> None: self._emit_download_start() output = await self._dst_slot.download_bytes(limit=self._limit) self._emit_download_end() - self._on_download(output) + await self._on_download(output) class _ReceiveJson(_ReceiveBytes): @@ -207,7 +207,7 @@ def __init__( self, storage: StorageProvider, src_path: str, - on_download: Callable[[Any], None], + on_download: Callable[[Any], Awaitable], limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, emitter: Optional[Callable[[StorageEvent], None]] = None, ): @@ -216,8 +216,8 @@ def __init__( ) @staticmethod - def __on_json_download(on_download: Callable[[bytes], None], content: bytes): - on_download(json.loads(content)) + async def __on_json_download(on_download: Callable[[bytes], Awaitable], content: bytes): + await on_download(json.loads(content)) class Steps(Work): From 6d171a3c6e1e51970c2e45fd9d9fb02c8d91b33a Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 7 Apr 2021 11:30:17 +0200 Subject: [PATCH 20/85] fix assert message --- yapapi/executor/ctx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index b8e2365cc..9cc61f038 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -149,7 +149,7 @@ async def prepare(self): self._dst_slot = await self._storage.new_destination(destination_file=self._dst_path) def register(self, commands: CommandContainer): - assert self._dst_slot, "_ReceiveFile command creation without prepare" + assert self._dst_slot, f"{self.__class__} command creation without prepare" self._idx = commands.transfer( _from=f"container:{self._src_path}", to=self._dst_slot.upload_url From d6e5a357d6d1fcd13fd34d2a84be7ab40cc4f4fc Mon Sep 17 00:00:00 2001 From: Filip Date: Wed, 7 Apr 2021 11:54:00 +0200 Subject: [PATCH 21/85] Set maximum client-side timeout to 5 seconds, continue on client-side and server-side timeouts. --- yapapi/rest/activity.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index ac463dfc1..ef0173b5d 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -167,11 +167,13 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: raise BatchTimeoutError() try: results: List[yaa.ExeScriptCommandResult] = await self._api.get_exec_batch_results( - self._activity_id, self._batch_id, _request_timeout=timeout + self._activity_id, self._batch_id, _request_timeout=min(timeout, 5) ) + except asyncio.TimeoutError: + continue except ApiException as err: if err.status == 408: - raise BatchTimeoutError() + continue raise any_new: bool = False results = results[last_idx:] From 10de416b9235917d1add4344bc9b85434593d494 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 7 Apr 2021 11:56:03 +0200 Subject: [PATCH 22/85] mypy --- yapapi/executor/ctx.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index 9cc61f038..777210957 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -3,6 +3,7 @@ import json from dataclasses import dataclass from datetime import timedelta +from os import PathLike from functools import partial from pathlib import Path from typing import Callable, Iterable, Optional, Dict, List, Tuple, Union, Any, Awaitable @@ -143,7 +144,7 @@ def __init__( self._idx: Optional[int] = None self._dst_slot: Optional[Destination] = None self._emitter: Optional[Callable[[StorageEvent], None]] = emitter - self._dst_path: Optional[str] = None + self._dst_path: Optional[PathLike] = None async def prepare(self): self._dst_slot = await self._storage.new_destination(destination_file=self._dst_path) @@ -178,6 +179,9 @@ def __init__( async def post(self) -> None: self._emit_download_start() + assert self._dst_path + assert self._dst_slot + await self._dst_slot.download_file(self._dst_path) self._emit_download_end() @@ -197,6 +201,8 @@ def __init__( async def post(self) -> None: self._emit_download_start() + assert self._dst_slot + output = await self._dst_slot.download_bytes(limit=self._limit) self._emit_download_end() await self._on_download(output) @@ -341,7 +347,7 @@ def download_file(self, src_path: str, dst_path: str): def download_bytes( self, src_path: str, - on_download: Callable[[bytes], None], + on_download: Callable[[bytes], Awaitable], limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, ): """Schedule downloading a remote file as bytes @@ -358,7 +364,7 @@ def download_bytes( def download_json( self, src_path: str, - on_download: Callable[[Any], None], + on_download: Callable[[Any], Awaitable], limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT, ): """Schedule downloading a remote file as JSON From 474565ba9d3b8b656318e7faca59f070009774cb Mon Sep 17 00:00:00 2001 From: Filip Date: Wed, 7 Apr 2021 12:00:00 +0200 Subject: [PATCH 23/85] Decrease timeout --- yapapi/rest/activity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index ef0173b5d..9074b200e 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -196,7 +196,7 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: if result.is_batch_finished: break if not any_new: - delay = min(10, max(0, self.seconds_left())) + delay = min(3, max(0, self.seconds_left())) await asyncio.sleep(delay) From 9afefd5b06d9c9f663077a739c40229b84bf8dfe Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 7 Apr 2021 13:17:58 +0200 Subject: [PATCH 24/85] make callbacks async in `simple_service.py` --- examples/service/simple_service.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py index 2f3a99066..ce27aaad3 100644 --- a/examples/service/simple_service.py +++ b/examples/service/simple_service.py @@ -56,17 +56,21 @@ async def service(ctx: WorkContext, tasks): # have a look at asyncio docs and figure out whether to leave the callback or replace it with something # more asyncio-ic - def on_plot(out: bytes): + async def on_plot(out: bytes): + nonlocal plots_to_download fname = json.loads(out.strip()) print(f"{TEXT_COLOR_CYAN}plot: {fname}{TEXT_COLOR_DEFAULT}") plots_to_download.append(fname) + async def on_stats(out: bytes): + print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}") + try: async for task in tasks: await asyncio.sleep(10) ctx.run("/bin/sh", "/golem/in/get_stats.sh") - ctx.download_bytes(STATS_PATH, lambda out: print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}")) + ctx.download_bytes(STATS_PATH, on_stats) ctx.run("/bin/sh", "/golem/in/get_plot.sh") ctx.download_bytes(PLOT_INFO_PATH, on_plot) yield ctx.commit() From 8f4690712a0c3ac8f386a0daf40d2ee5c798b507 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 8 Apr 2021 15:24:26 +0200 Subject: [PATCH 25/85] remove the non-working http server example --- examples/service/http_server.py | 148 ------------------ .../http_server/http_server.Dockerfile | 5 - .../service/http_server/run-http-server.sh | 1 - examples/service/http_test.py | 6 - examples/service/index.html | 1 - 5 files changed, 161 deletions(-) delete mode 100644 examples/service/http_server.py delete mode 100644 examples/service/http_server/http_server.Dockerfile delete mode 100644 examples/service/http_server/run-http-server.sh delete mode 100644 examples/service/http_test.py delete mode 100644 examples/service/index.html diff --git a/examples/service/http_server.py b/examples/service/http_server.py deleted file mode 100644 index 1e331223f..000000000 --- a/examples/service/http_server.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python3 -import asyncio -from datetime import datetime, timedelta -import pathlib -import sys - -from yapapi import ( - Executor, - NoPaymentAccountError, - Task, - __version__ as yapapi_version, - WorkContext, - windows_event_loop_fix, -) -from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa -from yapapi.package import vm -from yapapi.rest.activity import BatchTimeoutError - -examples_dir = pathlib.Path(__file__).resolve().parent.parent -sys.path.append(str(examples_dir)) - -from utils import ( - build_parser, - TEXT_COLOR_CYAN, - TEXT_COLOR_DEFAULT, - TEXT_COLOR_RED, - TEXT_COLOR_YELLOW, -) - - -async def main(subnet_tag, driver=None, network=None): - package = await vm.repo( - image_hash="54169fddccc723285789278e28899edab2bb3e73514aeae20f9fc3a2", - min_mem_gib=0.5, - min_storage_gib=2.0, - ) - - async def service(ctx: WorkContext, tasks): - script_dir = pathlib.Path(__file__).resolve().parent - ctx.send_file(script_dir / "index.html", "/golem/html/index.html") - ctx.send_file(script_dir / "http_test.py", "/golem/test/http_test.py") - - ctx.run("sh", "/golem/run/run-http-server.sh") - ctx.run("python", "/golem/out/http_test.py") - ctx.download_file("/golem/out/test", "test") - yield ctx.commit() - - async for task in tasks: - task.accept_result() - - # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) - # TODO: make this dynamic, e.g. depending on the size of files to transfer - init_overhead = 3 - # Providers will not accept work if the timeout is outside of the [5 min, 30min] range. - # We increase the lower bound to 6 min to account for the time needed for our demand to - # reach the providers. - min_timeout, max_timeout = 6, 30 - - timeout = timedelta(minutes=min_timeout) - - # By passing `event_consumer=log_summary()` we enable summary logging. - # See the documentation of the `yapapi.log` module on how to set - # the level of detail and format of the logged information. - async with Executor( - package=package, - max_workers=3, - budget=10.0, - timeout=timeout, - subnet_tag=subnet_tag, - driver=driver, - network=network, - event_consumer=log_summary(log_event_repr), - ) as executor: - - print( - f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" - f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " - f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " - f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" - ) - - num_tasks = 0 - start_time = datetime.now() - - async for task in executor.submit(service, [Task(data=None)]): - num_tasks += 1 - print( - f"{TEXT_COLOR_CYAN}" - f"Task computed: {task}, result: {task.result}, time: {task.running_time}" - f"{TEXT_COLOR_DEFAULT}" - ) - - print( - f"{TEXT_COLOR_CYAN}" - f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}" - f"{TEXT_COLOR_DEFAULT}" - ) - - -if __name__ == "__main__": - parser = build_parser("Test http") - parser.set_defaults(log_file="http-yapapi.log") - args = parser.parse_args() - - # This is only required when running on Windows with Python prior to 3.8: - windows_event_loop_fix() - - enable_default_logger( - log_file=args.log_file, - debug_activity_api=True, - debug_market_api=True, - debug_payment_api=True, - ) - - loop = asyncio.get_event_loop() - task = loop.create_task( - main(subnet_tag=args.subnet_tag, driver=args.driver, network=args.network) - ) - - try: - loop.run_until_complete(task) - except NoPaymentAccountError as e: - handbook_url = ( - "https://handbook.golem.network/requestor-tutorials/" - "flash-tutorial-of-requestor-development" - ) - print( - f"{TEXT_COLOR_RED}" - f"No payment account initialized for driver `{e.required_driver}` " - f"and network `{e.required_network}`.\n\n" - f"See {handbook_url} on how to initialize payment accounts for a requestor node." - f"{TEXT_COLOR_DEFAULT}" - ) - except KeyboardInterrupt: - print( - f"{TEXT_COLOR_YELLOW}" - "Shutting down gracefully, please wait a short while " - "or press Ctrl+C to exit immediately..." - f"{TEXT_COLOR_DEFAULT}" - ) - task.cancel() - try: - loop.run_until_complete(task) - print( - f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" - ) - except (asyncio.CancelledError, KeyboardInterrupt): - pass diff --git a/examples/service/http_server/http_server.Dockerfile b/examples/service/http_server/http_server.Dockerfile deleted file mode 100644 index dfa42dfdb..000000000 --- a/examples/service/http_server/http_server.Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.8-slim -VOLUME /golem/html /golem/out /golem/test -COPY run-http-server.sh /golem/run -EXPOSE 80 -ENTRYPOINT ["sh", "/golem/run/run-http-server.sh"] diff --git a/examples/service/http_server/run-http-server.sh b/examples/service/http_server/run-http-server.sh deleted file mode 100644 index d9e5e68d0..000000000 --- a/examples/service/http_server/run-http-server.sh +++ /dev/null @@ -1 +0,0 @@ -python3 -m http.serve --directory /golem/html 80 diff --git a/examples/service/http_test.py b/examples/service/http_test.py deleted file mode 100644 index 13fe92659..000000000 --- a/examples/service/http_test.py +++ /dev/null @@ -1,6 +0,0 @@ -from http.client import HTTPConnection - -h = HTTPConnection("127.0.0.1") -h.request("GET", "/") -with open("/golem/out/test", "w") as f: - f.write(h.getresponse().read()) diff --git a/examples/service/index.html b/examples/service/index.html deleted file mode 100644 index 47ff8f6dd..000000000 --- a/examples/service/index.html +++ /dev/null @@ -1 +0,0 @@ -Hello from a Golem Service. From 1ff0c4b1fa8c1e1c0a053e154925b95331d400d4 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 8 Apr 2021 16:25:57 +0200 Subject: [PATCH 26/85] some comments --- examples/service/simple_service.py | 3 +++ examples/service/simple_service/simple_service.py | 6 ++++++ examples/service/simple_service/simulate_observations.py | 9 +++++++++ .../service/simple_service/simulate_observations_ctl.py | 3 +++ 4 files changed, 21 insertions(+) diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py index ce27aaad3..221f501cd 100644 --- a/examples/service/simple_service.py +++ b/examples/service/simple_service.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +""" +the requestor agent controlling and interacting with the "simple service" +""" import asyncio from datetime import datetime, timedelta import itertools diff --git a/examples/service/simple_service/simple_service.py b/examples/service/simple_service/simple_service.py index c6c40dc04..3011896ce 100644 --- a/examples/service/simple_service/simple_service.py +++ b/examples/service/simple_service/simple_service.py @@ -1,4 +1,10 @@ #!/usr/local/bin/python +""" +a very basic "stub" that exposes a few commands of an imagined, very simple CLI-based +service that is able to accumulate some linear, time-based values and present it stats +(characteristics of the statistical distribution of the data collected so far) or provide +distribution and time-series plots of the collected data +""" import argparse from datetime import datetime import enum diff --git a/examples/service/simple_service/simulate_observations.py b/examples/service/simple_service/simulate_observations.py index b8c598abb..ad2324ca5 100644 --- a/examples/service/simple_service/simulate_observations.py +++ b/examples/service/simple_service/simulate_observations.py @@ -1,4 +1,12 @@ #!/usr/local/bin/python +""" +the "hello world" service here just adds randomized numbers with normal distribution + +in a real-world example, this could be e.g. a thermometer connected to the provider's +machine providing its inputs into the database or some other piece of information +from some external source that changes over time and which can be expressed as a +singular value +""" import os from pathlib import Path import random @@ -9,6 +17,7 @@ SERVICE_PATH = Path(__file__).absolute().parent / "simple_service.py" + while True: v = random.normalvariate(MU, SIGMA) os.system(f"{SERVICE_PATH} --add {v}") diff --git a/examples/service/simple_service/simulate_observations_ctl.py b/examples/service/simple_service/simulate_observations_ctl.py index 6a083dfe2..bcaffb5c9 100644 --- a/examples/service/simple_service/simulate_observations_ctl.py +++ b/examples/service/simple_service/simulate_observations_ctl.py @@ -1,4 +1,7 @@ #!/usr/local/bin/python +""" +a helper, control script that starts and stops our example `simulate_observations` service +""" import argparse import os import subprocess From 446738822eb2f36a9bb4965082643d732f41b223 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 13 Apr 2021 16:39:47 +0200 Subject: [PATCH 27/85] comment to the exception handler --- examples/service/simple_service.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py index 221f501cd..43116d226 100644 --- a/examples/service/simple_service.py +++ b/examples/service/simple_service.py @@ -49,8 +49,12 @@ async def service(ctx: WorkContext, tasks): SIMPLE_SERVICE = "/golem/run/simple_service.py" ctx.run("/golem/run/simulate_observations_ctl.py", "--start") - ctx.send_bytes("/golem/in/get_stats.sh", f"{SIMPLE_SERVICE} --stats > {STATS_PATH}".encode()) - ctx.send_bytes("/golem/in/get_plot.sh", f"{SIMPLE_SERVICE} --plot dist > {PLOT_INFO_PATH}".encode()) + ctx.send_bytes( + "/golem/in/get_stats.sh", f"{SIMPLE_SERVICE} --stats > {STATS_PATH}".encode() + ) + ctx.send_bytes( + "/golem/in/get_plot.sh", f"{SIMPLE_SERVICE} --plot dist > {PLOT_INFO_PATH}".encode() + ) yield ctx.commit() @@ -79,13 +83,17 @@ async def on_stats(out: bytes): yield ctx.commit() for plot in plots_to_download: - test_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" + test_filename = ( + "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" + ) ctx.download_file(plot, pathlib.Path(__file__).resolve().parent / test_filename) yield ctx.commit() task.accept_result() - except KeyboardInterrupt: + except (KeyboardInterrupt, asyncio.CancelledError): + # with the current implementation it won't work correctly + # because at this stage, the Executor is shutting down anyway ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") yield ctx.commit() raise From 196a60b102e0b7519a11a8924dafa2b60aa3beec Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 19 Apr 2021 19:20:48 +0200 Subject: [PATCH 28/85] add yapapi.payload deprecate yapapi.package + add "Deprecated" package to enable controlled deprecation of features --- README.md | 2 +- examples/blender/blender.py | 2 +- examples/yacat/yacat.py | 2 +- pyproject.toml | 1 + .../test_agreement_termination/requestor.py | 2 +- tests/{package => payload}/test_vm.py | 8 +- yapapi/_cli/market.py | 2 +- yapapi/package/__init__.py | 22 ++-- yapapi/package/vm.py | 83 ++------------ yapapi/payload/__init__.py | 11 ++ yapapi/payload/package.py | 17 +++ yapapi/payload/vm.py | 104 ++++++++++++++++++ yapapi/props/inf.py | 17 ++- 13 files changed, 168 insertions(+), 105 deletions(-) rename tests/{package => payload}/test_vm.py (72%) create mode 100644 yapapi/payload/__init__.py create mode 100644 yapapi/payload/package.py create mode 100644 yapapi/payload/vm.py diff --git a/README.md b/README.md index b54547e0b..2e8f50fe9 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ from yapapi import ( windows_event_loop_fix, ) from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa -from yapapi.package import vm +from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError diff --git a/examples/blender/blender.py b/examples/blender/blender.py index ca3858ce7..64c9b1c1c 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -13,7 +13,7 @@ windows_event_loop_fix, ) from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa -from yapapi.package import vm +from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError examples_dir = pathlib.Path(__file__).resolve().parent.parent diff --git a/examples/yacat/yacat.py b/examples/yacat/yacat.py index 71b6d97d9..59200b6c9 100644 --- a/examples/yacat/yacat.py +++ b/examples/yacat/yacat.py @@ -6,7 +6,7 @@ from yapapi import Executor, NoPaymentAccountError, Task, WorkContext, windows_event_loop_fix from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa -from yapapi.package import vm +from yapapi.payload import vm examples_dir = pathlib.Path(__file__).resolve().parent.parent sys.path.append(str(examples_dir)) diff --git a/pyproject.toml b/pyproject.toml index cda714912..6333d94c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ colorama = "^0.4.4" # Note that putting `goth` in `poetry.dev-dependencies` instead of `poetry.dependencies` # would not work: see https://github.com/python-poetry/poetry/issues/129. goth = { version = "^0.2.1", optional = true, python = "^3.8.0" } +Deprecated = "^1.2.12" [tool.poetry.extras] cli = ['fire', 'rich'] diff --git a/tests/goth/test_agreement_termination/requestor.py b/tests/goth/test_agreement_termination/requestor.py index b66d29d23..a0e12541a 100755 --- a/tests/goth/test_agreement_termination/requestor.py +++ b/tests/goth/test_agreement_termination/requestor.py @@ -6,7 +6,7 @@ from yapapi import Executor, Task, WorkContext from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa -from yapapi.package import vm +from yapapi.payload import vm async def main(): diff --git a/tests/package/test_vm.py b/tests/payload/test_vm.py similarity index 72% rename from tests/package/test_vm.py rename to tests/payload/test_vm.py index a65b7516e..400047666 100644 --- a/tests/package/test_vm.py +++ b/tests/payload/test_vm.py @@ -1,24 +1,22 @@ from dns.exception import DNSException -import pytest from srvresolver.srv_resolver import SRVRecord # type: ignore from unittest import mock -from yapapi.package import Package, PackageException -from yapapi.package.vm import resolve_repo_srv, _FALLBACK_REPO_URL +from yapapi.payload.vm import resolve_repo_srv, _FALLBACK_REPO_URL _MOCK_HOST = "non.existent.domain" _MOCK_PORT = 9999 @mock.patch( - "yapapi.package.vm.SRVResolver.resolve_random", + "yapapi.payload.vm.SRVResolver.resolve_random", mock.Mock(return_value=SRVRecord(host=_MOCK_HOST, port=_MOCK_PORT, weight=1, priority=1)), ) def test_resolve_srv(): assert resolve_repo_srv("") == f"http://{_MOCK_HOST}:{_MOCK_PORT}" -@mock.patch("yapapi.package.vm.SRVResolver.resolve_random", mock.Mock(side_effect=DNSException())) +@mock.patch("yapapi.payload.vm.SRVResolver.resolve_random", mock.Mock(side_effect=DNSException())) def test_resolve_srv_exception(): # should be: diff --git a/yapapi/_cli/market.py b/yapapi/_cli/market.py index 75047944e..27c942073 100644 --- a/yapapi/_cli/market.py +++ b/yapapi/_cli/market.py @@ -79,7 +79,7 @@ async def vm( min_storage_gib: float = 2.0, timeout=timedelta(minutes=5), ): - from yapapi.package import vm + from yapapi.payload import vm console = Console() now = datetime.now(timezone.utc) diff --git a/yapapi/package/__init__.py b/yapapi/package/__init__.py index 15217ca7e..354028fa4 100644 --- a/yapapi/package/__init__.py +++ b/yapapi/package/__init__.py @@ -1,21 +1,13 @@ import abc +from deprecated import deprecated +from yapapi.payload.package import Package as _Package, PackageException as _PackageException -from yapapi.props.builder import DemandBuilder - - -class PackageException(Exception): - """Exception raised on any problems related to the package repository.""" +@deprecated(version="0.6.0", reason="please use yapapi.payload.package.PackageException") +class PackageException(_PackageException): pass -class Package(abc.ABC): - """Information on task package to be used for running tasks on providers.""" - - @abc.abstractmethod - async def resolve_url(self) -> str: - """Return package URL.""" - - @abc.abstractmethod - async def decorate_demand(self, demand: DemandBuilder): - """Add package information to a Demand.""" +@deprecated(version="0.6.0", reason="please use yapapi.payload.package.Package") +class Package(_Package): + pass diff --git a/yapapi/package/vm.py b/yapapi/package/vm.py index e21580e2e..07039410f 100644 --- a/yapapi/package/vm.py +++ b/yapapi/package/vm.py @@ -1,63 +1,13 @@ -import aiohttp -from dns.exception import DNSException -from dataclasses import dataclass +from deprecated import deprecated import logging -from typing import Optional -from typing_extensions import Final -from srvresolver.srv_resolver import SRVResolver, SRVRecord # type: ignore -from yapapi.package import Package, PackageException -from yapapi.props.builder import DemandBuilder -from yapapi.props.inf import InfVmKeys, RuntimeType, VmRequest, VmPackageFormat - -_DEFAULT_REPO_SRV: Final = "_girepo._tcp.dev.golem.network" -_FALLBACK_REPO_URL: Final = "http://yacn2.dev.golem.network:8000" +from yapapi.package import Package +from yapapi.payload.vm import repo as _repo, resolve_repo_srv as _resolve_repo_srv, _FALLBACK_REPO_URL logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class _VmConstrains: - min_mem_gib: float - min_storage_gib: float - cores: int = 1 - - def __str__(self): - rules = "\n\t".join( - [ - f"({InfVmKeys.mem}>={self.min_mem_gib})", - f"({InfVmKeys.storage}>={self.min_storage_gib})", - # TODO: provider should report cores. - # - # f"({inf.InfVmKeys.cores}>={self.cores})", - f"({InfVmKeys.runtime}={RuntimeType.VM.value})", - ] - ) - return f"(&{rules})" - - -@dataclass -class _VmPackage(Package): - repo_url: str - image_hash: str - constraints: _VmConstrains - - async def resolve_url(self) -> str: - async with aiohttp.ClientSession() as client: - resp = await client.get(f"{self.repo_url}/image.{self.image_hash}.link") - if resp.status != 200: - resp.raise_for_status() - - image_url = await resp.text() - image_hash = self.image_hash - return f"hash:sha3:{image_hash}:{image_url}" - - async def decorate_demand(self, demand: DemandBuilder): - image_url = await self.resolve_url() - demand.ensure(str(self.constraints)) - demand.add(VmRequest(package_url=image_url, package_format=VmPackageFormat.GVMKIT_SQUASH)) - - +@deprecated(version="0.6.0", reason="moved to yapapi.payload.vm.repo") async def repo( *, image_hash: str, min_mem_gib: float = 0.5, min_storage_gib: float = 2.0 ) -> Package: @@ -69,14 +19,14 @@ async def repo( - *min_storage_gib* minimal disk storage to execute tasks. """ - - return _VmPackage( - repo_url=resolve_repo_srv(_DEFAULT_REPO_SRV), + return await _repo( image_hash=image_hash, - constraints=_VmConstrains(min_mem_gib, min_storage_gib), + min_mem_gib=min_mem_gib, + min_storage_gib=min_storage_gib ) +@deprecated(version="0.6.0", reason="moved to yapapi.payload.vm.resolve_repo_srv") def resolve_repo_srv(repo_srv, fallback_url=_FALLBACK_REPO_URL) -> str: """ Get the url of the package repository based on its SRV record address. @@ -86,19 +36,4 @@ def resolve_repo_srv(repo_srv, fallback_url=_FALLBACK_REPO_URL) -> str: :return: the url of the package repository containing the port :raises: PackageException if no valid service could be reached """ - try: - try: - srv: Optional[SRVRecord] = SRVResolver.resolve_random(repo_srv) - except DNSException as e: - raise PackageException(f"Could not resolve Golem package repository address [{e}].") - - if not srv: - raise PackageException("Golem package repository is currently unavailable.") - except Exception as e: - # this is a temporary fallback for a problem resolving the SRV record - logger.warning( - "Problem resolving %s, falling back to %s, exception: %s", repo_srv, fallback_url, e - ) - return fallback_url - - return f"http://{srv.host}:{srv.port}" + return _resolve_repo_srv(repo_srv=repo_srv, fallback_url=fallback_url) \ No newline at end of file diff --git a/yapapi/payload/__init__.py b/yapapi/payload/__init__.py new file mode 100644 index 000000000..2e514655b --- /dev/null +++ b/yapapi/payload/__init__.py @@ -0,0 +1,11 @@ +import abc + +from yapapi.props.builder import DemandBuilder + + +class Payload(abc.ABC): + """Base class for descriptions of the payload required by the requestor.""" + + @abc.abstractmethod + async def decorate_demand(self, demand: DemandBuilder): + """Hook that allows the specific payload to add appropriate properties and constraints to a Demand.""" diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py new file mode 100644 index 000000000..407b94a4c --- /dev/null +++ b/yapapi/payload/package.py @@ -0,0 +1,17 @@ +import abc + +from yapapi.payload import Payload + + +class PackageException(Exception): + """Exception raised on any problems related to the package repository.""" + + pass + + +class Package(Payload): + """Description of a task package (e.g. a VM image) deployed on the provider nodes""" + + @abc.abstractmethod + async def resolve_url(self) -> str: + """Return package URL.""" diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py new file mode 100644 index 000000000..32cc0bd10 --- /dev/null +++ b/yapapi/payload/vm.py @@ -0,0 +1,104 @@ +import aiohttp +from dns.exception import DNSException +from dataclasses import dataclass +import logging +from typing import Optional +from typing_extensions import Final +from srvresolver.srv_resolver import SRVResolver, SRVRecord # type: ignore + +from yapapi.payload.package import Package, PackageException +from yapapi.props.builder import DemandBuilder +from yapapi.props.inf import InfVmKeys, RUNTIME_VM, VmRequest, VmPackageFormat + +_DEFAULT_REPO_SRV: Final = "_girepo._tcp.dev.golem.network" +_FALLBACK_REPO_URL: Final = "http://yacn2.dev.golem.network:8000" + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _VmConstrains: + min_mem_gib: float + min_storage_gib: float + cores: int = 1 + + def __str__(self): + rules = "\n\t".join( + [ + f"({InfVmKeys.mem}>={self.min_mem_gib})", + f"({InfVmKeys.storage}>={self.min_storage_gib})", + # TODO: provider should report cores. + # + # f"({inf.InfVmKeys.cores}>={self.cores})", + f"({InfVmKeys.runtime}={RUNTIME_VM})", + ] + ) + return f"(&{rules})" + + +@dataclass +class _VmPackage(Package): + repo_url: str + image_hash: str + constraints: _VmConstrains + + async def resolve_url(self) -> str: + async with aiohttp.ClientSession() as client: + resp = await client.get(f"{self.repo_url}/image.{self.image_hash}.link") + if resp.status != 200: + resp.raise_for_status() + + image_url = await resp.text() + image_hash = self.image_hash + return f"hash:sha3:{image_hash}:{image_url}" + + async def decorate_demand(self, demand: DemandBuilder): + image_url = await self.resolve_url() + demand.ensure(str(self.constraints)) + demand.add(VmRequest(package_url=image_url, package_format=VmPackageFormat.GVMKIT_SQUASH)) + + +async def repo( + *, image_hash: str, min_mem_gib: float = 0.5, min_storage_gib: float = 2.0 +) -> Package: + """ + Build reference to application package. + + - *image_hash*: finds package by its contents hash. + - *min_mem_gib*: minimal memory required to execute application code. + - *min_storage_gib* minimal disk storage to execute tasks. + + """ + + return _VmPackage( + repo_url=resolve_repo_srv(_DEFAULT_REPO_SRV), + image_hash=image_hash, + constraints=_VmConstrains(min_mem_gib, min_storage_gib), + ) + + +def resolve_repo_srv(repo_srv, fallback_url=_FALLBACK_REPO_URL) -> str: + """ + Get the url of the package repository based on its SRV record address. + + :param repo_srv: the SRV domain name + :param fallback_url: temporary hardcoded fallback url in case there's a problem resolving SRV + :return: the url of the package repository containing the port + :raises: PackageException if no valid service could be reached + """ + try: + try: + srv: Optional[SRVRecord] = SRVResolver.resolve_random(repo_srv) + except DNSException as e: + raise PackageException(f"Could not resolve Golem package repository address [{e}].") + + if not srv: + raise PackageException("Golem package repository is currently unavailable.") + except Exception as e: + # this is a temporary fallback for a problem resolving the SRV record + logger.warning( + "Problem resolving %s, falling back to %s, exception: %s", repo_srv, fallback_url, e + ) + return fallback_url + + return f"http://{srv.host}:{srv.port}" diff --git a/yapapi/props/inf.py b/yapapi/props/inf.py index 71c1346e6..916985faf 100644 --- a/yapapi/props/inf.py +++ b/yapapi/props/inf.py @@ -2,6 +2,7 @@ from typing import Optional, List from dataclasses import dataclass, field +from deprecated import deprecated from enum import Enum from .base import Model @@ -10,13 +11,17 @@ INF_CORES: str = "golem.inf.cpu.cores" TRANSFER_CAPS: str = "golem.activity.caps.transfer.protocol" +RUNTIME_WASMTIME = "wasmtime" +RUNTIME_EMSCRIPTEN = "emscripten" +RUNTIME_VM = "vm" @dataclass +@deprecated(version="0.6.0", reason="please use yapapi.props.inf.RUNTIME_* constants directly", action="default") class RuntimeType(Enum): UNKNOWN = "" - WASMTIME = "wasmtime" - EMSCRIPTEN = "emscripten" - VM = "vm" + WASMTIME = RUNTIME_WASMTIME + EMSCRIPTEN = RUNTIME_EMSCRIPTEN + VM = RUNTIME_VM @dataclass @@ -27,16 +32,16 @@ class WasmInterface(Enum): @dataclass class InfBase(Model): - mem: float = field(metadata={"key": INF_MEM}) - runtime: RuntimeType = field(metadata={"key": "golem.runtime.name"}) + runtime: str = field(metadata={"key": "golem.runtime.name"}) + mem: float = field(metadata={"key": INF_MEM}) storage: Optional[float] = field(default=None, metadata={"key": INF_STORAGE}) transfers: Optional[List[str]] = field(default=None, metadata={"key": TRANSFER_CAPS}) @dataclass class InfVm(InfBase): - runtime = RuntimeType.VM + runtime = RUNTIME_VM cores: int = field(default=1, metadata={"key": INF_CORES}) From caf803b1562838eda0706217dd9a54747fb4b829 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 22 Apr 2021 17:41:38 +0200 Subject: [PATCH 29/85] * `turbogeth` example stub * + `prop` and `constraint` helpers * + functions to build Demand constraints from constraint fields of Models * + `DemandDecorator` interface + helper method on `DemandBuilder` that decorates the Demand with `DemandDecorator` objects * + `AutodecoratingModel` class that implements automatic building of properties / constraints from a model * remake `VmConstraints` using `prop` and `constraint` * update `DemandBuilder` to use the constraint helpers * deprecate a few more components --- examples/turbogeth/turbogeth.py | 45 +++++++++++++++ tests/rest/test_builder.py | 6 +- yapapi/package/__init__.py | 3 +- yapapi/package/vm.py | 4 +- yapapi/payload/__init__.py | 8 +-- yapapi/payload/vm.py | 59 ++++++++++++-------- yapapi/props/base.py | 99 ++++++++++++++++++++++++++++++--- yapapi/props/builder.py | 34 +++++++---- yapapi/props/inf.py | 28 ++++++---- 9 files changed, 222 insertions(+), 64 deletions(-) create mode 100644 examples/turbogeth/turbogeth.py diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py new file mode 100644 index 000000000..3476a0032 --- /dev/null +++ b/examples/turbogeth/turbogeth.py @@ -0,0 +1,45 @@ +import asyncio + +from dataclasses import dataclass + +from yapapi.props.builder import DemandBuilder, AutodecoratingModel +from yapapi.props.base import prop, constraint +from yapapi.props import inf + +from yapapi.payload import Payload + + +TURBOGETH_RUNTIME_NAME = "turbogeth-managed" +PROP_TURBOGETH_RPC_PORT = "golem.srv.app.eth.rpc-port" + + +@dataclass +class Turbogeth(Payload, AutodecoratingModel): + rpc_port: int = prop(PROP_TURBOGETH_RPC_PORT, default=None) + + runtime: str = constraint(inf.INF_RUNTIME_NAME, default=TURBOGETH_RUNTIME_NAME) + min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) + min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) + + +async def main(): + builder = DemandBuilder() + await builder.decorate(Turbogeth(rpc_port=1234)) + print(builder) + + +asyncio.run(main()) + +# notes for next steps +# +# -> Service class +# +# Executor.run_service(Service, num_instance=3) +# +# service instance -> instance of the Service class +# +# Service.shutdown should signal Executor to call Services' `exit` +# +# when "run" finishes, the service shuts down +# +# next: save/restore of the Service state \ No newline at end of file diff --git a/tests/rest/test_builder.py b/tests/rest/test_builder.py index 82a295452..798020efa 100644 --- a/tests/rest/test_builder.py +++ b/tests/rest/test_builder.py @@ -1,13 +1,13 @@ from yapapi import props -import yapapi.props.inf as inf +from yapapi.payload import vm from datetime import datetime, timezone, timedelta from yapapi.props.builder import DemandBuilder def test_builder(): - print("inf.cores=", inf.InfVmKeys.names()) + print("inf.cores=", vm.InfVmKeys.names()) b = DemandBuilder() e = datetime.now(timezone.utc) + timedelta(days=1) b.add(props.Activity(expiration=e)) - b.add(inf.VmRequest(package_url="", package_format=inf.VmPackageFormat.GVMKIT_SQUASH)) + b.add(vm.VmRequest(package_url="", package_format=vm.VmPackageFormat.GVMKIT_SQUASH)) print(b) diff --git a/yapapi/package/__init__.py b/yapapi/package/__init__.py index 354028fa4..98ca605b0 100644 --- a/yapapi/package/__init__.py +++ b/yapapi/package/__init__.py @@ -1,5 +1,4 @@ -import abc -from deprecated import deprecated +from deprecated import deprecated # type: ignore from yapapi.payload.package import Package as _Package, PackageException as _PackageException diff --git a/yapapi/package/vm.py b/yapapi/package/vm.py index 07039410f..72a425155 100644 --- a/yapapi/package/vm.py +++ b/yapapi/package/vm.py @@ -1,7 +1,7 @@ -from deprecated import deprecated +from deprecated import deprecated # type: ignore import logging -from yapapi.package import Package +from yapapi.payload.package import Package from yapapi.payload.vm import repo as _repo, resolve_repo_srv as _resolve_repo_srv, _FALLBACK_REPO_URL logger = logging.getLogger(__name__) diff --git a/yapapi/payload/__init__.py b/yapapi/payload/__init__.py index 2e514655b..4f352a136 100644 --- a/yapapi/payload/__init__.py +++ b/yapapi/payload/__init__.py @@ -1,11 +1,7 @@ import abc -from yapapi.props.builder import DemandBuilder +from yapapi.props.builder import DemandDecorator -class Payload(abc.ABC): +class Payload(DemandDecorator, abc.ABC): """Base class for descriptions of the payload required by the requestor.""" - - @abc.abstractmethod - async def decorate_demand(self, demand: DemandBuilder): - """Hook that allows the specific payload to add appropriate properties and constraints to a Demand.""" diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 32cc0bd10..54eaa5238 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -1,14 +1,17 @@ import aiohttp from dns.exception import DNSException -from dataclasses import dataclass +from dataclasses import dataclass, field +from enum import Enum import logging from typing import Optional from typing_extensions import Final from srvresolver.srv_resolver import SRVResolver, SRVRecord # type: ignore from yapapi.payload.package import Package, PackageException +from yapapi.props import base as prop_base from yapapi.props.builder import DemandBuilder -from yapapi.props.inf import InfVmKeys, RUNTIME_VM, VmRequest, VmPackageFormat +from yapapi.props import inf +from yapapi.props.inf import InfBase, INF_CORES, RUNTIME_VM, ExeUnitRequest _DEFAULT_REPO_SRV: Final = "_girepo._tcp.dev.golem.network" _FALLBACK_REPO_URL: Final = "http://yacn2.dev.golem.network:8000" @@ -16,31 +19,43 @@ logger = logging.getLogger(__name__) +@dataclass +class InfVm(InfBase): + runtime = RUNTIME_VM + cores: int = prop_base.prop(INF_CORES, default=1) + + +InfVmKeys = InfVm.keys() + + +class VmPackageFormat(Enum): + UNKNOWN = None + GVMKIT_SQUASH = "gvmkit-squash" + + +@dataclass +class VmRequest(ExeUnitRequest): + package_format: VmPackageFormat = prop_base.prop("golem.srv.comp.vm.package_format") + + @dataclass(frozen=True) -class _VmConstrains: - min_mem_gib: float - min_storage_gib: float - cores: int = 1 +class _VmConstraints: + min_mem_gib: float = prop_base.constraint(inf.INF_MEM, operator=">=") + min_storage_gib: float = prop_base.constraint(inf.INF_STORAGE, operator=">=") + # cores: int = prop_base.constraint(inf.INF_CORES, operator=">=") + runtime: str = prop_base.constraint(inf.INF_RUNTIME_NAME, operator="=", default=RUNTIME_VM) def __str__(self): - rules = "\n\t".join( - [ - f"({InfVmKeys.mem}>={self.min_mem_gib})", - f"({InfVmKeys.storage}>={self.min_storage_gib})", - # TODO: provider should report cores. - # - # f"({inf.InfVmKeys.cores}>={self.cores})", - f"({InfVmKeys.runtime}={RUNTIME_VM})", - ] + return prop_base.join_str_constraints( + prop_base.constraint_model_serialize(self) ) - return f"(&{rules})" @dataclass class _VmPackage(Package): repo_url: str image_hash: str - constraints: _VmConstrains + constraints: _VmConstraints async def resolve_url(self) -> str: async with aiohttp.ClientSession() as client: @@ -62,18 +77,18 @@ async def repo( *, image_hash: str, min_mem_gib: float = 0.5, min_storage_gib: float = 2.0 ) -> Package: """ - Build reference to application package. + Build a reference to application package. - - *image_hash*: finds package by its contents hash. - - *min_mem_gib*: minimal memory required to execute application code. - - *min_storage_gib* minimal disk storage to execute tasks. + - *image_hash*: hash of the package's image + - *min_mem_gib*: minimal memory required to execute application code + - *min_storage_gib* minimal disk storage to execute tasks """ return _VmPackage( repo_url=resolve_repo_srv(_DEFAULT_REPO_SRV), image_hash=image_hash, - constraints=_VmConstrains(min_mem_gib, min_storage_gib), + constraints=_VmConstraints(min_mem_gib, min_storage_gib), ) diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 0267451fd..4e46560e3 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -1,13 +1,22 @@ from typing import Dict, Type, Any, Union, List, cast, TypeVar +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal # type: ignore + import typing import abc import enum import json -from dataclasses import dataclass, fields, MISSING +from dataclasses import dataclass, fields, MISSING, field, Field from datetime import datetime, timezone Props = Dict[str, str] +PROP_KEY = "key" +PROP_OPERATOR = "operator" +PROP_MODEL_FIELD_TYPE = "model_field_type" + def as_list(data: Union[str, List[str]]) -> List[str]: if not isinstance(data, str): @@ -70,6 +79,7 @@ def __str__(self): return msg +@dataclass class Model(abc.ABC): """ Base class from which all property models inherit. @@ -85,6 +95,15 @@ def __init__(self, **kwargs): def _custom_mapping(cls, props: Props, data: Dict[str, Any]): pass + @classmethod + def property_fields(cls): + return ( + f + for f in fields(cls) + if PROP_KEY in f.metadata + and f.metadata.get(PROP_MODEL_FIELD_TYPE, ModelFieldType.property) == ModelFieldType.property + ) + @classmethod def from_properties(cls: Type[ME], props: Props) -> ME: """ @@ -99,11 +118,10 @@ def from_properties(cls: Type[ME], props: Props) -> ME: """ field_map = dict( ( - f.metadata["key"], + f.metadata[PROP_KEY], _PyField(name=f.name, type=f.type, required=f.default is MISSING), ) - for f in fields(cls) - if "key" in f.metadata + for f in cls.property_fields() ) data = dict( (field_map[key].encode(val) for (key, val) in props.items() if key in field_map) @@ -123,7 +141,7 @@ def from_properties(cls: Type[ME], props: Props) -> ME: raise InvalidPropertiesError(msg) from exc @classmethod - def keys(cls): + def keys(cls): # change to `property_keys` ? """ :return: a mapping between the model's field names and the property keys @@ -131,7 +149,7 @@ def keys(cls): ```python >>> import dataclasses >>> import typing - >>> from yapapi.properties.base import Model + >>> from yapapi.props.base import Model >>> @dataclasses.dataclass ... class NodeInfo(Model): ... name: typing.Optional[str] = \ @@ -149,7 +167,74 @@ def __init__(self, iter): def names(self): return self.__dict__.keys() - return _Keys((f.name, f.metadata["key"]) for f in fields(cls)) + return _Keys((f.name, f.metadata["key"]) for f in cls.property_fields()) + + +class ConstraintException(Exception): + pass + + +CONSTRAINT_VAL_ANY = "*" + +ConstraintOperator = Literal['=', ">=", "<="] +ConstraintGroupOperator = Literal["&", "|", "!"] + + +class ModelFieldType(enum.Enum): + constraint = "constraint" + property = "property" + + +def constraint(key: str, *, operator: ConstraintOperator = "=", default=MISSING): + """return a contraint-type dataclass field""" + return field( + default=default, + metadata={ + PROP_KEY: key, + PROP_OPERATOR: operator, + PROP_MODEL_FIELD_TYPE: ModelFieldType.constraint, + } + ) + + +def prop(key: str, *, default=MISSING): + """return a property-type dataclass field""" + return field( + default=default, + metadata={ + PROP_KEY: key, + PROP_MODEL_FIELD_TYPE: ModelFieldType.property + } + ) + + +def constraint_to_str(value, f: Field) -> str: + return f"({f.metadata[PROP_KEY]}{f.metadata[PROP_OPERATOR]}{value})" + + +def constraint_model_serialize(m: Model) -> List[str]: + return [ + constraint_to_str(getattr(m, f.name), f) + for f in fields(type(m)) + if f.metadata.get(PROP_MODEL_FIELD_TYPE, "") == ModelFieldType.constraint + ] + + +def join_str_constraints(constraints: List[str], operator: ConstraintGroupOperator = "&"): + if not constraints: + return "()" + + if operator == "!": + if len(constraints) == 1: + return f"({operator}({constraints[0]}))" + else: + raise ConstraintException(f"{operator} requires exactly one component.") + + if len(constraints) == 1: + return f"({constraints[0]})" + + rules = "\n\t".join(constraints) + return f"({operator}{rules})" __all__ = ("Model", "as_list", "Props") diff --git a/yapapi/props/builder.py b/yapapi/props/builder.py index 8123b053b..fb9b56699 100644 --- a/yapapi/props/builder.py +++ b/yapapi/props/builder.py @@ -1,3 +1,4 @@ +import abc import enum from datetime import datetime from typing import List @@ -6,6 +7,7 @@ from dataclasses import asdict from . import Model +from .base import join_str_constraints, constraint_model_serialize class DemandBuilder: @@ -50,18 +52,8 @@ def properties(self) -> dict: @property def constraints(self) -> str: - """List of constraints for this demand.""" - c_list = self._constraints - c_value: str - if not c_list: - c_value = "()" - elif len(c_list) == 1: - c_value = c_list[0] - else: - rules = "\n\t".join(c_list) - c_value = f"(&{rules})" - - return c_value + """Constraints definition for this demand.""" + return join_str_constraints(self._constraints) def ensure(self, constraint: str): """Add a constraint to the demand definition.""" @@ -87,3 +79,21 @@ def add(self, m: Model): async def subscribe(self, market: Market) -> Subscription: """Create a Demand on the market and subscribe to Offers that will match that Demand.""" return await market.subscribe(self._properties, self.constraints) + + async def decorate(self, *decorators: 'DemandDecorator'): + for decorator in decorators: + await decorator.decorate_demand(self) + + +class DemandDecorator(abc.ABC): + """An interface that specifies classes that can add properties and constraints through a DemandBuilder""" + + @abc.abstractmethod + async def decorate_demand(self, demand: DemandBuilder): + """Add appropriate properties and constraints to a Demand""" + + +class AutodecoratingModel(Model, DemandDecorator): + async def decorate_demand(self, demand: DemandBuilder): + demand.add(self) + demand.ensure(join_str_constraints(constraint_model_serialize(self))) diff --git a/yapapi/props/inf.py b/yapapi/props/inf.py index 916985faf..e5d047b0f 100644 --- a/yapapi/props/inf.py +++ b/yapapi/props/inf.py @@ -1,15 +1,16 @@ """Infrastructural properties.""" from typing import Optional, List -from dataclasses import dataclass, field -from deprecated import deprecated +from dataclasses import dataclass +from deprecated import deprecated # type: ignore from enum import Enum -from .base import Model +from .base import Model, prop INF_MEM: str = "golem.inf.mem.gib" INF_STORAGE: str = "golem.inf.storage.gib" INF_CORES: str = "golem.inf.cpu.cores" TRANSFER_CAPS: str = "golem.activity.caps.transfer.protocol" +INF_RUNTIME_NAME = "golem.runtime.name" RUNTIME_WASMTIME = "wasmtime" RUNTIME_EMSCRIPTEN = "emscripten" @@ -32,17 +33,22 @@ class WasmInterface(Enum): @dataclass class InfBase(Model): - runtime: str = field(metadata={"key": "golem.runtime.name"}) + runtime: str = prop(INF_RUNTIME_NAME) - mem: float = field(metadata={"key": INF_MEM}) - storage: Optional[float] = field(default=None, metadata={"key": INF_STORAGE}) - transfers: Optional[List[str]] = field(default=None, metadata={"key": TRANSFER_CAPS}) + mem: float = prop(INF_MEM) + storage: Optional[float] = prop(INF_STORAGE, default=None) + transfers: Optional[List[str]] = prop(TRANSFER_CAPS, default=None) @dataclass +@deprecated(version="0.6.0", reason="this is part of yapapi.payload.vm now") class InfVm(InfBase): runtime = RUNTIME_VM - cores: int = field(default=1, metadata={"key": INF_CORES}) + cores: int = prop(INF_CORES, default=1) + + @classmethod + def keys(cls): + return super().keys() InfVmKeys = InfVm.keys() @@ -50,14 +56,16 @@ class InfVm(InfBase): @dataclass class ExeUnitRequest(Model): - package_url: str = field(metadata={"key": "golem.srv.comp.task_package"}) + package_url: str = prop("golem.srv.comp.task_package") +@deprecated(version="0.6.0", reason="this is part of yapapi.payload.vm now") class VmPackageFormat(Enum): UNKNOWN = None GVMKIT_SQUASH = "gvmkit-squash" +@deprecated(version="0.6.0", reason="this is part of yapapi.payload.vm now") @dataclass class VmRequest(ExeUnitRequest): - package_format: VmPackageFormat = field(metadata={"key": "golem.srv.comp.vm.package_format"}) + package_format: VmPackageFormat = prop("golem.srv.comp.vm.package_format") From 75aa6533d61a993df1280bc35b9f98ee58731588 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 26 Apr 2021 14:36:23 +0200 Subject: [PATCH 30/85] dataclass info --- examples/turbogeth/turbogeth.py | 2 +- yapapi/payload/package.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 3476a0032..c22604ffd 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -42,4 +42,4 @@ async def main(): # # when "run" finishes, the service shuts down # -# next: save/restore of the Service state \ No newline at end of file +# next: save/restore of the Service state diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 407b94a4c..52e271440 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -1,4 +1,5 @@ import abc +from dataclasses import dataclass from yapapi.payload import Payload @@ -9,6 +10,7 @@ class PackageException(Exception): pass +@dataclass class Package(Payload): """Description of a task package (e.g. a VM image) deployed on the provider nodes""" From 2e7d5f60dc357be18ae6b86c52029c8418d15619 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 26 Apr 2021 15:21:49 +0200 Subject: [PATCH 31/85] limit workers to 1 --- examples/service/simple_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/service/simple_service.py b/examples/service/simple_service.py index 43116d226..1870aae8d 100644 --- a/examples/service/simple_service.py +++ b/examples/service/simple_service.py @@ -113,7 +113,7 @@ async def on_stats(out: bytes): # the level of detail and format of the logged information. async with Executor( package=package, - max_workers=3, + max_workers=1, budget=1.0, timeout=timeout, subnet_tag=subnet_tag, From a75a263afef278f3cdfe9d133e2c24644aa64069 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 27 Apr 2021 10:13:08 +0200 Subject: [PATCH 32/85] rename Executor.package to Executor.payload --- examples/blender/blender.py | 2 +- yapapi/executor/__init__.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 64c9b1c1c..15a9ed8ac 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -93,7 +93,7 @@ async def worker(ctx: WorkContext, tasks): # See the documentation of the `yapapi.log` module on how to set # the level of detail and format of the logged information. async with Executor( - package=package, + payload=package, max_workers=3, budget=10.0, timeout=timeout, diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 1fc91dd10..fa27c32d2 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -34,7 +34,7 @@ from . import events from .task import Task, TaskStatus from .utils import AsyncWrapper -from ..package import Package +from ..payload import Payload from ..props import Activity, com, NodeInfo, NodeInfoKeys from ..props.base import InvalidPropertiesError from ..props.builder import DemandBuilder @@ -114,7 +114,7 @@ class Executor(AsyncContextManager): def __init__( self, *, - package: Package, + package: Optional[Payload] = None, max_workers: int = 5, timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, budget: Union[float, Decimal], @@ -124,6 +124,7 @@ def __init__( network: Optional[str] = None, event_consumer: Optional[Callable[[Event], None]] = None, stream_output: bool = False, + payload: Optional[Payload] = None ): """Create a new executor. @@ -160,7 +161,16 @@ def __init__( self._strategy = strategy self._api_config = rest.Configuration() self._stack = AsyncExitStack() - self._package = package + + if package: + if payload: + raise TypeError("Cannot use `payload` and `package` at the same time") + logger.warning(f"`package` argument to `{self.__class__}` is deprecated, please use `payload` instead") + payload = package + if not payload: + raise TypeError("Executor `payload` must be specified") + + self._package = payload self._conf = _ExecutorConfig(max_workers, timeout) # TODO: setup precision self._budget_amount = Decimal(budget) From 3589478dc0f776db502c72608e1b891ca608bb78 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 27 Apr 2021 10:16:34 +0200 Subject: [PATCH 33/85] mock of the WorkContext change that makes deploy and start more explicit --- yapapi/executor/ctx.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index 777210957..a8387609b 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -51,9 +51,13 @@ def timeout(self) -> Optional[timedelta]: return None -class _InitStep(Work): +class _Deploy(Work): def register(self, commands: CommandContainer): commands.deploy() + + +class _Start(Work): + def register(self, commands: CommandContainer): commands.start() @@ -264,6 +268,7 @@ def __init__( node_info: NodeInfo, storage: StorageProvider, emitter: Optional[Callable[[StorageEvent], None]] = None, + implicit_init: bool = True, ): self.id = ctx_id self._node_info = node_info @@ -271,6 +276,7 @@ def __init__( self._pending_steps: List[Work] = [] self._started: bool = False self._emitter: Optional[Callable[[StorageEvent], None]] = emitter + self._implicit_init = implicit_init @property def provider_name(self) -> Optional[str]: @@ -278,13 +284,20 @@ def provider_name(self) -> Optional[str]: return self._node_info.name def __prepare(self): - if not self._started: - self._pending_steps.append(_InitStep()) + if not self._started and self._implicit_init: + self.deploy() + self.start() self._started = True def begin(self): pass + def deploy(self, on_deploy: Callable[[bytes], Awaitable]): + self._pending_steps.append(_Deploy()) + + def start(self, on_start: Callable[[bytes], Awaitable]): + self._pending_steps.append(_Start()) + def send_json(self, json_path: str, data: dict): """Schedule sending JSON data to the provider. From dafe1ed4871aea00f3dc8ce9cf625e0ad0619e38 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 27 Apr 2021 10:17:17 +0200 Subject: [PATCH 34/85] a very draft "strawman" try at the services API --- examples/turbogeth/turbogeth.py | 389 +++++++++++++++++++++++++++++++- 1 file changed, 382 insertions(+), 7 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index c22604ffd..01b9fc87b 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -1,20 +1,25 @@ import asyncio +import json +from datetime import timedelta +import typing from dataclasses import dataclass - +from yapapi.executor.ctx import WorkContext from yapapi.props.builder import DemandBuilder, AutodecoratingModel from yapapi.props.base import prop, constraint from yapapi.props import inf from yapapi.payload import Payload +from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa + TURBOGETH_RUNTIME_NAME = "turbogeth-managed" PROP_TURBOGETH_RPC_PORT = "golem.srv.app.eth.rpc-port" @dataclass -class Turbogeth(Payload, AutodecoratingModel): +class TurbogethPayload(Payload, AutodecoratingModel): rpc_port: int = prop(PROP_TURBOGETH_RPC_PORT, default=None) runtime: str = constraint(inf.INF_RUNTIME_NAME, default=TURBOGETH_RUNTIME_NAME) @@ -22,13 +27,145 @@ class Turbogeth(Payload, AutodecoratingModel): min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) -async def main(): - builder = DemandBuilder() - await builder.decorate(Turbogeth(rpc_port=1234)) - print(builder) +INSTANCES_NEEDED = 1 +EXECUTOR_TIMEOUT = timedelta(weeks=100) + + +class Service: + state: typing.Optional[str] = None + running = ('new', 'deployed', 'ready', 'shutdown') + + def __init__(self, ctx: WorkContext): + self.ctx = ctx + self.state = "new" # should state correspond with the ActivityState? + + async def on_deploy(self, out: bytes): + self.state = "deployed" + + async def on_start(self, out: bytes): + self.state = "ready" + + async def on_new(self): + self.ctx.deploy(on_deploy=self.on_deploy) + self.ctx.start(on_start=self.on_start) + yield self.ctx.commit() + + async def execute_batch(self, batch: Optional[Task]): + if batch: + executor.execute(batch) + + async def run(self): # some way to pass a signal into `run` ... or some other event handler inside `Service` + while self.state in self.running: + _handlers = { + 'new': self.on_new, + 'ready': self.on_ready, + 'shutdown': self.on_shutdown, + } + + handler = _handlers.get(self.state) + if handler: + async for batch in handler(): + await self.execute_batch(batch) + + async def on_ready(self): + while True: + print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") + await asyncio.sleep(10) + yield None + + async def on_shutdown(self): + yield None + + +class TurbogethService(Service): + def __init__(self, ctx: WorkContext): + super().__init__(ctx) + self.credentials = {} + + async def on_deploy(self, out: bytes): + print("deployed") + self.credentials = json.loads(out.decode("utf-8")) + + async def on_start(self, out: bytes): + print("started") + + async def on_shutdown(self): + self.ctx.download_file("some/service/state", "temp/path") + + +class Swarm: + def __init__(self, executor: "Executor", service: typing.Type[Service], payload: Payload): + self.executor = executor + self.service = service + self.payload = payload + self.instances: typing.List[Service] = [] + + async def _run_instance(self, ctx: WorkContext): + instance = Service(ctx) + self.instances.append(instance) + + print(f"{instance} started") + await instance.run() + print(f"{instance} finished") + + # pass `instance` to some loop in the executor + + async def spawn_instance( + self, + ): + act = await self.executor.get_activity(self.payload) + ctx = WorkContext(act.id) + await self._run_instance(ctx) + + +class Executor(typing.AsyncContextManager): + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self) -> "Executor": + print("start executor") + return self + async def __aexit__(self, *exc_info): + print("stop executor", exc_info) + return True -asyncio.run(main()) + def run_service( + self, + service: typing.Type[Service], + payload: Payload, + num_instances: int = 1, + ) -> Swarm: + swarm = Swarm(executor=self, service=service, payload=payload) + for i in range(num_instances): + asyncio.create_task(swarm.spawn_instance()) + return swarm + + +async def main(subnet_tag, driver=None, network=None): + + payload = TurbogethPayload(rpc_port=8888) + + async with Executor( + max_workers=INSTANCES_NEEDED, + budget=10.0, + subnet_tag=subnet_tag, + driver=driver, + network=network, + event_consumer=log_summary(log_event_repr), + ) as executor: + swarm = executor.run_service( + TurbogethService, + payload=payload, + num_instances=INSTANCES_NEEDED + ) + + while True: + print(f"{swarm} is running: {swarm.instances}") + await asyncio.sleep(10) + + +asyncio.run(main(None)) # notes for next steps # @@ -43,3 +180,241 @@ async def main(): # when "run" finishes, the service shuts down # # next: save/restore of the Service state + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# #!/usr/bin/env python3 +# """ +# the requestor agent controlling and interacting with the "simple service" +# """ +# import asyncio +# from datetime import datetime, timedelta +# import itertools +# import json +# import pathlib +# import random +# import string +# import sys +# +# +# from yapapi import ( +# Executor, +# NoPaymentAccountError, +# Task, +# __version__ as yapapi_version, +# WorkContext, +# windows_event_loop_fix, +# ) +# from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa +# from yapapi.package import vm +# from yapapi.rest.activity import BatchTimeoutError +# +# examples_dir = pathlib.Path(__file__).resolve().parent.parent +# sys.path.append(str(examples_dir)) +# +# from utils import ( +# build_parser, +# TEXT_COLOR_CYAN, +# TEXT_COLOR_DEFAULT, +# TEXT_COLOR_RED, +# TEXT_COLOR_YELLOW, +# ) +# + +# async def main(subnet_tag, driver=None, network=None): +# package = await vm.repo( +# image_hash="8b11df59f84358d47fc6776d0bb7290b0054c15ded2d6f54cf634488", +# min_mem_gib=0.5, +# min_storage_gib=2.0, +# ) +# +# async def service(ctx: WorkContext, tasks): +# STATS_PATH = "/golem/out/stats" +# PLOT_INFO_PATH = "/golem/out/plot" +# SIMPLE_SERVICE = "/golem/run/simple_service.py" +# +# ctx.run("/golem/run/simulate_observations_ctl.py", "--start") +# ctx.send_bytes( +# "/golem/in/get_stats.sh", f"{SIMPLE_SERVICE} --stats > {STATS_PATH}".encode() +# ) +# ctx.send_bytes( +# "/golem/in/get_plot.sh", f"{SIMPLE_SERVICE} --plot dist > {PLOT_INFO_PATH}".encode() +# ) +# +# yield ctx.commit() +# +# plots_to_download = [] +# +# # have a look at asyncio docs and figure out whether to leave the callback or replace it with something +# # more asyncio-ic +# +# async def on_plot(out: bytes): +# nonlocal plots_to_download +# fname = json.loads(out.strip()) +# print(f"{TEXT_COLOR_CYAN}plot: {fname}{TEXT_COLOR_DEFAULT}") +# plots_to_download.append(fname) +# +# async def on_stats(out: bytes): +# print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}") +# +# try: +# async for task in tasks: +# await asyncio.sleep(10) +# +# ctx.run("/bin/sh", "/golem/in/get_stats.sh") +# ctx.download_bytes(STATS_PATH, on_stats) +# ctx.run("/bin/sh", "/golem/in/get_plot.sh") +# ctx.download_bytes(PLOT_INFO_PATH, on_plot) +# yield ctx.commit() +# +# for plot in plots_to_download: +# test_filename = ( +# "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" +# ) +# ctx.download_file(plot, pathlib.Path(__file__).resolve().parent / test_filename) +# yield ctx.commit() +# +# task.accept_result() +# +# except (KeyboardInterrupt, asyncio.CancelledError): +# # with the current implementation it won't work correctly +# # because at this stage, the Executor is shutting down anyway +# ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") +# yield ctx.commit() +# raise +# +# # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) +# # TODO: make this dynamic, e.g. depending on the size of files to transfer +# init_overhead = 3 +# # Providers will not accept work if the timeout is outside of the [5 min, 30min] range. +# # We increase the lower bound to 6 min to account for the time needed for our demand to +# # reach the providers. +# min_timeout, max_timeout = 6, 30 +# +# timeout = timedelta(minutes=29) +# +# # By passing `event_consumer=log_summary()` we enable summary logging. +# # See the documentation of the `yapapi.log` module on how to set +# # the level of detail and format of the logged information. +# async with Executor( +# package=package, +# max_workers=1, +# budget=1.0, +# timeout=timeout, +# subnet_tag=subnet_tag, +# driver=driver, +# network=network, +# event_consumer=log_summary(log_event_repr), +# ) as executor: +# +# print( +# f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" +# f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " +# f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " +# f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" +# ) +# +# start_time = datetime.now() +# +# async for task in executor.submit(service, (Task(data=n) for n in itertools.count(1))): +# print( +# f"{TEXT_COLOR_CYAN}" +# f"Script executed: {task}, result: {task.result}, time: {task.running_time}" +# f"{TEXT_COLOR_DEFAULT}" +# ) +# +# print( +# f"{TEXT_COLOR_CYAN}" +# f"Service finished, total time: {datetime.now() - start_time}" +# f"{TEXT_COLOR_DEFAULT}" +# ) +# +# +# if __name__ == "__main__": +# parser = build_parser("Test http") +# parser.set_defaults(log_file="service-yapapi.log") +# args = parser.parse_args() +# +# # This is only required when running on Windows with Python prior to 3.8: +# windows_event_loop_fix() +# +# enable_default_logger( +# log_file=args.log_file, +# debug_activity_api=True, +# debug_market_api=True, +# debug_payment_api=True, +# ) +# +# loop = asyncio.get_event_loop() +# task = loop.create_task( +# main(subnet_tag=args.subnet_tag, driver=args.driver, network=args.network) +# ) +# +# try: +# loop.run_until_complete(task) +# except NoPaymentAccountError as e: +# handbook_url = ( +# "https://handbook.golem.network/requestor-tutorials/" +# "flash-tutorial-of-requestor-development" +# ) +# print( +# f"{TEXT_COLOR_RED}" +# f"No payment account initialized for driver `{e.required_driver}` " +# f"and network `{e.required_network}`.\n\n" +# f"See {handbook_url} on how to initialize payment accounts for a requestor node." +# f"{TEXT_COLOR_DEFAULT}" +# ) +# except KeyboardInterrupt: +# print( +# f"{TEXT_COLOR_YELLOW}" +# "Shutting down gracefully, please wait a short while " +# "or press Ctrl+C to exit immediately..." +# f"{TEXT_COLOR_DEFAULT}" +# ) +# task.cancel() +# try: +# loop.run_until_complete(task) +# print( +# f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" +# ) +# except (asyncio.CancelledError, KeyboardInterrupt): +# pass From 1904e4ae901da9cf5156161ae73e063e280e8e23 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 27 Apr 2021 10:58:13 +0200 Subject: [PATCH 35/85] - commented-out code --- examples/turbogeth/turbogeth.py | 238 -------------------------------- 1 file changed, 238 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 01b9fc87b..f144f46b2 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -180,241 +180,3 @@ async def main(subnet_tag, driver=None, network=None): # when "run" finishes, the service shuts down # # next: save/restore of the Service state - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# #!/usr/bin/env python3 -# """ -# the requestor agent controlling and interacting with the "simple service" -# """ -# import asyncio -# from datetime import datetime, timedelta -# import itertools -# import json -# import pathlib -# import random -# import string -# import sys -# -# -# from yapapi import ( -# Executor, -# NoPaymentAccountError, -# Task, -# __version__ as yapapi_version, -# WorkContext, -# windows_event_loop_fix, -# ) -# from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa -# from yapapi.package import vm -# from yapapi.rest.activity import BatchTimeoutError -# -# examples_dir = pathlib.Path(__file__).resolve().parent.parent -# sys.path.append(str(examples_dir)) -# -# from utils import ( -# build_parser, -# TEXT_COLOR_CYAN, -# TEXT_COLOR_DEFAULT, -# TEXT_COLOR_RED, -# TEXT_COLOR_YELLOW, -# ) -# - -# async def main(subnet_tag, driver=None, network=None): -# package = await vm.repo( -# image_hash="8b11df59f84358d47fc6776d0bb7290b0054c15ded2d6f54cf634488", -# min_mem_gib=0.5, -# min_storage_gib=2.0, -# ) -# -# async def service(ctx: WorkContext, tasks): -# STATS_PATH = "/golem/out/stats" -# PLOT_INFO_PATH = "/golem/out/plot" -# SIMPLE_SERVICE = "/golem/run/simple_service.py" -# -# ctx.run("/golem/run/simulate_observations_ctl.py", "--start") -# ctx.send_bytes( -# "/golem/in/get_stats.sh", f"{SIMPLE_SERVICE} --stats > {STATS_PATH}".encode() -# ) -# ctx.send_bytes( -# "/golem/in/get_plot.sh", f"{SIMPLE_SERVICE} --plot dist > {PLOT_INFO_PATH}".encode() -# ) -# -# yield ctx.commit() -# -# plots_to_download = [] -# -# # have a look at asyncio docs and figure out whether to leave the callback or replace it with something -# # more asyncio-ic -# -# async def on_plot(out: bytes): -# nonlocal plots_to_download -# fname = json.loads(out.strip()) -# print(f"{TEXT_COLOR_CYAN}plot: {fname}{TEXT_COLOR_DEFAULT}") -# plots_to_download.append(fname) -# -# async def on_stats(out: bytes): -# print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}") -# -# try: -# async for task in tasks: -# await asyncio.sleep(10) -# -# ctx.run("/bin/sh", "/golem/in/get_stats.sh") -# ctx.download_bytes(STATS_PATH, on_stats) -# ctx.run("/bin/sh", "/golem/in/get_plot.sh") -# ctx.download_bytes(PLOT_INFO_PATH, on_plot) -# yield ctx.commit() -# -# for plot in plots_to_download: -# test_filename = ( -# "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" -# ) -# ctx.download_file(plot, pathlib.Path(__file__).resolve().parent / test_filename) -# yield ctx.commit() -# -# task.accept_result() -# -# except (KeyboardInterrupt, asyncio.CancelledError): -# # with the current implementation it won't work correctly -# # because at this stage, the Executor is shutting down anyway -# ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") -# yield ctx.commit() -# raise -# -# # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) -# # TODO: make this dynamic, e.g. depending on the size of files to transfer -# init_overhead = 3 -# # Providers will not accept work if the timeout is outside of the [5 min, 30min] range. -# # We increase the lower bound to 6 min to account for the time needed for our demand to -# # reach the providers. -# min_timeout, max_timeout = 6, 30 -# -# timeout = timedelta(minutes=29) -# -# # By passing `event_consumer=log_summary()` we enable summary logging. -# # See the documentation of the `yapapi.log` module on how to set -# # the level of detail and format of the logged information. -# async with Executor( -# package=package, -# max_workers=1, -# budget=1.0, -# timeout=timeout, -# subnet_tag=subnet_tag, -# driver=driver, -# network=network, -# event_consumer=log_summary(log_event_repr), -# ) as executor: -# -# print( -# f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" -# f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " -# f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " -# f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" -# ) -# -# start_time = datetime.now() -# -# async for task in executor.submit(service, (Task(data=n) for n in itertools.count(1))): -# print( -# f"{TEXT_COLOR_CYAN}" -# f"Script executed: {task}, result: {task.result}, time: {task.running_time}" -# f"{TEXT_COLOR_DEFAULT}" -# ) -# -# print( -# f"{TEXT_COLOR_CYAN}" -# f"Service finished, total time: {datetime.now() - start_time}" -# f"{TEXT_COLOR_DEFAULT}" -# ) -# -# -# if __name__ == "__main__": -# parser = build_parser("Test http") -# parser.set_defaults(log_file="service-yapapi.log") -# args = parser.parse_args() -# -# # This is only required when running on Windows with Python prior to 3.8: -# windows_event_loop_fix() -# -# enable_default_logger( -# log_file=args.log_file, -# debug_activity_api=True, -# debug_market_api=True, -# debug_payment_api=True, -# ) -# -# loop = asyncio.get_event_loop() -# task = loop.create_task( -# main(subnet_tag=args.subnet_tag, driver=args.driver, network=args.network) -# ) -# -# try: -# loop.run_until_complete(task) -# except NoPaymentAccountError as e: -# handbook_url = ( -# "https://handbook.golem.network/requestor-tutorials/" -# "flash-tutorial-of-requestor-development" -# ) -# print( -# f"{TEXT_COLOR_RED}" -# f"No payment account initialized for driver `{e.required_driver}` " -# f"and network `{e.required_network}`.\n\n" -# f"See {handbook_url} on how to initialize payment accounts for a requestor node." -# f"{TEXT_COLOR_DEFAULT}" -# ) -# except KeyboardInterrupt: -# print( -# f"{TEXT_COLOR_YELLOW}" -# "Shutting down gracefully, please wait a short while " -# "or press Ctrl+C to exit immediately..." -# f"{TEXT_COLOR_DEFAULT}" -# ) -# task.cancel() -# try: -# loop.run_until_complete(task) -# print( -# f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" -# ) -# except (asyncio.CancelledError, KeyboardInterrupt): -# pass From cc2ca55abea961f7ffab4f243c3e95efc29b45f2 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 27 Apr 2021 11:10:45 +0200 Subject: [PATCH 36/85] comments --- examples/turbogeth/turbogeth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index f144f46b2..7efe32880 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -32,6 +32,7 @@ class TurbogethPayload(Payload, AutodecoratingModel): class Service: + """ THIS SHOULD BE PART OF THE API""" state: typing.Optional[str] = None running = ('new', 'deployed', 'ready', 'shutdown') @@ -94,6 +95,7 @@ async def on_shutdown(self): class Swarm: + """ THIS SHOULD BE PART OF THE API""" def __init__(self, executor: "Executor", service: typing.Type[Service], payload: Payload): self.executor = executor self.service = service @@ -119,6 +121,7 @@ async def spawn_instance( class Executor(typing.AsyncContextManager): + """ MOCK OF EXECUTOR JUST SO I COULD ILLUSTRATE THE NEW CALL""" def __init__(self, *args, **kwargs): pass From c70e8450dd702d1aaf7193b8bfffe968a22aa8db Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 28 Apr 2021 16:44:30 +0200 Subject: [PATCH 37/85] rename `Swarm` to `Cluster` some other minor stuff --- examples/turbogeth/turbogeth.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 7efe32880..c3b711988 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -51,9 +51,9 @@ async def on_new(self): self.ctx.start(on_start=self.on_start) yield self.ctx.commit() - async def execute_batch(self, batch: Optional[Task]): + async def execute_batch(self, batch: Optional[Work]): if batch: - executor.execute(batch) + executor.execute(batch) # some automagic of passing it for execution ;) async def run(self): # some way to pass a signal into `run` ... or some other event handler inside `Service` while self.state in self.running: @@ -68,7 +68,7 @@ async def run(self): # some way to pass a signal into `run` ... or some other e async for batch in handler(): await self.execute_batch(batch) - async def on_ready(self): + async def on_ready(self, *args, **kwargs): while True: print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") await asyncio.sleep(10) @@ -94,7 +94,7 @@ async def on_shutdown(self): self.ctx.download_file("some/service/state", "temp/path") -class Swarm: +class Cluster: """ THIS SHOULD BE PART OF THE API""" def __init__(self, executor: "Executor", service: typing.Type[Service], payload: Payload): self.executor = executor @@ -138,11 +138,11 @@ def run_service( service: typing.Type[Service], payload: Payload, num_instances: int = 1, - ) -> Swarm: - swarm = Swarm(executor=self, service=service, payload=payload) + ) -> Cluster: + cluster = Cluster(executor=self, service=service, payload=payload) for i in range(num_instances): - asyncio.create_task(swarm.spawn_instance()) - return swarm + asyncio.create_task(cluster.spawn_instance()) + return cluster async def main(subnet_tag, driver=None, network=None): From 789376136a73e230100f4f0d9bade733381d5f2f Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 5 May 2021 15:54:47 +0200 Subject: [PATCH 38/85] s/state str/state enum/ --- examples/turbogeth/turbogeth.py | 40 +++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index c3b711988..70d8a72c1 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -1,4 +1,5 @@ import asyncio +import enum import json from datetime import timedelta import typing @@ -31,22 +32,43 @@ class TurbogethPayload(Payload, AutodecoratingModel): EXECUTOR_TIMEOUT = timedelta(weeks=100) +class ServiceState(enum.Enum): + """ THIS SHOULD BE PART OF THE API""" + new = 'new' + deploying = 'deploying' + deployed = 'deployed' + ready = 'ready' + shutdown = 'shutting down' + terminated = 'terminated' + undefined = 'undefined' + + +# """ THIS SHOULD BE PART OF THE API""" +SERVICE_AVAILABLE = ( + ServiceState.new, + ServiceState.deploying, + ServiceState.deployed, + ServiceState.ready, + ServiceState.shutdown +) + + class Service: """ THIS SHOULD BE PART OF THE API""" - state: typing.Optional[str] = None - running = ('new', 'deployed', 'ready', 'shutdown') + state: typing.Optional[ServiceState] = None def __init__(self, ctx: WorkContext): self.ctx = ctx - self.state = "new" # should state correspond with the ActivityState? + self.state = ServiceState.new # should state correspond with the ActivityState? async def on_deploy(self, out: bytes): - self.state = "deployed" + self.state = ServiceState.deployed async def on_start(self, out: bytes): - self.state = "ready" + self.state = ServiceState.ready async def on_new(self): + self.state = ServiceState.deploying self.ctx.deploy(on_deploy=self.on_deploy) self.ctx.start(on_start=self.on_start) yield self.ctx.commit() @@ -56,11 +78,11 @@ async def execute_batch(self, batch: Optional[Work]): executor.execute(batch) # some automagic of passing it for execution ;) async def run(self): # some way to pass a signal into `run` ... or some other event handler inside `Service` - while self.state in self.running: + while self.state in SERVICE_AVAILABLE: _handlers = { - 'new': self.on_new, - 'ready': self.on_ready, - 'shutdown': self.on_shutdown, + ServiceState.new: self.on_new, + ServiceState.ready: self.on_ready, + ServiceState.shutdown: self.on_shutdown, } handler = _handlers.get(self.state) From ccb3076637d3866da9ee940a8b9bf7bd60d51ae5 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 5 May 2021 15:55:36 +0200 Subject: [PATCH 39/85] - some notes that were already completed --- examples/turbogeth/turbogeth.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 70d8a72c1..1c813fa40 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -194,12 +194,6 @@ async def main(subnet_tag, driver=None, network=None): # notes for next steps # -# -> Service class -# -# Executor.run_service(Service, num_instance=3) -# -# service instance -> instance of the Service class -# # Service.shutdown should signal Executor to call Services' `exit` # # when "run" finishes, the service shuts down From b40b7752d22f6ec1134bdfd68f389b799cc43731 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 6 May 2021 16:22:33 +0200 Subject: [PATCH 40/85] update with remarks from @stranger80 --- examples/turbogeth/turbogeth.py | 59 ++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 1c813fa40..58523358d 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -32,6 +32,11 @@ class TurbogethPayload(Payload, AutodecoratingModel): EXECUTOR_TIMEOUT = timedelta(weeks=100) +class ConfigurationError(Exception): + """ THIS SHOULD BE PART OF THE API""" + pass + + class ServiceState(enum.Enum): """ THIS SHOULD BE PART OF THE API""" new = 'new' @@ -44,7 +49,7 @@ class ServiceState(enum.Enum): # """ THIS SHOULD BE PART OF THE API""" -SERVICE_AVAILABLE = ( +SERVICE_STATE_AVAILABLE = ( ServiceState.new, ServiceState.deploying, ServiceState.deployed, @@ -61,7 +66,16 @@ def __init__(self, ctx: WorkContext): self.ctx = ctx self.state = ServiceState.new # should state correspond with the ActivityState? - async def on_deploy(self, out: bytes): + @staticmethod + def get_payload() -> typing.Optional[Payload]: + """Return the payload (runtime) definition for this service. + + If `get_payload` is not implemented, the payload will need to be provided in the + `Executor.run_service` call. + """ + pass + + async def on_deploy(self, out: bytes): # maybe `out` is a structure similar to what we get from the Activity API itself ? self.state = ServiceState.deployed async def on_start(self, out: bytes): @@ -73,12 +87,23 @@ async def on_new(self): self.ctx.start(on_start=self.on_start) yield self.ctx.commit() - async def execute_batch(self, batch: Optional[Work]): + async def execute_batch(self, batch: typing.Optional[Work]): if batch: executor.execute(batch) # some automagic of passing it for execution ;) - async def run(self): # some way to pass a signal into `run` ... or some other event handler inside `Service` - while self.state in SERVICE_AVAILABLE: + async def run(self): + # we need some way to pass a signal (or a queue of messages?) into `run` ... + # or some other event handler inside `Service` + + # the simplest could be some cancellation token (e.g. an asyncio.Event) + # but maybe we would like to pass some more data with it? + + # maybe the state of the service could be a pair or ServiceState + # plus some additional data object? + + # better yet -> maybe `run` could take an _input_ async generator + + while self.state in SERVICE_STATE_AVAILABLE: _handlers = { ServiceState.new: self.on_new, ServiceState.ready: self.on_ready, @@ -88,7 +113,11 @@ async def run(self): # some way to pass a signal into `run` ... or some other e handler = _handlers.get(self.state) if handler: async for batch in handler(): - await self.execute_batch(batch) + yield self.execute_batch(batch) + + # we could add something like e.g. Shutdown(Work) step + # that would signal the service executor to transition the service + # to the shutdown state async def on_ready(self, *args, **kwargs): while True: @@ -105,6 +134,9 @@ def __init__(self, ctx: WorkContext): super().__init__(ctx) self.credentials = {} + def get_payload(self): + return TurbogethPayload(rpc_port=8888) + async def on_deploy(self, out: bytes): print("deployed") self.credentials = json.loads(out.decode("utf-8")) @@ -121,6 +153,10 @@ class Cluster: def __init__(self, executor: "Executor", service: typing.Type[Service], payload: Payload): self.executor = executor self.service = service + + if not payload: + raise ConfigurationError("Payload must be defined when starting a cluster.") + self.payload = payload self.instances: typing.List[Service] = [] @@ -157,11 +193,12 @@ async def __aexit__(self, *exc_info): def run_service( self, - service: typing.Type[Service], - payload: Payload, + service_class: typing.Type[Service], num_instances: int = 1, + payload: typing.Optional[Payload] = None, ) -> Cluster: - cluster = Cluster(executor=self, service=service, payload=payload) + payload = payload or service_class.get_payload() + cluster = Cluster(executor=self, service=service_class, payload=payload) for i in range(num_instances): asyncio.create_task(cluster.spawn_instance()) return cluster @@ -169,8 +206,6 @@ def run_service( async def main(subnet_tag, driver=None, network=None): - payload = TurbogethPayload(rpc_port=8888) - async with Executor( max_workers=INSTANCES_NEEDED, budget=10.0, @@ -181,7 +216,7 @@ async def main(subnet_tag, driver=None, network=None): ) as executor: swarm = executor.run_service( TurbogethService, - payload=payload, + # payload=payload, num_instances=INSTANCES_NEEDED ) From 50ccdb6721a84f5ceffeaa4e36d7608a7a5ba204 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Fri, 7 May 2021 22:05:23 +0200 Subject: [PATCH 41/85] more or less _final_ draft --- examples/turbogeth/turbogeth.py | 455 +++++++++++++++++++++++--------- pyproject.toml | 1 + 2 files changed, 330 insertions(+), 126 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 58523358d..2491b4231 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -1,11 +1,14 @@ import asyncio -import enum -import json from datetime import timedelta import typing +from typing import Optional, Any +import random -from dataclasses import dataclass -from yapapi.executor.ctx import WorkContext +import statemachine + + +from dataclasses import dataclass, field +# from yapapi.executor.ctx import WorkContext from yapapi.props.builder import DemandBuilder, AutodecoratingModel from yapapi.props.base import prop, constraint from yapapi.props import inf @@ -15,144 +18,210 @@ from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa -TURBOGETH_RUNTIME_NAME = "turbogeth-managed" -PROP_TURBOGETH_RPC_PORT = "golem.srv.app.eth.rpc-port" +############################################################### +# # +# # +# MOCK API CODE # +# # +# # +############################################################### +_act_cnt = 0 + +def _act_id(): + global _act_cnt + _act_cnt += 1 + return _act_cnt @dataclass -class TurbogethPayload(Payload, AutodecoratingModel): - rpc_port: int = prop(PROP_TURBOGETH_RPC_PORT, default=None) +class Activity: + """ Mock Activity """ + payload: Payload + id: int = field(default_factory=_act_id) - runtime: str = constraint(inf.INF_RUNTIME_NAME, default=TURBOGETH_RUNTIME_NAME) - min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) - min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) +class Steps(list): + """ Mock Steps to illustrate the idea. """ + def __init__(self, *args, **kwargs): + self.ctx: WorkContext = kwargs.pop('ctx') + self.blocking: bool = kwargs.pop('blocking', True) + self.get_results: bool = kwargs.pop('get_results', False) -INSTANCES_NEEDED = 1 -EXECUTOR_TIMEOUT = timedelta(weeks=100) + super().__init__(*args, **kwargs) + def __repr__(self): + list_repr = super().__repr__() + return f"<{self.__class__.__name__}: {list_repr}, blocking: {self.blocking}, get_results: {self.get_results}" -class ConfigurationError(Exception): - """ THIS SHOULD BE PART OF THE API""" - pass +class WorkContext: + """ Mock WorkContext to illustrate the idea. """ + + def __init__(self, activity_id, provider_name): + self.id = activity_id + self.provider_name = provider_name + self._steps = Steps(ctx=self) + + def __repr__(self): + return f"<{self.__class__.__name__}: activity_id: {self.id}>" + + def __str__(self): + return self.__repr__() + + def commit(self, blocking: bool = True): + self._steps.blocking = blocking + steps = self._steps + self._steps = Steps(ctx=self) + return steps + + def commit_and_get_results(self, blocking: bool = True): + self._steps.blocking = blocking + self._steps.get_results = True + steps = self._steps + self._steps = Steps(ctx=self) + return steps + def __getattr__(self, item): + def add_step(*args, **kwargs) -> int: + idx = len(self._steps) + self._steps.append({item: (args, kwargs)}) + return idx -class ServiceState(enum.Enum): - """ THIS SHOULD BE PART OF THE API""" - new = 'new' - deploying = 'deploying' - deployed = 'deployed' - ready = 'ready' - shutdown = 'shutting down' - terminated = 'terminated' - undefined = 'undefined' + return add_step + + +@dataclass +class ServiceSignal: + """ THIS WOULD BE PART OF THE API CODE""" + message: Any + response_to: Optional["ServiceSignal"] = None + + +class ConfigurationError(Exception): + """ THIS WOULD BE PART OF THE API CODE""" + pass -# """ THIS SHOULD BE PART OF THE API""" -SERVICE_STATE_AVAILABLE = ( - ServiceState.new, - ServiceState.deploying, - ServiceState.deployed, - ServiceState.ready, - ServiceState.shutdown -) +class ServiceState(statemachine.StateMachine): + """ THIS WOULD BE PART OF THE API CODE""" + new = statemachine.State("new", initial=True) + deploying = statemachine.State("deploying") + deployed = statemachine.State("deployed") + starting = statemachine.State("starting") + running = statemachine.State("running") + stopping = statemachine.State("stopping") + stopped = statemachine.State("stopped") + terminated = statemachine.State("terminated") + unresponsive = statemachine.State("unresponsive") + + deploy = new.to(deploying) + end_deploy = deploying.to(deployed) + start = deployed.to(starting) + ready = running.from_(new, starting, deployed, deploying) + stop = running.to(stopping) + end_stop = stopping.to(stopped) + terminate = terminated.from_(new, deploying, deployed, starting, running, stopping, stopped) + mark_unresponsive = unresponsive.from_(new, deploying, deployed, starting, running, stopping, stopped) + + AVAILABLE = ( + new, deploying, deployed, starting, running, stopping + ) class Service: - """ THIS SHOULD BE PART OF THE API""" - state: typing.Optional[ServiceState] = None + """ THIS WOULD BE PART OF THE API CODE""" - def __init__(self, ctx: WorkContext): + def __init__(self, ctx: WorkContext, initial_state: statemachine.State = ServiceState.new): self.ctx = ctx - self.state = ServiceState.new # should state correspond with the ActivityState? - @staticmethod - def get_payload() -> typing.Optional[Payload]: - """Return the payload (runtime) definition for this service. + assert initial_state in ServiceState.states + self.__state: ServiceState = ServiceState(start_value=initial_state.value) - If `get_payload` is not implemented, the payload will need to be provided in the - `Executor.run_service` call. - """ - pass + self.__inqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() + self.__outqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() - async def on_deploy(self, out: bytes): # maybe `out` is a structure similar to what we get from the Activity API itself ? - self.state = ServiceState.deployed + def __repr__(self): + return f"<{self.__class__.__name__}: ctx: {self.ctx}, state: {self.state.value}>" - async def on_start(self, out: bytes): - self.state = ServiceState.ready + async def send_message(self, message: Any = None): + await self.__inqueue.put(ServiceSignal(message=message)) - async def on_new(self): - self.state = ServiceState.deploying - self.ctx.deploy(on_deploy=self.on_deploy) - self.ctx.start(on_start=self.on_start) - yield self.ctx.commit() + def send_message_nowait(self, message: Optional[Any] = None): + self.__inqueue.put_nowait(ServiceSignal(message=message)) - async def execute_batch(self, batch: typing.Optional[Work]): - if batch: - executor.execute(batch) # some automagic of passing it for execution ;) + async def receive_message(self) -> ServiceSignal: + return await self.__outqueue.get() - async def run(self): - # we need some way to pass a signal (or a queue of messages?) into `run` ... - # or some other event handler inside `Service` + def receive_message_nowait(self) -> Optional[ServiceSignal]: + try: + return self.__outqueue.get_nowait() + except asyncio.QueueEmpty: + pass - # the simplest could be some cancellation token (e.g. an asyncio.Event) - # but maybe we would like to pass some more data with it? + async def _listen(self) -> ServiceSignal: + return await self.__inqueue.get() - # maybe the state of the service could be a pair or ServiceState - # plus some additional data object? + def _listen_nowait(self) -> Optional[ServiceSignal]: + try: + return self.__inqueue.get_nowait() + except asyncio.QueueEmpty: + pass - # better yet -> maybe `run` could take an _input_ async generator + async def _respond(self, message: Optional[Any], response_to: Optional[ServiceSignal] = None): + await self.__outqueue.put(ServiceSignal(message=message, response_to=response_to)) - while self.state in SERVICE_STATE_AVAILABLE: - _handlers = { - ServiceState.new: self.on_new, - ServiceState.ready: self.on_ready, - ServiceState.shutdown: self.on_shutdown, - } + def _respond_nowait(self, message: Optional[Any], response_to: Optional[ServiceSignal] = None): + self.__outqueue.put_nowait(ServiceSignal(message=message, response_to=response_to)) - handler = _handlers.get(self.state) - if handler: - async for batch in handler(): - yield self.execute_batch(batch) + @staticmethod + def get_payload() -> typing.Optional[Payload]: + """Return the payload (runtime) definition for this service. - # we could add something like e.g. Shutdown(Work) step - # that would signal the service executor to transition the service - # to the shutdown state + If `get_payload` is not implemented, the payload will need to be provided in the + `Executor.run_service` call. + """ + pass - async def on_ready(self, *args, **kwargs): - while True: - print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") - await asyncio.sleep(10) - yield None + async def start(self): + self.state_transition(ServiceState.deploy) + self.ctx.deploy() + self.ctx.start() + yield self.ctx.commit() + self.state_transition(ServiceState.ready) - async def on_shutdown(self): - yield None + async def run(self): + yield + async def shutdown(self): + yield -class TurbogethService(Service): - def __init__(self, ctx: WorkContext): - super().__init__(ctx) - self.credentials = {} + @property + def state(self): + return self.__state.current_state - def get_payload(self): - return TurbogethPayload(rpc_port=8888) + def state_transition(self, transition: statemachine.Transition): + assert self.__state.get_transition(transition.identifier) + self.__state.run(transition.identifier) - async def on_deploy(self, out: bytes): - print("deployed") - self.credentials = json.loads(out.decode("utf-8")) + @property + def is_available(self): + return self.state in ServiceState.AVAILABLE - async def on_start(self, out: bytes): - print("started") + @property + def current_handler(self): + _handlers = { + ServiceState.new: self.start, + ServiceState.running: self.run, + ServiceState.stopping: self.shutdown, + } - async def on_shutdown(self): - self.ctx.download_file("some/service/state", "temp/path") + return _handlers.get(self.state, None) class Cluster: - """ THIS SHOULD BE PART OF THE API""" - def __init__(self, executor: "Executor", service: typing.Type[Service], payload: Payload): + """ THIS WOULD BE PART OF THE API CODE""" + def __init__(self, executor: "Executor", service_class: typing.Type[Service], payload: Payload): self.executor = executor - self.service = service + self.service_class = service_class if not payload: raise ConfigurationError("Payload must be defined when starting a cluster.") @@ -160,37 +229,90 @@ def __init__(self, executor: "Executor", service: typing.Type[Service], payload: self.payload = payload self.instances: typing.List[Service] = [] + def __repr__(self): + return f"Cluster " \ + f"[Service: {self.service_class.__name__}, " \ + f"Payload: {self.payload}]" + async def _run_instance(self, ctx: WorkContext): - instance = Service(ctx) - self.instances.append(instance) + service = self.service_class(ctx) + self.instances.append(service) - print(f"{instance} started") - await instance.run() - print(f"{instance} finished") + print(f"{service} commissioned") - # pass `instance` to some loop in the executor + while service.is_available: + handler = service.current_handler + if handler: + try: + gen = handler() + async for batch in gen: + r = yield batch + await gen.asend(r) + except StopAsyncIteration: + yield - async def spawn_instance( - self, - ): + print(f"{service} decomissioned") + + async def spawn_instance(self): act = await self.executor.get_activity(self.payload) - ctx = WorkContext(act.id) - await self._run_instance(ctx) + ctx = WorkContext(act.id, len(self.instances) + 1) + + gen = self._run_instance(ctx) + async for batch in gen: + r = yield batch + await gen.asend(r) class Executor(typing.AsyncContextManager): """ MOCK OF EXECUTOR JUST SO I COULD ILLUSTRATE THE NEW CALL""" def __init__(self, *args, **kwargs): - pass + print(f"Executor started with {args}, {kwargs}") async def __aenter__(self) -> "Executor": print("start executor") return self async def __aexit__(self, *exc_info): - print("stop executor", exc_info) + import traceback + tb = traceback.print_tb(exc_info[2]) if len(exc_info) > 2 else None + print("stop executor", exc_info, tb) return True + async def get_activity(self, payload): + await asyncio.sleep(random.randint(3, 7)) + return Activity(payload=payload) + + async def _run_batch(self, batch): + results = [] + print(f"EXESCRIPT EXECUTION: {batch} on {batch.ctx.id}") + for command in batch: + print(f"EXESCRIPT COMMAND {command}") + results.append({ + 'command': command, + 'message': 'some data here' + }) + + await asyncio.sleep(random.randint(1,7)) + return results + + async def _run_batches(self, batches: typing.AsyncGenerator): + try: + async for batch in batches: + if batch: + results = self._run_batch(batch) + + if not batch.blocking: + results = asyncio.create_task(results) + + if batch.get_results: + await batches.asend(results) + else: + await results + + except StopAsyncIteration: + print("RUN BATCHES - stop async iteration") + pass + def run_service( self, service_class: typing.Type[Service], @@ -198,12 +320,77 @@ def run_service( payload: typing.Optional[Payload] = None, ) -> Cluster: payload = payload or service_class.get_payload() - cluster = Cluster(executor=self, service=service_class, payload=payload) + cluster = Cluster(executor=self, service_class=service_class, payload=payload) + for i in range(num_instances): - asyncio.create_task(cluster.spawn_instance()) + asyncio.create_task(self._run_batches(cluster.spawn_instance())) return cluster +############################################################### +# # +# # +# CLIENT CODE # +# # +# # +############################################################### + + +TURBOGETH_RUNTIME_NAME = "turbogeth-managed" +PROP_TURBOGETH_RPC_PORT = "golem.srv.app.eth.rpc-port" + + +@dataclass +class TurbogethPayload(Payload, AutodecoratingModel): + rpc_port: int = prop(PROP_TURBOGETH_RPC_PORT, default=None) + + runtime: str = constraint(inf.INF_RUNTIME_NAME, default=TURBOGETH_RUNTIME_NAME) + min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) + min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) + + +INSTANCES_NEEDED = 3 +EXECUTOR_TIMEOUT = timedelta(weeks=100) + + +class TurbogethService(Service): + def __init__(self, ctx: WorkContext): + super().__init__(ctx) + self.credentials = {} + + def __repr__(self): + srv_repr = super().__repr__() + return f"{srv_repr}, credentials: {self.credentials}" + + @staticmethod + def get_payload(): + return TurbogethPayload(rpc_port=8888) + + async def start(self): + self.state_transition(ServiceState.deploy) + deploy_idx = self.ctx.deploy() + self.ctx.start() + future_results = yield self.ctx.commit_and_get_results() + results = await future_results + self.credentials = "RECEIVED" or results[deploy_idx] # (NORMALLY, WOULD BE PARSED) + self.state_transition(ServiceState.ready) + + async def run(self): + print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") + signal = self._listen_nowait() + if signal and signal.message == "go": + self.ctx.run("go!") + yield self.ctx.commit() + else: + await asyncio.sleep(1) + yield + + async def shutdown(self): + self.ctx.download_file("some/service/state", "temp/path") + yield self.ctx.commit() + self.state_transition(ServiceState.terminate) + + async def main(subnet_tag, driver=None, network=None): async with Executor( @@ -214,23 +401,39 @@ async def main(subnet_tag, driver=None, network=None): network=network, event_consumer=log_summary(log_event_repr), ) as executor: - swarm = executor.run_service( + cluster = executor.run_service( TurbogethService, # payload=payload, num_instances=INSTANCES_NEEDED ) - while True: - print(f"{swarm} is running: {swarm.instances}") - await asyncio.sleep(10) + def instances(): + return [{s.ctx.id, s.state.value} for s in cluster.instances] + + def still_running(): + return any([s for s in cluster.instances if s.is_available]) + + cnt = 0 + while cnt < 10: + print(f"instances: {instances()}") + await asyncio.sleep(3) + cnt += 1 + if cnt == 3: + if len(cluster.instances) > 1: + cluster.instances[0].send_message_nowait("go") + + for s in cluster.instances: + s.state_transition(ServiceState.stop) + + print(f"instances: {instances()}") + + cnt = 0 + while cnt < 10 and still_running(): + print(f"instances: {instances()}") + await asyncio.sleep(1) + + print(f"instances: {instances()}") asyncio.run(main(None)) -# notes for next steps -# -# Service.shutdown should signal Executor to call Services' `exit` -# -# when "run" finishes, the service shuts down -# -# next: save/restore of the Service state diff --git a/pyproject.toml b/pyproject.toml index 8142d16cf..38ead494f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ colorama = "^0.4.4" # would not work: see https://github.com/python-poetry/poetry/issues/129. goth = { version = "^0.2.1", optional = true, python = "^3.8.0" } Deprecated = "^1.2.12" +python-statemachine = "^0.8.0" [tool.poetry.extras] cli = ['fire', 'rich'] From 166c4d63cc85f20c0c7e14d9692ae33f14197df5 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Fri, 7 May 2021 22:09:03 +0200 Subject: [PATCH 42/85] remove callbacks from deploy/start --- yapapi/executor/ctx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index a8387609b..c11d92604 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -292,10 +292,10 @@ def __prepare(self): def begin(self): pass - def deploy(self, on_deploy: Callable[[bytes], Awaitable]): + def deploy(self): self._pending_steps.append(_Deploy()) - def start(self, on_start: Callable[[bytes], Awaitable]): + def start(self): self._pending_steps.append(_Start()) def send_json(self, json_path: str, data: dict): From a028595087ca04b6133caa554e6c39708d9a75f2 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 11 May 2021 13:33:14 +0200 Subject: [PATCH 43/85] - comment + rename `Model.keys` to `Model.property_keys` --- examples/turbogeth/turbogeth.py | 14 -------------- yapapi/payload/vm.py | 2 +- yapapi/props/__init__.py | 4 ++-- yapapi/props/base.py | 4 ++-- yapapi/props/builder.py | 2 +- yapapi/props/inf.py | 6 +----- 6 files changed, 7 insertions(+), 25 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index c22604ffd..791595c62 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -29,17 +29,3 @@ async def main(): asyncio.run(main()) - -# notes for next steps -# -# -> Service class -# -# Executor.run_service(Service, num_instance=3) -# -# service instance -> instance of the Service class -# -# Service.shutdown should signal Executor to call Services' `exit` -# -# when "run" finishes, the service shuts down -# -# next: save/restore of the Service state diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 54eaa5238..c1fe68c54 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -25,7 +25,7 @@ class InfVm(InfBase): cores: int = prop_base.prop(INF_CORES, default=1) -InfVmKeys = InfVm.keys() +InfVmKeys = InfVm.property_keys() class VmPackageFormat(Enum): diff --git a/yapapi/props/__init__.py b/yapapi/props/__init__.py index 32034fef0..c1616a871 100644 --- a/yapapi/props/__init__.py +++ b/yapapi/props/__init__.py @@ -16,7 +16,7 @@ class NodeInfo(Model): """the name of the subnet within which the Demands and Offers are matched""" -NodeInfoKeys = NodeInfo.keys() +NodeInfoKeys = NodeInfo.property_keys() @dataclass() @@ -60,4 +60,4 @@ class Activity(Model): """ -ActivityKeys = Activity.keys() +ActivityKeys = Activity.property_keys() diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 4e46560e3..99f75ebf0 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -141,7 +141,7 @@ def from_properties(cls: Type[ME], props: Props) -> ME: raise InvalidPropertiesError(msg) from exc @classmethod - def keys(cls): # change to `property_keys` ? + def property_keys(cls): """ :return: a mapping between the model's field names and the property keys @@ -155,7 +155,7 @@ def keys(cls): # change to `property_keys` ? ... name: typing.Optional[str] = \ ... dataclasses.field(default=None, metadata={"key": "golem.node.id.name"}) ... - >>> NodeInfo.keys().name + >>> NodeInfo.property_keys().name 'golem.node.id.name' ``` """ diff --git a/yapapi/props/builder.py b/yapapi/props/builder.py index fb9b56699..4a6b42bbd 100644 --- a/yapapi/props/builder.py +++ b/yapapi/props/builder.py @@ -61,7 +61,7 @@ def ensure(self, constraint: str): def add(self, m: Model): """Add properties from the specified model to this demand definition.""" - kv = m.keys() + kv = m.property_keys() base = asdict(m) for name in kv.names(): diff --git a/yapapi/props/inf.py b/yapapi/props/inf.py index e5d047b0f..231c33730 100644 --- a/yapapi/props/inf.py +++ b/yapapi/props/inf.py @@ -46,12 +46,8 @@ class InfVm(InfBase): runtime = RUNTIME_VM cores: int = prop(INF_CORES, default=1) - @classmethod - def keys(cls): - return super().keys() - -InfVmKeys = InfVm.keys() +InfVmKeys = InfVm.property_keys() @dataclass From 94df0ee6799210afdf4107858dce0bb89fdbace1 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 11 May 2021 13:41:50 +0200 Subject: [PATCH 44/85] make `Payload` an AutodecoratingModel --- examples/turbogeth/turbogeth.py | 6 +++--- yapapi/payload/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 791595c62..ef635e52c 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from yapapi.props.builder import DemandBuilder, AutodecoratingModel +from yapapi.props.builder import DemandBuilder from yapapi.props.base import prop, constraint from yapapi.props import inf @@ -14,7 +14,7 @@ @dataclass -class Turbogeth(Payload, AutodecoratingModel): +class TurbogethPayload(Payload): rpc_port: int = prop(PROP_TURBOGETH_RPC_PORT, default=None) runtime: str = constraint(inf.INF_RUNTIME_NAME, default=TURBOGETH_RUNTIME_NAME) @@ -24,7 +24,7 @@ class Turbogeth(Payload, AutodecoratingModel): async def main(): builder = DemandBuilder() - await builder.decorate(Turbogeth(rpc_port=1234)) + await builder.decorate(TurbogethPayload(rpc_port=1234)) print(builder) diff --git a/yapapi/payload/__init__.py b/yapapi/payload/__init__.py index 4f352a136..23819bf61 100644 --- a/yapapi/payload/__init__.py +++ b/yapapi/payload/__init__.py @@ -1,7 +1,7 @@ import abc -from yapapi.props.builder import DemandDecorator +from yapapi.props.builder import AutodecoratingModel -class Payload(DemandDecorator, abc.ABC): +class Payload(AutodecoratingModel, abc.ABC): """Base class for descriptions of the payload required by the requestor.""" From 8a5f0ef31c83200f975c40324cadbea23c7ff039 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 11 May 2021 13:47:01 +0200 Subject: [PATCH 45/85] black --- yapapi/executor/__init__.py | 6 ++++-- yapapi/package/vm.py | 12 +++++++----- yapapi/payload/vm.py | 4 +--- yapapi/props/base.py | 22 ++++++++++------------ yapapi/props/builder.py | 2 +- yapapi/props/inf.py | 7 ++++++- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 592aff0a2..7296698ac 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -123,7 +123,7 @@ def __init__( network: Optional[str] = None, event_consumer: Optional[Callable[[Event], None]] = None, stream_output: bool = False, - payload: Optional[Payload] = None + payload: Optional[Payload] = None, ): """Create a new executor. @@ -164,7 +164,9 @@ def __init__( if package: if payload: raise TypeError("Cannot use `payload` and `package` at the same time") - logger.warning(f"`package` argument to `{self.__class__}` is deprecated, please use `payload` instead") + logger.warning( + f"`package` argument to `{self.__class__}` is deprecated, please use `payload` instead" + ) payload = package if not payload: raise TypeError("Executor `payload` must be specified") diff --git a/yapapi/package/vm.py b/yapapi/package/vm.py index 72a425155..d68b944d4 100644 --- a/yapapi/package/vm.py +++ b/yapapi/package/vm.py @@ -2,7 +2,11 @@ import logging from yapapi.payload.package import Package -from yapapi.payload.vm import repo as _repo, resolve_repo_srv as _resolve_repo_srv, _FALLBACK_REPO_URL +from yapapi.payload.vm import ( + repo as _repo, + resolve_repo_srv as _resolve_repo_srv, + _FALLBACK_REPO_URL, +) logger = logging.getLogger(__name__) @@ -20,9 +24,7 @@ async def repo( """ return await _repo( - image_hash=image_hash, - min_mem_gib=min_mem_gib, - min_storage_gib=min_storage_gib + image_hash=image_hash, min_mem_gib=min_mem_gib, min_storage_gib=min_storage_gib ) @@ -36,4 +38,4 @@ def resolve_repo_srv(repo_srv, fallback_url=_FALLBACK_REPO_URL) -> str: :return: the url of the package repository containing the port :raises: PackageException if no valid service could be reached """ - return _resolve_repo_srv(repo_srv=repo_srv, fallback_url=fallback_url) \ No newline at end of file + return _resolve_repo_srv(repo_srv=repo_srv, fallback_url=fallback_url) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index c1fe68c54..b6e9d96fe 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -46,9 +46,7 @@ class _VmConstraints: runtime: str = prop_base.constraint(inf.INF_RUNTIME_NAME, operator="=", default=RUNTIME_VM) def __str__(self): - return prop_base.join_str_constraints( - prop_base.constraint_model_serialize(self) - ) + return prop_base.join_str_constraints(prop_base.constraint_model_serialize(self)) @dataclass diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 99f75ebf0..a68d64559 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -1,4 +1,5 @@ from typing import Dict, Type, Any, Union, List, cast, TypeVar + try: from typing import Literal except ImportError: @@ -101,7 +102,8 @@ def property_fields(cls): f for f in fields(cls) if PROP_KEY in f.metadata - and f.metadata.get(PROP_MODEL_FIELD_TYPE, ModelFieldType.property) == ModelFieldType.property + and f.metadata.get(PROP_MODEL_FIELD_TYPE, ModelFieldType.property) + == ModelFieldType.property ) @classmethod @@ -176,7 +178,7 @@ class ConstraintException(Exception): CONSTRAINT_VAL_ANY = "*" -ConstraintOperator = Literal['=', ">=", "<="] +ConstraintOperator = Literal["=", ">=", "<="] ConstraintGroupOperator = Literal["&", "|", "!"] @@ -193,18 +195,14 @@ def constraint(key: str, *, operator: ConstraintOperator = "=", default=MISSING) PROP_KEY: key, PROP_OPERATOR: operator, PROP_MODEL_FIELD_TYPE: ModelFieldType.constraint, - } + }, ) def prop(key: str, *, default=MISSING): """return a property-type dataclass field""" return field( - default=default, - metadata={ - PROP_KEY: key, - PROP_MODEL_FIELD_TYPE: ModelFieldType.property - } + default=default, metadata={PROP_KEY: key, PROP_MODEL_FIELD_TYPE: ModelFieldType.property} ) @@ -225,10 +223,10 @@ def join_str_constraints(constraints: List[str], operator: ConstraintGroupOperat return "()" if operator == "!": - if len(constraints) == 1: - return f"({operator}({constraints[0]}))" - else: - raise ConstraintException(f"{operator} requires exactly one component.") + if len(constraints) == 1: + return f"({operator}({constraints[0]}))" + else: + raise ConstraintException(f"{operator} requires exactly one component.") if len(constraints) == 1: return f"({constraints[0]})" diff --git a/yapapi/props/builder.py b/yapapi/props/builder.py index 4a6b42bbd..67d14c1e4 100644 --- a/yapapi/props/builder.py +++ b/yapapi/props/builder.py @@ -80,7 +80,7 @@ async def subscribe(self, market: Market) -> Subscription: """Create a Demand on the market and subscribe to Offers that will match that Demand.""" return await market.subscribe(self._properties, self.constraints) - async def decorate(self, *decorators: 'DemandDecorator'): + async def decorate(self, *decorators: "DemandDecorator"): for decorator in decorators: await decorator.decorate_demand(self) diff --git a/yapapi/props/inf.py b/yapapi/props/inf.py index 231c33730..4933fb66d 100644 --- a/yapapi/props/inf.py +++ b/yapapi/props/inf.py @@ -16,8 +16,13 @@ RUNTIME_EMSCRIPTEN = "emscripten" RUNTIME_VM = "vm" + @dataclass -@deprecated(version="0.6.0", reason="please use yapapi.props.inf.RUNTIME_* constants directly", action="default") +@deprecated( + version="0.6.0", + reason="please use yapapi.props.inf.RUNTIME_* constants directly", + action="default", +) class RuntimeType(Enum): UNKNOWN = "" WASMTIME = RUNTIME_WASMTIME From 621946ab1c3fdc45cd3f6fd7830b0c32da240efc Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 11 May 2021 15:18:48 +0200 Subject: [PATCH 46/85] update with @stranger80's remarks --- examples/turbogeth/turbogeth.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 2491b4231..05fd14686 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -104,26 +104,20 @@ class ConfigurationError(Exception): class ServiceState(statemachine.StateMachine): """ THIS WOULD BE PART OF THE API CODE""" new = statemachine.State("new", initial=True) - deploying = statemachine.State("deploying") - deployed = statemachine.State("deployed") starting = statemachine.State("starting") running = statemachine.State("running") stopping = statemachine.State("stopping") - stopped = statemachine.State("stopped") terminated = statemachine.State("terminated") unresponsive = statemachine.State("unresponsive") - deploy = new.to(deploying) - end_deploy = deploying.to(deployed) - start = deployed.to(starting) - ready = running.from_(new, starting, deployed, deploying) + start = new.to(starting) + ready = running.from_(new, starting) stop = running.to(stopping) - end_stop = stopping.to(stopped) - terminate = terminated.from_(new, deploying, deployed, starting, running, stopping, stopped) - mark_unresponsive = unresponsive.from_(new, deploying, deployed, starting, running, stopping, stopped) + terminate = terminated.from_(new, starting, running, stopping, terminated) + mark_unresponsive = unresponsive.from_(new, starting, running, stopping, terminated) AVAILABLE = ( - new, deploying, deployed, starting, running, stopping + new, starting, running, stopping ) @@ -182,7 +176,7 @@ def get_payload() -> typing.Optional[Payload]: pass async def start(self): - self.state_transition(ServiceState.deploy) + self.state_transition(ServiceState.start) self.ctx.deploy() self.ctx.start() yield self.ctx.commit() @@ -367,7 +361,7 @@ def get_payload(): return TurbogethPayload(rpc_port=8888) async def start(self): - self.state_transition(ServiceState.deploy) + self.state_transition(ServiceState.start) deploy_idx = self.ctx.deploy() self.ctx.start() future_results = yield self.ctx.commit_and_get_results() From b700dc6e517747856b7fa8fc0179dbb21b2abd4f Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 12 May 2021 01:37:23 +0200 Subject: [PATCH 47/85] address concerns from conversation with @stranger80 --- examples/turbogeth/turbogeth.py | 219 +++++++++++++++++++------------- 1 file changed, 134 insertions(+), 85 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 05fd14686..cf48147ad 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -1,4 +1,5 @@ import asyncio +import enum from datetime import timedelta import typing from typing import Optional, Any @@ -45,13 +46,12 @@ class Steps(list): def __init__(self, *args, **kwargs): self.ctx: WorkContext = kwargs.pop('ctx') self.blocking: bool = kwargs.pop('blocking', True) - self.get_results: bool = kwargs.pop('get_results', False) super().__init__(*args, **kwargs) def __repr__(self): list_repr = super().__repr__() - return f"<{self.__class__.__name__}: {list_repr}, blocking: {self.blocking}, get_results: {self.get_results}" + return f"<{self.__class__.__name__}: {list_repr}, blocking: {self.blocking}" class WorkContext: """ Mock WorkContext to illustrate the idea. """ @@ -73,13 +73,6 @@ def commit(self, blocking: bool = True): self._steps = Steps(ctx=self) return steps - def commit_and_get_results(self, blocking: bool = True): - self._steps.blocking = blocking - self._steps.get_results = True - steps = self._steps - self._steps = Steps(ctx=self) - return steps - def __getattr__(self, item): def add_step(*args, **kwargs) -> int: idx = len(self._steps) @@ -103,38 +96,44 @@ class ConfigurationError(Exception): class ServiceState(statemachine.StateMachine): """ THIS WOULD BE PART OF THE API CODE""" - new = statemachine.State("new", initial=True) - starting = statemachine.State("starting") + starting = statemachine.State("starting", initial=True) running = statemachine.State("running") stopping = statemachine.State("stopping") terminated = statemachine.State("terminated") unresponsive = statemachine.State("unresponsive") - start = new.to(starting) - ready = running.from_(new, starting) + ready = starting.to(running) stop = running.to(stopping) - terminate = terminated.from_(new, starting, running, stopping, terminated) - mark_unresponsive = unresponsive.from_(new, starting, running, stopping, terminated) + terminate = terminated.from_(starting, running, stopping, terminated) + mark_unresponsive = unresponsive.from_(starting, running, stopping, terminated) + + lifecycle = ready | stop | terminate AVAILABLE = ( - new, starting, running, stopping + starting, running, stopping ) class Service: """ THIS WOULD BE PART OF THE API CODE""" - def __init__(self, ctx: WorkContext, initial_state: statemachine.State = ServiceState.new): + def __init__(self, cluster: "Cluster", ctx: WorkContext): + self.cluster = cluster self.ctx = ctx - assert initial_state in ServiceState.states - self.__state: ServiceState = ServiceState(start_value=initial_state.value) - self.__inqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() self.__outqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() + self.post_init() + + def post_init(self): + pass + + @property + def id(self): + return self.ctx.id def __repr__(self): - return f"<{self.__class__.__name__}: ctx: {self.ctx}, state: {self.state.value}>" + return f"<{self.__class__.__name__}: {self.id}>" async def send_message(self, message: Any = None): await self.__inqueue.put(ServiceSignal(message=message)) @@ -176,43 +175,53 @@ def get_payload() -> typing.Optional[Payload]: pass async def start(self): - self.state_transition(ServiceState.start) self.ctx.deploy() self.ctx.start() yield self.ctx.commit() - self.state_transition(ServiceState.ready) async def run(self): - yield + while True: + yield async def shutdown(self): yield + @property + def is_available(self): + return self.cluster.get_state(self) in ServiceState.AVAILABLE + @property def state(self): - return self.__state.current_state + return self.cluster.get_state(self) - def state_transition(self, transition: statemachine.Transition): - assert self.__state.get_transition(transition.identifier) - self.__state.run(transition.identifier) - @property - def is_available(self): - return self.state in ServiceState.AVAILABLE +class ControlSignal(enum.Enum): + stop = "stop" + +@dataclass +class ServiceInstance: + """ THIS WOULD BE PART OF THE API CODE""" + service: Service + control_queue: "asyncio.Queue[ControlSignal]" = field(default_factory=asyncio.Queue) + service_state: ServiceState = field(default_factory=ServiceState) @property - def current_handler(self): - _handlers = { - ServiceState.new: self.start, - ServiceState.running: self.run, - ServiceState.stopping: self.shutdown, - } + def state(self) -> ServiceState: + return self.service_state.current_state - return _handlers.get(self.state, None) + def get_control_signal(self) -> typing.Optional[ControlSignal]: + try: + return self.control_queue.get_nowait() + except asyncio.QueueEmpty: + pass + + def send_control_signal(self, signal: ControlSignal): + self.control_queue.put_nowait(signal) class Cluster: """ THIS WOULD BE PART OF THE API CODE""" + def __init__(self, executor: "Executor", service_class: typing.Type[Service], payload: Payload): self.executor = executor self.service_class = service_class @@ -221,40 +230,89 @@ def __init__(self, executor: "Executor", service_class: typing.Type[Service], pa raise ConfigurationError("Payload must be defined when starting a cluster.") self.payload = payload - self.instances: typing.List[Service] = [] + self.__instances: typing.List[ServiceInstance] = [] def __repr__(self): return f"Cluster " \ f"[Service: {self.service_class.__name__}, " \ f"Payload: {self.payload}]" + @property + def instances(self) -> typing.List[Service]: + return [i.service for i in self.__instances] + + def __get_service_instance(self, service: Service) -> ServiceInstance: + for i in self.__instances: + if i.service == service: + return i + + def get_state(self, service: Service): + instance = self.__get_service_instance(service) + return instance.state + + @staticmethod + def _get_handler(instance: ServiceInstance): + _handlers = { + ServiceState.starting: instance.service.start, + ServiceState.running: instance.service.run, + ServiceState.stopping: instance.service.shutdown, + } + handler = _handlers.get(instance.state, None) + if handler: + return handler() + async def _run_instance(self, ctx: WorkContext): - service = self.service_class(ctx) - self.instances.append(service) + loop = asyncio.get_event_loop() + instance = ServiceInstance(service=self.service_class(self, ctx)) + self.__instances.append(instance) - print(f"{service} commissioned") + print(f"{instance.service} commissioned") - while service.is_available: - handler = service.current_handler - if handler: - try: - gen = handler() - async for batch in gen: - r = yield batch - await gen.asend(r) - except StopAsyncIteration: - yield + handler = self._get_handler(instance) + batch = None - print(f"{service} decomissioned") + while handler: + try: + if batch: + r = yield batch + fr = loop.create_future() + fr.set_result(await r) + batch = await handler.asend(fr) + else: + batch = await handler.__anext__() + except StopAsyncIteration: + instance.service_state.lifecycle() + handler = self._get_handler(instance) + batch = None + + ctl = instance.get_control_signal() + if ctl == ControlSignal.stop: + if instance.state == ServiceState.running: + instance.service_state.stop() + else: + instance.service_state.terminate() + + handler = self._get_handler(instance) + batch = None + + print(f"{instance.service} decomissioned") async def spawn_instance(self): act = await self.executor.get_activity(self.payload) ctx = WorkContext(act.id, len(self.instances) + 1) - gen = self._run_instance(ctx) - async for batch in gen: - r = yield batch - await gen.asend(r) + instance_batches = self._run_instance(ctx) + try: + batch = await instance_batches.__anext__() + while batch: + r = yield batch + batch = await instance_batches.asend(r) + except StopAsyncIteration: + pass + + def stop_instance(self, service: Service): + instance = self.__get_service_instance(service) + instance.send_control_signal(ControlSignal.stop) class Executor(typing.AsyncContextManager): @@ -287,22 +345,15 @@ async def _run_batch(self, batch): }) await asyncio.sleep(random.randint(1,7)) + print(f"RETURNING RESULTS: {batch} on {batch.ctx.id}") return results async def _run_batches(self, batches: typing.AsyncGenerator): try: - async for batch in batches: - if batch: - results = self._run_batch(batch) - - if not batch.blocking: - results = asyncio.create_task(results) - - if batch.get_results: - await batches.asend(results) - else: - await results - + batch = await batches.__anext__() + while batch: + results = self._run_batch(batch) + batch = await batches.asend(results) except StopAsyncIteration: print("RUN BATCHES - stop async iteration") pass @@ -348,8 +399,9 @@ class TurbogethPayload(Payload, AutodecoratingModel): class TurbogethService(Service): - def __init__(self, ctx: WorkContext): - super().__init__(ctx) + credentials = None + + def post_init(self): self.credentials = {} def __repr__(self): @@ -361,28 +413,26 @@ def get_payload(): return TurbogethPayload(rpc_port=8888) async def start(self): - self.state_transition(ServiceState.start) deploy_idx = self.ctx.deploy() self.ctx.start() - future_results = yield self.ctx.commit_and_get_results() + future_results = yield self.ctx.commit() results = await future_results self.credentials = "RECEIVED" or results[deploy_idx] # (NORMALLY, WOULD BE PARSED) - self.state_transition(ServiceState.ready) async def run(self): - print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") - signal = self._listen_nowait() - if signal and signal.message == "go": - self.ctx.run("go!") - yield self.ctx.commit() - else: - await asyncio.sleep(1) - yield + while True: + print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") + signal = self._listen_nowait() + if signal and signal.message == "go": + self.ctx.run("go!") + yield self.ctx.commit() + else: + await asyncio.sleep(1) + yield async def shutdown(self): self.ctx.download_file("some/service/state", "temp/path") yield self.ctx.commit() - self.state_transition(ServiceState.terminate) async def main(subnet_tag, driver=None, network=None): @@ -417,7 +467,7 @@ def still_running(): cluster.instances[0].send_message_nowait("go") for s in cluster.instances: - s.state_transition(ServiceState.stop) + cluster.stop_instance(s) print(f"instances: {instances()}") @@ -426,7 +476,6 @@ def still_running(): print(f"instances: {instances()}") await asyncio.sleep(1) - print(f"instances: {instances()}") asyncio.run(main(None)) From 377c29f9d2fc9a8b94d5db9ca81209870ba25e26 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 12 May 2021 01:52:33 +0200 Subject: [PATCH 48/85] DON'T busy-loop by default :/ --- examples/turbogeth/turbogeth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index cf48147ad..c3c1b7699 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -181,6 +181,7 @@ async def start(self): async def run(self): while True: + await asyncio.sleep(10) yield async def shutdown(self): From 18de3967f216695b6c339fef486ea722cb095839 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 12 May 2021 02:04:07 +0200 Subject: [PATCH 49/85] two more debug messages for illustration --- examples/turbogeth/turbogeth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index c3c1b7699..5e3fd37c8 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -285,6 +285,7 @@ async def _run_instance(self, ctx: WorkContext): instance.service_state.lifecycle() handler = self._get_handler(instance) batch = None + print(f"{instance.service} state changed to {instance.state.value}") ctl = instance.get_control_signal() if ctl == ControlSignal.stop: @@ -293,6 +294,8 @@ async def _run_instance(self, ctx: WorkContext): else: instance.service_state.terminate() + print(f"{instance.service} state changed to {instance.state.value}") + handler = self._get_handler(instance) batch = None From d750ffab1e4496ca70dafebb954f7fc1ab81a1d2 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 12 May 2021 09:31:13 +0200 Subject: [PATCH 50/85] actually, `Golem` --- examples/turbogeth/turbogeth.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 5e3fd37c8..89faa25fc 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -170,7 +170,7 @@ def get_payload() -> typing.Optional[Payload]: """Return the payload (runtime) definition for this service. If `get_payload` is not implemented, the payload will need to be provided in the - `Executor.run_service` call. + `Golem.run_service` call. """ pass @@ -223,7 +223,7 @@ def send_control_signal(self, signal: ControlSignal): class Cluster: """ THIS WOULD BE PART OF THE API CODE""" - def __init__(self, executor: "Executor", service_class: typing.Type[Service], payload: Payload): + def __init__(self, executor: "Golem", service_class: typing.Type[Service], payload: Payload): self.executor = executor self.service_class = service_class @@ -319,12 +319,12 @@ def stop_instance(self, service: Service): instance.send_control_signal(ControlSignal.stop) -class Executor(typing.AsyncContextManager): +class Golem(typing.AsyncContextManager): """ MOCK OF EXECUTOR JUST SO I COULD ILLUSTRATE THE NEW CALL""" def __init__(self, *args, **kwargs): - print(f"Executor started with {args}, {kwargs}") + print(f"Golem started with {args}, {kwargs}") - async def __aenter__(self) -> "Executor": + async def __aenter__(self) -> "Golem": print("start executor") return self @@ -441,15 +441,15 @@ async def shutdown(self): async def main(subnet_tag, driver=None, network=None): - async with Executor( + async with Golem( max_workers=INSTANCES_NEEDED, budget=10.0, subnet_tag=subnet_tag, driver=driver, network=network, event_consumer=log_summary(log_event_repr), - ) as executor: - cluster = executor.run_service( + ) as golem: + cluster = golem.run_service( TurbogethService, # payload=payload, num_instances=INSTANCES_NEEDED From 91f19a498e2b2bd8e95e211a2351dceac4a3b4c0 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 12 May 2021 10:05:20 +0200 Subject: [PATCH 51/85] + comment on a potential issue + solution --- examples/turbogeth/turbogeth.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index 89faa25fc..c461d337d 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -287,6 +287,13 @@ async def _run_instance(self, ctx: WorkContext): batch = None print(f"{instance.service} state changed to {instance.state.value}") + # two potential issues: + # * awaiting a batch makes use lose an ability to interpret a signal (await on generator won't return) + # * we may be losing a `batch` when we act on the control signal + # + # potential solution: + # * use `aiostream.stream.merge` + ctl = instance.get_control_signal() if ctl == ControlSignal.stop: if instance.state == ServiceState.running: From 4c10b73c9d1acb692ef0204f58b14b47d6bfcde6 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 12 May 2021 11:55:36 +0200 Subject: [PATCH 52/85] + `test_payload` --- tests/payload/test_payload.py | 24 ++++++++++++++++++++++++ yapapi/props/__init__.py | 2 +- yapapi/props/base.py | 11 ++++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tests/payload/test_payload.py diff --git a/tests/payload/test_payload.py b/tests/payload/test_payload.py new file mode 100644 index 000000000..ce5275531 --- /dev/null +++ b/tests/payload/test_payload.py @@ -0,0 +1,24 @@ +import pytest + +from dataclasses import dataclass +from yapapi.props import constraint, prop, inf +from yapapi.props.builder import DemandBuilder +from yapapi.payload import Payload + + +@dataclass +class _FooPayload(Payload): + port: int = prop("golem.srv.app.foo.port", default=None) + + runtime: str = constraint(inf.INF_RUNTIME_NAME, default="foo") + min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) + min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) + + +@pytest.mark.asyncio +async def test_payload(): + builder = DemandBuilder() + await builder.decorate(_FooPayload(port=1234, min_mem_gib=32)) + assert builder.properties == {'golem.srv.app.foo.port': 1234} + assert builder.constraints == '((&(golem.runtime.name=foo)\n\t(golem.inf.mem.gib>=32)\n\t(golem.inf.storage.gib>=1024)))' + diff --git a/yapapi/props/__init__.py b/yapapi/props/__init__.py index c1616a871..d5d8752b6 100644 --- a/yapapi/props/__init__.py +++ b/yapapi/props/__init__.py @@ -1,4 +1,4 @@ -from .base import InvalidPropertiesError, Model +from .base import InvalidPropertiesError, Model, prop, constraint from dataclasses import dataclass, field from typing import Optional from decimal import Decimal diff --git a/yapapi/props/base.py b/yapapi/props/base.py index a68d64559..6f6e3c1ae 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -235,4 +235,13 @@ def join_str_constraints(constraints: List[str], operator: ConstraintGroupOperat return f"({operator}{rules})" -__all__ = ("Model", "as_list", "Props") +__all__ = ( + "Model", + "as_list", + "Props", + "constraint", + "prop", + "constraint_to_str", + "constraint_model_serialize", + "join_str_constraints", +) From 47182bc12f95d2508450542bc0c0cba7fe8a81f5 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 12 May 2021 12:12:48 +0200 Subject: [PATCH 53/85] + tests for AutodecoratingModel --- tests/props/test_builder.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/props/test_builder.py diff --git a/tests/props/test_builder.py b/tests/props/test_builder.py new file mode 100644 index 000000000..b032f4f9c --- /dev/null +++ b/tests/props/test_builder.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +import pytest + +from yapapi.props import prop, constraint +from yapapi.props.builder import AutodecoratingModel, DemandBuilder + + +@pytest.mark.asyncio +async def test_autodecorating_model(): + @dataclass + class Foo(AutodecoratingModel): + bar: str = prop("some.bar") + max_baz: int = constraint("baz", operator="<=", default=100) + + foo = Foo(bar="a nice one", max_baz=50) + demand = DemandBuilder() + await foo.decorate_demand(demand) + assert demand.properties == {"some.bar": "a nice one"} + assert demand.constraints == "(((baz<=50)))" From 3c150f6afd2220b21813bd21fc686daa397e2f8b Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 12 May 2021 15:30:22 +0200 Subject: [PATCH 54/85] improve the payload definition ;) --- examples/turbogeth/turbogeth.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index faf940040..f2fa5f2cf 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -395,12 +395,12 @@ def run_service( TURBOGETH_RUNTIME_NAME = "turbogeth-managed" -PROP_TURBOGETH_RPC_PORT = "golem.srv.app.eth.rpc-port" +PROP_TURBOGETH_CHAIN = "golem.srv.app.eth.chain" @dataclass class TurbogethPayload(Payload): - rpc_port: int = prop(PROP_TURBOGETH_RPC_PORT, default=None) + chain: str = prop(PROP_TURBOGETH_CHAIN) runtime: str = constraint(inf.INF_RUNTIME_NAME, default=TURBOGETH_RUNTIME_NAME) min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) @@ -423,7 +423,7 @@ def __repr__(self): @staticmethod def get_payload(): - return TurbogethPayload(rpc_port=8888) + return TurbogethPayload(chain="rinkeby") async def start(self): deploy_idx = self.ctx.deploy() @@ -433,6 +433,7 @@ async def start(self): self.credentials = "RECEIVED" or results[deploy_idx] # (NORMALLY, WOULD BE PARSED) async def run(self): + while True: print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") signal = self._listen_nowait() From b1ab842bcca2a5f3cac78cf762b81ce1af094a12 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 13 May 2021 18:31:19 +0200 Subject: [PATCH 55/85] black --- examples/turbogeth/turbogeth.py | 43 +++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index f7853353e..b4f3353a5 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -8,6 +8,7 @@ import statemachine from dataclasses import dataclass, field + # from yapapi.executor.ctx import WorkContext from yapapi.props.builder import DemandBuilder @@ -29,23 +30,27 @@ _act_cnt = 0 + def _act_id(): global _act_cnt _act_cnt += 1 return _act_cnt + @dataclass class Activity: """ Mock Activity """ + payload: Payload id: int = field(default_factory=_act_id) class Steps(list): """ Mock Steps to illustrate the idea. """ + def __init__(self, *args, **kwargs): - self.ctx: WorkContext = kwargs.pop('ctx') - self.blocking: bool = kwargs.pop('blocking', True) + self.ctx: WorkContext = kwargs.pop("ctx") + self.blocking: bool = kwargs.pop("blocking", True) super().__init__(*args, **kwargs) @@ -53,6 +58,7 @@ def __repr__(self): list_repr = super().__repr__() return f"<{self.__class__.__name__}: {list_repr}, blocking: {self.blocking}" + class WorkContext: """ Mock WorkContext to illustrate the idea. """ @@ -85,17 +91,20 @@ def add_step(*args, **kwargs) -> int: @dataclass class ServiceSignal: """ THIS WOULD BE PART OF THE API CODE""" + message: Any response_to: Optional["ServiceSignal"] = None class ConfigurationError(Exception): """ THIS WOULD BE PART OF THE API CODE""" + pass class ServiceState(statemachine.StateMachine): """ THIS WOULD BE PART OF THE API CODE""" + starting = statemachine.State("starting", initial=True) running = statemachine.State("running") stopping = statemachine.State("stopping") @@ -109,9 +118,7 @@ class ServiceState(statemachine.StateMachine): lifecycle = ready | stop | terminate - AVAILABLE = ( - starting, running, stopping - ) + AVAILABLE = (starting, running, stopping) class Service: @@ -199,9 +206,11 @@ def state(self): class ControlSignal(enum.Enum): stop = "stop" + @dataclass class ServiceInstance: """ THIS WOULD BE PART OF THE API CODE""" + service: Service control_queue: "asyncio.Queue[ControlSignal]" = field(default_factory=asyncio.Queue) service_state: ServiceState = field(default_factory=ServiceState) @@ -234,9 +243,7 @@ def __init__(self, executor: "Golem", service_class: typing.Type[Service], paylo self.__instances: typing.List[ServiceInstance] = [] def __repr__(self): - return f"Cluster " \ - f"[Service: {self.service_class.__name__}, " \ - f"Payload: {self.payload}]" + return f"Cluster " f"[Service: {self.service_class.__name__}, " f"Payload: {self.payload}]" @property def instances(self) -> typing.List[Service]: @@ -328,6 +335,7 @@ def stop_instance(self, service: Service): class Golem(typing.AsyncContextManager): """ MOCK OF EXECUTOR JUST SO I COULD ILLUSTRATE THE NEW CALL""" + def __init__(self, *args, **kwargs): print(f"Golem started with {args}, {kwargs}") @@ -337,6 +345,7 @@ async def __aenter__(self) -> "Golem": async def __aexit__(self, *exc_info): import traceback + tb = traceback.print_tb(exc_info[2]) if len(exc_info) > 2 else None print("stop executor", exc_info, tb) return True @@ -353,12 +362,9 @@ async def _run_batch(self, batch): print(f"EXESCRIPT EXECUTION: {batch} on {batch.ctx.id}") for command in batch: print(f"EXESCRIPT COMMAND {command}") - results.append({ - 'command': command, - 'message': 'some data here' - }) + results.append({"command": command, "message": "some data here"}) - await asyncio.sleep(random.randint(1,7)) + await asyncio.sleep(random.randint(1, 7)) print(f"RETURNING RESULTS: {batch} on {batch.ctx.id}") return results @@ -373,10 +379,10 @@ async def _run_batches(self, batches: typing.AsyncGenerator): pass def run_service( - self, - service_class: typing.Type[Service], - num_instances: int = 1, - payload: typing.Optional[Payload] = None, + self, + service_class: typing.Type[Service], + num_instances: int = 1, + payload: typing.Optional[Payload] = None, ) -> Cluster: payload = payload or service_class.get_payload() cluster = Cluster(executor=self, service_class=service_class, payload=payload) @@ -463,7 +469,7 @@ async def main(subnet_tag, driver=None, network=None): cluster = golem.run_service( TurbogethService, # payload=payload, - num_instances=INSTANCES_NEEDED + num_instances=INSTANCES_NEEDED, ) def instances(): @@ -493,4 +499,5 @@ def still_running(): print(f"instances: {instances()}") + asyncio.run(main(None)) From 43885c511c43c0c5ccfeafab844008ba7ffd48c8 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 18 May 2021 18:14:43 +0200 Subject: [PATCH 56/85] move the easy part to `executor/services.py`, add comments --- examples/turbogeth/turbogeth.py | 309 -------------------------------- yapapi/executor/services.py | 267 +++++++++++++++++++++++++++ 2 files changed, 267 insertions(+), 309 deletions(-) create mode 100644 yapapi/executor/services.py diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py index b4f3353a5..6e2945b39 100644 --- a/examples/turbogeth/turbogeth.py +++ b/examples/turbogeth/turbogeth.py @@ -5,12 +5,8 @@ from typing import Optional, Any import random -import statemachine - from dataclasses import dataclass, field -# from yapapi.executor.ctx import WorkContext - from yapapi.props.builder import DemandBuilder from yapapi.props.base import prop, constraint from yapapi.props import inf @@ -28,311 +24,6 @@ # # ############################################################### -_act_cnt = 0 - - -def _act_id(): - global _act_cnt - _act_cnt += 1 - return _act_cnt - - -@dataclass -class Activity: - """ Mock Activity """ - - payload: Payload - id: int = field(default_factory=_act_id) - - -class Steps(list): - """ Mock Steps to illustrate the idea. """ - - def __init__(self, *args, **kwargs): - self.ctx: WorkContext = kwargs.pop("ctx") - self.blocking: bool = kwargs.pop("blocking", True) - - super().__init__(*args, **kwargs) - - def __repr__(self): - list_repr = super().__repr__() - return f"<{self.__class__.__name__}: {list_repr}, blocking: {self.blocking}" - - -class WorkContext: - """ Mock WorkContext to illustrate the idea. """ - - def __init__(self, activity_id, provider_name): - self.id = activity_id - self.provider_name = provider_name - self._steps = Steps(ctx=self) - - def __repr__(self): - return f"<{self.__class__.__name__}: activity_id: {self.id}>" - - def __str__(self): - return self.__repr__() - - def commit(self, blocking: bool = True): - self._steps.blocking = blocking - steps = self._steps - self._steps = Steps(ctx=self) - return steps - - def __getattr__(self, item): - def add_step(*args, **kwargs) -> int: - idx = len(self._steps) - self._steps.append({item: (args, kwargs)}) - return idx - - return add_step - - -@dataclass -class ServiceSignal: - """ THIS WOULD BE PART OF THE API CODE""" - - message: Any - response_to: Optional["ServiceSignal"] = None - - -class ConfigurationError(Exception): - """ THIS WOULD BE PART OF THE API CODE""" - - pass - - -class ServiceState(statemachine.StateMachine): - """ THIS WOULD BE PART OF THE API CODE""" - - starting = statemachine.State("starting", initial=True) - running = statemachine.State("running") - stopping = statemachine.State("stopping") - terminated = statemachine.State("terminated") - unresponsive = statemachine.State("unresponsive") - - ready = starting.to(running) - stop = running.to(stopping) - terminate = terminated.from_(starting, running, stopping, terminated) - mark_unresponsive = unresponsive.from_(starting, running, stopping, terminated) - - lifecycle = ready | stop | terminate - - AVAILABLE = (starting, running, stopping) - - -class Service: - """ THIS WOULD BE PART OF THE API CODE""" - - def __init__(self, cluster: "Cluster", ctx: WorkContext): - self.cluster = cluster - self.ctx = ctx - - self.__inqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() - self.__outqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() - self.post_init() - - def post_init(self): - pass - - @property - def id(self): - return self.ctx.id - - def __repr__(self): - return f"<{self.__class__.__name__}: {self.id}>" - - async def send_message(self, message: Any = None): - await self.__inqueue.put(ServiceSignal(message=message)) - - def send_message_nowait(self, message: Optional[Any] = None): - self.__inqueue.put_nowait(ServiceSignal(message=message)) - - async def receive_message(self) -> ServiceSignal: - return await self.__outqueue.get() - - def receive_message_nowait(self) -> Optional[ServiceSignal]: - try: - return self.__outqueue.get_nowait() - except asyncio.QueueEmpty: - pass - - async def _listen(self) -> ServiceSignal: - return await self.__inqueue.get() - - def _listen_nowait(self) -> Optional[ServiceSignal]: - try: - return self.__inqueue.get_nowait() - except asyncio.QueueEmpty: - pass - - async def _respond(self, message: Optional[Any], response_to: Optional[ServiceSignal] = None): - await self.__outqueue.put(ServiceSignal(message=message, response_to=response_to)) - - def _respond_nowait(self, message: Optional[Any], response_to: Optional[ServiceSignal] = None): - self.__outqueue.put_nowait(ServiceSignal(message=message, response_to=response_to)) - - @staticmethod - def get_payload() -> typing.Optional[Payload]: - """Return the payload (runtime) definition for this service. - - If `get_payload` is not implemented, the payload will need to be provided in the - `Golem.run_service` call. - """ - pass - - async def start(self): - self.ctx.deploy() - self.ctx.start() - yield self.ctx.commit() - - async def run(self): - while True: - await asyncio.sleep(10) - yield - - async def shutdown(self): - yield - - @property - def is_available(self): - return self.cluster.get_state(self) in ServiceState.AVAILABLE - - @property - def state(self): - return self.cluster.get_state(self) - - -class ControlSignal(enum.Enum): - stop = "stop" - - -@dataclass -class ServiceInstance: - """ THIS WOULD BE PART OF THE API CODE""" - - service: Service - control_queue: "asyncio.Queue[ControlSignal]" = field(default_factory=asyncio.Queue) - service_state: ServiceState = field(default_factory=ServiceState) - - @property - def state(self) -> ServiceState: - return self.service_state.current_state - - def get_control_signal(self) -> typing.Optional[ControlSignal]: - try: - return self.control_queue.get_nowait() - except asyncio.QueueEmpty: - pass - - def send_control_signal(self, signal: ControlSignal): - self.control_queue.put_nowait(signal) - - -class Cluster: - """ THIS WOULD BE PART OF THE API CODE""" - - def __init__(self, executor: "Golem", service_class: typing.Type[Service], payload: Payload): - self.executor = executor - self.service_class = service_class - - if not payload: - raise ConfigurationError("Payload must be defined when starting a cluster.") - - self.payload = payload - self.__instances: typing.List[ServiceInstance] = [] - - def __repr__(self): - return f"Cluster " f"[Service: {self.service_class.__name__}, " f"Payload: {self.payload}]" - - @property - def instances(self) -> typing.List[Service]: - return [i.service for i in self.__instances] - - def __get_service_instance(self, service: Service) -> ServiceInstance: - for i in self.__instances: - if i.service == service: - return i - - def get_state(self, service: Service): - instance = self.__get_service_instance(service) - return instance.state - - @staticmethod - def _get_handler(instance: ServiceInstance): - _handlers = { - ServiceState.starting: instance.service.start, - ServiceState.running: instance.service.run, - ServiceState.stopping: instance.service.shutdown, - } - handler = _handlers.get(instance.state, None) - if handler: - return handler() - - async def _run_instance(self, ctx: WorkContext): - loop = asyncio.get_event_loop() - instance = ServiceInstance(service=self.service_class(self, ctx)) - self.__instances.append(instance) - - print(f"{instance.service} commissioned") - - handler = self._get_handler(instance) - batch = None - - while handler: - try: - if batch: - r = yield batch - fr = loop.create_future() - fr.set_result(await r) - batch = await handler.asend(fr) - else: - batch = await handler.__anext__() - except StopAsyncIteration: - instance.service_state.lifecycle() - handler = self._get_handler(instance) - batch = None - print(f"{instance.service} state changed to {instance.state.value}") - - # two potential issues: - # * awaiting a batch makes use lose an ability to interpret a signal (await on generator won't return) - # * we may be losing a `batch` when we act on the control signal - # - # potential solution: - # * use `aiostream.stream.merge` - - ctl = instance.get_control_signal() - if ctl == ControlSignal.stop: - if instance.state == ServiceState.running: - instance.service_state.stop() - else: - instance.service_state.terminate() - - print(f"{instance.service} state changed to {instance.state.value}") - - handler = self._get_handler(instance) - batch = None - - print(f"{instance.service} decomissioned") - - async def spawn_instance(self): - act = await self.executor.get_activity(self.payload) - ctx = WorkContext(act.id, len(self.instances) + 1) - - instance_batches = self._run_instance(ctx) - try: - batch = await instance_batches.__anext__() - while batch: - r = yield batch - batch = await instance_batches.asend(r) - except StopAsyncIteration: - pass - - def stop_instance(self, service: Service): - instance = self.__get_service_instance(service) - instance.send_control_signal(ControlSignal.stop) - - class Golem(typing.AsyncContextManager): """ MOCK OF EXECUTOR JUST SO I COULD ILLUSTRATE THE NEW CALL""" diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py new file mode 100644 index 000000000..b97a8872c --- /dev/null +++ b/yapapi/executor/services.py @@ -0,0 +1,267 @@ +import asyncio +import typing +from dataclasses import dataclass, field +import enum +from typing import Any, List, Optional, Type +import statemachine + +from yapapi.executor.ctx import WorkContext +from yapapi.payload import Payload + +if typing.TYPE_CHECKING: + from yapapi.executor import Golem + + +class ServiceState(statemachine.StateMachine): + """ + State machine describing the state and lifecycle of a Service instance. + """ + + # states + starting = statemachine.State("starting", initial=True) + running = statemachine.State("running") + stopping = statemachine.State("stopping") + terminated = statemachine.State("terminated") + unresponsive = statemachine.State("unresponsive") + + # transitions + ready = starting.to(running) + stop = running.to(stopping) + terminate = terminated.from_(starting, running, stopping, terminated) + mark_unresponsive = unresponsive.from_(starting, running, stopping, terminated) + + lifecycle = ready | stop | terminate + + # just a helper set of states in which the service can be interacted-with + AVAILABLE = (starting, running, stopping) + + +@dataclass +class ServiceSignal: + """ + Simple container to carry information between the client code and the Service instance. + """ + + message: Any + response_to: Optional["ServiceSignal"] = None + + +class Service: + """ + Base Service class to be extended by application developers to define their own, + specialized Service specifications. + """ + + def __init__(self, cluster: "Cluster", ctx: WorkContext): + self.cluster = cluster + self.ctx = ctx + + self.__inqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() + self.__outqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() + self.post_init() + + def post_init(self): + pass + + @property + def id(self): + return self.ctx.id + + def __repr__(self): + return f"<{self.__class__.__name__}: {self.id}>" + + async def send_message(self, message: Any = None): + await self.__inqueue.put(ServiceSignal(message=message)) + + def send_message_nowait(self, message: Optional[Any] = None): + self.__inqueue.put_nowait(ServiceSignal(message=message)) + + async def receive_message(self) -> ServiceSignal: + return await self.__outqueue.get() + + def receive_message_nowait(self) -> Optional[ServiceSignal]: + try: + return self.__outqueue.get_nowait() + except asyncio.QueueEmpty: + pass + + async def _listen(self) -> ServiceSignal: + return await self.__inqueue.get() + + def _listen_nowait(self) -> Optional[ServiceSignal]: + try: + return self.__inqueue.get_nowait() + except asyncio.QueueEmpty: + pass + + async def _respond(self, message: Optional[Any], response_to: Optional[ServiceSignal] = None): + await self.__outqueue.put(ServiceSignal(message=message, response_to=response_to)) + + def _respond_nowait(self, message: Optional[Any], response_to: Optional[ServiceSignal] = None): + self.__outqueue.put_nowait(ServiceSignal(message=message, response_to=response_to)) + + @staticmethod + def get_payload() -> Optional[Payload]: + """Return the payload (runtime) definition for this service. + + If `get_payload` is not implemented, the payload will need to be provided in the + `Golem.run_service` call. + """ + pass + + async def start(self): + self.ctx.deploy() + self.ctx.start() + yield self.ctx.commit() + + async def run(self): + while True: + await asyncio.sleep(10) + yield + + async def shutdown(self): + yield + + @property + def is_available(self): + return self.cluster.get_state(self) in ServiceState.AVAILABLE + + @property + def state(self): + return self.cluster.get_state(self) + + +class ControlSignal(enum.Enum): + """ + Control signal, used to request an instance's state change from the controlling Cluster. + """ + stop = "stop" + + +@dataclass +class ServiceInstance: + """Cluster's service instance. + + A binding between the instance of the Service, its control queue and its state, + used by the Cluster to hold the complete state of each instance of a service. + """ + + service: Service + control_queue: "asyncio.Queue[ControlSignal]" = field(default_factory=asyncio.Queue) + service_state: ServiceState = field(default_factory=ServiceState) + + @property + def state(self) -> ServiceState: + return self.service_state.current_state + + def get_control_signal(self) -> Optional[ControlSignal]: + try: + return self.control_queue.get_nowait() + except asyncio.QueueEmpty: + pass + + def send_control_signal(self, signal: ControlSignal): + self.control_queue.put_nowait(signal) + + +class Cluster: + """ + Golem's sub-engine used to spawn and control instances of a single Service. + """ + + def __init__(self, engine: "Golem", service_class: Type[Service], payload: Payload): + self._engine = engine + self.service_class = service_class + + self.payload = payload + self.__instances: List[ServiceInstance] = [] + + def __repr__(self): + return f"Cluster " f"[Service: {self.service_class.__name__}, " f"Payload: {self.payload}]" + + @property + def instances(self) -> List[Service]: + return [i.service for i in self.__instances] + + def __get_service_instance(self, service: Service) -> ServiceInstance: + for i in self.__instances: + if i.service == service: + return i + + def get_state(self, service: Service): + instance = self.__get_service_instance(service) + return instance.state + + @staticmethod + def _get_handler(instance: ServiceInstance): + _handlers = { + ServiceState.starting: instance.service.start, + ServiceState.running: instance.service.run, + ServiceState.stopping: instance.service.shutdown, + } + handler = _handlers.get(instance.state, None) + if handler: + return handler() + + async def _run_instance(self, ctx: WorkContext): + loop = asyncio.get_event_loop() + instance = ServiceInstance(service=self.service_class(self, ctx)) + self.__instances.append(instance) + + print(f"{instance.service} commissioned") + + handler = self._get_handler(instance) + batch = None + + while handler: + try: + if batch: + r = yield batch + fr = loop.create_future() + fr.set_result(await r) + batch = await handler.asend(fr) + else: + batch = await handler.__anext__() + except StopAsyncIteration: + instance.service_state.lifecycle() + handler = self._get_handler(instance) + batch = None + print(f"{instance.service} state changed to {instance.state.value}") + + # two potential issues: + # * awaiting a batch makes us lose an ability to interpret a signal (await on generator won't return) + # * we may be losing a `batch` when we act on the control signal + # + # potential solution: + # * use `aiostream.stream.merge` + + ctl = instance.get_control_signal() + if ctl == ControlSignal.stop: + if instance.state == ServiceState.running: + instance.service_state.stop() + else: + instance.service_state.terminate() + + print(f"{instance.service} state changed to {instance.state.value}") + + handler = self._get_handler(instance) + batch = None + + print(f"{instance.service} decomissioned") + + async def spawn_instance(self): + act = await self._engine.get_activity(self.payload) + ctx = WorkContext(act.id, len(self.instances) + 1) + + instance_batches = self._run_instance(ctx) + try: + batch = await instance_batches.__anext__() + while batch: + r = yield batch + batch = await instance_batches.asend(r) + except StopAsyncIteration: + pass + + def stop_instance(self, service: Service): + instance = self.__get_service_instance(service) + instance.send_control_signal(ControlSignal.stop) From 49125cd7ee3dec7827a16b44eb4d2b016f5d6e4e Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 18 May 2021 18:18:45 +0200 Subject: [PATCH 57/85] unneeded `dataclass` x --- yapapi/props/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 413661a59..5edc823a2 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -81,7 +81,6 @@ def __str__(self): return msg -@dataclass class Model(abc.ABC): """ Base class from which all property models inherit. From 74774ac62f49c9a6603e1cffec82a72f657a0e66 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 16:13:53 +0200 Subject: [PATCH 58/85] implement the HL Services API --- examples/erigon/erigon.py | 108 ++++++++++++++++++ examples/turbogeth/turbogeth.py | 194 -------------------------------- yapapi/executor/__init__.py | 22 ++++ yapapi/executor/services.py | 158 +++++++++++++++++++++----- 4 files changed, 262 insertions(+), 220 deletions(-) create mode 100644 examples/erigon/erigon.py delete mode 100644 examples/turbogeth/turbogeth.py diff --git a/examples/erigon/erigon.py b/examples/erigon/erigon.py new file mode 100644 index 000000000..67ea0f133 --- /dev/null +++ b/examples/erigon/erigon.py @@ -0,0 +1,108 @@ +import asyncio + +from dataclasses import dataclass + +from yapapi.props.base import prop, constraint +from yapapi.props import inf + +from yapapi.payload import Payload +from yapapi.executor import Golem +from yapapi.executor.services import Service + +from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa + + +TURBOGETH_RUNTIME_NAME = "turbogeth-managed" +PROP_ERIGON_ETHEREUM_NETWORK = "golem.srv.app.eth.network" + + +@dataclass +class ErigonPayload(Payload): + network: str = prop(PROP_ERIGON_ETHEREUM_NETWORK) + + runtime: str = constraint(inf.INF_RUNTIME_NAME, default=TURBOGETH_RUNTIME_NAME) + min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) + min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) + + +class ErigonService(Service): + credentials = None + + def post_init(self): + self.credentials = {} + + def __repr__(self): + srv_repr = super().__repr__() + return f"{srv_repr}, credentials: {self.credentials}" + + @staticmethod + def get_payload(): + return ErigonPayload(network="rinkeby") + + async def start(self): + deploy_idx = self.ctx.deploy() + self.ctx.start() + future_results = yield self.ctx.commit() + results = await future_results + self.credentials = "RECEIVED" or results[deploy_idx] # (NORMALLY, WOULD BE PARSED) + + async def run(self): + + while True: + print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") + signal = self._listen_nowait() + if signal and signal.message == "go": + self.ctx.run("go!") + yield self.ctx.commit() + else: + await asyncio.sleep(1) + yield + + async def shutdown(self): + self.ctx.download_file("some/service/state", "temp/path") + yield self.ctx.commit() + + +async def main(subnet_tag, driver=None, network=None): + + async with Golem( + budget=10.0, + subnet_tag=subnet_tag, + driver=driver, + network=network, + event_consumer=log_summary(log_event_repr), + ) as golem: + cluster = await golem.run_service( + ErigonService, + num_instances=1, + ) + + def instances(): + return [{s.ctx.id, s.state.value} for s in cluster.instances] + + def still_running(): + return any([s for s in cluster.instances if s.is_available]) + + cnt = 0 + while cnt < 10: + print(f"instances: {instances()}") + await asyncio.sleep(3) + cnt += 1 + if cnt == 3: + if len(cluster.instances) > 1: + cluster.instances[0].send_message_nowait("go") + + for s in cluster.instances: + cluster.stop_instance(s) + + print(f"instances: {instances()}") + + cnt = 0 + while cnt < 10 and still_running(): + print(f"instances: {instances()}") + await asyncio.sleep(1) + + print(f"instances: {instances()}") + + +asyncio.run(main(None)) diff --git a/examples/turbogeth/turbogeth.py b/examples/turbogeth/turbogeth.py deleted file mode 100644 index 6e2945b39..000000000 --- a/examples/turbogeth/turbogeth.py +++ /dev/null @@ -1,194 +0,0 @@ -import asyncio -import enum -from datetime import timedelta -import typing -from typing import Optional, Any -import random - -from dataclasses import dataclass, field - -from yapapi.props.builder import DemandBuilder -from yapapi.props.base import prop, constraint -from yapapi.props import inf - -from yapapi.payload import Payload - -from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa - - -############################################################### -# # -# # -# MOCK API CODE # -# # -# # -############################################################### - -class Golem(typing.AsyncContextManager): - """ MOCK OF EXECUTOR JUST SO I COULD ILLUSTRATE THE NEW CALL""" - - def __init__(self, *args, **kwargs): - print(f"Golem started with {args}, {kwargs}") - - async def __aenter__(self) -> "Golem": - print("start executor") - return self - - async def __aexit__(self, *exc_info): - import traceback - - tb = traceback.print_tb(exc_info[2]) if len(exc_info) > 2 else None - print("stop executor", exc_info, tb) - return True - - async def get_activity(self, payload): - builder = DemandBuilder() - await builder.decorate(payload) - print(f"Commissioning an Activity for: {builder}") - await asyncio.sleep(random.randint(3, 7)) - return Activity(payload=payload) - - async def _run_batch(self, batch): - results = [] - print(f"EXESCRIPT EXECUTION: {batch} on {batch.ctx.id}") - for command in batch: - print(f"EXESCRIPT COMMAND {command}") - results.append({"command": command, "message": "some data here"}) - - await asyncio.sleep(random.randint(1, 7)) - print(f"RETURNING RESULTS: {batch} on {batch.ctx.id}") - return results - - async def _run_batches(self, batches: typing.AsyncGenerator): - try: - batch = await batches.__anext__() - while batch: - results = self._run_batch(batch) - batch = await batches.asend(results) - except StopAsyncIteration: - print("RUN BATCHES - stop async iteration") - pass - - def run_service( - self, - service_class: typing.Type[Service], - num_instances: int = 1, - payload: typing.Optional[Payload] = None, - ) -> Cluster: - payload = payload or service_class.get_payload() - cluster = Cluster(executor=self, service_class=service_class, payload=payload) - - for i in range(num_instances): - asyncio.create_task(self._run_batches(cluster.spawn_instance())) - return cluster - - -############################################################### -# # -# # -# CLIENT CODE # -# # -# # -############################################################### - - -TURBOGETH_RUNTIME_NAME = "turbogeth-managed" -PROP_TURBOGETH_CHAIN = "golem.srv.app.eth.chain" - - -@dataclass -class TurbogethPayload(Payload): - chain: str = prop(PROP_TURBOGETH_CHAIN) - - runtime: str = constraint(inf.INF_RUNTIME_NAME, default=TURBOGETH_RUNTIME_NAME) - min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) - min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) - - -INSTANCES_NEEDED = 3 -EXECUTOR_TIMEOUT = timedelta(weeks=100) - - -class TurbogethService(Service): - credentials = None - - def post_init(self): - self.credentials = {} - - def __repr__(self): - srv_repr = super().__repr__() - return f"{srv_repr}, credentials: {self.credentials}" - - @staticmethod - def get_payload(): - return TurbogethPayload(chain="rinkeby") - - async def start(self): - deploy_idx = self.ctx.deploy() - self.ctx.start() - future_results = yield self.ctx.commit() - results = await future_results - self.credentials = "RECEIVED" or results[deploy_idx] # (NORMALLY, WOULD BE PARSED) - - async def run(self): - - while True: - print(f"service {self.ctx.id} running on {self.ctx.provider_name} ... ") - signal = self._listen_nowait() - if signal and signal.message == "go": - self.ctx.run("go!") - yield self.ctx.commit() - else: - await asyncio.sleep(1) - yield - - async def shutdown(self): - self.ctx.download_file("some/service/state", "temp/path") - yield self.ctx.commit() - - -async def main(subnet_tag, driver=None, network=None): - - async with Golem( - max_workers=INSTANCES_NEEDED, - budget=10.0, - subnet_tag=subnet_tag, - driver=driver, - network=network, - event_consumer=log_summary(log_event_repr), - ) as golem: - cluster = golem.run_service( - TurbogethService, - # payload=payload, - num_instances=INSTANCES_NEEDED, - ) - - def instances(): - return [{s.ctx.id, s.state.value} for s in cluster.instances] - - def still_running(): - return any([s for s in cluster.instances if s.is_available]) - - cnt = 0 - while cnt < 10: - print(f"instances: {instances()}") - await asyncio.sleep(3) - cnt += 1 - if cnt == 3: - if len(cluster.instances) > 1: - cluster.instances[0].send_message_nowait("go") - - for s in cluster.instances: - cluster.stop_instance(s) - - print(f"instances: {instances()}") - - cnt = 0 - while cnt < 10 and still_running(): - print(f"instances: {instances()}") - await asyncio.sleep(1) - - print(f"instances: {instances()}") - - -asyncio.run(main(None)) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 9ab53a5d0..ef4744332 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -22,7 +22,9 @@ Optional, Set, Tuple, + Type, TypeVar, + TYPE_CHECKING, Union, cast, overload, @@ -47,6 +49,8 @@ from ..rest.market import OfferProposal, Subscription from ..storage import gftp from ._smartq import Consumer, Handle, SmartQueue +if TYPE_CHECKING: + from .services import Cluster, Service from .strategy import ( DecreaseScoreForUnconfirmedAgreement, LeastExpensiveLinearPayuMS, @@ -557,6 +561,24 @@ async def execute_tasks( async for t in executor.submit(worker, data): yield t + async def run_service( + self, + service_class: Type[Service], + num_instances: int = 1, + payload: Optional[Payload] = None, + expiration: Optional[datetime] = None, + ) -> Cluster: + from .services import Cluster # avoid circular dependency + + payload = payload or service_class.get_payload() + + if not payload: + raise ValueError(f"No payload returned from {service_class.__name__}.get_payload() nor given in the `payload` argument.") + + cluster = Cluster(engine=self, service_class=service_class, payload=payload, num_instances=num_instances, expiration=expiration) + await self._stack.enter_async_context(cluster) + cluster.spawn_instances() + return cluster class Job: diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index b97a8872c..67424640a 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -1,15 +1,27 @@ import asyncio -import typing from dataclasses import dataclass, field +from datetime import timedelta, datetime, timezone import enum -from typing import Any, List, Optional, Type +import logging +from typing import Any, AsyncContextManager, List, Optional, Set, Type, Final import statemachine +import sys -from yapapi.executor.ctx import WorkContext -from yapapi.payload import Payload +if sys.version_info >= (3, 7): + from contextlib import AsyncExitStack +else: + from async_exit_stack import AsyncExitStack # type: ignore -if typing.TYPE_CHECKING: - from yapapi.executor import Golem +from .. import rest +from ..executor import Golem, Job +from ..executor.ctx import WorkContext +from ..payload import Payload +from ..props import NodeInfo +from . import events + +logger = logging.getLogger(__name__) + +DEFAULT_SERVICE_EXPIRATION: Final[datetime] = datetime.now(timezone.utc) + timedelta(days=3650) class ServiceState(statemachine.StateMachine): @@ -164,20 +176,59 @@ def send_control_signal(self, signal: ControlSignal): self.control_queue.put_nowait(signal) -class Cluster: +class Cluster(AsyncContextManager): """ Golem's sub-engine used to spawn and control instances of a single Service. """ - def __init__(self, engine: "Golem", service_class: Type[Service], payload: Payload): + def __init__(self, engine: "Golem", service_class: Type[Service], payload: Payload, num_instances: int = 1, expiration: Optional[datetime] = None): self._engine = engine - self.service_class = service_class + self._service_class = service_class + self._payload = payload + self._num_instances = num_instances + self._expiration = expiration or DEFAULT_SERVICE_EXPIRATION - self.payload = payload self.__instances: List[ServiceInstance] = [] + """List of Service instances""" + + self._stack = AsyncExitStack() def __repr__(self): - return f"Cluster " f"[Service: {self.service_class.__name__}, " f"Payload: {self.payload}]" + return f"Cluster {self._num_instances} x [Service: {self._service_class.__name__}, Payload: {self._payload}]" + + def __aenter__(self): + self.__services: Set[asyncio.Task] = set() + """Asyncio tasks running within this cluster""" + + logger.debug("Starting new %s", self) + + self._job = Job(self._engine, expiration_time=self._expiration, payload=self._payload) + self._engine.add_job(self._job) + + loop = asyncio.get_event_loop() + self.__services.add(loop.create_task(self._job.find_offers())) + + async def agreements_pool_cycler(): + # shouldn't this be part of the Agreement pool itself? (or a task within Job?) + while True: + await asyncio.sleep(2) + await self._job.agreements_pool.cycle() + + self.__services.add(loop.create_task(agreements_pool_cycler())) + + def __aexit__(self, exc_type, exc_val, exc_tb): + logger.debug("%s is shutting down...", self) + + for task in self.__services: + if not task.done(): + logger.debug("Cancelling task: %s", task) + task.cancel() + await asyncio.gather(*self.__services, return_exceptions=True) + + self._engine.finalize_job(self._job) + + def emit(self, event: events.Event) -> None: + self._engine.emit(event) @property def instances(self) -> List[Service]: @@ -205,10 +256,10 @@ def _get_handler(instance: ServiceInstance): async def _run_instance(self, ctx: WorkContext): loop = asyncio.get_event_loop() - instance = ServiceInstance(service=self.service_class(self, ctx)) + instance = ServiceInstance(service=self._service_class(self, ctx)) self.__instances.append(instance) - print(f"{instance.service} commissioned") + logger.info(f"{instance.service} commissioned") handler = self._get_handler(instance) batch = None @@ -226,8 +277,10 @@ async def _run_instance(self, ctx: WorkContext): instance.service_state.lifecycle() handler = self._get_handler(instance) batch = None - print(f"{instance.service} state changed to {instance.state.value}") + logger.debug(f"{instance.service} state changed to {instance.state.value}") + # TODO + # # two potential issues: # * awaiting a batch makes us lose an ability to interpret a signal (await on generator won't return) # * we may be losing a `batch` when we act on the control signal @@ -242,26 +295,79 @@ async def _run_instance(self, ctx: WorkContext): else: instance.service_state.terminate() - print(f"{instance.service} state changed to {instance.state.value}") + logger.debug(f"{instance.service} state changed to {instance.state.value}") handler = self._get_handler(instance) batch = None - print(f"{instance.service} decomissioned") + logger.info(f"{instance.service} decomissioned") async def spawn_instance(self): - act = await self._engine.get_activity(self.payload) - ctx = WorkContext(act.id, len(self.instances) + 1) + spawned = False - instance_batches = self._run_instance(ctx) - try: - batch = await instance_batches.__anext__() - while batch: - r = yield batch - batch = await instance_batches.asend(r) - except StopAsyncIteration: - pass + async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> None: + nonlocal spawned + self.emit(events.WorkerStarted(agr_id=agreement.id)) + try: + act = await self._engine.create_activity(agreement.id) + except Exception: + self.emit( + events.ActivityCreateFailed( + agr_id=agreement.id, exc_info=sys.exc_info() # type: ignore + ) + ) + self.emit(events.WorkerFinished(agr_id=agreement.id)) + raise + + async with act: + spawned = True + self.emit(events.ActivityCreated(act_id=act.id, agr_id=agreement.id)) + self._engine.approve_agreement_payments(agreement.id) + work_context = WorkContext( + act.id, node_info, self._engine.storage_manager, emitter=self.emit + ) + + try: + instance_batches = self._run_instance(work_context) + try: + await self._engine.process_batches(agreement.id, act, instance_batches) + except StopAsyncIteration: + pass + self.emit(events.WorkerFinished(agr_id=agreement.id)) + except Exception: + self.emit( + events.WorkerFinished( + agr_id=agreement.id, exc_info=sys.exc_info() # type: ignore + ) + ) + raise + finally: + await self._engine.accept_payment_for_agreement(agreement.id) + + loop = asyncio.get_event_loop() + + while not spawned: + task = await self._job.agreements_pool.use_agreement( + lambda agreement, node: loop.create_task(start_worker(agreement, node)) + ) + await task def stop_instance(self, service: Service): instance = self.__get_service_instance(service) instance.send_control_signal(ControlSignal.stop) + + def spawn_instances(self, num_instances: Optional[int] = None) -> None: + """ + Spawn new instances within this Cluster. + + :param num_instances: number of instances to commission. + if not given, spawns the number that the Cluster has been initialized with. + """ + if num_instances: + self._num_instances += num_instances + else: + num_instances = self._num_instances + + loop = asyncio.get_event_loop() + for i in range(num_instances): + loop.create_task(self.spawn_instance()) From a3dcd7593b16527b5cc5772cad4ddfd7ff811621 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 16:27:17 +0200 Subject: [PATCH 59/85] fixes --- yapapi/executor/__init__.py | 4 ++-- yapapi/executor/services.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index ca9b7818c..56b12e7ef 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -580,11 +580,11 @@ async def execute_tasks( async def run_service( self, - service_class: Type[Service], + service_class: Type["Service"], num_instances: int = 1, payload: Optional[Payload] = None, expiration: Optional[datetime] = None, - ) -> Cluster: + ) -> "Cluster": from .services import Cluster # avoid circular dependency payload = payload or service_class.get_payload() diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index 67424640a..0cfa9c1a5 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -196,7 +196,7 @@ def __init__(self, engine: "Golem", service_class: Type[Service], payload: Paylo def __repr__(self): return f"Cluster {self._num_instances} x [Service: {self._service_class.__name__}, Payload: {self._payload}]" - def __aenter__(self): + async def __aenter__(self): self.__services: Set[asyncio.Task] = set() """Asyncio tasks running within this cluster""" @@ -216,7 +216,7 @@ async def agreements_pool_cycler(): self.__services.add(loop.create_task(agreements_pool_cycler())) - def __aexit__(self, exc_type, exc_val, exc_tb): + async def __aexit__(self, exc_type, exc_val, exc_tb): logger.debug("%s is shutting down...", self) for task in self.__services: @@ -347,10 +347,12 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> loop = asyncio.get_event_loop() while not spawned: + await asyncio.sleep(1.0) task = await self._job.agreements_pool.use_agreement( lambda agreement, node: loop.create_task(start_worker(agreement, node)) ) - await task + if task: + await task def stop_instance(self, service: Service): instance = self.__get_service_instance(service) From 9b00016e2f626ba45d775de250dad58d8c17066a Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 16:28:16 +0200 Subject: [PATCH 60/85] black --- yapapi/executor/__init__.py | 14 ++++++++++++-- yapapi/executor/services.py | 10 +++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 56b12e7ef..b129be39f 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -50,6 +50,7 @@ from ..rest.market import OfferProposal, Subscription from ..storage import gftp from ._smartq import Consumer, Handle, SmartQueue + if TYPE_CHECKING: from .services import Cluster, Service from .strategy import ( @@ -590,13 +591,22 @@ async def run_service( payload = payload or service_class.get_payload() if not payload: - raise ValueError(f"No payload returned from {service_class.__name__}.get_payload() nor given in the `payload` argument.") + raise ValueError( + f"No payload returned from {service_class.__name__}.get_payload() nor given in the `payload` argument." + ) - cluster = Cluster(engine=self, service_class=service_class, payload=payload, num_instances=num_instances, expiration=expiration) + cluster = Cluster( + engine=self, + service_class=service_class, + payload=payload, + num_instances=num_instances, + expiration=expiration, + ) await self._stack.enter_async_context(cluster) cluster.spawn_instances() return cluster + class Job: """Functionality related to a single job.""" diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index 0cfa9c1a5..a373f04f3 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -147,6 +147,7 @@ class ControlSignal(enum.Enum): """ Control signal, used to request an instance's state change from the controlling Cluster. """ + stop = "stop" @@ -181,7 +182,14 @@ class Cluster(AsyncContextManager): Golem's sub-engine used to spawn and control instances of a single Service. """ - def __init__(self, engine: "Golem", service_class: Type[Service], payload: Payload, num_instances: int = 1, expiration: Optional[datetime] = None): + def __init__( + self, + engine: "Golem", + service_class: Type[Service], + payload: Payload, + num_instances: int = 1, + expiration: Optional[datetime] = None, + ): self._engine = engine self._service_class = service_class self._payload = payload From 2a2717ddc427d0bca16ae559ca02371ee4524add Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 16:38:08 +0200 Subject: [PATCH 61/85] mv examples/service examples/simple-service-poc --- examples/{service => simple-service-poc}/simple_service.py | 0 .../simple_service/simple_service.Dockerfile | 0 .../simple_service/simple_service.py | 0 .../simple_service/simulate_observations.py | 0 .../simple_service/simulate_observations_ctl.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename examples/{service => simple-service-poc}/simple_service.py (100%) rename examples/{service => simple-service-poc}/simple_service/simple_service.Dockerfile (100%) rename examples/{service => simple-service-poc}/simple_service/simple_service.py (100%) rename examples/{service => simple-service-poc}/simple_service/simulate_observations.py (100%) rename examples/{service => simple-service-poc}/simple_service/simulate_observations_ctl.py (100%) diff --git a/examples/service/simple_service.py b/examples/simple-service-poc/simple_service.py similarity index 100% rename from examples/service/simple_service.py rename to examples/simple-service-poc/simple_service.py diff --git a/examples/service/simple_service/simple_service.Dockerfile b/examples/simple-service-poc/simple_service/simple_service.Dockerfile similarity index 100% rename from examples/service/simple_service/simple_service.Dockerfile rename to examples/simple-service-poc/simple_service/simple_service.Dockerfile diff --git a/examples/service/simple_service/simple_service.py b/examples/simple-service-poc/simple_service/simple_service.py similarity index 100% rename from examples/service/simple_service/simple_service.py rename to examples/simple-service-poc/simple_service/simple_service.py diff --git a/examples/service/simple_service/simulate_observations.py b/examples/simple-service-poc/simple_service/simulate_observations.py similarity index 100% rename from examples/service/simple_service/simulate_observations.py rename to examples/simple-service-poc/simple_service/simulate_observations.py diff --git a/examples/service/simple_service/simulate_observations_ctl.py b/examples/simple-service-poc/simple_service/simulate_observations_ctl.py similarity index 100% rename from examples/service/simple_service/simulate_observations_ctl.py rename to examples/simple-service-poc/simple_service/simulate_observations_ctl.py From 7277961e78dc6fc3685e1c9439505703a2d71ec1 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 17:20:42 +0200 Subject: [PATCH 62/85] make `get_payload` async --- examples/erigon/erigon.py | 2 +- yapapi/executor/__init__.py | 2 +- yapapi/executor/services.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/erigon/erigon.py b/examples/erigon/erigon.py index 67ea0f133..ede420648 100644 --- a/examples/erigon/erigon.py +++ b/examples/erigon/erigon.py @@ -36,7 +36,7 @@ def __repr__(self): return f"{srv_repr}, credentials: {self.credentials}" @staticmethod - def get_payload(): + async def get_payload(): return ErigonPayload(network="rinkeby") async def start(self): diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index b129be39f..195c3b3b4 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -588,7 +588,7 @@ async def run_service( ) -> "Cluster": from .services import Cluster # avoid circular dependency - payload = payload or service_class.get_payload() + payload = payload or await service_class.get_payload() if not payload: raise ValueError( diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index a373f04f3..83a0a58bc 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -113,7 +113,7 @@ def _respond_nowait(self, message: Optional[Any], response_to: Optional[ServiceS self.__outqueue.put_nowait(ServiceSignal(message=message, response_to=response_to)) @staticmethod - def get_payload() -> Optional[Payload]: + async def get_payload() -> Optional[Payload]: """Return the payload (runtime) definition for this service. If `get_payload` is not implemented, the payload will need to be provided in the From f809854f2dd6cc5eed43884ed32e399cbff8db38 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 17:50:13 +0200 Subject: [PATCH 63/85] if the user issues "start" or "deploy" explictly, disable "implicit init" --- yapapi/executor/ctx.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yapapi/executor/ctx.py b/yapapi/executor/ctx.py index a7c2eaa03..c4fbc15a1 100644 --- a/yapapi/executor/ctx.py +++ b/yapapi/executor/ctx.py @@ -306,9 +306,11 @@ def begin(self): pass def deploy(self): + self._implicit_init = False self._pending_steps.append(_Deploy()) def start(self): + self._implicit_init = False self._pending_steps.append(_Start()) def send_json(self, json_path: str, data: dict): From 9e64ffc898ae7399a3fdfec1212d8066249074ab Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 17:52:22 +0200 Subject: [PATCH 64/85] add `Cluster.stop` method --- yapapi/executor/services.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index 83a0a58bc..6937b4533 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -381,3 +381,8 @@ def spawn_instances(self, num_instances: Optional[int] = None) -> None: loop = asyncio.get_event_loop() for i in range(num_instances): loop.create_task(self.spawn_instance()) + + def stop(self): + """Signal the whole cluster to stop.""" + for s in self.instances: + self.stop_instance(s) From abf22114176bbb5c883595ccac3f0c2e993a07db Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 17:59:48 +0200 Subject: [PATCH 65/85] update the toy example with the HL Services API --- examples/simple-service-poc/simple_service.py | 156 ++++++++---------- 1 file changed, 73 insertions(+), 83 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 1870aae8d..77ca55546 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -4,7 +4,6 @@ """ import asyncio from datetime import datetime, timedelta -import itertools import json import pathlib import random @@ -13,16 +12,16 @@ from yapapi import ( - Executor, NoPaymentAccountError, - Task, __version__ as yapapi_version, WorkContext, windows_event_loop_fix, ) +from yapapi.executor import Golem +from yapapi.executor.services import Service + from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa from yapapi.package import vm -from yapapi.rest.activity import BatchTimeoutError examples_dir = pathlib.Path(__file__).resolve().parent.parent sys.path.append(str(examples_dir)) @@ -36,113 +35,104 @@ ) -async def main(subnet_tag, driver=None, network=None): - package = await vm.repo( +class SimpleService(Service): + STATS_PATH = "/golem/out/stats" + PLOT_INFO_PATH = "/golem/out/plot" + SIMPLE_SERVICE = "/golem/run/simple_service.py" + + plots_to_download = None + + def post_init(self): + self.plots_to_download = [] + + @staticmethod + async def get_payload(): + return await vm.repo( image_hash="8b11df59f84358d47fc6776d0bb7290b0054c15ded2d6f54cf634488", min_mem_gib=0.5, min_storage_gib=2.0, ) - async def service(ctx: WorkContext, tasks): - STATS_PATH = "/golem/out/stats" - PLOT_INFO_PATH = "/golem/out/plot" - SIMPLE_SERVICE = "/golem/run/simple_service.py" + async def on_plot(self, out: bytes): + fname = json.loads(out.strip()) + print(f"{TEXT_COLOR_CYAN}plot: {fname}{TEXT_COLOR_DEFAULT}") + self.plots_to_download.append(fname) - ctx.run("/golem/run/simulate_observations_ctl.py", "--start") - ctx.send_bytes( - "/golem/in/get_stats.sh", f"{SIMPLE_SERVICE} --stats > {STATS_PATH}".encode() + @staticmethod + async def on_stats(out: bytes): + print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}") + + async def start(self): + self.ctx.run("/golem/run/simulate_observations_ctl.py", "--start") + self.ctx.send_bytes( + "/golem/in/get_stats.sh", f"{self.SIMPLE_SERVICE} --stats > {self.STATS_PATH}".encode() ) - ctx.send_bytes( - "/golem/in/get_plot.sh", f"{SIMPLE_SERVICE} --plot dist > {PLOT_INFO_PATH}".encode() + self.ctx.send_bytes( + "/golem/in/get_plot.sh", f"{self.SIMPLE_SERVICE} --plot dist > {self.PLOT_INFO_PATH}".encode() ) + yield self.ctx.commit() - yield ctx.commit() + async def run(self): + while True: + await asyncio.sleep(10) - plots_to_download = [] + self.ctx.run("/bin/sh", "/golem/in/get_stats.sh") + self.ctx.download_bytes(self.STATS_PATH, self.on_stats) + self.ctx.run("/bin/sh", "/golem/in/get_plot.sh") + self.ctx.download_bytes(self.PLOT_INFO_PATH, self.on_plot) - # have a look at asyncio docs and figure out whether to leave the callback or replace it with something - # more asyncio-ic + for plot in self.plots_to_download: + test_filename = ( + "".join(random.choice(string.ascii_letters) for _ in + range(10)) + ".png" + ) + self.ctx.download_file(plot, str(pathlib.Path(__file__).resolve().parent / test_filename)) + yield self.ctx.commit() - async def on_plot(out: bytes): - nonlocal plots_to_download - fname = json.loads(out.strip()) - print(f"{TEXT_COLOR_CYAN}plot: {fname}{TEXT_COLOR_DEFAULT}") - plots_to_download.append(fname) + async def shutdown(self): + self.ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") + yield self.ctx.commit() - async def on_stats(out: bytes): - print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}") - try: - async for task in tasks: - await asyncio.sleep(10) - - ctx.run("/bin/sh", "/golem/in/get_stats.sh") - ctx.download_bytes(STATS_PATH, on_stats) - ctx.run("/bin/sh", "/golem/in/get_plot.sh") - ctx.download_bytes(PLOT_INFO_PATH, on_plot) - yield ctx.commit() - - for plot in plots_to_download: - test_filename = ( - "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" - ) - ctx.download_file(plot, pathlib.Path(__file__).resolve().parent / test_filename) - yield ctx.commit() - - task.accept_result() - - except (KeyboardInterrupt, asyncio.CancelledError): - # with the current implementation it won't work correctly - # because at this stage, the Executor is shutting down anyway - ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") - yield ctx.commit() - raise - - # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) - # TODO: make this dynamic, e.g. depending on the size of files to transfer - init_overhead = 3 - # Providers will not accept work if the timeout is outside of the [5 min, 30min] range. - # We increase the lower bound to 6 min to account for the time needed for our demand to - # reach the providers. - min_timeout, max_timeout = 6, 30 - - timeout = timedelta(minutes=29) - - # By passing `event_consumer=log_summary()` we enable summary logging. - # See the documentation of the `yapapi.log` module on how to set - # the level of detail and format of the logged information. - async with Executor( - package=package, - max_workers=1, +async def main(subnet_tag, driver=None, network=None): + async with Golem( budget=1.0, - timeout=timeout, subnet_tag=subnet_tag, driver=driver, network=network, event_consumer=log_summary(log_event_repr), - ) as executor: + ) as golem: print( f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " - f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " - f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" + f"payment driver: {TEXT_COLOR_YELLOW}{golem.driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{golem.network}{TEXT_COLOR_DEFAULT}\n" ) start_time = datetime.now() - async for task in executor.submit(service, (Task(data=n) for n in itertools.count(1))): - print( - f"{TEXT_COLOR_CYAN}" - f"Script executed: {task}, result: {task.result}, time: {task.running_time}" - f"{TEXT_COLOR_DEFAULT}" - ) + cluster = await golem.run_service(SimpleService) - print( - f"{TEXT_COLOR_CYAN}" - f"Service finished, total time: {datetime.now() - start_time}" - f"{TEXT_COLOR_DEFAULT}" - ) + def instances(): + return [{s.ctx.id, s.state.value} for s in cluster.instances] + + def still_running(): + return any([s for s in cluster.instances if s.is_available]) + + while datetime.now() < start_time + timedelta(minutes=3): + print(f"instances: {instances()}") + await asyncio.sleep(3) + + print("stopping instances") + cluster.stop() + + cnt = 0 + while cnt < 10 and still_running(): + print(f"instances: {instances()}") + await asyncio.sleep(1) + + print(f"instances: {instances()}") if __name__ == "__main__": From 9eee66fe5ce6b2e7fcf0e85d2e984b07fdd449fe Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 18:01:53 +0200 Subject: [PATCH 66/85] . --- examples/simple-service-poc/simple_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 77ca55546..aabcab45c 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -120,7 +120,7 @@ def instances(): def still_running(): return any([s for s in cluster.instances if s.is_available]) - while datetime.now() < start_time + timedelta(minutes=3): + while datetime.now() < start_time + timedelta(minutes=2): print(f"instances: {instances()}") await asyncio.sleep(3) From b821a556ac98a417cc68eab3fae3b0b65a2d2787 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 18:04:38 +0200 Subject: [PATCH 67/85] - deprecation warning --- examples/simple-service-poc/simple_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index aabcab45c..fdaa95ed2 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -14,14 +14,13 @@ from yapapi import ( NoPaymentAccountError, __version__ as yapapi_version, - WorkContext, windows_event_loop_fix, ) from yapapi.executor import Golem from yapapi.executor.services import Service from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa -from yapapi.package import vm +from yapapi.payload import vm examples_dir = pathlib.Path(__file__).resolve().parent.parent sys.path.append(str(examples_dir)) From 2e5f575119fac44f9ab613fd85a2d5db6ed2ec4b Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 18:05:44 +0200 Subject: [PATCH 68/85] make the log unique --- examples/simple-service-poc/simple_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index fdaa95ed2..1e3a53659 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -136,7 +136,8 @@ def still_running(): if __name__ == "__main__": parser = build_parser("Test http") - parser.set_defaults(log_file="service-yapapi.log") + now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") + parser.set_defaults(log_file=f"simple-service-yapapi-{now}.log") args = parser.parse_args() # This is only required when running on Windows with Python prior to 3.8: From 99a78456e465324b6b904f232bafe92ba6d18449 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 18:31:34 +0200 Subject: [PATCH 69/85] fix messages --- yapapi/executor/services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index 6937b4533..9b8bc6de1 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -202,7 +202,7 @@ def __init__( self._stack = AsyncExitStack() def __repr__(self): - return f"Cluster {self._num_instances} x [Service: {self._service_class.__name__}, Payload: {self._payload}]" + return f"Cluster: {self._num_instances} x [Service: {self._service_class.__name__}, Payload: {self._payload}]" async def __aenter__(self): self.__services: Set[asyncio.Task] = set() @@ -311,6 +311,7 @@ async def _run_instance(self, ctx: WorkContext): logger.info(f"{instance.service} decomissioned") async def spawn_instance(self): + logger.debug("spawning instance within %s", self) spawned = False async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> None: From e0535196a786df5b941d489bc0fa27cbbfc593b0 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 19 May 2021 18:31:52 +0200 Subject: [PATCH 70/85] ARGH ... short expiry workaround ... --- examples/simple-service-poc/simple_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 1e3a53659..92fdb26b1 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -3,7 +3,7 @@ the requestor agent controlling and interacting with the "simple service" """ import asyncio -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import json import pathlib import random @@ -111,7 +111,7 @@ async def main(subnet_tag, driver=None, network=None): start_time = datetime.now() - cluster = await golem.run_service(SimpleService) + cluster = await golem.run_service(SimpleService, expiration=datetime.now(timezone.utc) + timedelta(minutes=15)) def instances(): return [{s.ctx.id, s.state.value} for s in cluster.instances] From 0ee6c70e9a3b38e9a2d60f46f7df567928ffed30 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 07:35:37 +0200 Subject: [PATCH 71/85] make `ctx` and `cluster` private on `Service` --- examples/simple-service-poc/simple_service.py | 43 +++++++++++-------- yapapi/executor/services.py | 20 +++++---- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 92fdb26b1..58cc27a76 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -19,7 +19,7 @@ from yapapi.executor import Golem from yapapi.executor.services import Service -from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa +from yapapi.log import enable_default_logger, log_summary, log_event_repr, pluralize # noqa from yapapi.payload import vm examples_dir = pathlib.Path(__file__).resolve().parent.parent @@ -33,6 +33,8 @@ TEXT_COLOR_YELLOW, ) +NUM_INSTANCES = 1 + class SimpleService(Service): STATS_PATH = "/golem/out/stats" @@ -62,35 +64,35 @@ async def on_stats(out: bytes): print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}") async def start(self): - self.ctx.run("/golem/run/simulate_observations_ctl.py", "--start") - self.ctx.send_bytes( + self._ctx.run("/golem/run/simulate_observations_ctl.py", "--start") + self._ctx.send_bytes( "/golem/in/get_stats.sh", f"{self.SIMPLE_SERVICE} --stats > {self.STATS_PATH}".encode() ) - self.ctx.send_bytes( + self._ctx.send_bytes( "/golem/in/get_plot.sh", f"{self.SIMPLE_SERVICE} --plot dist > {self.PLOT_INFO_PATH}".encode() ) - yield self.ctx.commit() + yield self._ctx.commit() async def run(self): while True: await asyncio.sleep(10) - self.ctx.run("/bin/sh", "/golem/in/get_stats.sh") - self.ctx.download_bytes(self.STATS_PATH, self.on_stats) - self.ctx.run("/bin/sh", "/golem/in/get_plot.sh") - self.ctx.download_bytes(self.PLOT_INFO_PATH, self.on_plot) + self._ctx.run("/bin/sh", "/golem/in/get_stats.sh") + self._ctx.download_bytes(self.STATS_PATH, self.on_stats) + self._ctx.run("/bin/sh", "/golem/in/get_plot.sh") + self._ctx.download_bytes(self.PLOT_INFO_PATH, self.on_plot) for plot in self.plots_to_download: test_filename = ( "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" ) - self.ctx.download_file(plot, str(pathlib.Path(__file__).resolve().parent / test_filename)) - yield self.ctx.commit() + self._ctx.download_file(plot, str(pathlib.Path(__file__).resolve().parent / test_filename)) + yield self._ctx.commit() async def shutdown(self): - self.ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") - yield self.ctx.commit() + self._ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") + yield self._ctx.commit() async def main(subnet_tag, driver=None, network=None): @@ -111,25 +113,30 @@ async def main(subnet_tag, driver=None, network=None): start_time = datetime.now() - cluster = await golem.run_service(SimpleService, expiration=datetime.now(timezone.utc) + timedelta(minutes=15)) + print(TEXT_COLOR_YELLOW, f"starting {NUM_INSTANCES} ", pluralize(NUM_INSTANCES, "instance"), TEXT_COLOR_DEFAULT) + + cluster = await golem.run_service( + SimpleService, + num_instances=NUM_INSTANCES, + expiration=datetime.now(timezone.utc) + timedelta(minutes=15)) def instances(): - return [{s.ctx.id, s.state.value} for s in cluster.instances] + return [(s.provider_name, s.state.value) for s in cluster.instances] def still_running(): return any([s for s in cluster.instances if s.is_available]) while datetime.now() < start_time + timedelta(minutes=2): print(f"instances: {instances()}") - await asyncio.sleep(3) + await asyncio.sleep(5) - print("stopping instances") + print(f"{TEXT_COLOR_YELLOW}stopping instances{TEXT_COLOR_DEFAULT}") cluster.stop() cnt = 0 while cnt < 10 and still_running(): print(f"instances: {instances()}") - await asyncio.sleep(1) + await asyncio.sleep(5) print(f"instances: {instances()}") diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index 9b8bc6de1..75097ab43 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -65,8 +65,8 @@ class Service: """ def __init__(self, cluster: "Cluster", ctx: WorkContext): - self.cluster = cluster - self.ctx = ctx + self._cluster: "Cluster" = cluster + self._ctx: WorkContext = ctx self.__inqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() self.__outqueue: asyncio.Queue[ServiceSignal] = asyncio.Queue() @@ -77,7 +77,11 @@ def post_init(self): @property def id(self): - return self.ctx.id + return self._ctx.id + + @property + def provider_name(self): + return self._ctx.provider_name def __repr__(self): return f"<{self.__class__.__name__}: {self.id}>" @@ -122,9 +126,9 @@ async def get_payload() -> Optional[Payload]: pass async def start(self): - self.ctx.deploy() - self.ctx.start() - yield self.ctx.commit() + self._ctx.deploy() + self._ctx.start() + yield self._ctx.commit() async def run(self): while True: @@ -136,11 +140,11 @@ async def shutdown(self): @property def is_available(self): - return self.cluster.get_state(self) in ServiceState.AVAILABLE + return self._cluster.get_state(self) in ServiceState.AVAILABLE @property def state(self): - return self.cluster.get_state(self) + return self._cluster.get_state(self) class ControlSignal(enum.Enum): From 033e5db0df629662c5671565c1b43ae230e6e65a Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 10:08:24 +0200 Subject: [PATCH 72/85] fix plot downloading in simple_service --- examples/simple-service-poc/simple_service.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 58cc27a76..7ae635402 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -6,11 +6,13 @@ from datetime import datetime, timedelta, timezone import json import pathlib +import queue import random import string import sys + from yapapi import ( NoPaymentAccountError, __version__ as yapapi_version, @@ -44,7 +46,7 @@ class SimpleService(Service): plots_to_download = None def post_init(self): - self.plots_to_download = [] + self.plots_to_download = queue.Queue() @staticmethod async def get_payload(): @@ -56,8 +58,7 @@ async def get_payload(): async def on_plot(self, out: bytes): fname = json.loads(out.strip()) - print(f"{TEXT_COLOR_CYAN}plot: {fname}{TEXT_COLOR_DEFAULT}") - self.plots_to_download.append(fname) + self.plots_to_download.put_nowait(fname) @staticmethod async def on_stats(out: bytes): @@ -82,12 +83,18 @@ async def run(self): self._ctx.run("/bin/sh", "/golem/in/get_plot.sh") self._ctx.download_bytes(self.PLOT_INFO_PATH, self.on_plot) - for plot in self.plots_to_download: - test_filename = ( - "".join(random.choice(string.ascii_letters) for _ in - range(10)) + ".png" - ) - self._ctx.download_file(plot, str(pathlib.Path(__file__).resolve().parent / test_filename)) + while True: + try: + plot = self.plots_to_download.get_nowait() + plot_filename = ( + "".join(random.choice(string.ascii_letters) for _ in + range(10)) + ".png" + ) + print(f"{TEXT_COLOR_CYAN}downloading plot: {plot_filename}{TEXT_COLOR_DEFAULT}") + self._ctx.download_file(plot, str(pathlib.Path(__file__).resolve().parent / plot_filename)) + except queue.Empty: + break + yield self._ctx.commit() async def shutdown(self): @@ -113,7 +120,7 @@ async def main(subnet_tag, driver=None, network=None): start_time = datetime.now() - print(TEXT_COLOR_YELLOW, f"starting {NUM_INSTANCES} ", pluralize(NUM_INSTANCES, "instance"), TEXT_COLOR_DEFAULT) + print(f"{TEXT_COLOR_YELLOW}starting {pluralize(NUM_INSTANCES, 'instance')}{TEXT_COLOR_DEFAULT}") cluster = await golem.run_service( SimpleService, From f2de8256a7c4dea7441b31155a41b8ac822c7659 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 13:29:16 +0200 Subject: [PATCH 73/85] fix logging for services --- yapapi/executor/services.py | 31 +++++++++++++++++++++++++++---- yapapi/log.py | 2 +- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index 6937b4533..9338a2694 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -1,4 +1,5 @@ import asyncio +import itertools from dataclasses import dataclass, field from datetime import timedelta, datetime, timezone import enum @@ -13,7 +14,7 @@ from async_exit_stack import AsyncExitStack # type: ignore from .. import rest -from ..executor import Golem, Job +from ..executor import Golem, Job, Task from ..executor.ctx import WorkContext from ..payload import Payload from ..props import NodeInfo @@ -21,7 +22,10 @@ logger = logging.getLogger(__name__) -DEFAULT_SERVICE_EXPIRATION: Final[datetime] = datetime.now(timezone.utc) + timedelta(days=3650) +# current default for yagna providers as of yagna 0.6.x +DEFAULT_SERVICE_EXPIRATION: Final[timedelta] = timedelta(minutes=175) + +cluster_ids = itertools.count(1) class ServiceState(statemachine.StateMachine): @@ -190,19 +194,23 @@ def __init__( num_instances: int = 1, expiration: Optional[datetime] = None, ): + self.id = str(next(cluster_ids)) + self._engine = engine self._service_class = service_class self._payload = payload self._num_instances = num_instances - self._expiration = expiration or DEFAULT_SERVICE_EXPIRATION + self._expiration = expiration or datetime.now(timezone.utc) + DEFAULT_SERVICE_EXPIRATION self.__instances: List[ServiceInstance] = [] """List of Service instances""" + self._task_ids = itertools.count(1) + self._stack = AsyncExitStack() def __repr__(self): - return f"Cluster {self._num_instances} x [Service: {self._service_class.__name__}, Payload: {self._payload}]" + return f"Cluster {self.id}: {self._num_instances}x[Service: {self._service_class.__name__}, Payload: {self._payload}]" async def __aenter__(self): self.__services: Set[asyncio.Task] = set() @@ -334,6 +342,14 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> work_context = WorkContext( act.id, node_info, self._engine.storage_manager, emitter=self.emit ) + task_id = f"{self.id}{next(self._task_ids)}" + self.emit( + events.TaskStarted( + agr_id=agreement.id, + task_id=task_id, + task_data=f"Service: {self._service_class.__name__}" + ) + ) try: instance_batches = self._run_instance(work_context) @@ -341,6 +357,13 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> await self._engine.process_batches(agreement.id, act, instance_batches) except StopAsyncIteration: pass + + self.emit( + events.TaskFinished( + agr_id=agreement.id, + task_id=task_id, + ) + ) self.emit(events.WorkerFinished(agr_id=agreement.id)) except Exception: self.emit( diff --git a/yapapi/log.py b/yapapi/log.py index 739a24adf..0b7644456 100644 --- a/yapapi/log.py +++ b/yapapi/log.py @@ -417,7 +417,7 @@ def _handle(self, event: events.Event): provider_info = self.agreement_provider_info[event.agr_id] data = self.task_data[event.task_id] self.logger.info( - "Task computed by provider '%s', task data: %s", + "Task finished by provider '%s', task data: %s", provider_info.name, str_capped(data, 200), ) From cdadb3066b2e8c67944e4352b733c7c57742568f Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 15:23:19 +0200 Subject: [PATCH 74/85] optimize the example to eliminate the callbacks --- examples/simple-service-poc/simple_service.py | 55 ++++++------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 7ae635402..beb6f4e74 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -43,11 +43,6 @@ class SimpleService(Service): PLOT_INFO_PATH = "/golem/out/plot" SIMPLE_SERVICE = "/golem/run/simple_service.py" - plots_to_download = None - - def post_init(self): - self.plots_to_download = queue.Queue() - @staticmethod async def get_payload(): return await vm.repo( @@ -56,46 +51,32 @@ async def get_payload(): min_storage_gib=2.0, ) - async def on_plot(self, out: bytes): - fname = json.loads(out.strip()) - self.plots_to_download.put_nowait(fname) - - @staticmethod - async def on_stats(out: bytes): - print(f"{TEXT_COLOR_CYAN}stats: {out}{TEXT_COLOR_DEFAULT}") - async def start(self): self._ctx.run("/golem/run/simulate_observations_ctl.py", "--start") - self._ctx.send_bytes( - "/golem/in/get_stats.sh", f"{self.SIMPLE_SERVICE} --stats > {self.STATS_PATH}".encode() - ) - self._ctx.send_bytes( - "/golem/in/get_plot.sh", f"{self.SIMPLE_SERVICE} --plot dist > {self.PLOT_INFO_PATH}".encode() - ) yield self._ctx.commit() async def run(self): while True: await asyncio.sleep(10) + self._ctx.run(self.SIMPLE_SERVICE, "--stats") # idx 0 + self._ctx.run(self.SIMPLE_SERVICE, "--plot", "dist") # idx 1 + + future_results = yield self._ctx.commit() + results = await future_results + stats = results[0].stdout.strip() + plot = results[1].stdout.strip().strip('"') + + print(f"{TEXT_COLOR_CYAN}stats: {stats}{TEXT_COLOR_DEFAULT}") + + plot_filename = ( + "".join(random.choice(string.ascii_letters) for _ in + range(10)) + ".png" + ) + print(f"{TEXT_COLOR_CYAN}downloading plot: {plot} to {plot_filename}{TEXT_COLOR_DEFAULT}") + self._ctx.download_file(plot, str(pathlib.Path(__file__).resolve().parent / plot_filename)) - self._ctx.run("/bin/sh", "/golem/in/get_stats.sh") - self._ctx.download_bytes(self.STATS_PATH, self.on_stats) - self._ctx.run("/bin/sh", "/golem/in/get_plot.sh") - self._ctx.download_bytes(self.PLOT_INFO_PATH, self.on_plot) - - while True: - try: - plot = self.plots_to_download.get_nowait() - plot_filename = ( - "".join(random.choice(string.ascii_letters) for _ in - range(10)) + ".png" - ) - print(f"{TEXT_COLOR_CYAN}downloading plot: {plot_filename}{TEXT_COLOR_DEFAULT}") - self._ctx.download_file(plot, str(pathlib.Path(__file__).resolve().parent / plot_filename)) - except queue.Empty: - break - - yield self._ctx.commit() + steps = self._ctx.commit() + yield steps async def shutdown(self): self._ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") From 487d80e55a2805daec6c2afedf53ae702924b31a Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 15:24:06 +0200 Subject: [PATCH 75/85] fix the task id for services --- yapapi/executor/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index 9338a2694..2e3e555cf 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -342,7 +342,7 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> work_context = WorkContext( act.id, node_info, self._engine.storage_manager, emitter=self.emit ) - task_id = f"{self.id}{next(self._task_ids)}" + task_id = f"{self.id}:{next(self._task_ids)}" self.emit( events.TaskStarted( agr_id=agreement.id, From 11a3655ab70afe8e3106f8147b3c9d85679812f4 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Wed, 26 May 2021 12:36:27 +0200 Subject: [PATCH 76/85] Shutdown Golem's services & jobs as part of closing the exit stack --- yapapi/executor/__init__.py | 78 +++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 182db37f2..cd2ead46d 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -264,57 +264,61 @@ def report_shutdown(*exc_info): self._storage_manager = await stack.enter_async_context(gftp.provider()) + stack.push_async_exit(self._shutdown) + return self except: await self.__aexit__(*sys.exc_info()) raise - async def __aexit__(self, exc_type, exc_val, exc_tb): + async def _shutdown(self, *exc_info): + """Shutdown this Golem instance.""" + # Importing this at the beginning would cause circular dependencies from ..log import pluralize - try: - logger.debug("Golem is shutting down...") - # Wait until all computations are finished - await asyncio.gather(*[job.finished.wait() for job in self._jobs]) + logger.info("Golem is shutting down...") + # Wait until all computations are finished + await asyncio.gather(*[job.finished.wait() for job in self._jobs]) + logger.info("All jobs have finished") - self._payment_closing = True + self._payment_closing = True - for task in self._services: - if task is not self._process_invoices_job: - task.cancel() + for task in self._services: + if task is not self._process_invoices_job: + task.cancel() - if self._process_invoices_job and not any( - True for job in self._jobs if job.agreements_pool.confirmed > 0 - ): - logger.debug("No need to wait for invoices.") - self._process_invoices_job.cancel() + if self._process_invoices_job and not any( + True for job in self._jobs if job.agreements_pool.confirmed > 0 + ): + logger.debug("No need to wait for invoices.") + self._process_invoices_job.cancel() - try: - logger.info("Waiting for Golem services to finish...") - _, pending = await asyncio.wait( - self._services, timeout=10, return_when=asyncio.ALL_COMPLETED - ) - if pending: - logger.debug( - "%s still running: %s", pluralize(len(pending), "service"), pending - ) - except Exception: - logger.debug("Got error when waiting for services to finish", exc_info=True) + try: + logger.info("Waiting for Golem services to finish...") + _, pending = await asyncio.wait( + self._services, timeout=10, return_when=asyncio.ALL_COMPLETED + ) + if pending: + logger.debug("%s still running: %s", pluralize(len(pending), "service"), pending) + except Exception: + logger.debug("Got error when waiting for services to finish", exc_info=True) + + if self._agreements_to_pay and self._process_invoices_job: + logger.info( + "%s still unpaid, waiting for invoices...", + pluralize(len(self._agreements_to_pay), "agreement"), + ) + await asyncio.wait( + {self._process_invoices_job}, timeout=30, return_when=asyncio.ALL_COMPLETED + ) + if self._agreements_to_pay: + logger.warning("Unpaid agreements: %s", self._agreements_to_pay) - if self._agreements_to_pay and self._process_invoices_job: - logger.info( - "%s still unpaid, waiting for invoices...", - pluralize(len(self._agreements_to_pay), "agreement"), - ) - await asyncio.wait( - {self._process_invoices_job}, timeout=30, return_when=asyncio.ALL_COMPLETED - ) - if self._agreements_to_pay: - logger.warning("Unpaid agreements: %s", self._agreements_to_pay) + await asyncio.gather(*[job.finished.wait() for job in self._jobs]) - finally: - await self._stack.aclose() + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self._stack.aclose() async def _create_allocations(self) -> rest.payment.MarketDecoration: From dbefecd9f372beb0f5e09df2a6d70b41ca5634d2 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Wed, 26 May 2021 14:33:34 +0200 Subject: [PATCH 77/85] Terminate agreements in Cluster.__aexit__() --- yapapi/executor/services.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index 2e3e555cf..a187217aa 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -235,6 +235,18 @@ async def agreements_pool_cycler(): async def __aexit__(self, exc_type, exc_val, exc_tb): logger.debug("%s is shutting down...", self) + # TODO: should be different if we stop due to an error + termination_reason = { + "message": "Successfully finished all work", + "golem.requestor.code": "Success", + } + + try: + logger.debug("Terminating agreements...") + await self._job.agreements_pool.terminate_all(reason=termination_reason) + except Exception: + logger.debug("Couldn't terminate agreements", exc_info=True) + for task in self.__services: if not task.done(): logger.debug("Cancelling task: %s", task) @@ -347,7 +359,7 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> events.TaskStarted( agr_id=agreement.id, task_id=task_id, - task_data=f"Service: {self._service_class.__name__}" + task_data=f"Service: {self._service_class.__name__}", ) ) From 30074a4595901426d055e9542d4cb9728e528af6 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Wed, 26 May 2021 16:01:37 +0200 Subject: [PATCH 78/85] Fix mypy issues --- yapapi/executor/services.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/yapapi/executor/services.py b/yapapi/executor/services.py index a187217aa..012dbdb1d 100644 --- a/yapapi/executor/services.py +++ b/yapapi/executor/services.py @@ -4,8 +4,8 @@ from datetime import timedelta, datetime, timezone import enum import logging -from typing import Any, AsyncContextManager, List, Optional, Set, Type, Final -import statemachine +from typing import Any, AsyncContextManager, List, Optional, Set, Type +import statemachine # type: ignore import sys if sys.version_info >= (3, 7): @@ -13,6 +13,12 @@ else: from async_exit_stack import AsyncExitStack # type: ignore +if sys.version_info >= (3, 8): + from typing import Final +else: + from typing_extensions import Final + + from .. import rest from ..executor import Golem, Job, Task from ..executor.ctx import WorkContext @@ -99,7 +105,7 @@ def receive_message_nowait(self) -> Optional[ServiceSignal]: try: return self.__outqueue.get_nowait() except asyncio.QueueEmpty: - pass + return None async def _listen(self) -> ServiceSignal: return await self.__inqueue.get() @@ -108,7 +114,7 @@ def _listen_nowait(self) -> Optional[ServiceSignal]: try: return self.__inqueue.get_nowait() except asyncio.QueueEmpty: - pass + return None async def _respond(self, message: Optional[Any], response_to: Optional[ServiceSignal] = None): await self.__outqueue.put(ServiceSignal(message=message, response_to=response_to)) @@ -175,7 +181,7 @@ def get_control_signal(self) -> Optional[ControlSignal]: try: return self.control_queue.get_nowait() except asyncio.QueueEmpty: - pass + return None def send_control_signal(self, signal: ControlSignal): self.control_queue.put_nowait(signal) @@ -266,8 +272,9 @@ def __get_service_instance(self, service: Service) -> ServiceInstance: for i in self.__instances: if i.service == service: return i + assert False, f"No instance found for {service}" - def get_state(self, service: Service): + def get_state(self, service: Service) -> ServiceState: instance = self.__get_service_instance(service) return instance.state From 0ed9598e799d7a3cc9882ba3ca970f6f2d6c5e82 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 27 May 2021 12:02:48 +0200 Subject: [PATCH 79/85] more sensible example behavior (waiting for start, etc) --- examples/simple-service-poc/simple_service.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index beb6f4e74..311480bba 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -19,7 +19,7 @@ windows_event_loop_fix, ) from yapapi.executor import Golem -from yapapi.executor.services import Service +from yapapi.executor.services import Service, ServiceState from yapapi.log import enable_default_logger, log_summary, log_event_repr, pluralize # noqa from yapapi.payload import vm @@ -36,6 +36,7 @@ ) NUM_INSTANCES = 1 +STARTING_TIMEOUT = timedelta(minutes=10) class SimpleService(Service): @@ -99,14 +100,14 @@ async def main(subnet_tag, driver=None, network=None): f"and network: {TEXT_COLOR_YELLOW}{golem.network}{TEXT_COLOR_DEFAULT}\n" ) - start_time = datetime.now() + commissioning_time = datetime.now() print(f"{TEXT_COLOR_YELLOW}starting {pluralize(NUM_INSTANCES, 'instance')}{TEXT_COLOR_DEFAULT}") cluster = await golem.run_service( SimpleService, num_instances=NUM_INSTANCES, - expiration=datetime.now(timezone.utc) + timedelta(minutes=15)) + expiration=datetime.now(timezone.utc) + timedelta(minutes=120)) def instances(): return [(s.provider_name, s.state.value) for s in cluster.instances] @@ -114,6 +115,21 @@ def instances(): def still_running(): return any([s for s in cluster.instances if s.is_available]) + def still_starting(): + return len(cluster.instances) < NUM_INSTANCES or \ + any([s for s in cluster.instances if s.state == ServiceState.running]) + + while still_starting() and datetime.now() < commissioning_time + STARTING_TIMEOUT: + print(f"instances: {instances()}") + await asyncio.sleep(5) + + if still_starting(): + raise Exception(f"Failed to start instances before {STARTING_TIMEOUT} elapsed :( ...") + + print("All instances started :)") + + start_time = datetime.now() + while datetime.now() < start_time + timedelta(minutes=2): print(f"instances: {instances()}") await asyncio.sleep(5) From ae7b8c0e6cd0c40c66d1a3651028b8272b2317fd Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 27 May 2021 14:15:02 +0200 Subject: [PATCH 80/85] add README.md to the Docker image directory, add appropriate annotations to .py files included within to make it obvious that these files are not part of the requestor agent --- .../simple_service/README.md | 38 +++++++++++++++++++ .../simple_service/simple_service.py | 4 +- .../simple_service/simulate_observations.py | 2 + .../simulate_observations_ctl.py | 2 + 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 examples/simple-service-poc/simple_service/README.md diff --git a/examples/simple-service-poc/simple_service/README.md b/examples/simple-service-poc/simple_service/README.md new file mode 100644 index 000000000..692f5156a --- /dev/null +++ b/examples/simple-service-poc/simple_service/README.md @@ -0,0 +1,38 @@ +This directory contains files used to construct the application Docker image +that's then converted to a GVMI file (a Golem Virtual Machine Image file) and uploaded +to the Yagna image repository. + +All Python scripts here are run within a VM on the Provider's end. + +The example (`../simple_service.py`) already contains the appropriate image hash +but if you'd like to experiment with it, feel free to re-build it. + +## Building the image + +You'll need: + +* Docker: https://www.docker.com/products/docker-desktop +* gvmkit-build: `pip install gvmkit-build` + +Once you have those installed, run the following from this directory: + +```bash +docker build -f simple_service.Dockerfile -t simple-service . +gvmkit-build simple-service:latest +gvmkit-build simple-service:latest --push +``` + +Note the hash link that's presented after the upload finishes. + +e.g. `b742b6cb04123d07bacb36a2462f8b2347b20c32223c1ac49664635f` + +and update the service's `get_payload` method to point to this image: + +```python + async def get_payload(): + return await vm.repo( + image_hash="b742b6cb04123d07bacb36a2462f8b2347b20c32223c1ac49664635f", + min_mem_gib=0.5, + min_storage_gib=2.0, + ) +``` diff --git a/examples/simple-service-poc/simple_service/simple_service.py b/examples/simple-service-poc/simple_service/simple_service.py index 3011896ce..cdde5cf98 100644 --- a/examples/simple-service-poc/simple_service/simple_service.py +++ b/examples/simple-service-poc/simple_service/simple_service.py @@ -3,7 +3,9 @@ a very basic "stub" that exposes a few commands of an imagined, very simple CLI-based service that is able to accumulate some linear, time-based values and present it stats (characteristics of the statistical distribution of the data collected so far) or provide -distribution and time-series plots of the collected data +distribution and time-series plots of the collected data. + +[ part of the VM image that's deployed by the runtime on the Provider's end. ] """ import argparse from datetime import datetime diff --git a/examples/simple-service-poc/simple_service/simulate_observations.py b/examples/simple-service-poc/simple_service/simulate_observations.py index ad2324ca5..430807dc1 100644 --- a/examples/simple-service-poc/simple_service/simulate_observations.py +++ b/examples/simple-service-poc/simple_service/simulate_observations.py @@ -6,6 +6,8 @@ machine providing its inputs into the database or some other piece of information from some external source that changes over time and which can be expressed as a singular value + +[ part of the VM image that's deployed by the runtime on the Provider's end. ] """ import os from pathlib import Path diff --git a/examples/simple-service-poc/simple_service/simulate_observations_ctl.py b/examples/simple-service-poc/simple_service/simulate_observations_ctl.py index bcaffb5c9..274063a54 100644 --- a/examples/simple-service-poc/simple_service/simulate_observations_ctl.py +++ b/examples/simple-service-poc/simple_service/simulate_observations_ctl.py @@ -1,6 +1,8 @@ #!/usr/local/bin/python """ a helper, control script that starts and stops our example `simulate_observations` service + +[ part of the VM image that's deployed by the runtime on the Provider's end. ] """ import argparse import os From 57f2c4882b078a0dbed67da6a9c856996deb2957 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 27 May 2021 15:13:29 +0200 Subject: [PATCH 81/85] prevent the example from finishing before we've established whether the services have started --- examples/simple-service-poc/simple_service.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 311480bba..7b4d1c069 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -4,9 +4,7 @@ """ import asyncio from datetime import datetime, timedelta, timezone -import json import pathlib -import queue import random import string import sys @@ -36,7 +34,7 @@ ) NUM_INSTANCES = 1 -STARTING_TIMEOUT = timedelta(minutes=10) +STARTING_TIMEOUT = timedelta(minutes=4) class SimpleService(Service): @@ -117,7 +115,7 @@ def still_running(): def still_starting(): return len(cluster.instances) < NUM_INSTANCES or \ - any([s for s in cluster.instances if s.state == ServiceState.running]) + any([s for s in cluster.instances if s.state == ServiceState.starting]) while still_starting() and datetime.now() < commissioning_time + STARTING_TIMEOUT: print(f"instances: {instances()}") @@ -146,7 +144,7 @@ def still_starting(): if __name__ == "__main__": - parser = build_parser("Test http") + parser = build_parser("A very simple / POC example of a service running on Golem, utilizing the VM runtime") now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") parser.set_defaults(log_file=f"simple-service-yapapi-{now}.log") args = parser.parse_args() From ab9d2780b3b25fd3060125e2d8cebbe4ad94fb15 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 27 May 2021 15:20:54 +0200 Subject: [PATCH 82/85] some comments to clarify the service example --- examples/simple-service-poc/simple_service.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 7b4d1c069..c02d9a8d6 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -102,11 +102,15 @@ async def main(subnet_tag, driver=None, network=None): print(f"{TEXT_COLOR_YELLOW}starting {pluralize(NUM_INSTANCES, 'instance')}{TEXT_COLOR_DEFAULT}") + # start the service + cluster = await golem.run_service( SimpleService, num_instances=NUM_INSTANCES, expiration=datetime.now(timezone.utc) + timedelta(minutes=120)) + # helper functions to display / filter instances + def instances(): return [(s.provider_name, s.state.value) for s in cluster.instances] @@ -117,6 +121,8 @@ def still_starting(): return len(cluster.instances) < NUM_INSTANCES or \ any([s for s in cluster.instances if s.state == ServiceState.starting]) + # wait until instances are started + while still_starting() and datetime.now() < commissioning_time + STARTING_TIMEOUT: print(f"instances: {instances()}") await asyncio.sleep(5) @@ -126,6 +132,9 @@ def still_starting(): print("All instances started :)") + # allow the service to run for a short while + # (and allowing its requestor-end handlers to interact with it) + start_time = datetime.now() while datetime.now() < start_time + timedelta(minutes=2): @@ -135,6 +144,8 @@ def still_starting(): print(f"{TEXT_COLOR_YELLOW}stopping instances{TEXT_COLOR_DEFAULT}") cluster.stop() + # wait for instances to stop + cnt = 0 while cnt < 10 and still_running(): print(f"instances: {instances()}") From 44f826e0780c89cea4f1f5aaf2e0f0f9be349569 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 27 May 2021 15:26:16 +0200 Subject: [PATCH 83/85] make the ctl script a constant, add some comments to the handlers --- examples/simple-service-poc/simple_service.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index c02d9a8d6..80146d63c 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -41,6 +41,7 @@ class SimpleService(Service): STATS_PATH = "/golem/out/stats" PLOT_INFO_PATH = "/golem/out/plot" SIMPLE_SERVICE = "/golem/run/simple_service.py" + SIMPLE_SERVICE_CTL = "/golem/run/simulate_observations_ctl.py" @staticmethod async def get_payload(): @@ -51,10 +52,12 @@ async def get_payload(): ) async def start(self): - self._ctx.run("/golem/run/simulate_observations_ctl.py", "--start") + # handler responsible for starting the service + self._ctx.run(self.SIMPLE_SERVICE_CTL, "--start") yield self._ctx.commit() async def run(self): + # handler responsible for providing the required interactions while the service is running while True: await asyncio.sleep(10) self._ctx.run(self.SIMPLE_SERVICE, "--stats") # idx 0 @@ -78,7 +81,8 @@ async def run(self): yield steps async def shutdown(self): - self._ctx.run("/golem/run/simulate_observations_ctl.py", "--stop") + # handler reponsible for executing operations on shutdown + self._ctx.run(self.SIMPLE_SERVICE_CTL, "--stop") yield self._ctx.commit() From 73b30366adf687e3402095c692e281e9c495a671 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Thu, 27 May 2021 16:53:21 +0200 Subject: [PATCH 84/85] Adjust expected log messages in blender/yacat integration tests --- tests/goth/test_run_blender.py | 2 +- tests/goth/test_run_yacat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/goth/test_run_blender.py b/tests/goth/test_run_blender.py index a5167ab93..24c005fca 100644 --- a/tests/goth/test_run_blender.py +++ b/tests/goth/test_run_blender.py @@ -51,7 +51,7 @@ async def assert_all_tasks_started(output_lines: EventStream[str]): async def assert_all_tasks_computed(output_lines: EventStream[str]): """Assert that for every task a line with `Task computed by provider` will appear.""" - await assert_all_tasks_processed("computed by provider", output_lines) + await assert_all_tasks_processed("finished by provider", output_lines) async def assert_all_invoices_accepted(output_lines: EventStream[str]): diff --git a/tests/goth/test_run_yacat.py b/tests/goth/test_run_yacat.py index d5f758758..d92c71265 100644 --- a/tests/goth/test_run_yacat.py +++ b/tests/goth/test_run_yacat.py @@ -51,7 +51,7 @@ async def assert_all_tasks_started(output_lines: EventStream[str]): async def assert_all_tasks_computed(output_lines: EventStream[str]): """Assert that for every task a line with `Task computed by provider` will appear.""" - await assert_all_tasks_processed("computed by provider", output_lines) + await assert_all_tasks_processed("finished by provider", output_lines) async def assert_all_invoices_accepted(output_lines: EventStream[str]): From 71a2542dd3b0fc32d3d3b9b233e3d5bb9fa7faaf Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 27 May 2021 17:13:30 +0200 Subject: [PATCH 85/85] black --- examples/simple-service-poc/simple_service.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 80146d63c..2a5fa1fd9 100644 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -10,7 +10,6 @@ import sys - from yapapi import ( NoPaymentAccountError, __version__ as yapapi_version, @@ -46,10 +45,10 @@ class SimpleService(Service): @staticmethod async def get_payload(): return await vm.repo( - image_hash="8b11df59f84358d47fc6776d0bb7290b0054c15ded2d6f54cf634488", - min_mem_gib=0.5, - min_storage_gib=2.0, - ) + image_hash="8b11df59f84358d47fc6776d0bb7290b0054c15ded2d6f54cf634488", + min_mem_gib=0.5, + min_storage_gib=2.0, + ) async def start(self): # handler responsible for starting the service @@ -70,12 +69,13 @@ async def run(self): print(f"{TEXT_COLOR_CYAN}stats: {stats}{TEXT_COLOR_DEFAULT}") - plot_filename = ( - "".join(random.choice(string.ascii_letters) for _ in - range(10)) + ".png" + plot_filename = "".join(random.choice(string.ascii_letters) for _ in range(10)) + ".png" + print( + f"{TEXT_COLOR_CYAN}downloading plot: {plot} to {plot_filename}{TEXT_COLOR_DEFAULT}" + ) + self._ctx.download_file( + plot, str(pathlib.Path(__file__).resolve().parent / plot_filename) ) - print(f"{TEXT_COLOR_CYAN}downloading plot: {plot} to {plot_filename}{TEXT_COLOR_DEFAULT}") - self._ctx.download_file(plot, str(pathlib.Path(__file__).resolve().parent / plot_filename)) steps = self._ctx.commit() yield steps @@ -104,14 +104,17 @@ async def main(subnet_tag, driver=None, network=None): commissioning_time = datetime.now() - print(f"{TEXT_COLOR_YELLOW}starting {pluralize(NUM_INSTANCES, 'instance')}{TEXT_COLOR_DEFAULT}") + print( + f"{TEXT_COLOR_YELLOW}starting {pluralize(NUM_INSTANCES, 'instance')}{TEXT_COLOR_DEFAULT}" + ) # start the service cluster = await golem.run_service( SimpleService, num_instances=NUM_INSTANCES, - expiration=datetime.now(timezone.utc) + timedelta(minutes=120)) + expiration=datetime.now(timezone.utc) + timedelta(minutes=120), + ) # helper functions to display / filter instances @@ -122,8 +125,9 @@ def still_running(): return any([s for s in cluster.instances if s.is_available]) def still_starting(): - return len(cluster.instances) < NUM_INSTANCES or \ - any([s for s in cluster.instances if s.state == ServiceState.starting]) + return len(cluster.instances) < NUM_INSTANCES or any( + [s for s in cluster.instances if s.state == ServiceState.starting] + ) # wait until instances are started @@ -159,7 +163,9 @@ def still_starting(): if __name__ == "__main__": - parser = build_parser("A very simple / POC example of a service running on Golem, utilizing the VM runtime") + parser = build_parser( + "A very simple / POC example of a service running on Golem, utilizing the VM runtime" + ) now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") parser.set_defaults(log_file=f"simple-service-yapapi-{now}.log") args = parser.parse_args()