From f4812bd758e27c8138a146d77ec4b42fbe2eab69 Mon Sep 17 00:00:00 2001 From: maaktweluit Date: Tue, 21 Sep 2021 09:47:36 +0200 Subject: [PATCH 1/2] Added power outage test --- tests/goth_tests/test_power_outage.py | 111 ++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/goth_tests/test_power_outage.py diff --git a/tests/goth_tests/test_power_outage.py b/tests/goth_tests/test_power_outage.py new file mode 100644 index 000000000..0e536b681 --- /dev/null +++ b/tests/goth_tests/test_power_outage.py @@ -0,0 +1,111 @@ +import logging +import os +from pathlib import Path +from typing import List + +import pytest + +from goth.assertions import EventStream +from goth.configuration import load_yaml, Override +from goth.runner.log import configure_logging +from goth.runner import Runner +from goth.runner.probe import RequestorProbe, ProviderProbe + +from yapapi.log import SummaryLogger + +from .assertions import assert_no_errors, assert_tasks_processed + + +logger = logging.getLogger("goth.test.power_outage") + +ALL_TASKS = {"0", "10", "20", "30", "40", "50"} + + +# Temporal assertions expressing properties of sequences of "events". In this case, each "event" +# is just a line of output from `blender.py`. + + +async def assert_all_tasks_started(output_lines: EventStream[str]): + """Assert that for every task a line with `Task started on provider` will appear.""" + await assert_tasks_processed(ALL_TASKS, "started on provider", output_lines) + + +async def assert_all_tasks_computed(output_lines: EventStream[str]): + """Assert that for every task a line with `Task computed by provider` will appear.""" + await assert_tasks_processed(ALL_TASKS, "finished by provider", output_lines) + + +""" Test when provider has a power outage. + + Start a regular blender task, when the first task is started on the provider it gets + shut down. The requestor is supposed to send all tasks to the other machine and complete + the task. The invoice for the power-outage provider will not be paid. """ + + +@pytest.mark.asyncio +async def test_power_outage( + project_dir: Path, log_dir: Path, goth_config_path: Path, config_overrides: List[Override] +) -> None: + + # This is the default configuration with 2 wasm/VM providers + goth_config = load_yaml(goth_config_path, config_overrides) + + blender_path = project_dir / "examples" / "blender" / "blender.py" + + configure_logging(log_dir) + + runner = Runner( + base_log_dir=log_dir, + compose_config=goth_config.compose_config, + ) + + async with runner(goth_config.containers): + + requestor = runner.get_probes(probe_type=RequestorProbe)[0] + provider_1 = runner.get_probes(probe_type=ProviderProbe)[0] + + async with requestor.run_command_on_host( + f"{blender_path} --subnet-tag goth", + env=os.environ, + ) as (_cmd_task, cmd_monitor): + + # Add assertions to the command output monitor `cmd_monitor`: + cmd_monitor.add_assertion(assert_no_errors) + all_sent = cmd_monitor.add_assertion(assert_all_tasks_started) + all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed) + + await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ", timeout=20) + logger.info("Received proposals") + + await cmd_monitor.wait_for_pattern(".*Agreement proposed ", timeout=10) + logger.info("Agreement proposed") + + await cmd_monitor.wait_for_pattern(".*Agreement confirmed ", timeout=10) + logger.info("Agreement confirmed") + + await cmd_monitor.wait_for_pattern( + ".*Task started on provider 'provider-1'.*", timeout=10 + ) + + logger.debug("Stopping provider 1") + await provider_1.stop() + provider_1.container.stop() + logger.info("Provider 1 stopped.") + + await cmd_monitor.wait_for_pattern(".*Terminated agreement with provider-1") + logger.info("Agreement properly terminated") + + await all_sent.wait_for_result(timeout=120) + logger.info("All tasks sent") + + await all_computed.wait_for_result(timeout=120) + logger.info("All tasks computed, waiting for Golem shutdown") + + await cmd_monitor.wait_for_pattern(".*Unpaid agreements: .*") + logger.info("Expected 1 unpaid agreement") + + await cmd_monitor.wait_for_pattern( + f".*{SummaryLogger.GOLEM_SHUTDOWN_SUCCESSFUL_MESSAGE}", timeout=120 + ) + + logger.info("Requestor script finished") From 326f23ec43c269def0abeb20b60e2bea78589889 Mon Sep 17 00:00:00 2001 From: maaktweluit Date: Fri, 24 Sep 2021 14:18:30 +0200 Subject: [PATCH 2/2] Start machine again --- tests/goth_tests/test_power_outage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/goth_tests/test_power_outage.py b/tests/goth_tests/test_power_outage.py index 0e536b681..c563b33da 100644 --- a/tests/goth_tests/test_power_outage.py +++ b/tests/goth_tests/test_power_outage.py @@ -37,7 +37,7 @@ async def assert_all_tasks_computed(output_lines: EventStream[str]): """ Test when provider has a power outage. - Start a regular blender task, when the first task is started on the provider it gets + Start a regular blender task, when the first task is started on the provider it gets shut down. The requestor is supposed to send all tasks to the other machine and complete the task. The invoice for the power-outage provider will not be paid. """ @@ -91,6 +91,7 @@ async def test_power_outage( await provider_1.stop() provider_1.container.stop() logger.info("Provider 1 stopped.") + provider_1.container.restart() await cmd_monitor.wait_for_pattern(".*Terminated agreement with provider-1") logger.info("Agreement properly terminated")