From fe7cb1b3cf35dad142b55b41816d288ad3348d9f Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 5 May 2021 15:27:40 +0200 Subject: [PATCH 01/29] + comment to "decrease-scores" strategy --- yapapi/executor/strategy.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/yapapi/executor/strategy.py b/yapapi/executor/strategy.py index 1e59875eb..db660369a 100644 --- a/yapapi/executor/strategy.py +++ b/yapapi/executor/strategy.py @@ -165,6 +165,11 @@ class DecreaseScoreForUnconfirmedAgreement(MarketStrategy): factor: float def __init__(self, base_strategy, factor): + """ + :param base_strategy: the base strategy around which this strategy is wrapped + :param factor: the factor by which the score of an offer for a provider which + failed to confirm the last agreement proposed to them will be multiplied + """ self.base_strategy = base_strategy self.factor = factor self._logger = logging.getLogger(f"{__name__}.{type(self).__name__}") From 6da6bff3e8319dfaaef7e211e0b6be39001ba719 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 5 May 2021 15:28:39 +0200 Subject: [PATCH 02/29] splitting executor... --- yapapi/executor/__init__.py | 649 +++++++++++++++++++++--------------- 1 file changed, 384 insertions(+), 265 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 869722d58..fd54e6185 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -23,6 +23,7 @@ cast, ) import traceback +import warnings from dataclasses import dataclass, field @@ -91,31 +92,14 @@ def __str__(self) -> str: ) -@dataclass -class _ExecutorConfig: - max_workers: int = 5 - timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT - get_offers_timeout: timedelta = timedelta(seconds=20) - traceback: bool = bool(os.getenv("YAPAPI_TRACEBACK", 0)) - - D = TypeVar("D") # Type var for task data R = TypeVar("R") # Type var for task result -class Executor(AsyncContextManager): - """ - Task executor. - - Used to run tasks using the specified application package within providers' execution units. - """ - +class Engine(AsyncContextManager): def __init__( self, *, - package: Package, - max_workers: int = 5, - timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, budget: Union[float, Decimal], strategy: Optional[MarketStrategy] = None, subnet_tag: Optional[str] = None, @@ -123,13 +107,11 @@ def __init__( network: Optional[str] = None, event_consumer: Optional[Callable[[Event], None]] = None, stream_output: bool = False, + app_key: Optional[str] = None ): - """Create a new executor. + """ + Base execution engine containing functions common to all modes of operation - :param package: a package common for all tasks; vm.repo() function may be - used to return package from a repository - :param max_workers: maximum number of workers doing the computation - :param timeout: timeout for the whole computation :param budget: maximum budget for payments :param strategy: market strategy used to select providers from the market (e.g. LeastExpensiveLinearPayuMS or DummyMS) @@ -141,13 +123,14 @@ def __init__( :param event_consumer: a callable that processes events related to the computation; by default it is a function that logs all events :param stream_output: stream computation output from providers + :param app_key: optional Yagna application key. If not provided, the default is to + get the value from `YAGNA_APPKEY` environment variable """ - logger.debug("Creating Executor instance; parameters: %s", locals()) + self._init_api(app_key) + + self._budget_amount = Decimal(budget) + self._budget_allocations: List[rest.payment.Allocation] = [] - self._subnet: Optional[str] = subnet_tag - self._driver = driver.lower() if driver else DEFAULT_DRIVER - self._network = network.lower() if network else DEFAULT_NETWORK - self._stream_output = stream_output if not strategy: strategy = LeastExpensiveLinearPayuMS( max_fixed_price=Decimal("1.0"), @@ -157,13 +140,10 @@ def __init__( # the last agreement proposed to them will have it's score multiplied by 0.5. strategy = DecreaseScoreForUnconfirmedAgreement(strategy, 0.5) self._strategy = strategy - self._api_config = rest.Configuration() - self._stack = AsyncExitStack() - self._package = package - self._conf = _ExecutorConfig(max_workers, timeout) - # TODO: setup precision - self._budget_amount = Decimal(budget) - self._budget_allocations: List[rest.payment.Allocation] = [] + + self._subnet: Optional[str] = subnet_tag + self._driver = driver.lower() if driver else DEFAULT_DRIVER + self._network = network.lower() if network else DEFAULT_NETWORK if not event_consumer: # Use local import to avoid cyclic imports when yapapi.log @@ -175,9 +155,17 @@ def __init__( # Add buffering to the provided event emitter to make sure # that emitting events will not block self._wrapped_consumer = AsyncWrapper(event_consumer) - # Each call to `submit()` will add an instance of `asyncio.Event` to this set. - # This event can be used to wait until the call to `submit()` is finished. - self._active_computations: Set[asyncio.Event] = set() + + self._stream_output = stream_output + + self._stack = AsyncExitStack() + + def _init_api(self, app_key: Optional[str] = None): + """ + initialize the REST (low-level) API + :param app_key: (optional) yagna daemon application key + """ + self._api_config = rest.Configuration(app_key) @property def driver(self) -> str: @@ -191,6 +179,356 @@ def network(self) -> str: def strategy(self) -> MarketStrategy: return self._strategy + def emit(self, *args, **kwargs) -> None: + self._wrapped_consumer.async_call(*args, **kwargs) + + async def __aenter__(self): + stack = self._stack + + market_client = await stack.enter_async_context(self._api_config.market()) + self._market_api = rest.Market(market_client) + + activity_client = await stack.enter_async_context(self._api_config.activity()) + self._activity_api = rest.Activity(activity_client) + + payment_client = await stack.enter_async_context(self._api_config.payment()) + self._payment_api = rest.Payment(payment_client) + + # a set of `asyncio.Event` instances used to track jobs - computations or services - started + # those events can be used to wait until all jobs are finished + self._active_jobs: Set[asyncio.Event] = set() + + self.payment_decoration = Engine.PaymentDecoration(await self._create_allocations()) + + async def __aexit__(self, exc_type, exc_val, exc_tb): + logger.debug("Engine is shutting down...") + # Wait until all computations are finished + await asyncio.gather(*[event.wait() for event in self._active_jobs]) + # TODO: prevent new computations at this point (if it's even possible to start one) + try: + await self._stack.aclose() + self.emit(events.ShutdownFinished()) + except Exception: + self.emit(events.ShutdownFinished(exc_info=sys.exc_info())) + finally: + await self._wrapped_consumer.stop() + + async def _create_allocations(self) -> rest.payment.MarketDecoration: + + if not self._budget_allocations: + async for account in self._payment_api.accounts(): + driver = account.driver.lower() + network = account.network.lower() + if (driver, network) != (self._driver, self._network): + logger.debug( + f"Not using payment platform `%s`, platform's driver/network " + f"`%s`/`%s` is different than requested driver/network `%s`/`%s`", + account.platform, + driver, + network, + self._driver, + self._network, + ) + continue + logger.debug("Creating allocation using payment platform `%s`", account.platform) + allocation = cast( + rest.payment.Allocation, + await self._stack.enter_async_context( + self._payment_api.new_allocation( + self._budget_amount, + payment_platform=account.platform, + payment_address=account.address, + # TODO what do to with this? + # expires=self._expires + CFG_INVOICE_TIMEOUT, + ) + ), + ) + self._budget_allocations.append(allocation) + + if not self._budget_allocations: + raise NoPaymentAccountError(self._driver, self._network) + + allocation_ids = [allocation.id for allocation in self._budget_allocations] + return await self._payment_api.decorate_demand(allocation_ids) + + def _get_allocation( + self, item: Union[rest.payment.DebitNote, rest.payment.Invoice] + ) -> rest.payment.Allocation: + try: + return next( + allocation + for allocation in self._budget_allocations + if allocation.payment_address == item.payer_addr + and allocation.payment_platform == item.payment_platform + ) + except: + raise ValueError(f"No allocation for {item.payment_platform} {item.payer_addr}.") + + @dataclass + class PaymentDecoration: # TODO add: (DemandDecorator) + market_decoration: rest.payment.MarketDecoration + + async def decorate_demand(self, demand: DemandBuilder): + for constraint in self.market_decoration.constraints: + demand.ensure(constraint) + demand.properties.update({p.key: p.value for p in self.market_decoration.properties}) + + class Job: + """Functionality related to a single job.""" + + def __init__( + self, + engine: "Engine", + builder: DemandBuilder, + agreements_pool: AgreementsPool, + ): + self.engine = engine + self.builder = builder + self.agreements_pool = agreements_pool + + self.offers_collected: int = 0 + self.proposals_confirmed: int = 0 + + async def _handle_proposal( + self, + proposal: OfferProposal, + ) -> events.Event: + """Handle a single `OfferProposal`. + + A `proposal` is scored and then can be rejected, responded with + a counter-proposal or stored in an agreements pool to be used + for negotiating an agreement. + """ + + async def reject_proposal(reason: str) -> events.ProposalRejected: + """Reject `proposal` due to given `reason`.""" + await proposal.reject(reason) + return events.ProposalRejected(prop_id=proposal.id, reason=reason) + + score = await self.engine._strategy.score_offer(proposal, self.agreements_pool) + logger.debug( + "Scored offer %s, provider: %s, strategy: %s, score: %f", + proposal.id, + proposal.props.get("golem.node.id.name"), + type(self.engine._strategy).__name__, + score, + ) + + if score < SCORE_NEUTRAL: + return await reject_proposal("Score too low") + + if not proposal.is_draft: + # Proposal is not yet a draft of an agreement + + # Check if any of the supported payment platforms matches the proposal + common_platforms = self._get_common_payment_platforms(proposal) + if common_platforms: + self.builder.properties["golem.com.payment.chosen-platform"] = next( + iter(common_platforms) + ) + else: + # reject proposal if there are no common payment platforms + return await reject_proposal("No common payment platform") + + # Check if the timeout for debit note acceptance is not too low + timeout = proposal.props.get(DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP) + if timeout: + if timeout < DEBIT_NOTE_MIN_TIMEOUT: + return await reject_proposal("Debit note acceptance timeout is too short") + else: + self.builder.properties[DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP] = timeout + + await proposal.respond(self.builder.properties, self.builder.constraints) + return events.ProposalResponded(prop_id=proposal.id) + + else: + # It's a draft agreement + await self.agreements_pool.add_proposal(score, proposal) + return events.ProposalConfirmed(prop_id=proposal.id) + + async def _find_offers_for_subscription( + self, subscription: Subscription + ) -> None: + """Create a market subscription and repeatedly collect offer proposals for it. + + Collected proposals are processed concurrently using a bounded number + of asyncio tasks. + + :param state: A state related to a call to `Executor.submit()` + """ + max_number_of_tasks = 5 + + try: + proposals = subscription.events() + except Exception as ex: + self.engine.emit(events.CollectFailed(sub_id=subscription.id, reason=str(ex))) + raise + + # A semaphore is used to limit the number of handler tasks + semaphore = asyncio.Semaphore(max_number_of_tasks) + + async for proposal in proposals: + + self.engine.emit(events.ProposalReceived(prop_id=proposal.id, provider_id=proposal.issuer)) + self.offers_collected += 1 + + async def handler(proposal_): + """A coroutine that wraps `_handle_proposal()` method with error handling.""" + try: + event = await self._handle_proposal(proposal_) + assert isinstance(event, events.ProposalEvent) + self.engine.emit(event) + if isinstance(event, events.ProposalConfirmed): + self.proposals_confirmed += 1 + except CancelledError: + raise + except Exception: + with contextlib.suppress(Exception): + self.engine.emit( + events.ProposalFailed( + prop_id=proposal_.id, exc_info=sys.exc_info() # type: ignore + ) + ) + finally: + semaphore.release() + + # Create a new handler task + await semaphore.acquire() + asyncio.get_event_loop().create_task(handler(proposal)) + + async def _find_offers(self) -> None: + """Create demand subscription and process offers. + When the subscription expires, create a new one. And so on... + """ + + while True: + try: + subscription = await self.builder.subscribe(self.engine._market_api) + self.engine.emit(events.SubscriptionCreated(sub_id=subscription.id)) + except Exception as ex: + self.engine.emit(events.SubscriptionFailed(reason=str(ex))) + raise + async with subscription: + await self._find_offers_for_subscription(subscription) + + def _get_common_payment_platforms(self, proposal: rest.market.OfferProposal) -> Set[str]: + prov_platforms = { + property.split(".")[4] + for property in proposal.props + if property.startswith("golem.com.payment.platform.") and property is not None + } + if not prov_platforms: + prov_platforms = {"NGNT"} + req_platforms = { + allocation.payment_platform + for allocation in self.engine._budget_allocations + if allocation.payment_platform is not None + } + return req_platforms.intersection(prov_platforms) + + + + async def execute_task( + self, + worker: Callable[[WorkContext, AsyncIterator[Task[D, R]]], AsyncGenerator[Work, None]], + data: Iterable[Task[D, R]], + payload: Package, + max_workers: Optional[int] = None, + timeout: Optional[timedelta] = None, + ) -> AsyncIterator[Task[D, R]]: + + kwargs = { + 'package': payload + } + if max_workers: + kwargs['max_workers'] = max_workers + if timeout: + kwargs['timeout'] = timeout + + async with Executor(_engine=self, **kwargs) as executor: + async for t in executor.submit(worker, data): + yield t + + + +@dataclass +class _ExecutorConfig: + max_workers: int = 5 + timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT + get_offers_timeout: timedelta = timedelta(seconds=20) + traceback: bool = bool(os.getenv("YAPAPI_TRACEBACK", 0)) + + +class Executor(AsyncContextManager): + """ + Task executor. + + Used to run batch tasks using the specified application package within providers' execution units. + """ + + def __init__( + self, + *, + budget: Union[float, Decimal], + strategy: Optional[MarketStrategy] = None, + subnet_tag: Optional[str] = None, + driver: Optional[str] = None, + network: Optional[str] = None, + event_consumer: Optional[Callable[[Event], None]] = None, + stream_output: bool = False, + max_workers: int = 5, + timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, + package: Package, + _engine: Optional[Engine] = None + ): + """Create a new executor. + + :param budget: [DEPRECATED use `Engine` instead] maximum budget for payments + :param strategy: [DEPRECATED use `Engine` instead] market strategy used to + select providers from the market (e.g. LeastExpensiveLinearPayuMS or DummyMS) + :param subnet_tag: [DEPRECATED use `Engine` instead] use only providers in the + subnet with the subnet_tag name + :param driver: [DEPRECATED use `Engine` instead] name of the payment driver + to use or `None` to use the default driver; + only payment platforms with the specified driver will be used + :param network: [DEPRECATED use `Engine` instead] name of the network + to use or `None` to use the default network; + only payment platforms with the specified network will be used + :param event_consumer: [DEPRECATED use `Engine` instead] a callable that + processes events related to the computation; + by default it is a function that logs all events + :param stream_output: [DEPRECATED use `Engine` instead] + stream computation output from providers + + :param max_workers: maximum number of workers performing the computation + :param timeout: timeout for the whole computation + :param package: a package common for all tasks; vm.repo() function may be used + to return package from a repository + """ + logger.debug("Creating Executor instance; parameters: %s", locals()) + self.__standalone = False + + if not _engine: + warnings.warn( + "stand-alone usage is deprecated, please `Engine.execute_task` class instead ", + DeprecationWarning + ) + self._engine = Engine( + budget = budget, + strategy = strategy, + subnet_tag = subnet_tag, + driver = driver, + network = network, + event_consumer = event_consumer, + stream_output=stream_output + ) + + self._package = package + self._conf = _ExecutorConfig(max_workers, timeout) + # TODO: setup precision + + self._stack = AsyncExitStack() + async def submit( self, worker: Callable[[WorkContext, AsyncIterator[Task[D, R]]], AsyncGenerator[Work, None]], @@ -205,7 +543,7 @@ async def submit( """ computation_finished = asyncio.Event() - self._active_computations.add(computation_finished) + self._engine._active_jobs.add(computation_finished) services: Set[asyncio.Task] = set() workers: Set[asyncio.Task] = set() @@ -231,147 +569,7 @@ async def submit( await asyncio.gather(*all_tasks, return_exceptions=True) # Signal that this computation is finished computation_finished.set() - self._active_computations.remove(computation_finished) - - @dataclass - class SubmissionState: - """Variables related to a single call to `Executor.submit()`. - - One day this will be a fully-fledged and mature class with - methods of its own. But for now it only includes the variables - required by `find_offers()`. - """ - - builder: DemandBuilder - agreements_pool: AgreementsPool - offers_collected: int = field(default=0) - proposals_confirmed: int = field(default=0) - - async def _handle_proposal( - self, - state: "Executor.SubmissionState", - proposal: OfferProposal, - ) -> events.Event: - """Handle a single `OfferProposal`. - - A `proposal` is scored and then can be rejected, responded with - a counter-proposal or stored in an agreements pool to be used - for negotiating an agreement. - """ - - async def reject_proposal(reason: str) -> events.ProposalRejected: - """Reject `proposal` due to given `reason`.""" - await proposal.reject(reason) - return events.ProposalRejected(prop_id=proposal.id, reason=reason) - - score = await self._strategy.score_offer(proposal, state.agreements_pool) - logger.debug( - "Scored offer %s, provider: %s, strategy: %s, score: %f", - proposal.id, - proposal.props.get("golem.node.id.name"), - type(self._strategy).__name__, - score, - ) - - if score < SCORE_NEUTRAL: - return await reject_proposal("Score too low") - - if not proposal.is_draft: - # Proposal is not yet a draft of an agreement - - # Check if any of the supported payment platforms matches the proposal - common_platforms = self._get_common_payment_platforms(proposal) - if common_platforms: - state.builder.properties["golem.com.payment.chosen-platform"] = next( - iter(common_platforms) - ) - else: - # reject proposal if there are no common payment platforms - return await reject_proposal("No common payment platform") - - # Check if the timeout for debit note acceptance is not too low - timeout = proposal.props.get(DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP) - if timeout: - if timeout < DEBIT_NOTE_MIN_TIMEOUT: - return await reject_proposal("Debit note acceptance timeout is too short") - else: - state.builder.properties[DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP] = timeout - - await proposal.respond(state.builder.properties, state.builder.constraints) - return events.ProposalResponded(prop_id=proposal.id) - - else: - # It's a draft agreement - await state.agreements_pool.add_proposal(score, proposal) - return events.ProposalConfirmed(prop_id=proposal.id) - - async def _find_offers_for_subscription( - self, state: "Executor.SubmissionState", subscription: Subscription - ) -> None: - """Create a market subscription and repeatedly collect offer proposals for it. - - Collected proposals are processed concurrently using a bounded number - of asyncio tasks. - - :param state: A state related to a call to `Executor.submit()` - """ - max_number_of_tasks = 5 - - emit = cast(Callable[[Event], None], self._wrapped_consumer.async_call) - - try: - proposals = subscription.events() - except Exception as ex: - emit(events.CollectFailed(sub_id=subscription.id, reason=str(ex))) - raise - - # A semaphore is used to limit the number of handler tasks - semaphore = asyncio.Semaphore(max_number_of_tasks) - - async for proposal in proposals: - - emit(events.ProposalReceived(prop_id=proposal.id, provider_id=proposal.issuer)) - state.offers_collected += 1 - - async def handler(proposal_): - """A coroutine that wraps `_handle_proposal()` method with error handling.""" - try: - event = await self._handle_proposal(state, proposal_) - assert isinstance(event, events.ProposalEvent) - emit(event) - if isinstance(event, events.ProposalConfirmed): - state.proposals_confirmed += 1 - except CancelledError: - raise - except Exception: - with contextlib.suppress(Exception): - emit( - events.ProposalFailed( - prop_id=proposal_.id, exc_info=sys.exc_info() # type: ignore - ) - ) - finally: - semaphore.release() - - # Create a new handler task - await semaphore.acquire() - asyncio.get_event_loop().create_task(handler(proposal)) - - async def _find_offers(self, state: "Executor.SubmissionState") -> None: - """Create demand subscription and process offers. - When the subscription expires, create a new one. And so on... - """ - emit = cast(Callable[[Event], None], self._wrapped_consumer.async_call) - - while True: - try: - subscription = await state.builder.subscribe(self._market_api) - emit(events.SubscriptionCreated(sub_id=subscription.id)) - except Exception as ex: - emit(events.SubscriptionFailed(reason=str(ex))) - raise - async with subscription: - await self._find_offers_for_subscription(state, subscription) + self._engine._active_jobs.remove(computation_finished) async def _submit( self, @@ -381,11 +579,7 @@ async def _submit( workers: Set[asyncio.Task], ) -> AsyncGenerator[Task[D, R], None]: - emit = cast(Callable[[Event], None], self._wrapped_consumer.async_call) - - multi_payment_decoration = await self._create_allocations() - - emit(events.ComputationStarted(self._expires)) + self._engine.emit(events.ComputationStarted(self._expires)) # Building offer builder = DemandBuilder() @@ -393,15 +587,13 @@ async def _submit( builder.add(NodeInfo(subnet_tag=self._subnet)) if self._subnet: builder.ensure(f"({NodeInfoKeys.subnet_tag}={self._subnet})") - for constraint in multi_payment_decoration.constraints: - builder.ensure(constraint) - builder.properties.update({p.key: p.value for p in multi_payment_decoration.properties}) - await self._package.decorate_demand(builder) + await self._engine.payment_decoration.decorate_demand(builder) await self._strategy.decorate_demand(builder) + await self._package.decorate_demand(builder) - agreements_pool = AgreementsPool(emit) + agreements_pool = AgreementsPool(self._engine.emit) - state = Executor.SubmissionState(builder, agreements_pool) + state = Executor.Job(builder, agreements_pool) market_api = self._market_api activity_api: rest.Activity = self._activity_api @@ -575,7 +767,6 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> await accept_payment_for_agreement(agreement.id, partial=True) except Exception: - try: await command_generator.athrow(*sys.exc_info()) except Exception: @@ -743,84 +934,12 @@ async def worker_starter() -> None: if agreements_to_pay: logger.warning("Unpaid agreements: %s", agreements_to_pay) - async def _create_allocations(self) -> rest.payment.MarketDecoration: - - if not self._budget_allocations: - async for account in self._payment_api.accounts(): - driver = account.driver.lower() - network = account.network.lower() - if (driver, network) != (self._driver, self._network): - logger.debug( - f"Not using payment platform `%s`, platform's driver/network " - f"`%s`/`%s` is different than requested driver/network `%s`/`%s`", - account.platform, - driver, - network, - self._driver, - self._network, - ) - continue - logger.debug("Creating allocation using payment platform `%s`", account.platform) - allocation = cast( - rest.payment.Allocation, - await self._stack.enter_async_context( - self._payment_api.new_allocation( - self._budget_amount, - payment_platform=account.platform, - payment_address=account.address, - expires=self._expires + CFG_INVOICE_TIMEOUT, - ) - ), - ) - self._budget_allocations.append(allocation) - - if not self._budget_allocations: - raise NoPaymentAccountError(self._driver, self._network) - - allocation_ids = [allocation.id for allocation in self._budget_allocations] - return await self._payment_api.decorate_demand(allocation_ids) - - def _get_common_payment_platforms(self, proposal: rest.market.OfferProposal) -> Set[str]: - prov_platforms = { - property.split(".")[4] - for property in proposal.props - if property.startswith("golem.com.payment.platform.") and property is not None - } - if not prov_platforms: - prov_platforms = {"NGNT"} - req_platforms = { - allocation.payment_platform - for allocation in self._budget_allocations - if allocation.payment_platform is not None - } - return req_platforms.intersection(prov_platforms) - - def _get_allocation( - self, item: Union[rest.payment.DebitNote, rest.payment.Invoice] - ) -> rest.payment.Allocation: - try: - return next( - allocation - for allocation in self._budget_allocations - if allocation.payment_address == item.payer_addr - and allocation.payment_platform == item.payment_platform - ) - except: - raise ValueError(f"No allocation for {item.payment_platform} {item.payer_addr}.") async def __aenter__(self) -> "Executor": stack = self._stack # TODO: Cleanup on exception here. self._expires = datetime.now(timezone.utc) + self._conf.timeout - market_client = await stack.enter_async_context(self._api_config.market()) - self._market_api = rest.Market(market_client) - - activity_client = await stack.enter_async_context(self._api_config.activity()) - self._activity_api = rest.Activity(activity_client) - - payment_client = await stack.enter_async_context(self._api_config.payment()) - self._payment_api = rest.Payment(payment_client) return self @@ -828,7 +947,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): logger.debug("Executor is shutting down...") emit = cast(Callable[[Event], None], self._wrapped_consumer.async_call) # Wait until all computations are finished - await asyncio.gather(*[event.wait() for event in self._active_computations]) + await asyncio.gather(*[event.wait() for event in self._active_jobs]) # TODO: prevent new computations at this point (if it's even possible to start one) try: await self._stack.aclose() From cea1705e6c1aed4e53bde5f90e92262d48c2c6d4 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 13 May 2021 15:09:14 +0200 Subject: [PATCH 03/29] s/Engine/Golem/ --- yapapi/executor/__init__.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 9190515ef..f5a09bd97 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -96,7 +96,7 @@ def __str__(self) -> str: R = TypeVar("R") # Type var for task result -class Engine(AsyncContextManager): +class Golem(AsyncContextManager): def __init__( self, *, @@ -198,10 +198,10 @@ async def __aenter__(self): # those events can be used to wait until all jobs are finished self._active_jobs: Set[asyncio.Event] = set() - self.payment_decoration = Engine.PaymentDecoration(await self._create_allocations()) + self.payment_decoration = Golem.PaymentDecoration(await self._create_allocations()) async def __aexit__(self, exc_type, exc_val, exc_tb): - logger.debug("Engine is shutting down...") + logger.debug("Golem is shutting down...") # Wait until all computations are finished await asyncio.gather(*[event.wait() for event in self._active_jobs]) # TODO: prevent new computations at this point (if it's even possible to start one) @@ -278,7 +278,7 @@ class Job: def __init__( self, - engine: "Engine", + engine: "Golem", builder: DemandBuilder, agreements_pool: AgreementsPool, ): @@ -479,25 +479,25 @@ def __init__( max_workers: int = 5, timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, package: Package, - _engine: Optional[Engine] = None + _engine: Optional[Golem] = None ): """Create a new executor. - :param budget: [DEPRECATED use `Engine` instead] maximum budget for payments - :param strategy: [DEPRECATED use `Engine` instead] market strategy used to + :param budget: [DEPRECATED use `Golem` instead] maximum budget for payments + :param strategy: [DEPRECATED use `Golem` instead] market strategy used to select providers from the market (e.g. LeastExpensiveLinearPayuMS or DummyMS) - :param subnet_tag: [DEPRECATED use `Engine` instead] use only providers in the + :param subnet_tag: [DEPRECATED use `Golem` instead] use only providers in the subnet with the subnet_tag name - :param driver: [DEPRECATED use `Engine` instead] name of the payment driver + :param driver: [DEPRECATED use `Golem` instead] name of the payment driver to use or `None` to use the default driver; only payment platforms with the specified driver will be used - :param network: [DEPRECATED use `Engine` instead] name of the network + :param network: [DEPRECATED use `Golem` instead] name of the network to use or `None` to use the default network; only payment platforms with the specified network will be used - :param event_consumer: [DEPRECATED use `Engine` instead] a callable that + :param event_consumer: [DEPRECATED use `Golem` instead] a callable that processes events related to the computation; by default it is a function that logs all events - :param stream_output: [DEPRECATED use `Engine` instead] + :param stream_output: [DEPRECATED use `Golem` instead] stream computation output from providers :param max_workers: maximum number of workers performing the computation @@ -510,10 +510,10 @@ def __init__( if not _engine: warnings.warn( - "stand-alone usage is deprecated, please `Engine.execute_task` class instead ", + "stand-alone usage is deprecated, please `Golem.execute_task` class instead ", DeprecationWarning ) - self._engine = Engine( + self._engine = Golem( budget = budget, strategy = strategy, subnet_tag = subnet_tag, From a617f40a4c399b9c2a49157e82de7159aba39402 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Mon, 17 May 2021 08:57:14 +0200 Subject: [PATCH 04/29] Add method Golem.create_demand_builder() --- yapapi/executor/__init__.py | 49 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 524c0f672..b5f341633 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -165,6 +165,15 @@ def __init__( self._stack = AsyncExitStack() + def create_demand_builder(self) -> DemandBuilder: + """Create a `DemandBuilder`.""" + builder = DemandBuilder() + builder.add(NodeInfo(subnet_tag=self._subnet)) + if self._subnet: + builder.ensure(f"({NodeInfoKeys.subnet_tag}={self._subnet})") + await builder.decorate(self.payment_decoration, self.strategy) + return builder + def _init_api(self, app_key: Optional[str] = None): """ initialize the REST (low-level) API @@ -284,16 +293,21 @@ class Job: def __init__( self, engine: "Golem", - builder: DemandBuilder, agreements_pool: AgreementsPool, + expiration: datetime, + payload: Payload, + ): self.engine = engine - self.builder = builder self.agreements_pool = agreements_pool self.offers_collected: int = 0 self.proposals_confirmed: int = 0 + self.builder = engine.create_demand_builder() + self.builder.add(Activity(expiration=expiration, multi_activity=True)) + self.builder.decorate(payload) + async def _handle_proposal( self, proposal: OfferProposal, @@ -455,15 +469,6 @@ async def execute_task( yield t - -@dataclass -class _ExecutorConfig: - max_workers: int = 5 - timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT - get_offers_timeout: timedelta = timedelta(seconds=20) - traceback: bool = bool(os.getenv("YAPAPI_TRACEBACK", 0)) - - class Executor(AsyncContextManager): """ Task executor. @@ -540,7 +545,8 @@ def __init__( raise ValueError("Executor `payload` must be specified") self._payload = payload - self._conf = _ExecutorConfig(max_workers, timeout) + # self._conf = _ExecutorConfig(max_workers, timeout) + self._max_workers = max_workers # TODO: setup precision self._stack = AsyncExitStack() @@ -603,21 +609,14 @@ async def _submit( self._engine.emit(events.ComputationStarted(self._expires)) - # Building offer - builder = DemandBuilder() - builder.add(Activity(expiration=self._expires, multi_activity=True)) - builder.add(NodeInfo(subnet_tag=self._engine._subnet)) - if self._engine._subnet: - builder.ensure(f"({NodeInfoKeys.subnet_tag}={self._engine._subnet})") - await builder.decorate( - self._engine.payment_decoration, - self._engine.strategy, - self._payload, - ) - agreements_pool = AgreementsPool(self._engine.emit) - state = Golem.Job(builder, agreements_pool) + state = Golem.Job( + self._engine, + agreements_pool, + expiration=self._expires, + payload=self._payload + ) activity_api: rest.Activity = self._activity_api From 91ad481e5da4b934e996e6ee6b31680c40dcbe27 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Mon, 17 May 2021 09:08:11 +0200 Subject: [PATCH 05/29] Make Job a toplevel class --- yapapi/executor/__init__.py | 302 ++++++++++++++++++------------------ 1 file changed, 149 insertions(+), 153 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index b5f341633..c7c90373a 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -165,13 +165,14 @@ def __init__( self._stack = AsyncExitStack() - def create_demand_builder(self) -> DemandBuilder: - """Create a `DemandBuilder`.""" + def create_demand_builder(self, expiration_date: datetime, payload: Payload) -> DemandBuilder: + """Create a `DemandBuilder` for given `payload` and `expiration_date`.""" builder = DemandBuilder() + builder.add(Activity(expiration=expiration_date, multi_activity=True)) builder.add(NodeInfo(subnet_tag=self._subnet)) if self._subnet: builder.ensure(f"({NodeInfoKeys.subnet_tag}={self._subnet})") - await builder.decorate(self.payment_decoration, self.strategy) + await builder.decorate(self.payment_decoration, self.strategy, payload) return builder def _init_api(self, app_key: Optional[str] = None): @@ -287,186 +288,181 @@ async def decorate_demand(self, demand: DemandBuilder): demand.ensure(constraint) demand.properties.update({p.key: p.value for p in self.market_decoration.properties}) - class Job: - """Functionality related to a single job.""" + async def execute_task( + self, + worker: Callable[[WorkContext, AsyncIterator[Task[D, R]]], AsyncGenerator[Work, None]], + data: Iterable[Task[D, R]], + payload: Payload, + max_workers: Optional[int] = None, + timeout: Optional[timedelta] = None, + ) -> AsyncIterator[Task[D, R]]: - def __init__( - self, - engine: "Golem", - agreements_pool: AgreementsPool, - expiration: datetime, - payload: Payload, + kwargs = { + 'package': payload + } + if max_workers: + kwargs['max_workers'] = max_workers + if timeout: + kwargs['timeout'] = timeout - ): - self.engine = engine - self.agreements_pool = agreements_pool + async with Executor(_engine=self, **kwargs) as executor: + async for t in executor.submit(worker, data): + yield t - self.offers_collected: int = 0 - self.proposals_confirmed: int = 0 - self.builder = engine.create_demand_builder() - self.builder.add(Activity(expiration=expiration, multi_activity=True)) - self.builder.decorate(payload) +class Job: + """Functionality related to a single job.""" - async def _handle_proposal( + def __init__( self, - proposal: OfferProposal, - ) -> events.Event: - """Handle a single `OfferProposal`. + engine: Golem, + agreements_pool: AgreementsPool, + expiration_date: datetime, + payload: Payload, + ): + self.engine = engine + self.agreements_pool = agreements_pool - A `proposal` is scored and then can be rejected, responded with - a counter-proposal or stored in an agreements pool to be used - for negotiating an agreement. - """ + self.offers_collected: int = 0 + self.proposals_confirmed: int = 0 + self.builder = engine.create_demand_builder(expiration_date, payload) - async def reject_proposal(reason: str) -> events.ProposalRejected: - """Reject `proposal` due to given `reason`.""" - await proposal.reject(reason) - return events.ProposalRejected(prop_id=proposal.id, reason=reason) - - score = await self.engine._strategy.score_offer(proposal, self.agreements_pool) - logger.debug( - "Scored offer %s, provider: %s, strategy: %s, score: %f", - proposal.id, - proposal.props.get("golem.node.id.name"), - type(self.engine._strategy).__name__, - score, - ) + async def _handle_proposal( + self, + proposal: OfferProposal, + ) -> events.Event: + """Handle a single `OfferProposal`. - if score < SCORE_NEUTRAL: - return await reject_proposal("Score too low") + A `proposal` is scored and then can be rejected, responded with + a counter-proposal or stored in an agreements pool to be used + for negotiating an agreement. + """ - if not proposal.is_draft: - # Proposal is not yet a draft of an agreement + async def reject_proposal(reason: str) -> events.ProposalRejected: + """Reject `proposal` due to given `reason`.""" + await proposal.reject(reason) + return events.ProposalRejected(prop_id=proposal.id, reason=reason) + + score = await self.engine._strategy.score_offer(proposal, self.agreements_pool) + logger.debug( + "Scored offer %s, provider: %s, strategy: %s, score: %f", + proposal.id, + proposal.props.get("golem.node.id.name"), + type(self.engine._strategy).__name__, + score, + ) - # Check if any of the supported payment platforms matches the proposal - common_platforms = self._get_common_payment_platforms(proposal) - if common_platforms: - self.builder.properties["golem.com.payment.chosen-platform"] = next( - iter(common_platforms) - ) - else: - # reject proposal if there are no common payment platforms - return await reject_proposal("No common payment platform") - - # Check if the timeout for debit note acceptance is not too low - timeout = proposal.props.get(DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP) - if timeout: - if timeout < DEBIT_NOTE_MIN_TIMEOUT: - return await reject_proposal("Debit note acceptance timeout is too short") - else: - self.builder.properties[DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP] = timeout + if score < SCORE_NEUTRAL: + return await reject_proposal("Score too low") - await proposal.respond(self.builder.properties, self.builder.constraints) - return events.ProposalResponded(prop_id=proposal.id) + if not proposal.is_draft: + # Proposal is not yet a draft of an agreement + # Check if any of the supported payment platforms matches the proposal + common_platforms = self._get_common_payment_platforms(proposal) + if common_platforms: + self.builder.properties["golem.com.payment.chosen-platform"] = next( + iter(common_platforms) + ) else: - # It's a draft agreement - await self.agreements_pool.add_proposal(score, proposal) - return events.ProposalConfirmed(prop_id=proposal.id) - - async def _find_offers_for_subscription( - self, subscription: Subscription - ) -> None: - """Create a market subscription and repeatedly collect offer proposals for it. + # reject proposal if there are no common payment platforms + return await reject_proposal("No common payment platform") + + # Check if the timeout for debit note acceptance is not too low + timeout = proposal.props.get(DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP) + if timeout: + if timeout < DEBIT_NOTE_MIN_TIMEOUT: + return await reject_proposal("Debit note acceptance timeout is too short") + else: + self.builder.properties[DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP] = timeout - Collected proposals are processed concurrently using a bounded number - of asyncio tasks. + await proposal.respond(self.builder.properties, self.builder.constraints) + return events.ProposalResponded(prop_id=proposal.id) - :param state: A state related to a call to `Executor.submit()` - """ - max_number_of_tasks = 5 + else: + # It's a draft agreement + await self.agreements_pool.add_proposal(score, proposal) + return events.ProposalConfirmed(prop_id=proposal.id) - try: - proposals = subscription.events() - except Exception as ex: - self.engine.emit(events.CollectFailed(sub_id=subscription.id, reason=str(ex))) - raise + async def _find_offers_for_subscription( + self, subscription: Subscription + ) -> None: + """Create a market subscription and repeatedly collect offer proposals for it. - # A semaphore is used to limit the number of handler tasks - semaphore = asyncio.Semaphore(max_number_of_tasks) + Collected proposals are processed concurrently using a bounded number + of asyncio tasks. - async for proposal in proposals: + :param state: A state related to a call to `Executor.submit()` + """ + max_number_of_tasks = 5 - self.engine.emit(events.ProposalReceived(prop_id=proposal.id, provider_id=proposal.issuer)) - self.offers_collected += 1 + try: + proposals = subscription.events() + except Exception as ex: + self.engine.emit(events.CollectFailed(sub_id=subscription.id, reason=str(ex))) + raise - async def handler(proposal_): - """A coroutine that wraps `_handle_proposal()` method with error handling.""" - try: - event = await self._handle_proposal(proposal_) - assert isinstance(event, events.ProposalEvent) - self.engine.emit(event) - if isinstance(event, events.ProposalConfirmed): - self.proposals_confirmed += 1 - except CancelledError: - raise - except Exception: - with contextlib.suppress(Exception): - self.engine.emit( - events.ProposalFailed( - prop_id=proposal_.id, exc_info=sys.exc_info() # type: ignore - ) - ) - finally: - semaphore.release() + # A semaphore is used to limit the number of handler tasks + semaphore = asyncio.Semaphore(max_number_of_tasks) - # Create a new handler task - await semaphore.acquire() - asyncio.get_event_loop().create_task(handler(proposal)) + async for proposal in proposals: - async def _find_offers(self) -> None: - """Create demand subscription and process offers. - When the subscription expires, create a new one. And so on... - """ + self.engine.emit(events.ProposalReceived(prop_id=proposal.id, provider_id=proposal.issuer)) + self.offers_collected += 1 - while True: + async def handler(proposal_): + """A coroutine that wraps `_handle_proposal()` method with error handling.""" try: - subscription = await self.builder.subscribe(self.engine._market_api) - self.engine.emit(events.SubscriptionCreated(sub_id=subscription.id)) - except Exception as ex: - self.engine.emit(events.SubscriptionFailed(reason=str(ex))) + event = await self._handle_proposal(proposal_) + assert isinstance(event, events.ProposalEvent) + self.engine.emit(event) + if isinstance(event, events.ProposalConfirmed): + self.proposals_confirmed += 1 + except CancelledError: raise - async with subscription: - await self._find_offers_for_subscription(subscription) - - def _get_common_payment_platforms(self, proposal: rest.market.OfferProposal) -> Set[str]: - prov_platforms = { - property.split(".")[4] - for property in proposal.props - if property.startswith("golem.com.payment.platform.") and property is not None - } - if not prov_platforms: - prov_platforms = {"NGNT"} - req_platforms = { - allocation.payment_platform - for allocation in self.engine._budget_allocations - if allocation.payment_platform is not None - } - return req_platforms.intersection(prov_platforms) - + except Exception: + with contextlib.suppress(Exception): + self.engine.emit( + events.ProposalFailed( + prop_id=proposal_.id, exc_info=sys.exc_info() # type: ignore + ) + ) + finally: + semaphore.release() + # Create a new handler task + await semaphore.acquire() + asyncio.get_event_loop().create_task(handler(proposal)) - async def execute_task( - self, - worker: Callable[[WorkContext, AsyncIterator[Task[D, R]]], AsyncGenerator[Work, None]], - data: Iterable[Task[D, R]], - payload: Payload, - max_workers: Optional[int] = None, - timeout: Optional[timedelta] = None, - ) -> AsyncIterator[Task[D, R]]: + async def _find_offers(self) -> None: + """Create demand subscription and process offers. + When the subscription expires, create a new one. And so on... + """ - kwargs = { - 'package': payload + while True: + try: + subscription = await self.builder.subscribe(self.engine._market_api) + self.engine.emit(events.SubscriptionCreated(sub_id=subscription.id)) + except Exception as ex: + self.engine.emit(events.SubscriptionFailed(reason=str(ex))) + raise + async with subscription: + await self._find_offers_for_subscription(subscription) + + def _get_common_payment_platforms(self, proposal: rest.market.OfferProposal) -> Set[str]: + prov_platforms = { + property.split(".")[4] + for property in proposal.props + if property.startswith("golem.com.payment.platform.") and property is not None } - if max_workers: - kwargs['max_workers'] = max_workers - if timeout: - kwargs['timeout'] = timeout - - async with Executor(_engine=self, **kwargs) as executor: - async for t in executor.submit(worker, data): - yield t + if not prov_platforms: + prov_platforms = {"NGNT"} + req_platforms = { + allocation.payment_platform + for allocation in self.engine._budget_allocations + if allocation.payment_platform is not None + } + return req_platforms.intersection(prov_platforms) class Executor(AsyncContextManager): From d240e14add8b624dca4d5a88cf2afccb9c0e07c3 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Mon, 17 May 2021 09:26:16 +0200 Subject: [PATCH 06/29] Add Golem.create_activity(), add Activity._stream_events field --- yapapi/executor/__init__.py | 48 +++++++++++++++++++++---------------- yapapi/rest/activity.py | 19 +++++++++------ 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index c7c90373a..6368f47ef 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -165,10 +165,10 @@ def __init__( self._stack = AsyncExitStack() - def create_demand_builder(self, expiration_date: datetime, payload: Payload) -> DemandBuilder: - """Create a `DemandBuilder` for given `payload` and `expiration_date`.""" + def create_demand_builder(self, expiration_time: datetime, payload: Payload) -> DemandBuilder: + """Create a `DemandBuilder` for given `payload` and `expiration_time`.""" builder = DemandBuilder() - builder.add(Activity(expiration=expiration_date, multi_activity=True)) + builder.add(Activity(expiration=expiration_time, multi_activity=True)) builder.add(NodeInfo(subnet_tag=self._subnet)) if self._subnet: builder.ensure(f"({NodeInfoKeys.subnet_tag}={self._subnet})") @@ -309,6 +309,12 @@ async def execute_task( async for t in executor.submit(worker, data): yield t + async def create_activity(self, agreement_id: str) -> Activity: + return await self._activity_api.new_activity( + agreement_id, + stream_events=self._stream_output + ) + class Job: """Functionality related to a single job.""" @@ -317,7 +323,7 @@ def __init__( self, engine: Golem, agreements_pool: AgreementsPool, - expiration_date: datetime, + expiration_time: datetime, payload: Payload, ): self.engine = engine @@ -325,7 +331,7 @@ def __init__( self.offers_collected: int = 0 self.proposals_confirmed: int = 0 - self.builder = engine.create_demand_builder(expiration_date, payload) + self.builder = engine.create_demand_builder(expiration_time, payload) async def _handle_proposal( self, @@ -449,6 +455,7 @@ async def _find_offers(self) -> None: async with subscription: await self._find_offers_for_subscription(subscription) + # TODO: move to Golem def _get_common_payment_platforms(self, proposal: rest.market.OfferProposal) -> Set[str]: prov_platforms = { property.split(".")[4] @@ -571,15 +578,16 @@ async def submit( try: generator = self._submit(worker, data, services, workers) - async for result in generator: - yield result - except GeneratorExit: - logger.debug("Early exit from submit(), cancelling the computation") try: - # Cancel `generator`. It should then exit by raising `StopAsyncIteration`. - await generator.athrow(CancelledError) - except StopAsyncIteration: - pass + async for result in generator: + yield result + except GeneratorExit: + logger.debug("Early exit from submit(), cancelling the computation") + try: + # Cancel `generator`. It should then exit by raising `StopAsyncIteration`. + await generator.athrow(CancelledError) + except StopAsyncIteration: + pass finally: # Cancel and gather all tasks to make sure all exceptions are retrieved. all_tasks = workers.union(services) @@ -603,21 +611,19 @@ async def _submit( workers: Set[asyncio.Task], ) -> AsyncGenerator[Task[D, R], None]: - self._engine.emit(events.ComputationStarted(self._expires)) + emit = self._engine.emit + emit(events.ComputationStarted(self._expires)) agreements_pool = AgreementsPool(self._engine.emit) - state = Golem.Job( + state = Job( self._engine, agreements_pool, - expiration=self._expires, + expiration_time=self._expires, payload=self._payload ) - activity_api: rest.Activity = self._activity_api - done_queue: asyncio.Queue[Task[D, R]] = asyncio.Queue() - stream_output = self._stream_output def on_task_done(task: Task[D, R], status: TaskStatus) -> None: """Callback run when `task` is accepted or rejected.""" @@ -776,7 +782,7 @@ async def process_batches( await batch.prepare() cc = CommandContainer() batch.register(cc) - remote = await activity.send(cc.commands(), stream_output, deadline=batch_deadline) + remote = await activity.send(cc.commands(), deadline=batch_deadline) cmds = cc.commands() emit(events.ScriptSent(agr_id=agreement_id, task_id=task_id, cmds=cmds)) @@ -825,7 +831,7 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> emit(events.WorkerStarted(agr_id=agreement.id)) try: - act = await activity_api.new_activity(agreement.id) + act = await self._engine.create_activity(agreement.id) except Exception: emit( events.ActivityCreateFailed( diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index 9074b200e..8cb37bd1a 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -33,7 +33,7 @@ def __init__(self, api_client: ApiClient): self._api = RequestorControlApi(api_client) self._state = RequestorStateApi(api_client) - async def new_activity(self, agreement_id: str) -> "Activity": + async def new_activity(self, agreement_id: str, stream_events: bool = False) -> "Activity": """Create an activity within bounds of the specified agreement. :return: the object that represents the Activity @@ -41,16 +41,23 @@ async def new_activity(self, agreement_id: str) -> "Activity": :rtype: Activity """ activity_id = await self._api.create_activity(agreement_id) - return Activity(self._api, self._state, activity_id) + return Activity(self._api, self._state, activity_id, stream_events) class Activity(AsyncContextManager["Activity"]): """Mid-level wrapper for REST's Activity endpoint""" - def __init__(self, _api: RequestorControlApi, _state: RequestorStateApi, activity_id: str): + def __init__( + self, + _api: RequestorControlApi, + _state: RequestorStateApi, + activity_id: str, + stream_events: bool, + ): self._api: RequestorControlApi = _api self._state: RequestorStateApi = _state self._id: str = activity_id + self._stream_events = stream_events @property def id(self) -> str: @@ -61,14 +68,12 @@ async def state(self) -> yaa.ActivityState: state: yaa.ActivityState = await self._state.get_activity_state(self._id) return state - async def send( - self, script: List[dict], stream: bool = False, deadline: Optional[datetime] = None - ): + async def send(self, script: List[dict], deadline: Optional[datetime] = None) -> "Batch": """Send the execution script to the provider's execution unit.""" script_txt = json.dumps(script) batch_id = await self._api.call_exec(self._id, yaa.ExeScriptRequest(text=script_txt)) - if stream: + if self._stream_events: return StreamingBatch(self._api, self._id, batch_id, len(script), deadline) return PollingBatch(self._api, self._id, batch_id, len(script), deadline) From 933710756ddb5770e7af8ac98b38e15a51a91df3 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 17 May 2021 10:31:35 +0200 Subject: [PATCH 07/29] move `agreements_pool` to Job --- yapapi/executor/__init__.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 6368f47ef..bb836d744 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -322,17 +322,16 @@ class Job: def __init__( self, engine: Golem, - agreements_pool: AgreementsPool, expiration_time: datetime, payload: Payload, ): self.engine = engine - self.agreements_pool = agreements_pool - self.offers_collected: int = 0 self.proposals_confirmed: int = 0 self.builder = engine.create_demand_builder(expiration_time, payload) + self.agreements_pool = AgreementsPool(self.engine.emit) + async def _handle_proposal( self, proposal: OfferProposal, @@ -614,11 +613,9 @@ async def _submit( emit = self._engine.emit emit(events.ComputationStarted(self._expires)) - agreements_pool = AgreementsPool(self._engine.emit) - state = Job( + job = Job( self._engine, - agreements_pool, expiration_time=self._expires, payload=self._payload ) @@ -873,14 +870,14 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> async def worker_starter() -> None: while True: await asyncio.sleep(2) - await agreements_pool.cycle() + await job.agreements_pool.cycle() if ( len(workers) < self._conf.max_workers and await work_queue.has_unassigned_items() ): new_task = None try: - new_task = await agreements_pool.use_agreement( + new_task = await job.agreements_pool.use_agreement( lambda agreement, node: loop.create_task(start_worker(agreement, node)) ) if new_task is None: @@ -896,7 +893,7 @@ async def worker_starter() -> None: logger.debug("There was a problem during use_agreement", exc_info=True) loop = asyncio.get_event_loop() - find_offers_task = loop.create_task(self._find_offers(state)) + find_offers_task = loop.create_task(self._find_offers(job)) process_invoices_job = loop.create_task(process_invoices()) wait_until_done = loop.create_task(work_queue.wait_until_done()) worker_starter_task = loop.create_task(worker_starter()) @@ -922,10 +919,10 @@ async def worker_starter() -> None: now = datetime.now(timezone.utc) if now > self._expires: raise TimeoutError(f"Computation timed out after {self._conf.timeout}") - if now > get_offers_deadline and state.proposals_confirmed == 0: + if now > get_offers_deadline and job.proposals_confirmed == 0: emit( events.NoProposalsConfirmed( - num_offers=state.offers_collected, timeout=self._conf.get_offers_timeout + num_offers=job.offers_collected, timeout=self._conf.get_offers_timeout ) ) get_offers_deadline += self._conf.get_offers_timeout @@ -972,7 +969,7 @@ async def worker_starter() -> None: for task in services: if task is not process_invoices_job: task.cancel() - if agreements_pool.confirmed == 0: + if job.agreements_pool.confirmed == 0: # No need to wait for invoices process_invoices_job.cancel() if cancelled: @@ -995,7 +992,7 @@ async def worker_starter() -> None: try: logger.debug("Terminating agreements...") - await agreements_pool.terminate_all(reason=reason) + await job.agreements_pool.terminate_all(reason=reason) except Exception: logger.debug("Problem with agreements termination", exc_info=True) if self._conf.traceback: From 38a0148a9361e916c4c76c3e78b14aeb2e37c2d6 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Mon, 17 May 2021 12:06:47 +0200 Subject: [PATCH 08/29] Working blender example (without payments-related features) --- examples/blender/blender.py | 21 +++++---- yapapi/executor/__init__.py | 92 ++++++++++++++++++++++--------------- 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 9f2a6310a..b17ecf899 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -12,6 +12,7 @@ WorkContext, windows_event_loop_fix, ) +from yapapi.executor import Golem from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError @@ -93,28 +94,32 @@ async def worker(ctx: WorkContext, tasks): # By passing `event_consumer=log_summary()` we enable summary logging. # See the documentation of the `yapapi.log` module on how to set # the level of detail and format of the logged information. - async with Executor( - payload=package, - max_workers=3, + async with Golem( budget=10.0, - timeout=timeout, subnet_tag=subnet_tag, driver=driver, network=network, event_consumer=log_summary(log_event_repr), - ) as executor: + ) as golem: print( f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " - f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " - f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" + f"payment driver: {TEXT_COLOR_YELLOW}{driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{network}{TEXT_COLOR_DEFAULT}\n" ) num_tasks = 0 start_time = datetime.now() - async for task in executor.submit(worker, [Task(data=frame) for frame in frames]): + completed_tasks = golem.execute_task( + worker, + [Task(data=frame) for frame in frames], + payload=package, + max_workers=3, + timeout=timeout, + ) + async for task in completed_tasks: num_tasks += 1 print( f"{TEXT_COLOR_CYAN}" diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 6368f47ef..264303a22 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -7,7 +7,6 @@ from datetime import datetime, timedelta, timezone from decimal import Decimal import logging -import os import sys from typing import ( AsyncContextManager, @@ -28,7 +27,7 @@ import warnings -from dataclasses import dataclass, field +from dataclasses import dataclass from yapapi.executor.agreements_pool import AgreementsPool from typing_extensions import Final, AsyncGenerator @@ -165,14 +164,18 @@ def __init__( self._stack = AsyncExitStack() - def create_demand_builder(self, expiration_time: datetime, payload: Payload) -> DemandBuilder: + async def create_demand_builder( + self, expiration_time: datetime, payload: Payload + ) -> DemandBuilder: """Create a `DemandBuilder` for given `payload` and `expiration_time`.""" builder = DemandBuilder() builder.add(Activity(expiration=expiration_time, multi_activity=True)) builder.add(NodeInfo(subnet_tag=self._subnet)) if self._subnet: builder.ensure(f"({NodeInfoKeys.subnet_tag}={self._subnet})") - await builder.decorate(self.payment_decoration, self.strategy, payload) + await builder.decorate(self.payment_decoration) + await builder.decorate(self.strategy) + await builder.decorate(payload) return builder def _init_api(self, app_key: Optional[str] = None): @@ -197,7 +200,7 @@ def strategy(self) -> MarketStrategy: def emit(self, *args, **kwargs) -> None: self._wrapped_consumer.async_call(*args, **kwargs) - async def __aenter__(self): + async def __aenter__(self) -> "Golem": stack = self._stack market_client = await stack.enter_async_context(self._api_config.market()) @@ -215,6 +218,8 @@ async def __aenter__(self): self.payment_decoration = Golem.PaymentDecoration(await self._create_allocations()) + return self + async def __aexit__(self, exc_type, exc_val, exc_tb): logger.debug("Golem is shutting down...") # Wait until all computations are finished @@ -295,6 +300,7 @@ async def execute_task( payload: Payload, max_workers: Optional[int] = None, timeout: Optional[timedelta] = None, + budget: Optional[Union[float, Decimal]] = None, ) -> AsyncIterator[Task[D, R]]: kwargs = { @@ -304,6 +310,7 @@ async def execute_task( kwargs['max_workers'] = max_workers if timeout: kwargs['timeout'] = timeout + kwargs['budget'] = budget if budget is not None else self._budget_amount async with Executor(_engine=self, **kwargs) as executor: async for t in executor.submit(worker, data): @@ -331,11 +338,13 @@ def __init__( self.offers_collected: int = 0 self.proposals_confirmed: int = 0 - self.builder = engine.create_demand_builder(expiration_time, payload) + self.expiration_time: datetime = expiration_time + self.payload: Payload = payload async def _handle_proposal( self, proposal: OfferProposal, + demand_builder: DemandBuilder, ) -> events.Event: """Handle a single `OfferProposal`. @@ -367,7 +376,7 @@ async def reject_proposal(reason: str) -> events.ProposalRejected: # Check if any of the supported payment platforms matches the proposal common_platforms = self._get_common_payment_platforms(proposal) if common_platforms: - self.builder.properties["golem.com.payment.chosen-platform"] = next( + demand_builder.properties["golem.com.payment.chosen-platform"] = next( iter(common_platforms) ) else: @@ -380,9 +389,9 @@ async def reject_proposal(reason: str) -> events.ProposalRejected: if timeout < DEBIT_NOTE_MIN_TIMEOUT: return await reject_proposal("Debit note acceptance timeout is too short") else: - self.builder.properties[DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP] = timeout + demand_builder.properties[DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP] = timeout - await proposal.respond(self.builder.properties, self.builder.constraints) + await proposal.respond(demand_builder.properties, demand_builder.constraints) return events.ProposalResponded(prop_id=proposal.id) else: @@ -391,7 +400,7 @@ async def reject_proposal(reason: str) -> events.ProposalRejected: return events.ProposalConfirmed(prop_id=proposal.id) async def _find_offers_for_subscription( - self, subscription: Subscription + self, subscription: Subscription, demand_builder: DemandBuilder, ) -> None: """Create a market subscription and repeatedly collect offer proposals for it. @@ -419,7 +428,7 @@ async def _find_offers_for_subscription( async def handler(proposal_): """A coroutine that wraps `_handle_proposal()` method with error handling.""" try: - event = await self._handle_proposal(proposal_) + event = await self._handle_proposal(proposal_, demand_builder) assert isinstance(event, events.ProposalEvent) self.engine.emit(event) if isinstance(event, events.ProposalConfirmed): @@ -440,20 +449,24 @@ async def handler(proposal_): await semaphore.acquire() asyncio.get_event_loop().create_task(handler(proposal)) - async def _find_offers(self) -> None: + async def find_offers(self) -> None: """Create demand subscription and process offers. When the subscription expires, create a new one. And so on... """ + builder = await self.engine.create_demand_builder( + self.expiration_time, self.payload + ) + while True: try: - subscription = await self.builder.subscribe(self.engine._market_api) + subscription = await builder.subscribe(self.engine._market_api) self.engine.emit(events.SubscriptionCreated(sub_id=subscription.id)) except Exception as ex: self.engine.emit(events.SubscriptionFailed(reason=str(ex))) raise async with subscription: - await self._find_offers_for_subscription(subscription) + await self._find_offers_for_subscription(subscription, builder) # TODO: move to Golem def _get_common_payment_platforms(self, proposal: rest.market.OfferProposal) -> Set[str]: @@ -472,6 +485,9 @@ def _get_common_payment_platforms(self, proposal: rest.market.OfferProposal) -> return req_platforms.intersection(prov_platforms) +DEFAULT_GET_OFFERS_TIMEOUT = timedelta(seconds=20) + + class Executor(AsyncContextManager): """ Task executor. @@ -522,7 +538,9 @@ def __init__( logger.debug("Creating Executor instance; parameters: %s", locals()) self.__standalone = False - if not _engine: + if _engine: + self._engine = _engine + else: warnings.warn( "stand-alone usage is deprecated, please `Golem.execute_task` class instead ", DeprecationWarning @@ -548,6 +566,7 @@ def __init__( raise ValueError("Executor `payload` must be specified") self._payload = payload + self._timeout: timedelta = timeout # self._conf = _ExecutorConfig(max_workers, timeout) self._max_workers = max_workers # TODO: setup precision @@ -649,7 +668,7 @@ async def input_tasks() -> AsyncIterator[Task[D, R]]: invoices: Dict[str, rest.payment.Invoice] = dict() payment_closing: bool = False - async def process_invoices() -> None: + async def process_invoices(agreements_to_pay: Set[str]) -> None: async for invoice in self._payment_api.incoming_invoices(): if invoice.agreement_id in agreements_to_pay: emit( @@ -875,7 +894,7 @@ async def worker_starter() -> None: await asyncio.sleep(2) await agreements_pool.cycle() if ( - len(workers) < self._conf.max_workers + len(workers) < self._max_workers and await work_queue.has_unassigned_items() ): new_task = None @@ -889,29 +908,34 @@ async def worker_starter() -> None: except CancelledError: raise except Exception: - if self._conf.traceback: - traceback.print_exc() if new_task: new_task.cancel() logger.debug("There was a problem during use_agreement", exc_info=True) + async def _dummy_job(): + while True: + await asyncio.sleep(1.0) + loop = asyncio.get_event_loop() - find_offers_task = loop.create_task(self._find_offers(state)) - process_invoices_job = loop.create_task(process_invoices()) + find_offers_task = loop.create_task(state.find_offers()) + # process_invoices_job = loop.create_task(process_invoices()) + process_invoices_job = loop.create_task(_dummy_job()) wait_until_done = loop.create_task(work_queue.wait_until_done()) worker_starter_task = loop.create_task(worker_starter()) - debit_notes_job = loop.create_task(process_debit_notes()) + # debit_notes_job = loop.create_task(process_debit_notes()) + debit_nodes_job = loop.create_task(_dummy_job()) + # Py38: find_offers_task.set_name('find_offers_task') - get_offers_deadline = datetime.now(timezone.utc) + self._conf.get_offers_timeout + get_offers_deadline = datetime.now(timezone.utc) + DEFAULT_GET_OFFERS_TIMEOUT get_done_task: Optional[asyncio.Task] = None services.update( { find_offers_task, - process_invoices_job, + # process_invoices_job, wait_until_done, worker_starter_task, - debit_notes_job, + # debit_notes_job, } ) cancelled = False @@ -921,14 +945,14 @@ async def worker_starter() -> None: now = datetime.now(timezone.utc) if now > self._expires: - raise TimeoutError(f"Computation timed out after {self._conf.timeout}") + raise TimeoutError(f"Computation timed out after {self._timeout}") if now > get_offers_deadline and state.proposals_confirmed == 0: emit( events.NoProposalsConfirmed( - num_offers=state.offers_collected, timeout=self._conf.get_offers_timeout + num_offers=state.offers_collected, timeout=DEFAULT_GET_OFFERS_TIMEOUT ) ) - get_offers_deadline += self._conf.get_offers_timeout + get_offers_deadline += DEFAULT_GET_OFFERS_TIMEOUT if not get_done_task: get_done_task = loop.create_task(done_queue.get()) @@ -946,8 +970,7 @@ async def worker_starter() -> None: try: await task except Exception: - if self._conf.traceback: - traceback.print_exc() + pass workers -= done services -= done @@ -998,8 +1021,6 @@ async def worker_starter() -> None: await agreements_pool.terminate_all(reason=reason) except Exception: logger.debug("Problem with agreements termination", exc_info=True) - if self._conf.traceback: - traceback.print_exc() try: logger.info("Waiting for all services to finish...") @@ -1011,8 +1032,8 @@ async def worker_starter() -> None: "%s still running: %s", pluralize(len(pending), "service"), pending ) except Exception: - if self._conf.traceback: - traceback.print_exc() + # TODO: add message + logger.debug("TODO", exc_info=True) if agreements_to_pay: logger.info( @@ -1026,10 +1047,9 @@ async def worker_starter() -> None: logger.warning("Unpaid agreements: %s", agreements_to_pay) async def __aenter__(self) -> "Executor": - stack = self._stack # TODO: Cleanup on exception here. - self._expires = datetime.now(timezone.utc) + self._conf.timeout + self._expires = datetime.now(timezone.utc) + self._timeout return self From feb956db402d665a8d7ed6db6a60d34114206f36 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 17 May 2021 12:18:44 +0200 Subject: [PATCH 09/29] move payment methods to `Golem` --- yapapi/executor/__init__.py | 179 ++++++++++++++++++------------------ 1 file changed, 92 insertions(+), 87 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index bb836d744..c86f24adb 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -163,8 +163,16 @@ def __init__( self._stream_output = stream_output + # initialize the payment structures + self.agreements_to_pay: Set[str] = set() + self.agreements_accepting_debit_notes: Set[str] = set() + self.invoices: Dict[str, rest.payment.Invoice] = dict() + self._payment_closing: bool = False + self._stack = AsyncExitStack() + + def create_demand_builder(self, expiration_time: datetime, payload: Payload) -> DemandBuilder: """Create a `DemandBuilder` for given `payload` and `expiration_time`.""" builder = DemandBuilder() @@ -279,6 +287,86 @@ def _get_allocation( except: raise ValueError(f"No allocation for {item.payment_platform} {item.payer_addr}.") + async def process_invoices(self) -> None: + async for invoice in self._payment_api.incoming_invoices(): + if invoice.agreement_id in self.agreements_to_pay: + self.emit( + events.InvoiceReceived( + agr_id=invoice.agreement_id, + inv_id=invoice.invoice_id, + amount=invoice.amount, + ) + ) + try: + allocation = self._get_allocation(invoice) + await invoice.accept(amount=invoice.amount, allocation=allocation) + except CancelledError: + raise + except Exception: + self.emit( + events.PaymentFailed( + agr_id=invoice.agreement_id, exc_info=sys.exc_info() # type: ignore + ) + ) + else: + self.agreements_to_pay.remove(invoice.agreement_id) + assert invoice.agreement_id in self.agreements_accepting_debit_notes + self.agreements_accepting_debit_notes.remove(invoice.agreement_id) + self.emit( + events.PaymentAccepted( + agr_id=invoice.agreement_id, + inv_id=invoice.invoice_id, + amount=invoice.amount, + ) + ) + else: + self.invoices[invoice.agreement_id] = invoice + if self._payment_closing and not self.agreements_to_pay: + break + + # TODO Consider processing invoices and debit notes together + async def process_debit_notes(self) -> None: + async for debit_note in self._payment_api.incoming_debit_notes(): + if debit_note.agreement_id in self.agreements_accepting_debit_notes: + self.emit( + events.DebitNoteReceived( + agr_id=debit_note.agreement_id, + amount=debit_note.total_amount_due, + note_id=debit_note.debit_note_id, + ) + ) + try: + allocation = self._get_allocation(debit_note) + await debit_note.accept( + amount=debit_note.total_amount_due, allocation=allocation + ) + except CancelledError: + raise + except Exception: + self.emit( + events.PaymentFailed( + agr_id=debit_note.agreement_id, exc_info=sys.exc_info() # type: ignore + ) + ) + if self._payment_closing and not self.agreements_to_pay: + break + + async def accept_payment_for_agreement(self, agreement_id: str, *, partial: bool = False) -> None: + self.emit(events.PaymentPrepared(agr_id=agreement_id)) + inv = self.invoices.get(agreement_id) + if inv is None: + self.agreements_to_pay.add(agreement_id) + self.emit(events.PaymentQueued(agr_id=agreement_id)) + return + del self.invoices[agreement_id] + allocation = self._get_allocation(inv) + await inv.accept(amount=inv.amount, allocation=allocation) + self.emit( + events.PaymentAccepted( + agr_id=agreement_id, inv_id=inv.invoice_id, amount=inv.amount + ) + ) + @dataclass class PaymentDecoration(DemandDecorator): market_decoration: rest.payment.MarketDecoration @@ -613,7 +701,6 @@ async def _submit( emit = self._engine.emit emit(events.ComputationStarted(self._expires)) - job = Job( self._engine, expiration_time=self._expires, @@ -641,90 +728,8 @@ async def input_tasks() -> AsyncIterator[Task[D, R]]: last_wid = 0 - agreements_to_pay: Set[str] = set() - agreements_accepting_debit_notes: Set[str] = set() - invoices: Dict[str, rest.payment.Invoice] = dict() payment_closing: bool = False - async def process_invoices() -> None: - async for invoice in self._payment_api.incoming_invoices(): - if invoice.agreement_id in agreements_to_pay: - emit( - events.InvoiceReceived( - agr_id=invoice.agreement_id, - inv_id=invoice.invoice_id, - amount=invoice.amount, - ) - ) - try: - allocation = self._get_allocation(invoice) - await invoice.accept(amount=invoice.amount, allocation=allocation) - except CancelledError: - raise - except Exception: - emit( - events.PaymentFailed( - agr_id=invoice.agreement_id, exc_info=sys.exc_info() # type: ignore - ) - ) - else: - agreements_to_pay.remove(invoice.agreement_id) - assert invoice.agreement_id in agreements_accepting_debit_notes - agreements_accepting_debit_notes.remove(invoice.agreement_id) - emit( - events.PaymentAccepted( - agr_id=invoice.agreement_id, - inv_id=invoice.invoice_id, - amount=invoice.amount, - ) - ) - else: - invoices[invoice.agreement_id] = invoice - if payment_closing and not agreements_to_pay: - break - - # TODO Consider processing invoices and debit notes together - async def process_debit_notes() -> None: - async for debit_note in self._payment_api.incoming_debit_notes(): - if debit_note.agreement_id in agreements_accepting_debit_notes: - emit( - events.DebitNoteReceived( - agr_id=debit_note.agreement_id, - amount=debit_note.total_amount_due, - note_id=debit_note.debit_note_id, - ) - ) - try: - allocation = self._get_allocation(debit_note) - await debit_note.accept( - amount=debit_note.total_amount_due, allocation=allocation - ) - except CancelledError: - raise - except Exception: - emit( - events.PaymentFailed( - agr_id=debit_note.agreement_id, exc_info=sys.exc_info() # type: ignore - ) - ) - if payment_closing and not agreements_to_pay: - break - - async def accept_payment_for_agreement(agreement_id: str, *, partial: bool = False) -> None: - emit(events.PaymentPrepared(agr_id=agreement_id)) - inv = invoices.get(agreement_id) - if inv is None: - agreements_to_pay.add(agreement_id) - emit(events.PaymentQueued(agr_id=agreement_id)) - return - del invoices[agreement_id] - allocation = self._get_allocation(inv) - await inv.accept(amount=inv.amount, allocation=allocation) - emit( - events.PaymentAccepted( - agr_id=agreement_id, inv_id=inv.invoice_id, amount=inv.amount - ) - ) storage_manager = await self._stack.enter_async_context(gftp.provider()) @@ -795,7 +800,7 @@ async def get_batch_results() -> List[events.CommandEvent]: emit(events.GettingResults(agr_id=agreement_id, task_id=task_id)) await batch.post() emit(events.ScriptFinished(agr_id=agreement_id, task_id=task_id)) - await accept_payment_for_agreement(agreement_id, partial=True) + await self._engine.accept_payment_for_agreement(agreement_id, partial=True) return results loop = asyncio.get_event_loop() @@ -894,10 +899,10 @@ async def worker_starter() -> None: loop = asyncio.get_event_loop() find_offers_task = loop.create_task(self._find_offers(job)) - process_invoices_job = loop.create_task(process_invoices()) + process_invoices_job = loop.create_task(self._engine.process_invoices()) wait_until_done = loop.create_task(work_queue.wait_until_done()) worker_starter_task = loop.create_task(worker_starter()) - debit_notes_job = loop.create_task(process_debit_notes()) + debit_notes_job = loop.create_task(self._engine.process_debit_notes()) # Py38: find_offers_task.set_name('find_offers_task') get_offers_deadline = datetime.now(timezone.utc) + self._conf.get_offers_timeout @@ -965,7 +970,7 @@ async def worker_starter() -> None: # Importing this at the beginning would cause circular dependencies from ..log import pluralize - payment_closing = True + self._engine._payment_closing = True for task in services: if task is not process_invoices_job: task.cancel() From 9c3e6af97e027a26bf5b5f1f3f59ee118567d242 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 17 May 2021 15:13:18 +0200 Subject: [PATCH 10/29] move debit note and invoice processing to `Golem` --- yapapi/executor/__init__.py | 109 ++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 41 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 70aa9bdd8..4596a41a4 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -167,6 +167,8 @@ def __init__( self._invoices: Dict[str, rest.payment.Invoice] = dict() self._payment_closing: bool = False + self._services: Set[asyncio.Task] = set() + self._stack = AsyncExitStack() async def create_demand_builder( @@ -217,18 +219,61 @@ async def __aenter__(self) -> "Golem": payment_client = await stack.enter_async_context(self._api_config.payment()) self._payment_api = rest.Payment(payment_client) - # a set of `asyncio.Event` instances used to track jobs - computations or services - started - # those events can be used to wait until all jobs are finished - self._active_jobs: Set[asyncio.Event] = set() + # a set of `Job` instances used to track jobs - computations or services - started + # it can be used to wait until all jobs are finished + self._jobs: Set[Job] = set() self.payment_decoration = Golem.PaymentDecoration(await self._create_allocations()) + loop = asyncio.get_event_loop() + self._process_invoices_job = loop.create_task(self.process_invoices()) + self._services.add(self._process_invoices_job) + self._services.add(loop.create_task(self.process_debit_notes())) + return self async def __aexit__(self, exc_type, exc_val, exc_tb): + # Importing this at the beginning would cause circular dependencies + from ..log import pluralize + logger.debug("Golem is shutting down...") # Wait until all computations are finished - await asyncio.gather(*[event.wait() for event in self._active_jobs]) + await asyncio.gather(*[job.finished.wait() for job in self._jobs]) + + self._payment_closing = True + + for task in self._services: + if task is not self._process_invoices_job: + task.cancel() + + if not any(True for job in self._jobs if job.agreements_pool.confirmed > 0): + logger.debug("No need to wait for invoices.") + self._process_invoices_job.cancel() + + try: + logger.info("Waiting for Golem services to finish...") + _, pending = await asyncio.wait( + self._services, timeout=10, return_when=asyncio.ALL_COMPLETED + ) + if pending: + logger.debug( + "%s still running: %s", pluralize(len(pending), "service"), pending + ) + except Exception: + # TODO: add message + logger.debug("TODO", exc_info=True) + + if self._agreements_to_pay: + logger.info( + "%s still unpaid, waiting for invoices...", + pluralize(len(self._agreements_to_pay), "agreement"), + ) + await asyncio.wait( + {self._process_invoices_job}, timeout=30, return_when=asyncio.ALL_COMPLETED + ) + if self._agreements_to_pay: + logger.warning("Unpaid agreements: %s", self._agreements_to_pay) + # TODO: prevent new computations at this point (if it's even possible to start one) try: await self._stack.aclose() @@ -372,6 +417,13 @@ async def accept_payment_for_agreement(self, agreement_id: str, *, partial: bool def approve_agreement_payments(self, agreement_id): self._agreements_accepting_debit_notes.add(agreement_id) + def add_job(self, job: "Job"): + self._jobs.add(job) + + @staticmethod + def finalize_job(job: "Job"): + job.finished.set() + @dataclass class PaymentDecoration(DemandDecorator): market_decoration: rest.payment.MarketDecoration @@ -427,6 +479,7 @@ def __init__( self.payload: Payload = payload self.agreements_pool = AgreementsPool(self.engine.emit) + self.finished = asyncio.Event() async def _handle_proposal( self, @@ -676,14 +729,18 @@ async def submit( :return: yields computation progress events """ - computation_finished = asyncio.Event() - self._engine._active_jobs.add(computation_finished) + job = Job( + self._engine, + expiration_time=self._expires, + payload=self._payload + ) + self._engine.add_job(job) services: Set[asyncio.Task] = set() workers: Set[asyncio.Task] = set() try: - generator = self._submit(worker, data, services, workers) + generator = self._submit(worker, data, services, workers, job) try: async for result in generator: yield result @@ -703,8 +760,7 @@ async def submit( task.cancel() await asyncio.gather(*all_tasks, return_exceptions=True) # Signal that this computation is finished - computation_finished.set() - self._engine._active_jobs.remove(computation_finished) + self._engine.finalize_job(job) async def _submit( self, @@ -715,17 +771,12 @@ async def _submit( data: Union[AsyncIterator[Task[D, R]], Iterable[Task[D, R]]], services: Set[asyncio.Task], workers: Set[asyncio.Task], + job: Job, ) -> AsyncGenerator[Task[D, R], None]: emit = self._engine.emit emit(events.ComputationStarted(self._expires)) - job = Job( - self._engine, - expiration_time=self._expires, - payload=self._payload - ) - done_queue: asyncio.Queue[Task[D, R]] = asyncio.Queue() def on_task_done(task: Task[D, R], status: TaskStatus) -> None: @@ -911,19 +962,12 @@ async def worker_starter() -> None: new_task.cancel() logger.debug("There was a problem during use_agreement", exc_info=True) - async def _dummy_job(): - while True: - await asyncio.sleep(1.0) - loop = asyncio.get_event_loop() find_offers_task = loop.create_task(job.find_offers()) wait_until_done = loop.create_task(work_queue.wait_until_done()) worker_starter_task = loop.create_task(worker_starter()) - process_invoices_job = loop.create_task(self._engine.process_invoices()) - debit_notes_job = loop.create_task(self._engine.process_debit_notes()) - # Py38: find_offers_task.set_name('find_offers_task') get_offers_deadline = datetime.now(timezone.utc) + DEFAULT_GET_OFFERS_TIMEOUT @@ -931,10 +975,8 @@ async def _dummy_job(): services.update( { find_offers_task, - process_invoices_job, wait_until_done, worker_starter_task, - debit_notes_job, } ) cancelled = False @@ -992,11 +1034,7 @@ async def _dummy_job(): self._engine._payment_closing = True for task in services: - if task is not process_invoices_job: - task.cancel() - if job.agreements_pool.confirmed == 0: - # No need to wait for invoices - process_invoices_job.cancel() + task.cancel() if cancelled: reason = {"message": "Work cancelled", "golem.requestor.code": "Cancelled"} for worker_task in workers: @@ -1022,7 +1060,7 @@ async def _dummy_job(): logger.debug("Problem with agreements termination", exc_info=True) try: - logger.info("Waiting for all services to finish...") + logger.info("Waiting for Executor services to finish...") _, pending = await asyncio.wait( workers.union(services), timeout=10, return_when=asyncio.ALL_COMPLETED ) @@ -1034,17 +1072,6 @@ async def _dummy_job(): # TODO: add message logger.debug("TODO", exc_info=True) - if self._engine._agreements_to_pay: - logger.info( - "%s still unpaid, waiting for invoices...", - pluralize(len(self._engine._agreements_to_pay), "agreement"), - ) - await asyncio.wait( - {process_invoices_job}, timeout=30, return_when=asyncio.ALL_COMPLETED - ) - if self._engine._agreements_to_pay: - logger.warning("Unpaid agreements: %s", self._engine._agreements_to_pay) - async def __aenter__(self) -> "Executor": # TODO: Cleanup on exception here. From 07949471ba2da4011e6ca597facbf1181a8ef2ee Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Mon, 17 May 2021 15:17:51 +0200 Subject: [PATCH 11/29] use `payload` instead of `package` when instantiating the Executor --- yapapi/executor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 4596a41a4..13205c12f 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -444,7 +444,7 @@ async def execute_task( ) -> AsyncIterator[Task[D, R]]: kwargs = { - 'package': payload + 'payload': payload } if max_workers: kwargs['max_workers'] = max_workers From 79d4e8913695b576c78fe941398586fd0097c633 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Mon, 17 May 2021 17:33:19 +0200 Subject: [PATCH 12/29] Move `process_batches()` from `Executor` to `Golem` --- yapapi/executor/__init__.py | 237 +++++++++++++++++++----------------- 1 file changed, 126 insertions(+), 111 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 13205c12f..f81b492aa 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -6,15 +6,18 @@ import contextlib from datetime import datetime, timedelta, timezone from decimal import Decimal +import itertools import logging import sys from typing import ( + Any, AsyncContextManager, AsyncIterator, Awaitable, Callable, Dict, Iterable, + Iterator, List, Optional, Set, @@ -95,6 +98,20 @@ def __str__(self) -> str: WorkItem = Union[Work, Tuple[Work, ExecOptions]] """The type of items yielded by a generator created by the `worker` function supplied by user.""" + +def unpack_work_item(item: WorkItem) -> Tuple[Work, ExecOptions]: + """Extract `Work` object and options from a work item. + If the item does not specify options, default ones are provided. + """ + if isinstance(item, tuple): + return item + else: + return item, ExecOptions() + + +batch_ids: Iterator[int] = itertools.count(1) + + D = TypeVar("D") # Type var for task data R = TypeVar("R") # Type var for task result @@ -435,7 +452,10 @@ async def decorate_demand(self, demand: DemandBuilder): async def execute_task( self, - worker: Callable[[WorkContext, AsyncIterator[Task[D, R]]], AsyncGenerator[Work, None]], + worker: Callable[ + [WorkContext, AsyncIterator[Task[D, R]]], + AsyncGenerator[Work, Awaitable[List[events.CommandEvent]]], + ], data: Iterable[Task[D, R]], payload: Payload, max_workers: Optional[int] = None, @@ -443,7 +463,7 @@ async def execute_task( budget: Optional[Union[float, Decimal]] = None, ) -> AsyncIterator[Task[D, R]]: - kwargs = { + kwargs: Dict[str, Any] = { 'payload': payload } if max_workers: @@ -456,12 +476,87 @@ async def execute_task( async for t in executor.submit(worker, data): yield t - async def create_activity(self, agreement_id: str) -> Activity: + async def create_activity(self, agreement_id: str): return await self._activity_api.new_activity( agreement_id, stream_events=self._stream_output ) + async def process_batches( + self, + agreement_id: str, + activity: rest.activity.Activity, + command_generator: AsyncGenerator[Work, Awaitable[List[events.CommandEvent]]], + ) -> None: + """Send command batches produced by `command_generator` to `activity`.""" + + item = await command_generator.__anext__() + + while True: + + batch, exec_options = unpack_work_item(item) + # TODO: `task_id` should really be `batch_id`, but then we should also rename + # `task_id` field of several events (e.g. `ScriptSent`) + task_id = str(next(batch_ids)) + + if batch.timeout: + if exec_options.batch_timeout: + logger.warning( + "Overriding batch timeout set with commit(batch_timeout)" + "by the value set in exec options" + ) + else: + exec_options.batch_timeout = batch.timeout + + batch_deadline = ( + datetime.now(timezone.utc) + exec_options.batch_timeout + if exec_options.batch_timeout + else None + ) + + await batch.prepare() + cc = CommandContainer() + batch.register(cc) + remote = await activity.send(cc.commands(), deadline=batch_deadline) + cmds = cc.commands() + self.emit(events.ScriptSent(agr_id=agreement_id, task_id=task_id, cmds=cmds)) + + async def get_batch_results() -> List[events.CommandEvent]: + results = [] + async for evt_ctx in remote: + evt = evt_ctx.event(agr_id=agreement_id, task_id=task_id, cmds=cmds) + self.emit(evt) + results.append(evt) + if isinstance(evt, events.CommandExecuted) and not evt.success: + raise CommandExecutionError(evt.command, evt.message) + + self.emit(events.GettingResults(agr_id=agreement_id, task_id=task_id)) + await batch.post() + self.emit(events.ScriptFinished(agr_id=agreement_id, task_id=task_id)) + await self.accept_payment_for_agreement(agreement_id, partial=True) + return results + + loop = asyncio.get_event_loop() + + if exec_options.wait_for_results: + # Block until the results are available + try: + future_results = loop.create_future() + results = await get_batch_results() + future_results.set_result(results) + item = await command_generator.asend(future_results) + except StopAsyncIteration: + raise + except Exception: + # Raise the exception in `command_generator` (the `worker` coroutine). + # If the client code is able to handle it then we'll proceed with + # subsequent batches. Otherwise the worker finishes with error. + item = await command_generator.athrow(*sys.exc_info()) + else: + # Schedule the coroutine in a separate asyncio task + future_results = loop.create_task(get_batch_results()) + item = await command_generator.asend(future_results) + class Job: """Functionality related to a single job.""" @@ -713,6 +808,9 @@ def __init__( self._stack = AsyncExitStack() + def emit(self, event: events.Event) -> None: + self._engine.emit(event) + async def submit( self, worker: Callable[ @@ -774,8 +872,7 @@ async def _submit( job: Job, ) -> AsyncGenerator[Task[D, R], None]: - emit = self._engine.emit - emit(events.ComputationStarted(self._expires)) + self.emit(events.ComputationStarted(self._expires)) done_queue: asyncio.Queue[Task[D, R]] = asyncio.Queue() @@ -800,137 +897,55 @@ async def input_tasks() -> AsyncIterator[Task[D, R]]: storage_manager = await self._stack.enter_async_context(gftp.provider()) - def unpack_work_item(item: WorkItem) -> Tuple[Work, ExecOptions]: - """Extract `Work` object and options from a work item. - If the item does not specify options, default ones are provided. - """ - if isinstance(item, tuple): - return item - else: - return item, ExecOptions() - - async def process_batches( - agreement_id: str, - activity: rest.activity.Activity, - command_generator: AsyncGenerator[Work, Awaitable[List[events.CommandEvent]]], - consumer: Consumer[Task[D, R]], - ) -> None: - """Send command batches produced by `command_generator` to `activity`.""" - - item = await command_generator.__anext__() - - while True: - - batch, exec_options = unpack_work_item(item) - if batch.timeout: - if exec_options.batch_timeout: - logger.warning( - "Overriding batch timeout set with commit(batch_timeout)" - "by the value set in exec options" - ) - else: - exec_options.batch_timeout = batch.timeout - - batch_deadline = ( - datetime.now(timezone.utc) + exec_options.batch_timeout - if exec_options.batch_timeout - else None - ) - - current_worker_task = consumer.current_item - if current_worker_task: - emit( - events.TaskStarted( - agr_id=agreement_id, - task_id=current_worker_task.id, - task_data=current_worker_task.data, - ) - ) - task_id = current_worker_task.id if current_worker_task else None - - await batch.prepare() - cc = CommandContainer() - batch.register(cc) - remote = await activity.send(cc.commands(), deadline=batch_deadline) - cmds = cc.commands() - emit(events.ScriptSent(agr_id=agreement_id, task_id=task_id, cmds=cmds)) - - async def get_batch_results() -> List[events.CommandEvent]: - results = [] - async for evt_ctx in remote: - evt = evt_ctx.event(agr_id=agreement_id, task_id=task_id, cmds=cmds) - emit(evt) - results.append(evt) - if isinstance(evt, events.CommandExecuted) and not evt.success: - raise CommandExecutionError(evt.command, evt.message) - - emit(events.GettingResults(agr_id=agreement_id, task_id=task_id)) - await batch.post() - emit(events.ScriptFinished(agr_id=agreement_id, task_id=task_id)) - await self._engine.accept_payment_for_agreement(agreement_id, partial=True) - return results - - loop = asyncio.get_event_loop() - - if exec_options.wait_for_results: - # Block until the results are available - try: - future_results = loop.create_future() - results = await get_batch_results() - future_results.set_result(results) - item = await command_generator.asend(future_results) - except StopAsyncIteration: - raise - except Exception: - # Raise the exception in `command_generator` (the `worker` coroutine). - # If the client code is able to handle it then we'll proceed with - # subsequent batches. Otherwise the worker finishes with error. - item = await command_generator.athrow(*sys.exc_info()) - else: - # Schedule the coroutine in a separate asyncio task - future_results = loop.create_task(get_batch_results()) - item = await command_generator.asend(future_results) - async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> None: nonlocal last_wid wid = last_wid last_wid += 1 - emit(events.WorkerStarted(agr_id=agreement.id)) + self.emit(events.WorkerStarted(agr_id=agreement.id)) try: act = await self._engine.create_activity(agreement.id) except Exception: - emit( + self.emit( events.ActivityCreateFailed( agr_id=agreement.id, exc_info=sys.exc_info() # type: ignore ) ) - emit(events.WorkerFinished(agr_id=agreement.id)) + self.emit(events.WorkerFinished(agr_id=agreement.id)) raise async with act: - emit(events.ActivityCreated(act_id=act.id, agr_id=agreement.id)) + self.emit(events.ActivityCreated(act_id=act.id, agr_id=agreement.id)) self._engine.approve_agreement_payments(agreement.id) work_context = WorkContext( - f"worker-{wid}", node_info, storage_manager, emitter=emit + f"worker-{wid}", node_info, storage_manager, emitter=self.emit ) with work_queue.new_consumer() as consumer: try: - tasks = ( - Task.for_handle(handle, work_queue, emit) async for handle in consumer - ) - batch_generator = worker(work_context, tasks) + async def task_generator() -> AsyncIterator[Task[D, R]]: + async for handle in consumer: + task = Task.for_handle(handle, work_queue, self.emit) + self._engine.emit( + events.TaskStarted( + agr_id=agreement.id, + task_id=handle.data.id, + task_data=handle.data.data + ) + ) + yield task + + batch_generator = worker(work_context, task_generator()) try: - await process_batches(agreement.id, act, batch_generator, consumer) + await self._engine.process_batches(agreement.id, act, batch_generator) except StopAsyncIteration: pass - emit(events.WorkerFinished(agr_id=agreement.id)) + self.emit(events.WorkerFinished(agr_id=agreement.id)) except Exception: - emit( + self.emit( events.WorkerFinished( agr_id=agreement.id, exc_info=sys.exc_info() # type: ignore ) @@ -988,7 +1003,7 @@ async def worker_starter() -> None: if now > self._expires: raise TimeoutError(f"Computation timed out after {self._timeout}") if now > get_offers_deadline and job.proposals_confirmed == 0: - emit( + self.emit( events.NoProposalsConfirmed( num_offers=job.offers_collected, timeout=DEFAULT_GET_OFFERS_TIMEOUT ) @@ -1022,10 +1037,10 @@ async def worker_starter() -> None: assert get_done_task not in services get_done_task = None - emit(events.ComputationFinished()) + self.emit(events.ComputationFinished()) except (Exception, CancelledError, KeyboardInterrupt): - emit(events.ComputationFinished(exc_info=sys.exc_info())) # type: ignore + self.emit(events.ComputationFinished(exc_info=sys.exc_info())) # type: ignore cancelled = True finally: From 5eafa0144dcae8324949358e13d88eb4ad5c1e9b Mon Sep 17 00:00:00 2001 From: azawlocki Date: Tue, 18 May 2021 09:40:40 +0200 Subject: [PATCH 13/29] Make Executor backward compatible, so that original blender.py works --- examples/blender/blender-golem.py | 186 ++++++++++++++++++++++++++++++ examples/blender/blender.py | 21 ++-- yapapi/executor/__init__.py | 70 ++++++----- 3 files changed, 233 insertions(+), 44 deletions(-) create mode 100755 examples/blender/blender-golem.py diff --git a/examples/blender/blender-golem.py b/examples/blender/blender-golem.py new file mode 100755 index 000000000..b17ecf899 --- /dev/null +++ b/examples/blender/blender-golem.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +import asyncio +from datetime import datetime, timedelta +import pathlib +import sys + +from yapapi import ( + Executor, + NoPaymentAccountError, + Task, + __version__ as yapapi_version, + WorkContext, + windows_event_loop_fix, +) +from yapapi.executor import Golem +from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa +from yapapi.payload import vm +from yapapi.rest.activity import BatchTimeoutError + +examples_dir = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(str(examples_dir)) + +from utils import ( + build_parser, + TEXT_COLOR_CYAN, + TEXT_COLOR_DEFAULT, + TEXT_COLOR_RED, + TEXT_COLOR_YELLOW, +) + + +async def main(subnet_tag, driver=None, network=None): + package = await vm.repo( + image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + min_mem_gib=0.5, + min_storage_gib=2.0, + ) + + async def worker(ctx: WorkContext, tasks): + script_dir = pathlib.Path(__file__).resolve().parent + scene_path = str(script_dir / "cubes.blend") + ctx.send_file(scene_path, "/golem/resource/scene.blend") + async for task in tasks: + frame = task.data + crops = [{"outfilebasename": "out", "borders_x": [0.0, 1.0], "borders_y": [0.0, 1.0]}] + ctx.send_json( + "/golem/work/params.json", + { + "scene_file": "/golem/resource/scene.blend", + "resolution": (400, 300), + "use_compositing": False, + "crops": crops, + "samples": 100, + "frames": [frame], + "output_format": "PNG", + "RESOURCES_DIR": "/golem/resources", + "WORK_DIR": "/golem/work", + "OUTPUT_DIR": "/golem/output", + }, + ) + ctx.run("/golem/entrypoints/run-blender.sh") + output_file = f"output_{frame}.png" + ctx.download_file(f"/golem/output/out{frame:04d}.png", output_file) + try: + # Set timeout for executing the script on the provider. Usually, 30 seconds + # should be more than enough for computing a single frame, however a provider + # may require more time for the first task if it needs to download a VM image + # first. Once downloaded, the VM image will be cached and other tasks that use + # that image will be computed faster. + yield ctx.commit(timeout=timedelta(minutes=10)) + # TODO: Check if job results are valid + # and reject by: task.reject_task(reason = 'invalid file') + task.accept_result(result=output_file) + except BatchTimeoutError: + print( + f"{TEXT_COLOR_RED}" + f"Task {task} timed out on {ctx.provider_name}, time: {task.running_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + raise + + # Iterator over the frame indices that we want to render + frames: range = range(0, 60, 10) + # Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.) + # TODO: make this dynamic, e.g. depending on the size of files to transfer + init_overhead = 3 + # Providers will not accept work if the timeout is outside of the [5 min, 30min] range. + # We increase the lower bound to 6 min to account for the time needed for our demand to + # reach the providers. + min_timeout, max_timeout = 6, 30 + + timeout = timedelta(minutes=max(min(init_overhead + len(frames) * 2, max_timeout), min_timeout)) + + # By passing `event_consumer=log_summary()` we enable summary logging. + # See the documentation of the `yapapi.log` module on how to set + # the level of detail and format of the logged information. + async with Golem( + budget=10.0, + subnet_tag=subnet_tag, + driver=driver, + network=network, + event_consumer=log_summary(log_event_repr), + ) as golem: + + print( + f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" + f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " + f"payment driver: {TEXT_COLOR_YELLOW}{driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{network}{TEXT_COLOR_DEFAULT}\n" + ) + + num_tasks = 0 + start_time = datetime.now() + + completed_tasks = golem.execute_task( + worker, + [Task(data=frame) for frame in frames], + payload=package, + max_workers=3, + timeout=timeout, + ) + async for task in completed_tasks: + num_tasks += 1 + print( + f"{TEXT_COLOR_CYAN}" + f"Task computed: {task}, result: {task.result}, time: {task.running_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + + print( + f"{TEXT_COLOR_CYAN}" + f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}" + f"{TEXT_COLOR_DEFAULT}" + ) + + +if __name__ == "__main__": + parser = build_parser("Render a Blender scene") + now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") + parser.set_defaults(log_file=f"blender-yapapi-{now}.log") + args = parser.parse_args() + + # This is only required when running on Windows with Python prior to 3.8: + windows_event_loop_fix() + + enable_default_logger( + log_file=args.log_file, + debug_activity_api=True, + debug_market_api=True, + debug_payment_api=True, + ) + + loop = asyncio.get_event_loop() + task = loop.create_task( + main(subnet_tag=args.subnet_tag, driver=args.driver, network=args.network) + ) + + try: + loop.run_until_complete(task) + except NoPaymentAccountError as e: + handbook_url = ( + "https://handbook.golem.network/requestor-tutorials/" + "flash-tutorial-of-requestor-development" + ) + print( + f"{TEXT_COLOR_RED}" + f"No payment account initialized for driver `{e.required_driver}` " + f"and network `{e.required_network}`.\n\n" + f"See {handbook_url} on how to initialize payment accounts for a requestor node." + f"{TEXT_COLOR_DEFAULT}" + ) + except KeyboardInterrupt: + print( + f"{TEXT_COLOR_YELLOW}" + "Shutting down gracefully, please wait a short while " + "or press Ctrl+C to exit immediately..." + f"{TEXT_COLOR_DEFAULT}" + ) + task.cancel() + try: + loop.run_until_complete(task) + print( + f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" + ) + except (asyncio.CancelledError, KeyboardInterrupt): + pass diff --git a/examples/blender/blender.py b/examples/blender/blender.py index b17ecf899..9f2a6310a 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -12,7 +12,6 @@ WorkContext, windows_event_loop_fix, ) -from yapapi.executor import Golem from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError @@ -94,32 +93,28 @@ async def worker(ctx: WorkContext, tasks): # By passing `event_consumer=log_summary()` we enable summary logging. # See the documentation of the `yapapi.log` module on how to set # the level of detail and format of the logged information. - async with Golem( + async with Executor( + payload=package, + max_workers=3, budget=10.0, + timeout=timeout, subnet_tag=subnet_tag, driver=driver, network=network, event_consumer=log_summary(log_event_repr), - ) as golem: + ) as executor: print( f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " - f"payment driver: {TEXT_COLOR_YELLOW}{driver}{TEXT_COLOR_DEFAULT}, " - f"and network: {TEXT_COLOR_YELLOW}{network}{TEXT_COLOR_DEFAULT}\n" + f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" ) num_tasks = 0 start_time = datetime.now() - completed_tasks = golem.execute_task( - worker, - [Task(data=frame) for frame in frames], - payload=package, - max_workers=3, - timeout=timeout, - ) - async for task in completed_tasks: + async for task in executor.submit(worker, [Task(data=frame) for frame in frames]): num_tasks += 1 print( f"{TEXT_COLOR_CYAN}" diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index f81b492aa..7399a1c49 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -25,6 +25,7 @@ TypeVar, Union, cast, + overload, ) import warnings @@ -733,18 +734,18 @@ class Executor(AsyncContextManager): def __init__( self, *, - budget: Union[float, Decimal], - strategy: Optional[MarketStrategy] = None, - subnet_tag: Optional[str] = None, - driver: Optional[str] = None, - network: Optional[str] = None, - event_consumer: Optional[Callable[[Event], None]] = None, - stream_output: bool = False, - max_workers: int = 5, - timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, - package: Optional[Payload] = None, - payload: Optional[Payload] = None, - _engine: Optional[Golem] = None + budget: Union[float, Decimal], + strategy: Optional[MarketStrategy] = None, + subnet_tag: Optional[str] = None, + driver: Optional[str] = None, + network: Optional[str] = None, + event_consumer: Optional[Callable[[Event], None]] = None, + stream_output: bool = False, + max_workers: int = 5, + timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, + package: Optional[Payload] = None, + payload: Optional[Payload] = None, + _engine: Optional[Golem] = None, ): """Create a new executor. @@ -781,14 +782,15 @@ def __init__( DeprecationWarning ) self._engine = Golem( - budget = budget, - strategy = strategy, - subnet_tag = subnet_tag, - driver = driver, - network = network, - event_consumer = event_consumer, + budget=budget, + strategy=strategy, + subnet_tag=subnet_tag, + driver=driver, + network=network, + event_consumer=event_consumer, stream_output=stream_output ) + self.__standalone = True if package: if payload: @@ -802,12 +804,28 @@ def __init__( self._payload = payload self._timeout: timedelta = timeout - # self._conf = _ExecutorConfig(max_workers, timeout) self._max_workers = max_workers - # TODO: setup precision - self._stack = AsyncExitStack() + @property + def driver(self) -> str: + return self._engine.driver + + @property + def network(self) -> str: + return self._engine.network + + async def __aenter__(self) -> "Executor": + self._expires = datetime.now(timezone.utc) + self._timeout + if self.__standalone: + await self._engine.__aenter__() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self._stack.aclose() + if self.__standalone: + await self._engine.__aexit__(exc_type, exc_val, exc_tb) + def emit(self, event: events.Event) -> None: self._engine.emit(event) @@ -1086,13 +1104,3 @@ async def worker_starter() -> None: except Exception: # TODO: add message logger.debug("TODO", exc_info=True) - - async def __aenter__(self) -> "Executor": - - # TODO: Cleanup on exception here. - self._expires = datetime.now(timezone.utc) + self._timeout - - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self._stack.aclose() From e1315480a19c5e6700dcfb4402c9a15aff21684f Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 18 May 2021 10:41:17 +0200 Subject: [PATCH 14/29] move `storage_manager` to Golem --- yapapi/executor/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 7399a1c49..7281f3210 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -218,6 +218,10 @@ def driver(self) -> str: def network(self) -> str: return self._network + @property + def storage_manager(self): + return self._storage_manager + @property def strategy(self) -> MarketStrategy: return self._strategy @@ -248,6 +252,8 @@ async def __aenter__(self) -> "Golem": self._services.add(self._process_invoices_job) self._services.add(loop.create_task(self.process_debit_notes())) + self._storage_manager = await self._stack.enter_async_context(gftp.provider()) + return self async def __aexit__(self, exc_type, exc_val, exc_tb): @@ -913,7 +919,6 @@ async def input_tasks() -> AsyncIterator[Task[D, R]]: last_wid = 0 - storage_manager = await self._stack.enter_async_context(gftp.provider()) async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> None: @@ -939,7 +944,7 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> self.emit(events.ActivityCreated(act_id=act.id, agr_id=agreement.id)) self._engine.approve_agreement_payments(agreement.id) work_context = WorkContext( - f"worker-{wid}", node_info, storage_manager, emitter=self.emit + f"worker-{wid}", node_info, self._engine.storage_manager, emitter=self.emit ) with work_queue.new_consumer() as consumer: From de300fc2236f1cbcdb6a40759a6c8bc776f807ab Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 18 May 2021 12:05:57 +0200 Subject: [PATCH 15/29] bug --- yapapi/executor/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 7281f3210..70c692bb8 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -1070,7 +1070,6 @@ async def worker_starter() -> None: # Importing this at the beginning would cause circular dependencies from ..log import pluralize - self._engine._payment_closing = True for task in services: task.cancel() if cancelled: From e51ef681d4d2a11d53c97b6ee392865ae3e52d71 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 18 May 2021 12:43:07 +0200 Subject: [PATCH 16/29] rename the "old" `blender.py` to `blener-deprecated.py` rename the example based on the new API to `blender.py` --- ...blender-golem.py => blender-deprecated.py} | 21 +++++++------------ examples/blender/blender.py | 21 ++++++++++++------- 2 files changed, 21 insertions(+), 21 deletions(-) rename examples/blender/{blender-golem.py => blender-deprecated.py} (92%) diff --git a/examples/blender/blender-golem.py b/examples/blender/blender-deprecated.py similarity index 92% rename from examples/blender/blender-golem.py rename to examples/blender/blender-deprecated.py index b17ecf899..9f2a6310a 100755 --- a/examples/blender/blender-golem.py +++ b/examples/blender/blender-deprecated.py @@ -12,7 +12,6 @@ WorkContext, windows_event_loop_fix, ) -from yapapi.executor import Golem from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError @@ -94,32 +93,28 @@ async def worker(ctx: WorkContext, tasks): # By passing `event_consumer=log_summary()` we enable summary logging. # See the documentation of the `yapapi.log` module on how to set # the level of detail and format of the logged information. - async with Golem( + async with Executor( + payload=package, + max_workers=3, budget=10.0, + timeout=timeout, subnet_tag=subnet_tag, driver=driver, network=network, event_consumer=log_summary(log_event_repr), - ) as golem: + ) as executor: print( f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " - f"payment driver: {TEXT_COLOR_YELLOW}{driver}{TEXT_COLOR_DEFAULT}, " - f"and network: {TEXT_COLOR_YELLOW}{network}{TEXT_COLOR_DEFAULT}\n" + f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" ) num_tasks = 0 start_time = datetime.now() - completed_tasks = golem.execute_task( - worker, - [Task(data=frame) for frame in frames], - payload=package, - max_workers=3, - timeout=timeout, - ) - async for task in completed_tasks: + async for task in executor.submit(worker, [Task(data=frame) for frame in frames]): num_tasks += 1 print( f"{TEXT_COLOR_CYAN}" diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 9f2a6310a..b17ecf899 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -12,6 +12,7 @@ WorkContext, windows_event_loop_fix, ) +from yapapi.executor import Golem from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError @@ -93,28 +94,32 @@ async def worker(ctx: WorkContext, tasks): # By passing `event_consumer=log_summary()` we enable summary logging. # See the documentation of the `yapapi.log` module on how to set # the level of detail and format of the logged information. - async with Executor( - payload=package, - max_workers=3, + async with Golem( budget=10.0, - timeout=timeout, subnet_tag=subnet_tag, driver=driver, network=network, event_consumer=log_summary(log_event_repr), - ) as executor: + ) as golem: print( f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " - f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, " - f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n" + f"payment driver: {TEXT_COLOR_YELLOW}{driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{network}{TEXT_COLOR_DEFAULT}\n" ) num_tasks = 0 start_time = datetime.now() - async for task in executor.submit(worker, [Task(data=frame) for frame in frames]): + completed_tasks = golem.execute_task( + worker, + [Task(data=frame) for frame in frames], + payload=package, + max_workers=3, + timeout=timeout, + ) + async for task in completed_tasks: num_tasks += 1 print( f"{TEXT_COLOR_CYAN}" From 7429ed9ad6d8f5db22f1afaffaa389c0134abf78 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Tue, 18 May 2021 13:08:19 +0200 Subject: [PATCH 17/29] Golem.execute_task() -> Golem.execute_tasks() & other minor changes --- examples/blender/blender-golem.py | 6 +-- yapapi/executor/__init__.py | 71 ++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/examples/blender/blender-golem.py b/examples/blender/blender-golem.py index b17ecf899..0949fe3b0 100755 --- a/examples/blender/blender-golem.py +++ b/examples/blender/blender-golem.py @@ -105,14 +105,14 @@ async def worker(ctx: WorkContext, tasks): print( f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " - f"payment driver: {TEXT_COLOR_YELLOW}{driver}{TEXT_COLOR_DEFAULT}, " - f"and network: {TEXT_COLOR_YELLOW}{network}{TEXT_COLOR_DEFAULT}\n" + f"payment driver: {TEXT_COLOR_YELLOW}{golem.driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{golem.network}{TEXT_COLOR_DEFAULT}\n" ) num_tasks = 0 start_time = datetime.now() - completed_tasks = golem.execute_task( + completed_tasks = golem.execute_tasks( worker, [Task(data=frame) for frame in frames], payload=package, diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 7399a1c49..813020bdd 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -198,9 +198,9 @@ async def create_demand_builder( builder.add(NodeInfo(subnet_tag=self._subnet)) if self._subnet: builder.ensure(f"({NodeInfoKeys.subnet_tag}={self._subnet})") - await builder.decorate(self.payment_decoration) - await builder.decorate(self.strategy) - await builder.decorate(payload) + await builder.decorate( + self.payment_decoration, self.strategy, payload + ) return builder def _init_api(self, app_key: Optional[str] = None): @@ -451,13 +451,13 @@ async def decorate_demand(self, demand: DemandBuilder): demand.ensure(constraint) demand.properties.update({p.key: p.value for p in self.market_decoration.properties}) - async def execute_task( + async def execute_tasks( self, worker: Callable[ [WorkContext, AsyncIterator[Task[D, R]]], AsyncGenerator[Work, Awaitable[List[events.CommandEvent]]], ], - data: Iterable[Task[D, R]], + data: Union[AsyncIterator[Task[D, R]], Iterable[Task[D, R]]], payload: Payload, max_workers: Optional[int] = None, timeout: Optional[timedelta] = None, @@ -731,6 +731,56 @@ class Executor(AsyncContextManager): Used to run batch tasks using the specified application package within providers' execution units. """ + @overload + def __init__( + self, + *, + budget: Union[float, Decimal], + payload: Optional[Payload] = None, + max_workers: int = 5, + timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, + _engine: Golem + ): + # A variant with explicit `_engine` + ... + + @overload + def __init__( + self, + *, + budget: Union[float, Decimal], + strategy: Optional[MarketStrategy] = None, + subnet_tag: Optional[str] = None, + driver: Optional[str] = None, + network: Optional[str] = None, + event_consumer: Optional[Callable[[Event], None]] = None, + stream_output: bool = False, + payload: Optional[Payload] = None, + max_workers: int = 5, + timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, + ): + # Standalone usage, with `payload` parameter + ... + + @overload + def __init__( + self, + *, + budget: Union[float, Decimal], + strategy: Optional[MarketStrategy] = None, + subnet_tag: Optional[str] = None, + driver: Optional[str] = None, + network: Optional[str] = None, + event_consumer: Optional[Callable[[Event], None]] = None, + stream_output: bool = False, + max_workers: int = 5, + timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, + package: Optional[Payload] = None, + ): + # Standalone usage, with `package` parameter + ... + + def __init__( self, *, @@ -816,15 +866,14 @@ def network(self) -> str: return self._engine.network async def __aenter__(self) -> "Executor": - self._expires = datetime.now(timezone.utc) + self._timeout if self.__standalone: - await self._engine.__aenter__() + await self._stack.enter_async_context(self._engine) + + self._expires = datetime.now(timezone.utc) + self._timeout return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self._stack.aclose() - if self.__standalone: - await self._engine.__aexit__(exc_type, exc_val, exc_tb) def emit(self, event: events.Event) -> None: self._engine.emit(event) @@ -950,8 +999,8 @@ async def task_generator() -> AsyncIterator[Task[D, R]]: self._engine.emit( events.TaskStarted( agr_id=agreement.id, - task_id=handle.data.id, - task_data=handle.data.data + task_id=task.id, + task_data=task.data ) ) yield task From 142d3a3add39c18d64a46298f0a2d25ad57ee370 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 18 May 2021 13:20:42 +0200 Subject: [PATCH 18/29] black --- yapapi/executor/__init__.py | 100 ++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 57 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index c7b6bdbbf..bea31bb0c 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -128,7 +128,7 @@ def __init__( network: Optional[str] = None, event_consumer: Optional[Callable[[Event], None]] = None, stream_output: bool = False, - app_key: Optional[str] = None + app_key: Optional[str] = None, ): """ Base execution engine containing functions common to all modes of operation @@ -198,9 +198,7 @@ async def create_demand_builder( builder.add(NodeInfo(subnet_tag=self._subnet)) if self._subnet: builder.ensure(f"({NodeInfoKeys.subnet_tag}={self._subnet})") - await builder.decorate( - self.payment_decoration, self.strategy, payload - ) + await builder.decorate(self.payment_decoration, self.strategy, payload) return builder def _init_api(self, app_key: Optional[str] = None): @@ -280,9 +278,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): self._services, timeout=10, return_when=asyncio.ALL_COMPLETED ) if pending: - logger.debug( - "%s still running: %s", pluralize(len(pending), "service"), pending - ) + logger.debug("%s still running: %s", pluralize(len(pending), "service"), pending) except Exception: # TODO: add message logger.debug("TODO", exc_info=True) @@ -422,7 +418,9 @@ async def process_debit_notes(self) -> None: if self._payment_closing and not self._agreements_to_pay: break - async def accept_payment_for_agreement(self, agreement_id: str, *, partial: bool = False) -> None: + async def accept_payment_for_agreement( + self, agreement_id: str, *, partial: bool = False + ) -> None: self.emit(events.PaymentPrepared(agr_id=agreement_id)) inv = self._invoices.get(agreement_id) if inv is None: @@ -433,9 +431,7 @@ async def accept_payment_for_agreement(self, agreement_id: str, *, partial: bool allocation = self._get_allocation(inv) await inv.accept(amount=inv.amount, allocation=allocation) self.emit( - events.PaymentAccepted( - agr_id=agreement_id, inv_id=inv.invoice_id, amount=inv.amount - ) + events.PaymentAccepted(agr_id=agreement_id, inv_id=inv.invoice_id, amount=inv.amount) ) def approve_agreement_payments(self, agreement_id): @@ -458,26 +454,24 @@ async def decorate_demand(self, demand: DemandBuilder): demand.properties.update({p.key: p.value for p in self.market_decoration.properties}) async def execute_tasks( - self, - worker: Callable[ - [WorkContext, AsyncIterator[Task[D, R]]], - AsyncGenerator[Work, Awaitable[List[events.CommandEvent]]], - ], - data: Union[AsyncIterator[Task[D, R]], Iterable[Task[D, R]]], - payload: Payload, - max_workers: Optional[int] = None, - timeout: Optional[timedelta] = None, - budget: Optional[Union[float, Decimal]] = None, + self, + worker: Callable[ + [WorkContext, AsyncIterator[Task[D, R]]], + AsyncGenerator[Work, Awaitable[List[events.CommandEvent]]], + ], + data: Union[AsyncIterator[Task[D, R]], Iterable[Task[D, R]]], + payload: Payload, + max_workers: Optional[int] = None, + timeout: Optional[timedelta] = None, + budget: Optional[Union[float, Decimal]] = None, ) -> AsyncIterator[Task[D, R]]: - kwargs: Dict[str, Any] = { - 'payload': payload - } + kwargs: Dict[str, Any] = {"payload": payload} if max_workers: - kwargs['max_workers'] = max_workers + kwargs["max_workers"] = max_workers if timeout: - kwargs['timeout'] = timeout - kwargs['budget'] = budget if budget is not None else self._budget_amount + kwargs["timeout"] = timeout + kwargs["budget"] = budget if budget is not None else self._budget_amount async with Executor(_engine=self, **kwargs) as executor: async for t in executor.submit(worker, data): @@ -485,8 +479,7 @@ async def execute_tasks( async def create_activity(self, agreement_id: str): return await self._activity_api.new_activity( - agreement_id, - stream_events=self._stream_output + agreement_id, stream_events=self._stream_output ) async def process_batches( @@ -569,10 +562,10 @@ class Job: """Functionality related to a single job.""" def __init__( - self, - engine: Golem, - expiration_time: datetime, - payload: Payload, + self, + engine: Golem, + expiration_time: datetime, + payload: Payload, ): self.engine = engine self.offers_collected: int = 0 @@ -584,9 +577,9 @@ def __init__( self.finished = asyncio.Event() async def _handle_proposal( - self, - proposal: OfferProposal, - demand_builder: DemandBuilder, + self, + proposal: OfferProposal, + demand_builder: DemandBuilder, ) -> events.Event: """Handle a single `OfferProposal`. @@ -642,7 +635,9 @@ async def reject_proposal(reason: str) -> events.ProposalRejected: return events.ProposalConfirmed(prop_id=proposal.id) async def _find_offers_for_subscription( - self, subscription: Subscription, demand_builder: DemandBuilder, + self, + subscription: Subscription, + demand_builder: DemandBuilder, ) -> None: """Create a market subscription and repeatedly collect offer proposals for it. @@ -664,7 +659,9 @@ async def _find_offers_for_subscription( async for proposal in proposals: - self.engine.emit(events.ProposalReceived(prop_id=proposal.id, provider_id=proposal.issuer)) + self.engine.emit( + events.ProposalReceived(prop_id=proposal.id, provider_id=proposal.issuer) + ) self.offers_collected += 1 async def handler(proposal_): @@ -696,9 +693,7 @@ async def find_offers(self) -> None: When the subscription expires, create a new one. And so on... """ - builder = await self.engine.create_demand_builder( - self.expiration_time, self.payload - ) + builder = await self.engine.create_demand_builder(self.expiration_time, self.payload) while True: try: @@ -745,7 +740,7 @@ def __init__( payload: Optional[Payload] = None, max_workers: int = 5, timeout: timedelta = DEFAULT_EXECUTOR_TIMEOUT, - _engine: Golem + _engine: Golem, ): # A variant with explicit `_engine` ... @@ -786,7 +781,6 @@ def __init__( # Standalone usage, with `package` parameter ... - def __init__( self, *, @@ -835,7 +829,7 @@ def __init__( else: warnings.warn( "stand-alone usage is deprecated, please `Golem.execute_task` class instead ", - DeprecationWarning + DeprecationWarning, ) self._engine = Golem( budget=budget, @@ -844,7 +838,7 @@ def __init__( driver=driver, network=network, event_consumer=event_consumer, - stream_output=stream_output + stream_output=stream_output, ) self.__standalone = True @@ -900,11 +894,7 @@ async def submit( :return: yields computation progress events """ - job = Job( - self._engine, - expiration_time=self._expires, - payload=self._payload - ) + job = Job(self._engine, expiration_time=self._expires, payload=self._payload) self._engine.add_job(job) services: Set[asyncio.Task] = set() @@ -997,14 +987,13 @@ async def start_worker(agreement: rest.market.Agreement, node_info: NodeInfo) -> with work_queue.new_consumer() as consumer: try: + async def task_generator() -> AsyncIterator[Task[D, R]]: async for handle in consumer: task = Task.for_handle(handle, work_queue, self.emit) self._engine.emit( events.TaskStarted( - agr_id=agreement.id, - task_id=task.id, - task_data=task.data + agr_id=agreement.id, task_id=task.id, task_data=task.data ) ) yield task @@ -1029,10 +1018,7 @@ async def worker_starter() -> None: while True: await asyncio.sleep(2) await job.agreements_pool.cycle() - if ( - len(workers) < self._max_workers - and await work_queue.has_unassigned_items() - ): + if len(workers) < self._max_workers and await work_queue.has_unassigned_items(): new_task = None try: new_task = await job.agreements_pool.use_agreement( From 9d8a9a325b240b3ee2fa0af8dd4108dd0ad7e235 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 18 May 2021 18:27:01 +0200 Subject: [PATCH 19/29] move `execute_tasks` to the end of `Golem` --- yapapi/executor/__init__.py | 49 +++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index bea31bb0c..9ab53a5d0 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -453,30 +453,6 @@ async def decorate_demand(self, demand: DemandBuilder): demand.ensure(constraint) demand.properties.update({p.key: p.value for p in self.market_decoration.properties}) - async def execute_tasks( - self, - worker: Callable[ - [WorkContext, AsyncIterator[Task[D, R]]], - AsyncGenerator[Work, Awaitable[List[events.CommandEvent]]], - ], - data: Union[AsyncIterator[Task[D, R]], Iterable[Task[D, R]]], - payload: Payload, - max_workers: Optional[int] = None, - timeout: Optional[timedelta] = None, - budget: Optional[Union[float, Decimal]] = None, - ) -> AsyncIterator[Task[D, R]]: - - kwargs: Dict[str, Any] = {"payload": payload} - if max_workers: - kwargs["max_workers"] = max_workers - if timeout: - kwargs["timeout"] = timeout - kwargs["budget"] = budget if budget is not None else self._budget_amount - - async with Executor(_engine=self, **kwargs) as executor: - async for t in executor.submit(worker, data): - yield t - async def create_activity(self, agreement_id: str): return await self._activity_api.new_activity( agreement_id, stream_events=self._stream_output @@ -557,6 +533,31 @@ async def get_batch_results() -> List[events.CommandEvent]: future_results = loop.create_task(get_batch_results()) item = await command_generator.asend(future_results) + async def execute_tasks( + self, + worker: Callable[ + [WorkContext, AsyncIterator[Task[D, R]]], + AsyncGenerator[Work, Awaitable[List[events.CommandEvent]]], + ], + data: Union[AsyncIterator[Task[D, R]], Iterable[Task[D, R]]], + payload: Payload, + max_workers: Optional[int] = None, + timeout: Optional[timedelta] = None, + budget: Optional[Union[float, Decimal]] = None, + ) -> AsyncIterator[Task[D, R]]: + + kwargs: Dict[str, Any] = {"payload": payload} + if max_workers: + kwargs["max_workers"] = max_workers + if timeout: + kwargs["timeout"] = timeout + kwargs["budget"] = budget if budget is not None else self._budget_amount + + async with Executor(_engine=self, **kwargs) as executor: + async for t in executor.submit(worker, data): + yield t + + class Job: """Functionality related to a single job.""" From 8c13ca08b6f10a9b39bcb78b98163cbcfd4c8804 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Tue, 18 May 2021 18:48:36 +0200 Subject: [PATCH 20/29] Fixes in executor/__init__.py to pass unit tests --- tests/executor/test_payment_platforms.py | 18 +++---- tests/executor/test_strategy.py | 10 ++-- yapapi/executor/__init__.py | 63 +++++++++++++++--------- 3 files changed, 53 insertions(+), 38 deletions(-) diff --git a/tests/executor/test_payment_platforms.py b/tests/executor/test_payment_platforms.py index 81cb90262..5c2c83efd 100644 --- a/tests/executor/test_payment_platforms.py +++ b/tests/executor/test_payment_platforms.py @@ -70,8 +70,8 @@ async def test_no_accounts_raises(monkeypatch): monkeypatch.setattr(Payment, "accounts", _mock_accounts_iterator()) - async with Executor(package=mock.Mock(), budget=10.0) as executor: - with pytest.raises(NoPaymentAccountError): + with pytest.raises(NoPaymentAccountError): + async with Executor(package=mock.Mock(), budget=10.0) as executor: async for _ in executor.submit(worker=mock.Mock(), data=mock.Mock()): pass @@ -90,15 +90,15 @@ async def test_no_matching_account_raises(monkeypatch): ), ) - async with Executor( - package=mock.Mock(), budget=10.0, driver="matching-driver", network="matching-network" - ) as executor: - with pytest.raises(NoPaymentAccountError) as exc_info: + with pytest.raises(NoPaymentAccountError) as exc_info: + async with Executor( + package=mock.Mock(), budget=10.0, driver="matching-driver", network="matching-network" + ) as executor: async for _ in executor.submit(worker=mock.Mock(), data=mock.Mock()): pass - exc = exc_info.value - assert exc.required_driver == "matching-driver" - assert exc.required_network == "matching-network" + exc = exc_info.value + assert exc.required_driver == "matching-driver" + assert exc.required_network == "matching-network" @pytest.mark.asyncio diff --git a/tests/executor/test_strategy.py b/tests/executor/test_strategy.py index a06ebd39b..5851dc400 100644 --- a/tests/executor/test_strategy.py +++ b/tests/executor/test_strategy.py @@ -3,7 +3,7 @@ import pytest from unittest.mock import Mock -from yapapi.executor import Executor +from yapapi.executor import Golem from yapapi.executor.strategy import ( DecreaseScoreForUnconfirmedAgreement, LeastExpensiveLinearPayuMS, @@ -130,8 +130,8 @@ async def test_default_strategy_type(monkeypatch): monkeypatch.setattr(yapapi.rest, "Configuration", Mock) - executor = Executor(package=Mock(), budget=1.0) - default_strategy = executor.strategy + golem = Golem(budget=1.0) + default_strategy = golem.strategy assert isinstance(default_strategy, DecreaseScoreForUnconfirmedAgreement) assert isinstance(default_strategy.base_strategy, LeastExpensiveLinearPayuMS) @@ -143,8 +143,8 @@ async def test_user_strategy_not_modified(monkeypatch): monkeypatch.setattr(yapapi.rest, "Configuration", Mock) user_strategy = Mock() - executor = Executor(package=Mock(), budget=1.0, strategy=user_strategy) - assert executor.strategy == user_strategy + golem = Golem(budget=1.0, strategy=user_strategy) + assert golem.strategy == user_strategy class TestLeastExpensiveLinearPayuMS: diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index bea31bb0c..6e603ce4f 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -175,7 +175,11 @@ def __init__( # Add buffering to the provided event emitter to make sure # that emitting events will not block - self._wrapped_consumer = AsyncWrapper(event_consumer) + # TODO: make AsyncWrapper an AsyncContextManager and start it in + # in __aenter__(); if it's started here then there's no guarantee that + # it will be cancelled properly + self._wrapped_consumer: Optional[AsyncWrapper] = None + self._event_consumer = event_consumer self._stream_output = stream_output @@ -185,8 +189,12 @@ def __init__( self._invoices: Dict[str, rest.payment.Invoice] = dict() self._payment_closing: bool = False - self._services: Set[asyncio.Task] = set() + # a set of `Job` instances used to track jobs - computations or services - started + # it can be used to wait until all jobs are finished + self._jobs: Set[Job] = set() + self._process_invoices_job: Optional[asyncio.Task] = None + self._services: Set[asyncio.Task] = set() self._stack = AsyncExitStack() async def create_demand_builder( @@ -225,34 +233,39 @@ def strategy(self) -> MarketStrategy: return self._strategy def emit(self, *args, **kwargs) -> None: - self._wrapped_consumer.async_call(*args, **kwargs) + if self._wrapped_consumer: + self._wrapped_consumer.async_call(*args, **kwargs) async def __aenter__(self) -> "Golem": - stack = self._stack + try: + stack = self._stack - market_client = await stack.enter_async_context(self._api_config.market()) - self._market_api = rest.Market(market_client) + self._wrapped_consumer = AsyncWrapper(self._event_consumer) - activity_client = await stack.enter_async_context(self._api_config.activity()) - self._activity_api = rest.Activity(activity_client) + market_client = await stack.enter_async_context(self._api_config.market()) + self._market_api = rest.Market(market_client) - payment_client = await stack.enter_async_context(self._api_config.payment()) - self._payment_api = rest.Payment(payment_client) + activity_client = await stack.enter_async_context(self._api_config.activity()) + self._activity_api = rest.Activity(activity_client) - # a set of `Job` instances used to track jobs - computations or services - started - # it can be used to wait until all jobs are finished - self._jobs: Set[Job] = set() + payment_client = await stack.enter_async_context(self._api_config.payment()) + self._payment_api = rest.Payment(payment_client) - self.payment_decoration = Golem.PaymentDecoration(await self._create_allocations()) + self.payment_decoration = Golem.PaymentDecoration(await self._create_allocations()) - loop = asyncio.get_event_loop() - self._process_invoices_job = loop.create_task(self.process_invoices()) - self._services.add(self._process_invoices_job) - self._services.add(loop.create_task(self.process_debit_notes())) + # TODO: make the method starting the process_invoices() task an async context manager + # to simplify code in __aexit__() + loop = asyncio.get_event_loop() + self._process_invoices_job = loop.create_task(self.process_invoices()) + self._services.add(self._process_invoices_job) + self._services.add(loop.create_task(self.process_debit_notes())) - self._storage_manager = await self._stack.enter_async_context(gftp.provider()) + self._storage_manager = await self._stack.enter_async_context(gftp.provider()) - return self + return self + except: + await self.__aexit__(*sys.exc_info()) + raise async def __aexit__(self, exc_type, exc_val, exc_tb): # Importing this at the beginning would cause circular dependencies @@ -268,7 +281,9 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): if task is not self._process_invoices_job: task.cancel() - if not any(True for job in self._jobs if job.agreements_pool.confirmed > 0): + if self._process_invoices_job and not any( + True for job in self._jobs if job.agreements_pool.confirmed > 0 + ): logger.debug("No need to wait for invoices.") self._process_invoices_job.cancel() @@ -283,7 +298,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): # TODO: add message logger.debug("TODO", exc_info=True) - if self._agreements_to_pay: + if self._agreements_to_pay and self._process_invoices_job: logger.info( "%s still unpaid, waiting for invoices...", pluralize(len(self._agreements_to_pay), "agreement"), @@ -301,7 +316,8 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): except Exception: self.emit(events.ShutdownFinished(exc_info=sys.exc_info())) finally: - await self._wrapped_consumer.stop() + if self._wrapped_consumer: + await self._wrapped_consumer.stop() async def _create_allocations(self) -> rest.payment.MarketDecoration: @@ -868,7 +884,6 @@ def network(self) -> str: async def __aenter__(self) -> "Executor": if self.__standalone: await self._stack.enter_async_context(self._engine) - self._expires = datetime.now(timezone.utc) + self._timeout return self From b0df49c7aab199e3f660cbfe1c250d0be17cf344 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Wed, 19 May 2021 16:10:37 +0200 Subject: [PATCH 21/29] Changes in events/SummaryLogger to run yacat withour errors --- examples/blender/blender.py | 5 ++--- yapapi/executor/__init__.py | 8 +++++--- yapapi/executor/events.py | 9 +++++++-- yapapi/log.py | 29 ++++++++++++++--------------- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 21019e157..d1d8104d3 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -5,7 +5,6 @@ import sys from yapapi import ( - Executor, NoPaymentAccountError, Task, __version__ as yapapi_version, @@ -105,8 +104,8 @@ async def worker(ctx: WorkContext, tasks): print( f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, " - f"payment driver: {TEXT_COLOR_YELLOW}{driver}{TEXT_COLOR_DEFAULT}, " - f"and network: {TEXT_COLOR_YELLOW}{network}{TEXT_COLOR_DEFAULT}\n" + f"payment driver: {TEXT_COLOR_YELLOW}{golem.driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{golem.network}{TEXT_COLOR_DEFAULT}\n" ) num_tasks = 0 diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 75156ad6f..d540427ff 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -27,6 +27,7 @@ cast, overload, ) +import uuid import warnings @@ -583,6 +584,7 @@ def __init__( expiration_time: datetime, payload: Payload, ): + self.id = str(uuid.uuid4()) self.engine = engine self.offers_collected: int = 0 self.proposals_confirmed: int = 0 @@ -950,7 +952,7 @@ async def _submit( job: Job, ) -> AsyncGenerator[Task[D, R], None]: - self.emit(events.ComputationStarted(self._expires)) + self.emit(events.ComputationStarted(job.id, self._expires)) done_queue: asyncio.Queue[Task[D, R]] = asyncio.Queue() @@ -1109,10 +1111,10 @@ async def worker_starter() -> None: assert get_done_task not in services get_done_task = None - self.emit(events.ComputationFinished()) + self.emit(events.ComputationFinished(job.id)) except (Exception, CancelledError, KeyboardInterrupt): - self.emit(events.ComputationFinished(exc_info=sys.exc_info())) # type: ignore + self.emit(events.ComputationFinished(job.id, exc_info=sys.exc_info())) # type: ignore cancelled = True finally: diff --git a/yapapi/executor/events.py b/yapapi/executor/events.py index b0a234d9c..28252c5e9 100644 --- a/yapapi/executor/events.py +++ b/yapapi/executor/events.py @@ -45,13 +45,18 @@ def extract_exc_info(self) -> Tuple[Optional[ExcInfo], "Event"]: return exc_info, me +@dataclass(init=False) +class ComputationEvent(Event): + job_id: str + + @dataclass -class ComputationStarted(Event): +class ComputationStarted(ComputationEvent): expires: datetime @dataclass -class ComputationFinished(HasExcInfo): +class ComputationFinished(HasExcInfo, ComputationEvent): """Indicates successful completion if `exc_info` is `None` and a failure otherwise.""" diff --git a/yapapi/log.py b/yapapi/log.py index a42c4b854..920d77076 100644 --- a/yapapi/log.py +++ b/yapapi/log.py @@ -232,8 +232,8 @@ class SummaryLogger: # Generates subsequent numbers, for use in generated provider names numbers: Iterator[int] - # Start time of the computation - start_time: float + # Start time of the computation (indexed by job_id) + start_time: Dict[str, float] # Maps received proposal ids to provider ids received_proposals: Dict[str, str] @@ -273,24 +273,19 @@ def __init__(self, wrapped_emitter: Optional[Callable[[events.Event], None]] = N self._wrapped_emitter = wrapped_emitter self.numbers: Iterator[int] = itertools.count(1) - self.provider_cost = {} - self._reset() - - def _reset(self) -> None: - """Reset all information aggregated by this logger related to a single computation. - - Here "computation" means an interval of time between the events - `ComputationStarted` and `ComputationFinished`. + self._reset_counters() - Note that the `provider_cost` is not reset here, it is zeroed on `ExecutorShutdown`. - """ + def _reset_counters(self): + """Reset all information aggregated by this logger related to a single Executor instance.""" - self.start_time = time.time() + self.provider_cost = {} + self.start_time = {} self.received_proposals = {} self.confirmed_proposals = set() self.agreement_provider_info = {} self.confirmed_agreements = set() self.task_data = {} + self.provider_cost = {} self.provider_tasks = defaultdict(list) self.provider_failures = Counter() self.cancelled = False @@ -298,6 +293,10 @@ def _reset(self) -> None: self.error_occurred = False self.time_waiting_for_proposals = timedelta(0) + def _register_job(self, job_id: str) -> None: + """Initialize counters for a new job.""" + self.start_time[job_id] = time.time() + def _print_summary(self) -> None: """Print a summary at the end of computation.""" @@ -343,7 +342,7 @@ def log(self, event: events.Event) -> None: def _handle(self, event: events.Event): if isinstance(event, events.ComputationStarted): - self._reset() + self._register_job(event.job_id) if self.provider_cost: # This means another computation run in the current Executor instance. self._print_total_cost(partial=True) @@ -459,7 +458,7 @@ def _handle(self, event: events.Event): elif isinstance(event, events.ComputationFinished): if not event.exc_info: - total_time = time.time() - self.start_time + total_time = time.time() - self.start_time[event.job_id] self.logger.info(f"Computation finished in {total_time:.1f}s") self.finished = True else: From b1627def481598a54568dd03672701e3a88913f3 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 06:50:34 +0200 Subject: [PATCH 22/29] Update yapapi/executor/__init__.py Co-authored-by: Kuba Mazurek --- yapapi/executor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index d540427ff..f6843d217 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -846,7 +846,7 @@ def __init__( self._engine = _engine else: warnings.warn( - "stand-alone usage is deprecated, please `Golem.execute_task` class instead ", + "stand-alone usage is deprecated, please use `Golem.execute_task` class instead.", DeprecationWarning, ) self._engine = Golem( From 99de16396bee13012a55c5b6b2c9bc00bea07dce Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 07:41:03 +0200 Subject: [PATCH 23/29] fix tests --- tests/test_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_log.py b/tests/test_log.py index a8ab37462..b2cf32447 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -43,7 +43,7 @@ def test_log_event_emit_traceback(): try: raise Exception("Hello!") except: - log_event(ComputationFinished(exc_info=sys.exc_info())) + log_event(ComputationFinished(exc_info=sys.exc_info(), job_id="42")) def test_log_event_repr_emit_traceback(): @@ -52,4 +52,4 @@ def test_log_event_repr_emit_traceback(): try: raise Exception("Hello!") except: - log_event_repr(ComputationFinished(exc_info=sys.exc_info())) + log_event_repr(ComputationFinished(exc_info=sys.exc_info(), job_id="42")) From 61a28960ac4ad51a854d788cf6e8ee6e1773b299 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 12:50:01 +0200 Subject: [PATCH 24/29] fix script vs task events confusion --- yapapi/executor/__init__.py | 17 +++++++++++------ yapapi/executor/events.py | 11 ++++++++--- yapapi/log.py | 34 ++++++++++++++++++++++++++-------- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index f6843d217..dc773d313 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -111,7 +111,7 @@ def unpack_work_item(item: WorkItem) -> Tuple[Work, ExecOptions]: return item, ExecOptions() -batch_ids: Iterator[int] = itertools.count(1) +exescript_ids: Iterator[int] = itertools.count(1) D = TypeVar("D") # Type var for task data @@ -490,7 +490,7 @@ async def process_batches( batch, exec_options = unpack_work_item(item) # TODO: `task_id` should really be `batch_id`, but then we should also rename # `task_id` field of several events (e.g. `ScriptSent`) - task_id = str(next(batch_ids)) + script_id = str(next(exescript_ids)) if batch.timeout: if exec_options.batch_timeout: @@ -512,20 +512,20 @@ async def process_batches( batch.register(cc) remote = await activity.send(cc.commands(), deadline=batch_deadline) cmds = cc.commands() - self.emit(events.ScriptSent(agr_id=agreement_id, task_id=task_id, cmds=cmds)) + self.emit(events.ScriptSent(agr_id=agreement_id, script_id=script_id, cmds=cmds)) async def get_batch_results() -> List[events.CommandEvent]: results = [] async for evt_ctx in remote: - evt = evt_ctx.event(agr_id=agreement_id, task_id=task_id, cmds=cmds) + evt = evt_ctx.event(agr_id=agreement_id, script_id=script_id, cmds=cmds) self.emit(evt) results.append(evt) if isinstance(evt, events.CommandExecuted) and not evt.success: raise CommandExecutionError(evt.command, evt.stderr) - self.emit(events.GettingResults(agr_id=agreement_id, task_id=task_id)) + self.emit(events.GettingResults(agr_id=agreement_id, script_id=script_id)) await batch.post() - self.emit(events.ScriptFinished(agr_id=agreement_id, task_id=task_id)) + self.emit(events.ScriptFinished(agr_id=agreement_id, script_id=script_id)) await self.accept_payment_for_agreement(agreement_id, partial=True) return results @@ -1014,6 +1014,11 @@ async def task_generator() -> AsyncIterator[Task[D, R]]: ) ) yield task + self._engine.emit( + events.TaskFinished( + agr_id=agreement.id, task_id=task.id + ) + ) batch_generator = worker(work_context, task_generator()) try: diff --git a/yapapi/executor/events.py b/yapapi/executor/events.py index 28252c5e9..11204054b 100644 --- a/yapapi/executor/events.py +++ b/yapapi/executor/events.py @@ -196,6 +196,11 @@ class TaskStarted(AgreementEvent, TaskEvent): task_data: Any +@dataclass +class TaskFinished(AgreementEvent, TaskEvent): + pass + + @dataclass class WorkerFinished(HasExcInfo, AgreementEvent): """Indicates successful completion if `exc_info` is `None` and a failure otherwise.""" @@ -203,7 +208,7 @@ class WorkerFinished(HasExcInfo, AgreementEvent): @dataclass(init=False) class ScriptEvent(AgreementEvent): - task_id: Optional[str] + script_id: Optional[str] @dataclass @@ -284,8 +289,8 @@ def computation_finished(self, last_idx: int) -> bool: self.kwargs["cmd_idx"] >= last_idx or not self.kwargs["success"] ) - def event(self, agr_id: str, task_id: str, cmds: List) -> CommandEvent: - kwargs = dict(agr_id=agr_id, task_id=task_id, **self.kwargs) + def event(self, agr_id: str, script_id: str, cmds: List) -> CommandEvent: + kwargs = dict(agr_id=agr_id, script_id=script_id, **self.kwargs) if self.evt_cls is CommandExecuted: kwargs["command"] = cmds[self.kwargs["cmd_idx"]] kwargs["stdout"] = kwargs["stderr"] = None diff --git a/yapapi/log.py b/yapapi/log.py index 920d77076..739a24adf 100644 --- a/yapapi/log.py +++ b/yapapi/log.py @@ -139,6 +139,7 @@ def enable_default_logger( events.ActivityCreated: "Activity created on provider", events.ActivityCreateFailed: "Failed to create activity", events.TaskStarted: "Task started", + events.TaskFinished: "Task finished", events.ScriptSent: "Script sent to provider", events.CommandStarted: "Command started", events.CommandStdOut: "Command stdout", @@ -285,6 +286,7 @@ def _reset_counters(self): self.agreement_provider_info = {} self.confirmed_agreements = set() self.task_data = {} + self.script_cmds = {} self.provider_cost = {} self.provider_tasks = defaultdict(list) self.provider_failures = Counter() @@ -403,20 +405,17 @@ def _handle(self, event: events.Event): self.confirmed_agreements.add(event.agr_id) elif isinstance(event, events.TaskStarted): - self.task_data[event.task_id] = event.task_data - - elif isinstance(event, events.ScriptSent): provider_info = self.agreement_provider_info[event.agr_id] - data = self.task_data[event.task_id] if event.task_id else "" + self.task_data[event.task_id] = event.task_data self.logger.info( - "Task sent to provider '%s', task data: %s", + "Task started on provider '%s', task data: %s", provider_info.name, - str_capped(data, 200), + str_capped(event.task_data, 200), ) - elif isinstance(event, events.ScriptFinished): + elif isinstance(event, events.TaskFinished): provider_info = self.agreement_provider_info[event.agr_id] - data = self.task_data[event.task_id] if event.task_id else "" + data = self.task_data[event.task_id] self.logger.info( "Task computed by provider '%s', task data: %s", provider_info.name, @@ -425,6 +424,25 @@ def _handle(self, event: events.Event): if event.task_id: self.provider_tasks[provider_info].append(event.task_id) + elif isinstance(event, events.ScriptSent): + provider_info = self.agreement_provider_info[event.agr_id] + self.script_cmds[event.script_id] = event.cmds + cmds = ", ".join([", ".join(cmd.keys()) for cmd in event.cmds]) + self.logger.debug( + "Script '%s' sent to provider '%s', cmds: %s", + event.script_id, + provider_info.name, + str_capped(cmds, 200), + ) + + elif isinstance(event, events.ScriptFinished): + provider_info = self.agreement_provider_info[event.agr_id] + self.logger.debug( + "Script '%s' finished on provider '%s'", + event.script_id, + provider_info.name, + ) + elif isinstance(event, events.PaymentAccepted): provider_info = self.agreement_provider_info[event.agr_id] cost = self.provider_cost.get(provider_info, Decimal(0)) From 68da31a4d46dc2733cca2f0396bcf81e0c1cbf25 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Thu, 20 May 2021 12:57:22 +0200 Subject: [PATCH 25/29] fix test, black --- tests/executor/test_events.py | 2 +- yapapi/executor/__init__.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/executor/test_events.py b/tests/executor/test_events.py index b4a466f78..05ed6a632 100644 --- a/tests/executor/test_events.py +++ b/tests/executor/test_events.py @@ -29,7 +29,7 @@ def test_command_executed_stdout_stderr( kwargs = {"cmd_idx": 1, "command": "mock-command", "message": message} ctx = CommandEventContext(CommandExecuted, kwargs) - e = ctx.event(agr_id="2c3b6f473b86fd923591ec568df4797", task_id="2", cmds=[1, 2, 3, 4, 5]) + e = ctx.event(agr_id="2c3b6f473b86fd923591ec568df4797", script_id="2", cmds=[1, 2, 3, 4, 5]) assert isinstance(e, CommandExecuted) assert e.stdout == expected_stdout diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index dc773d313..6d53a8159 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -1015,9 +1015,7 @@ async def task_generator() -> AsyncIterator[Task[D, R]]: ) yield task self._engine.emit( - events.TaskFinished( - agr_id=agreement.id, task_id=task.id - ) + events.TaskFinished(agr_id=agreement.id, task_id=task.id) ) batch_generator = worker(work_context, task_generator()) From be30dbf69b151253ab29239a7cb52bac28cf7cee Mon Sep 17 00:00:00 2001 From: azawlocki Date: Thu, 20 May 2021 16:17:22 +0200 Subject: [PATCH 26/29] Add `message` field to `CommandExecuted` event --- tests/executor/test_events.py | 36 ----------------------------------- yapapi/executor/__init__.py | 2 +- yapapi/executor/events.py | 17 +---------------- yapapi/log.py | 12 +++++++++--- yapapi/rest/activity.py | 24 +++++++++++++---------- 5 files changed, 25 insertions(+), 66 deletions(-) delete mode 100644 tests/executor/test_events.py diff --git a/tests/executor/test_events.py b/tests/executor/test_events.py deleted file mode 100644 index 05ed6a632..000000000 --- a/tests/executor/test_events.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Unit tests for the `yapapi.executor.events` module.""" -from typing import Any - -import pytest - -from yapapi.executor.events import CommandEventContext, CommandExecuted - - -@pytest.mark.parametrize( - "message, expected_stdout, expected_stderr", - [ - ('{"stdout": null, "stderr": null}', None, None), - ('{"stderr":null, "stdout": "some\\noutput"}', "some\noutput", None), - ('{"stdout": null, "stderr": "stderr", "extra": "extra"}', None, "stderr"), - # Error values - (None, None, None), - ("[not a valid JSON...", None, None), - ('{"stdout": "some output", "stderr": error}', None, None), - ('{"std_out": "some output", "stderr": "error"}', None, None), - ('{"stdout": "just some output"}', "just some output", None), - ], -) -def test_command_executed_stdout_stderr( - message: Any, - expected_stdout: str, - expected_stderr: str, -) -> None: - """Check if command's stdout and stderr are correctly extracted from `message` argument.""" - - kwargs = {"cmd_idx": 1, "command": "mock-command", "message": message} - ctx = CommandEventContext(CommandExecuted, kwargs) - e = ctx.event(agr_id="2c3b6f473b86fd923591ec568df4797", script_id="2", cmds=[1, 2, 3, 4, 5]) - - assert isinstance(e, CommandExecuted) - assert e.stdout == expected_stdout - assert e.stderr == expected_stderr diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 6d53a8159..803bec07b 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -521,7 +521,7 @@ async def get_batch_results() -> List[events.CommandEvent]: self.emit(evt) results.append(evt) if isinstance(evt, events.CommandExecuted) and not evt.success: - raise CommandExecutionError(evt.command, evt.stderr) + raise CommandExecutionError(evt.command, evt.message, evt.stderr) self.emit(events.GettingResults(agr_id=agreement_id, script_id=script_id)) await batch.post() diff --git a/yapapi/executor/events.py b/yapapi/executor/events.py index 11204054b..e95e6b2b7 100644 --- a/yapapi/executor/events.py +++ b/yapapi/executor/events.py @@ -235,6 +235,7 @@ class CommandEvent(ScriptEvent): class CommandExecuted(CommandEvent): command: Any success: bool = dataclasses.field(default=True) + message: Optional[str] = dataclasses.field(default=None) stdout: Optional[str] = dataclasses.field(default=None) stderr: Optional[str] = dataclasses.field(default=None) @@ -293,20 +294,4 @@ def event(self, agr_id: str, script_id: str, cmds: List) -> CommandEvent: kwargs = dict(agr_id=agr_id, script_id=script_id, **self.kwargs) if self.evt_cls is CommandExecuted: kwargs["command"] = cmds[self.kwargs["cmd_idx"]] - kwargs["stdout"] = kwargs["stderr"] = None - try: - # Replace the "message" kwarg with "stdout" and "stderr" - message = self.kwargs["message"] - del kwargs["message"] - if message is not None: - message_dict = json.loads(message) - kwargs["stdout"] = message_dict["stdout"] - kwargs["stderr"] = message_dict["stderr"] - except (json.JSONDecodeError, KeyError, TypeError) as e: - logger.warning( - 'Missing or invalid field "message" in CommandExecuted event; ' - "kwargs: %s; error: %r", - self.kwargs, - e, - ) return self.evt_cls(**kwargs) diff --git a/yapapi/log.py b/yapapi/log.py index 739a24adf..adc8172f4 100644 --- a/yapapi/log.py +++ b/yapapi/log.py @@ -55,6 +55,7 @@ import yapapi.executor.events as events from yapapi import __version__ as yapapi_version +from yapapi.rest.activity import CommandExecutionError event_logger = logging.getLogger("yapapi.events") executor_logger = logging.getLogger("yapapi.executor") @@ -470,9 +471,14 @@ def _handle(self, event: events.Event): provider_info = self.agreement_provider_info[event.agr_id] self.provider_failures[provider_info] += 1 reason = str(exc) or repr(exc) or "unexpected error" - self.logger.warning( - "Activity failed on provider '%s', reason: %s", provider_info.name, reason - ) + if isinstance(exc, CommandExecutionError): + self.logger.warning( + "Activity failed on provider '%s'; reason: %s", provider_info.name, reason + ) + else: + self.logger.warning( + "Worker for provider '%s' failed; reason: %s", provider_info.name, reason + ) elif isinstance(event, events.ComputationFinished): if not event.exc_info: diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index 8cb37bd1a..d4abdbb97 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -108,16 +108,22 @@ class CommandExecutionError(Exception): """The command that failed.""" message: Optional[str] - """The command's output, if any.""" + """Optional error message from exe unit.""" - def __init__(self, command: str, message: Optional[str] = None): + stderr: Optional[str] + """Stderr produced by the command running on provider.""" + + def __init__(self, command: str, message: Optional[str] = None, stderr: Optional[str] = None): self.command = command self.message = message + self.stderr = stderr def __str__(self) -> str: msg = f"Command '{self.command}' failed on provider" if self.message: - msg += f" with message '{self.message}'" + msg += f"; message: '{self.message}'" + if self.stderr: + msg += f"; stderr: '{self.stderr}'" return msg @@ -186,14 +192,12 @@ async def __aiter__(self) -> AsyncIterator[events.CommandEventContext]: any_new = True assert last_idx == result.index, f"Expected {last_idx}, got {result.index}" - message = None - if result.message: - message = result.message - elif result.stdout or result.stderr: - message = json.dumps({"stdout": result.stdout, "stderr": result.stderr}) - kwargs = dict( - cmd_idx=result.index, message=message, success=(result.result.lower() == "ok") + cmd_idx=result.index, + message=result.message, + stdout=result.stdout, + stderr=result.stderr, + success=(result.result.lower() == "ok"), ) yield events.CommandEventContext(evt_cls=events.CommandExecuted, kwargs=kwargs) From 37e4c01c920bc91556625210d128407566798b8f Mon Sep 17 00:00:00 2001 From: azawlocki Date: Thu, 20 May 2021 16:53:49 +0200 Subject: [PATCH 27/29] Adjust integration tests to changed summary logger messages --- tests/goth/test_run_blender.py | 12 ++++++------ tests/goth/test_run_yacat.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/goth/test_run_blender.py b/tests/goth/test_run_blender.py index 673ee305f..a5167ab93 100644 --- a/tests/goth/test_run_blender.py +++ b/tests/goth/test_run_blender.py @@ -44,14 +44,14 @@ async def assert_all_tasks_processed(status: str, output_lines: EventStream[str] raise AssertionError(f"Tasks not {status}: {remaining_tasks}") -async def assert_all_tasks_sent(output_lines: EventStream[str]): - """Assert that for every task a line with `Task sent` will appear.""" - await assert_all_tasks_processed("sent", output_lines) +async def assert_all_tasks_started(output_lines: EventStream[str]): + """Assert that for every task a line with `Task started on provider` will appear.""" + await assert_all_tasks_processed("started on provider", output_lines) async def assert_all_tasks_computed(output_lines: EventStream[str]): - """Assert that for every task a line with `Task computed` will appear.""" - await assert_all_tasks_processed("computed", output_lines) + """Assert that for every task a line with `Task computed by provider` will appear.""" + await assert_all_tasks_processed("computed by provider", output_lines) async def assert_all_invoices_accepted(output_lines: EventStream[str]): @@ -103,7 +103,7 @@ async def test_run_blender(log_dir: Path, project_dir: Path, config_overrides) - # Add assertions to the command output monitor `cmd_monitor`: cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) - all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent) + all_sent = cmd_monitor.add_assertion(assert_all_tasks_started) all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed) await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ", timeout=20) diff --git a/tests/goth/test_run_yacat.py b/tests/goth/test_run_yacat.py index be41596b6..d5f758758 100644 --- a/tests/goth/test_run_yacat.py +++ b/tests/goth/test_run_yacat.py @@ -44,14 +44,14 @@ async def assert_all_tasks_processed(status: str, output_lines: EventStream[str] raise AssertionError(f"Tasks not {status}: {remaining_tasks}") -async def assert_all_tasks_sent(output_lines: EventStream[str]): - """Assert that for every task a line with `Task sent` will appear.""" - await assert_all_tasks_processed("sent", output_lines) +async def assert_all_tasks_started(output_lines: EventStream[str]): + """Assert that for every task a line with `Task started on provider` will appear.""" + await assert_all_tasks_processed("started on provider", output_lines) async def assert_all_tasks_computed(output_lines: EventStream[str]): - """Assert that for every task a line with `Task computed` will appear.""" - await assert_all_tasks_processed("computed", output_lines) + """Assert that for every task a line with `Task computed by provider` will appear.""" + await assert_all_tasks_processed("computed by provider", output_lines) async def assert_all_invoices_accepted(output_lines: EventStream[str]): @@ -103,7 +103,7 @@ async def test_run_yacat(log_dir: Path, project_dir: Path, config_overrides) -> # Add assertions to the command output monitor `cmd_monitor`: cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) - all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent) + all_sent = cmd_monitor.add_assertion(assert_all_tasks_started) all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed) await cmd_monitor.wait_for_pattern(".*The keyspace size is 95", timeout=120) From 1973569a0ded18b5e6720c664dfdfc0303e720d8 Mon Sep 17 00:00:00 2001 From: azawlocki Date: Fri, 21 May 2021 11:21:52 +0200 Subject: [PATCH 28/29] Make `AsyncWrapper` an async context manager --- tests/executor/test_async_wrapper.py | 129 +++++++++++++---------- tests/executor/test_payment_platforms.py | 4 + yapapi/executor/__init__.py | 93 ++++++++-------- yapapi/executor/utils.py | 39 ++++--- 4 files changed, 148 insertions(+), 117 deletions(-) diff --git a/tests/executor/test_async_wrapper.py b/tests/executor/test_async_wrapper.py index 0d7332b4b..9f2a42823 100644 --- a/tests/executor/test_async_wrapper.py +++ b/tests/executor/test_async_wrapper.py @@ -5,35 +5,54 @@ from yapapi.executor.utils import AsyncWrapper -def test_keyboard_interrupt(event_loop): +def test_async_wrapper_ordering(): + """Test if AsyncWrapper preserves order of calls.""" + + input_ = list(range(10)) + output = [] + + def func(n): + output.append(n) + + async def main(): + async with AsyncWrapper(func) as wrapper: + for n in input_: + wrapper.async_call(n) + + asyncio.get_event_loop().run_until_complete(main()) + assert output == input_ + + +def test_keyboard_interrupt(): """Test if AsyncWrapper handles KeyboardInterrupt by passing it to the event loop.""" def func(interrupt): if interrupt: raise KeyboardInterrupt - wrapper = AsyncWrapper(func, event_loop) - async def main(): - for _ in range(100): - wrapper.async_call(False) - # This will raise KeyboardInterrupt in the wrapper's worker task - wrapper.async_call(True) - await asyncio.sleep(0.01) + async with AsyncWrapper(func) as wrapper: + for _ in range(100): + wrapper.async_call(False) + # This will raise KeyboardInterrupt in the wrapper's worker task + wrapper.async_call(True) + await asyncio.sleep(0.01) - task = event_loop.create_task(main()) + loop = asyncio.get_event_loop() + task = loop.create_task(main()) with pytest.raises(KeyboardInterrupt): - event_loop.run_until_complete(task) + loop.run_until_complete(task) # Make sure the main task did not get KeyboardInterrupt assert not task.done() - # Make sure the wrapper can still make calls, it's worker task shouldn't exit - wrapper.async_call(False) + with pytest.raises(asyncio.CancelledError): + task.cancel() + loop.run_until_complete(task) -def test_stop_doesnt_deadlock(event_loop): - """Test if the AsyncWrapper.stop() coroutine completes after an AsyncWrapper is interrupted. +def test_aexit_doesnt_deadlock(): + """Test if the AsyncWrapper.__aexit__() completes after an AsyncWrapper is interrupted. See https://github.com/golemfactory/yapapi/issues/238. """ @@ -46,55 +65,57 @@ def func(interrupt): async def main(): """"This coroutine mimics how an AsyncWrapper is used in an Executor.""" - wrapper = AsyncWrapper(func, event_loop) - try: - # Queue some calls - for _ in range(10): - wrapper.async_call(False) - wrapper.async_call(True) - for _ in range(10): - wrapper.async_call(False) - # Sleep until cancelled - await asyncio.sleep(30) - assert False, "Sleep should be cancelled" - except asyncio.CancelledError: - # This call should exit without timeout - await asyncio.wait_for(wrapper.stop(), timeout=30.0) - - task = event_loop.create_task(main()) + async with AsyncWrapper(func) as wrapper: + try: + # Queue some calls + for _ in range(10): + wrapper.async_call(False) + wrapper.async_call(True) + for _ in range(10): + wrapper.async_call(False) + # Sleep until cancelled + await asyncio.sleep(30) + assert False, "Sleep should be cancelled" + except asyncio.CancelledError: + pass + + loop = asyncio.get_event_loop() + task = loop.create_task(main()) try: - event_loop.run_until_complete(task) + loop.run_until_complete(task) assert False, "Expected KeyboardInterrupt" except KeyboardInterrupt: task.cancel() - event_loop.run_until_complete(task) + loop.run_until_complete(task) -def test_stop_doesnt_wait(event_loop): - """Test if the AsyncWrapper.stop() coroutine prevents new calls from be queued.""" +def test_cancel_doesnt_wait(): + """Test if the AsyncWrapper stops processing calls when it's cancelled.""" - def func(): - time.sleep(0.1) - pass + num_calls = 0 - wrapper = AsyncWrapper(func, event_loop) + def func(d): + print("Calling func()") + nonlocal num_calls + num_calls += 1 + time.sleep(d) async def main(): - with pytest.raises(RuntimeError): - for n in range(100): - wrapper.async_call() - await asyncio.sleep(0.01) - # wrapper should be stopped before all calls are made - assert False, "Should raise RuntimeError" - - async def stop(): - await asyncio.sleep(0.1) - await wrapper.stop() + try: + async with AsyncWrapper(func) as wrapper: + for _ in range(10): + wrapper.async_call(0.1) + except asyncio.CancelledError: + pass - task = event_loop.create_task(main()) - event_loop.create_task(stop()) - try: - event_loop.run_until_complete(task) - except KeyboardInterrupt: + async def cancel(): + await asyncio.sleep(0.05) + print("Cancelling!") task.cancel() - event_loop.run_until_complete(task) + + loop = asyncio.get_event_loop() + task = loop.create_task(main()) + loop.create_task(cancel()) + loop.run_until_complete(task) + + assert num_calls < 10 diff --git a/tests/executor/test_payment_platforms.py b/tests/executor/test_payment_platforms.py index 5c2c83efd..ac3f8b57b 100644 --- a/tests/executor/test_payment_platforms.py +++ b/tests/executor/test_payment_platforms.py @@ -123,7 +123,11 @@ async def mock_create_allocation(_self, model): create_allocation_args.append(model) return mock.Mock() + async def mock_release_allocation(*args, **kwargs): + pass + monkeypatch.setattr(RequestorApi, "create_allocation", mock_create_allocation) + monkeypatch.setattr(RequestorApi, "release_allocation", mock_release_allocation) with pytest.raises(_StopExecutor): async with Executor( diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 803bec07b..ab5b56f4d 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -176,11 +176,7 @@ def __init__( # Add buffering to the provided event emitter to make sure # that emitting events will not block - # TODO: make AsyncWrapper an AsyncContextManager and start it in - # in __aenter__(); if it's started here then there's no guarantee that - # it will be cancelled properly - self._wrapped_consumer: Optional[AsyncWrapper] = None - self._event_consumer = event_consumer + self._wrapped_consumer = AsyncWrapper(event_consumer) self._stream_output = stream_output @@ -241,7 +237,15 @@ async def __aenter__(self) -> "Golem": try: stack = self._stack - self._wrapped_consumer = AsyncWrapper(self._event_consumer) + await stack.enter_async_context(self._wrapped_consumer) + + def report_shutdown(*exc_info): + if any(item for item in exc_info): + self.emit(events.ShutdownFinished(exc_info=exc_info)) # noqa + else: + self.emit(events.ShutdownFinished()) + + stack.push(report_shutdown) market_client = await stack.enter_async_context(self._api_config.market()) self._market_api = rest.Market(market_client) @@ -272,53 +276,48 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): # Importing this at the beginning would cause circular dependencies from ..log import pluralize - logger.debug("Golem is shutting down...") - # Wait until all computations are finished - await asyncio.gather(*[job.finished.wait() for job in self._jobs]) + try: + logger.debug("Golem is shutting down...") + # Wait until all computations are finished + await asyncio.gather(*[job.finished.wait() for job in self._jobs]) - self._payment_closing = True + self._payment_closing = True - for task in self._services: - if task is not self._process_invoices_job: - task.cancel() + for task in self._services: + if task is not self._process_invoices_job: + task.cancel() - if self._process_invoices_job and not any( - True for job in self._jobs if job.agreements_pool.confirmed > 0 - ): - logger.debug("No need to wait for invoices.") - self._process_invoices_job.cancel() + if self._process_invoices_job and not any( + True for job in self._jobs if job.agreements_pool.confirmed > 0 + ): + logger.debug("No need to wait for invoices.") + self._process_invoices_job.cancel() - try: - logger.info("Waiting for Golem services to finish...") - _, pending = await asyncio.wait( - self._services, timeout=10, return_when=asyncio.ALL_COMPLETED - ) - if pending: - logger.debug("%s still running: %s", pluralize(len(pending), "service"), pending) - except Exception: - # TODO: add message - logger.debug("TODO", exc_info=True) - - if self._agreements_to_pay and self._process_invoices_job: - logger.info( - "%s still unpaid, waiting for invoices...", - pluralize(len(self._agreements_to_pay), "agreement"), - ) - await asyncio.wait( - {self._process_invoices_job}, timeout=30, return_when=asyncio.ALL_COMPLETED - ) - if self._agreements_to_pay: - logger.warning("Unpaid agreements: %s", self._agreements_to_pay) + try: + logger.info("Waiting for Golem services to finish...") + _, pending = await asyncio.wait( + self._services, timeout=10, return_when=asyncio.ALL_COMPLETED + ) + if pending: + logger.debug( + "%s still running: %s", pluralize(len(pending), "service"), pending + ) + except Exception: + logger.debug("Got error when waiting for services to finish", exc_info=True) + + if self._agreements_to_pay and self._process_invoices_job: + logger.info( + "%s still unpaid, waiting for invoices...", + pluralize(len(self._agreements_to_pay), "agreement"), + ) + await asyncio.wait( + {self._process_invoices_job}, timeout=30, return_when=asyncio.ALL_COMPLETED + ) + if self._agreements_to_pay: + logger.warning("Unpaid agreements: %s", self._agreements_to_pay) - # TODO: prevent new computations at this point (if it's even possible to start one) - try: - await self._stack.aclose() - self.emit(events.ShutdownFinished()) - except Exception: - self.emit(events.ShutdownFinished(exc_info=sys.exc_info())) finally: - if self._wrapped_consumer: - await self._wrapped_consumer.stop() + await self._stack.aclose() async def _create_allocations(self) -> rest.payment.MarketDecoration: diff --git a/yapapi/executor/utils.py b/yapapi/executor/utils.py index 04d70b2bd..13622705b 100644 --- a/yapapi/executor/utils.py +++ b/yapapi/executor/utils.py @@ -1,13 +1,13 @@ """Utility functions and classes used within the `yapapi.executor` package.""" import asyncio import logging -from typing import Callable, Optional +from typing import AsyncContextManager, Callable, Optional logger = logging.getLogger(__name__) -class AsyncWrapper: +class AsyncWrapper(AsyncContextManager): """Wraps a given callable to provide asynchronous calls. Example usage: @@ -25,11 +25,27 @@ class AsyncWrapper: _args_buffer: asyncio.Queue _task: Optional[asyncio.Task] - def __init__(self, wrapped: Callable, event_loop: Optional[asyncio.AbstractEventLoop] = None): + def __init__(self, wrapped: Callable): self._wrapped = wrapped # type: ignore # suppress mypy issue #708 self._args_buffer = asyncio.Queue() - loop = event_loop or asyncio.get_event_loop() - self._task = loop.create_task(self._worker()) + self._loop = asyncio.get_event_loop() + self._task = None + + async def __aenter__(self) -> "AsyncWrapper": + self._task = self._loop.create_task(self._worker()) + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> Optional[bool]: + """Stop the wrapper, process queued calls but do not accept any new ones.""" + if self._task: + # Set self._task to None so we don't accept any more calls in `async_call()` + worker_task = self._task + self._task = None + await self._args_buffer.join() + worker_task.cancel() + await asyncio.gather(worker_task, return_exceptions=True) + # Don't suppress the exception (if any), so return a non-True value + return None async def _worker(self) -> None: while True: @@ -39,6 +55,7 @@ async def _worker(self) -> None: self._wrapped(*args, **kwargs) finally: self._args_buffer.task_done() + await asyncio.sleep(0) except KeyboardInterrupt as ke: # Don't stop on KeyboardInterrupt, but pass it to the event loop logger.debug("Caught KeybordInterrupt in AsyncWrapper's worker task") @@ -46,23 +63,13 @@ async def _worker(self) -> None: def raise_interrupt(ke_): raise ke_ - asyncio.get_event_loop().call_soon(raise_interrupt, ke) + self._loop.call_soon(raise_interrupt, ke) except asyncio.CancelledError: logger.debug("AsyncWrapper's worker task cancelled") break except Exception: logger.exception("Unhandled exception in wrapped callable") - async def stop(self) -> None: - """Stop the wrapper, process queued calls but do not accept any new ones.""" - if self._task: - # Set self._task to None so we don't accept any more calls in `async_call()` - worker_task = self._task - self._task = None - await self._args_buffer.join() - worker_task.cancel() - await asyncio.gather(worker_task, return_exceptions=True) - def async_call(self, *args, **kwargs) -> None: """Schedule an asynchronous call to the wrapped callable.""" if not self._task or self._task.done(): From a2e5c2d0ac8cf50318a5ffd6a101aac04e61724c Mon Sep 17 00:00:00 2001 From: azawlocki Date: Fri, 21 May 2021 13:10:04 +0200 Subject: [PATCH 29/29] Address CR suggestions --- examples/blender/blender.py | 2 +- yapapi/__init__.py | 1 + yapapi/executor/__init__.py | 17 ++++------------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index d1d8104d3..b019fc714 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -5,13 +5,13 @@ import sys from yapapi import ( + Golem, NoPaymentAccountError, Task, __version__ as yapapi_version, WorkContext, windows_event_loop_fix, ) -from yapapi.executor import Golem from yapapi.log import enable_default_logger, log_summary, log_event_repr # noqa from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError diff --git a/yapapi/__init__.py b/yapapi/__init__.py index cdc3e4d35..c27800f8e 100644 --- a/yapapi/__init__.py +++ b/yapapi/__init__.py @@ -10,6 +10,7 @@ CaptureContext, ExecOptions, Executor, + Golem, NoPaymentAccountError, Task, WorkContext, diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index ab5b56f4d..013aa4e66 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -112,7 +112,7 @@ def unpack_work_item(item: WorkItem) -> Tuple[Work, ExecOptions]: exescript_ids: Iterator[int] = itertools.count(1) - +"""An iterator providing unique ids used to correlate events related to a single exe script.""" D = TypeVar("D") # Type var for task data R = TypeVar("R") # Type var for task result @@ -148,8 +148,7 @@ def __init__( :param app_key: optional Yagna application key. If not provided, the default is to get the value from `YAGNA_APPKEY` environment variable """ - self._init_api(app_key) - + self._api_config = rest.Configuration(app_key) self._budget_amount = Decimal(budget) self._budget_allocations: List[rest.payment.Allocation] = [] @@ -206,13 +205,6 @@ async def create_demand_builder( await builder.decorate(self.payment_decoration, self.strategy, payload) return builder - def _init_api(self, app_key: Optional[str] = None): - """ - initialize the REST (low-level) API - :param app_key: (optional) yagna daemon application key - """ - self._api_config = rest.Configuration(app_key) - @property def driver(self) -> str: return self._driver @@ -265,7 +257,7 @@ def report_shutdown(*exc_info): self._services.add(self._process_invoices_job) self._services.add(loop.create_task(self.process_debit_notes())) - self._storage_manager = await self._stack.enter_async_context(gftp.provider()) + self._storage_manager = await stack.enter_async_context(gftp.provider()) return self except: @@ -1159,5 +1151,4 @@ async def worker_starter() -> None: "%s still running: %s", pluralize(len(pending), "service"), pending ) except Exception: - # TODO: add message - logger.debug("TODO", exc_info=True) + logger.debug("Got error when waiting for services to finish", exc_info=True)