Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prevent double file open in yacat example #464

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions examples/yacat/yacat.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
import asyncio
from datetime import datetime, timedelta
import math
import os
from pathlib import Path
import sys
from tempfile import NamedTemporaryFile
import tempfile
from typing import AsyncIterable, List, Optional

from yapapi import Golem, NoPaymentAccountError, Task, WorkContext, windows_event_loop_fix
Expand Down Expand Up @@ -104,15 +105,16 @@ async def perform_mask_attack(ctx: WorkContext, tasks: AsyncIterable[Task]):
limit = skip + args.chunk_size
worker_output_path = f"/golem/output/hashcat_{skip}.potfile"

ctx.run(f"/bin/sh", "-c", _make_attack_command(skip, limit, worker_output_path))
output_file = NamedTemporaryFile()
ctx.download_file(worker_output_path, output_file.name)
ctx.run("/bin/sh", "-c", _make_attack_command(skip, limit, worker_output_path))
output_file = Path(tempfile.gettempdir()) / os.urandom(16).hex()
ctx.download_file(worker_output_path, output_file)

yield ctx.commit(timeout=MASK_ATTACK_TIMEOUT)

result = output_file.file.readline()
task.accept_result(result)
output_file.close()
with output_file.open() as f:
result = f.readline()
task.accept_result(result)
os.remove(str(output_file))


def _make_attack_command(skip: int, limit: int, output_path: str) -> str:
Expand Down
6 changes: 3 additions & 3 deletions yapapi/ctx.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from os import PathLike
from functools import partial
from pathlib import Path
from typing import Callable, Iterable, Optional, Dict, List, Tuple, Union, Any, Awaitable
from typing import BinaryIO, Callable, Iterable, Optional, Dict, List, Tuple, Union, Any, Awaitable

from yapapi.events import DownloadStarted, DownloadFinished
from yapapi.props import NodeInfo
Expand Down Expand Up @@ -189,7 +189,7 @@ def __init__(
self,
storage: StorageProvider,
src_path: str,
dst_path: str,
dst_path: Union[PathLike, BinaryIO],
emitter: Optional[Callable[[StorageEvent], None]] = None,
):
super().__init__(storage, src_path, emitter)
Expand Down Expand Up @@ -385,7 +385,7 @@ def run(
self.__prepare()
self._pending_steps.append(_Run(cmd, *args, env=env, stdout=stdout, stderr=stderr))

def download_file(self, src_path: str, dst_path: str):
def download_file(self, src_path: str, dst_path: Union[PathLike, BinaryIO]):
"""Schedule downloading remote file from the provider.

:param src_path: remote (provider) path
Expand Down
10 changes: 7 additions & 3 deletions yapapi/storage/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import pathlib
import io
from os import PathLike
from typing import AsyncIterator, NamedTuple, Union, Optional
from typing import AsyncIterator, BinaryIO, NamedTuple, Union, Optional
import asyncio
import aiohttp

Expand Down Expand Up @@ -65,9 +65,13 @@ async def download_bytes(self, limit: int = DOWNLOAD_BYTES_LIMIT_DEFAULT):

return output

async def download_file(self, destination_file: PathLike):
async def download_file(self, destination_file: Union[PathLike, BinaryIO]):
content = await self.download_stream()
with open(destination_file, "wb") as f:

if isinstance(destination_file, PathLike):
destination_file = open(destination_file, "wb")

with destination_file as f:
async for chunk in content.stream:
f.write(chunk)

Expand Down