Refactor yacat example

examples/utils.py

@@ -1,5 +1,9 @@
 """Utilities for yapapi example scripts."""
 import argparse
+from datetime import datetime, timezone
+from pathlib import Path
+import tempfile
+
 
 import colorama  # type: ignore
 
@@ -18,13 +22,18 @@
 
 
 def build_parser(description: str):
+    current_time_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z")
+    default_log_path = Path(tempfile.gettempdir()) / f"yapapi_{current_time_str}.log"
+
     parser = argparse.ArgumentParser(description=description)
     parser.add_argument("--driver", help="Payment driver name, for example `zksync`")
     parser.add_argument("--network", help="Network name, for example `rinkeby`")
     parser.add_argument(
         "--subnet-tag", default="devnet-beta.2", help="Subnet name; default: %(default)s"
     )
     parser.add_argument(
-        "--log-file", default=None, help="Log file for YAPAPI; default: %(default)s"
+        "--log-file",
+        default=str(default_log_path),
+        help="Log file for YAPAPI; default: %(default)s",
     )
     return parser

examples/yacat/yacat.Dockerfile

@@ -1,52 +1,4 @@
-FROM golemfactory/base:1.5
+FROM dizcza/docker-hashcat:intel-cpu
 
-MAINTAINER Radek Tereszczuk <[email protected]>
-
-
-RUN apt-get update && apt-get install -y alien clinfo
-
-# Install Intel OpenCL driver
-
-#ENV INTEL_OPENCL_URL=http://registrationcenter-download.intel.com/akdlm/irc_nas/vcp/13793/l_opencl_p_18.1.0.013.tgz
-ENV INTEL_OPENCL_URL=http://registrationcenter-download.intel.com/akdlm/irc_nas/9019/opencl_runtime_16.1.1_x64_ubuntu_6.4.0.25.tgz
-
-RUN mkdir -p /tmp/opencl-driver-intel
-WORKDIR /tmp/opencl-driver-intel
-RUN curl -O $INTEL_OPENCL_URL; \
-    tar -xzf $(basename $INTEL_OPENCL_URL); \
-    for i in $(basename $INTEL_OPENCL_URL .tgz)/rpm/*.rpm; do alien --to-deb $i; done; \
-    dpkg -i *.deb; \
-    mkdir -p /etc/OpenCL/vendors; \
-    echo /opt/intel/*/lib64/libintelocl.so > /etc/OpenCL/vendors/intel.icd; \
-    rm -rf *
-
-ENV HASHCAT_VERSION        hashcat-5.1.0
-ENV HASHCAT_UTILS_VERSION  1.9
-
-# Update & install packages for installing hashcat
-RUN apt-get update && \
-    apt-get install -y wget p7zip make build-essential git libcurl4-openssl-dev libssl-dev zlib1g-dev
-
-RUN mkdir /golem/yacat
-
-WORKDIR /golem/yacat
-RUN wget --no-check-certificate https://hashcat.net/files/${HASHCAT_VERSION}.7z && \
-    7zr x ${HASHCAT_VERSION}.7z && \
-    rm ${HASHCAT_VERSION}.7z
-
-RUN wget --no-check-certificate https://github.com/hashcat/hashcat-utils/releases/download/v${HASHCAT_UTILS_VERSION}/hashcat-utils-${HASHCAT_UTILS_VERSION}.7z && \
-    7zr x hashcat-utils-${HASHCAT_UTILS_VERSION}.7z && \
-    rm hashcat-utils-${HASHCAT_UTILS_VERSION}.7z
-
-#Add link for binary
-RUN ln -s /golem/yacat/${HASHCAT_VERSION}/hashcat64.bin /usr/bin/hashcat
-RUN ln -s /golem/yacat/hashcat-utils-${HASHCAT_UTILS_VERSION}/bin/cap2hccapx.bin /usr/bin/cap2hccapx
-
-RUN cp /golem/yacat/${HASHCAT_VERSION}/hashcat.hcstat2 /golem/yacat
-RUN chmod -R 777 /golem/yacat
-
-RUN apt clean
-
-WORKDIR /golem/work
-
-VOLUME /golem/work /golem/output /golem/resource
+VOLUME /golem/input /golem/output
+WORKDIR /golem/entrypoint

examples/yacat/yacat.py

@@ -1,14 +1,20 @@
 #!/usr/bin/env python3
+import argparse
 import asyncio
 from datetime import datetime, timedelta
-import pathlib
+import math
+from pathlib import Path
 import sys
+from tempfile import NamedTemporaryFile
+from typing import AsyncIterable, List, Optional
 
-from yapapi import Executor, NoPaymentAccountError, Task, WorkContext, windows_event_loop_fix
+from yapapi import Golem, NoPaymentAccountError, Task, WorkContext, windows_event_loop_fix
+from yapapi.executor.events import CommandExecuted
 from yapapi.log import enable_default_logger, log_summary, log_event_repr  # noqa
 from yapapi.payload import vm
+from yapapi.rest.activity import CommandExecutionError
 
-examples_dir = pathlib.Path(__file__).resolve().parent.parent
+examples_dir = Path(__file__).resolve().parent.parent
 sys.path.append(str(examples_dir))
 
 from utils import (
@@ -20,146 +26,192 @@
     TEXT_COLOR_YELLOW,
 )
 
+HASHCAT_ATTACK_MODE = 3  # stands for mask attack, hashcat -a option
+KEYSPACE_OUTPUT_PATH = Path("/golem/output/keyspace")
 
-def write_hash(hash):
-    with open("in.hash", "w") as f:
-        f.write(hash)
+# Ideally, this value should depend on the chunk size
+MASK_ATTACK_TIMEOUT: timedelta = timedelta(minutes=30)
+KEYSPACE_TIMEOUT: timedelta = timedelta(minutes=10)
 
+arg_parser = build_parser("Run a hashcat attack (mask mode) on Golem network.")
+arg_parser.epilog = (
+    "Example invocation: ./yacat.py --mask '?a?a?a' --hash '$P$5ZDzPE45CLLhEx/72qt3NehVzwN2Ry/'"
+)
+arg_parser.add_argument("--hash", type=str, help="Target hash to be cracked", required=True)
+arg_parser.add_argument(
+    "--mask",
+    type=str,
+    help="Hashcat mask to be used for the attack. Example: a value of '?a?a?a' will "
+    "try all 3-character combinations, where each character is mixalpha-numeric "
+    "(lower and upper-case letters + digits) or a special character",
+    required=True,
+)
+arg_parser.add_argument(
+    "--chunk-size",  # affects skip and limit hashcat parameters
+    type=int,
+    help="Limit for the number of words to be checked as part of a single activity",
+    default=4096,
+)
+arg_parser.add_argument(
+    "--hash-type",
+    type=int,
+    help="Type of hashing algorithm to use (hashcat -m option). Default: 400 (phpass)",
+    default=400,
+)
+arg_parser.add_argument(
+    "--max-workers",
+    type=int,
+    help="The maximum number of nodes we want to perform the attack on (default is dynamic)",
+    default=None,
+)
+
+# Container object for parsed arguments
+args = argparse.Namespace()
+
+
+async def compute_keyspace(context: WorkContext, tasks: AsyncIterable[Task]):
+    """Worker script which computes the size of the keyspace for the mask attack.
+
+    This function is used as the `worker` parameter to `Golem#execute_tasks`.
+    It represents a sequence of commands to be executed on a remote provider node.
+    """
+    async for task in tasks:
+        cmd = f"hashcat --keyspace " f"-a {HASHCAT_ATTACK_MODE} -m {args.hash_type} {args.mask}"
+        context.run("/bin/bash", "-c", cmd)
 
-def write_keyspace_check_script(mask):
-    with open("keyspace.sh", "w") as f:
-        f.write(f"hashcat --keyspace -a 3 {mask} -m 400 > /golem/work/keyspace.txt")
+        try:
+            future_result = yield context.commit(timeout=KEYSPACE_TIMEOUT)
+
+            # each item is the result of a single command on the provider (including setup commands)
+            result: List[CommandExecuted] = await future_result
+            # we take the last item since it's the last command that was executed on the provider
+            cmd_result: CommandExecuted = result[-1]
+
+            keyspace = int(cmd_result.stdout)
+            task.accept_result(result=keyspace)
+        except CommandExecutionError as e:
+            raise RuntimeError(f"Failed to compute attack keyspace: {e}")
+
+
+async def perform_mask_attack(ctx: WorkContext, tasks: AsyncIterable[Task]):
+    """Worker script which performs a hashcat mask attack against a target hash.
 
+    This function is used as the `worker` parameter to `Golem#execute_tasks`.
+    It represents a sequence of commands to be executed on a remote provider node.
+    """
+    async for task in tasks:
+        skip = task.data
+        limit = skip + args.chunk_size
+        worker_output_path = f"/golem/output/hashcat_{skip}.potfile"
 
-def read_keyspace():
-    with open("keyspace.txt", "r") as f:
-        return int(f.readline())
+        ctx.run(f"/bin/sh", "-c", _make_attack_command(skip, limit, worker_output_path))
+        output_file = NamedTemporaryFile()
+        ctx.download_file(worker_output_path, output_file.name)
 
+        yield ctx.commit(timeout=MASK_ATTACK_TIMEOUT)
+
+        result = output_file.file.readline()
+        task.accept_result(result)
+        output_file.close()
+
+
+def _make_attack_command(skip: int, limit: int, output_path: str) -> str:
+    return (
+        f"touch {output_path}; "
+        f"hashcat -a {HASHCAT_ATTACK_MODE} -m {args.hash_type} "
+        f"--self-test-disable --potfile-disable "
+        f"--skip={skip} --limit={limit} -o {output_path} "
+        f"'{args.hash}' '{args.mask}' || true"
+    )
 
-def read_password(ranges):
-    for r in ranges:
-        path = pathlib.Path(f"hashcat_{r}.potfile")
-        if not path.is_file():
-            continue
-        with open(path, "r") as f:
-            line = f.readline()
-        split_list = line.split(":")
-        if len(split_list) >= 2:
-            return split_list[1]
+
+def _parse_result(potfile_line: bytes) -> Optional[str]:
+    """Helper function which parses a single .potfile line and returns the password part.
+
+    Hashcat uses its .potfile format to report results. In this format, each line consists of the
+    hash and its matching word, separated with a colon (e.g. `asdf1234:password`).
+    """
+    potfile_line = potfile_line.decode("utf-8")
+    if potfile_line:
+        return potfile_line.split(":")[-1].strip()
     return None
 
 
 async def main(args):
     package = await vm.repo(
-        image_hash="2c17589f1651baff9b82aa431850e296455777be265c2c5446c902e9",
+        image_hash="055911c811e56da4d75ffc928361a78ed13077933ffa8320fb1ec2db",
         min_mem_gib=0.5,
         min_storage_gib=2.0,
     )
 
-    async def worker_check_keyspace(ctx: WorkContext, tasks):
-        async for task in tasks:
-            keyspace_sh_filename = "keyspace.sh"
-            ctx.send_file(keyspace_sh_filename, "/golem/work/keyspace.sh")
-            ctx.run("/bin/sh", "/golem/work/keyspace.sh")
-            output_file = "keyspace.txt"
-            ctx.download_file("/golem/work/keyspace.txt", output_file)
-            yield ctx.commit(timeout=timedelta(minutes=10))
-            task.accept_result()
-
-    async def worker_find_password(ctx: WorkContext, tasks):
-        ctx.send_file("in.hash", "/golem/work/in.hash")
-
-        async for task in tasks:
-            skip = task.data
-            limit = skip + step
-
-            # Commands to be run on the provider
-            commands = (
-                "rm -f /golem/work/*.potfile ~/.hashcat/hashcat.potfile; "
-                f"touch /golem/work/hashcat_{skip}.potfile; "
-                f"hashcat -a 3 -m 400 /golem/work/in.hash {args.mask} --skip={skip} --limit={limit} --self-test-disable -o /golem/work/hashcat_{skip}.potfile || true"
-            )
-            ctx.run(f"/bin/sh", "-c", commands)
-
-            output_file = f"hashcat_{skip}.potfile"
-            ctx.download_file(f"/golem/work/hashcat_{skip}.potfile", output_file)
-            yield ctx.commit(timeout=timedelta(minutes=10))
-            task.accept_result(result=output_file)
-
-    # beginning of the main flow
-
-    write_hash(args.hash)
-    write_keyspace_check_script(args.mask)
-
     # By passing `event_consumer=log_summary()` we enable summary logging.
     # See the documentation of the `yapapi.log` module on how to set
     # the level of detail and format of the logged information.
-    async with Executor(
-        package=package,
-        max_workers=args.number_of_providers,
+    async with Golem(
         budget=10.0,
-        # timeout should be keyspace / number of providers dependent
-        timeout=timedelta(minutes=30),
         subnet_tag=args.subnet_tag,
         driver=args.driver,
         network=args.network,
         event_consumer=log_summary(log_event_repr),
-    ) as executor:
+    ) as golem:
 
         print(
             f"Using subnet: {TEXT_COLOR_YELLOW}{args.subnet_tag}{TEXT_COLOR_DEFAULT}, "
-            f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, "
-            f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n"
+            f"payment driver: {TEXT_COLOR_YELLOW}{golem.driver}{TEXT_COLOR_DEFAULT}, "
+            f"and network: {TEXT_COLOR_YELLOW}{golem.network}{TEXT_COLOR_DEFAULT}\n"
         )
 
-        keyspace_computed = False
         start_time = datetime.now()
 
-        # This is not a typical use of executor.submit as there is only one task, with no data:
-        async for _task in executor.submit(worker_check_keyspace, [Task(data=None)]):
-            keyspace_computed = True
-
-        if not keyspace_computed:
-            # Assume the errors have been already reported and we may return quietly.
-            return
+        completed = golem.execute_tasks(
+            compute_keyspace,
+            [Task(data="compute_keyspace")],
+            payload=package,
+            max_workers=1,
+            timeout=KEYSPACE_TIMEOUT,
+        )
 
-        keyspace = read_keyspace()
+        async for task in completed:
+            keyspace = task.result
 
         print(
             f"{TEXT_COLOR_CYAN}"
             f"Task computed: keyspace size count. The keyspace size is {keyspace}"
             f"{TEXT_COLOR_DEFAULT}"
         )
 
-        step = int(keyspace / args.number_of_providers) + 1
+        data = [Task(data=c) for c in range(0, keyspace, args.chunk_size)]
+        max_workers = args.max_workers or math.ceil(keyspace / args.chunk_size) // 2
 
-        ranges = range(0, keyspace, step)
+        completed = golem.execute_tasks(
+            perform_mask_attack,
+            data,
+            payload=package,
+            max_workers=max_workers,
+            timeout=MASK_ATTACK_TIMEOUT,
+        )
+
+        password = None
 
-        async for task in executor.submit(
-            worker_find_password, [Task(data=range) for range in ranges]
-        ):
+        async for task in completed:
             print(
                 f"{TEXT_COLOR_CYAN}Task computed: {task}, result: {task.result}{TEXT_COLOR_DEFAULT}"
             )
 
-        password = read_password(ranges)
+            result = _parse_result(task.result)
+            if result:
+                password = result
 
-        if password is None:
-            print(f"{TEXT_COLOR_RED}No password found{TEXT_COLOR_DEFAULT}")
-        else:
+        if password:
             print(f"{TEXT_COLOR_GREEN}Password found: {password}{TEXT_COLOR_DEFAULT}")
+        else:
+            print(f"{TEXT_COLOR_RED}No password found{TEXT_COLOR_DEFAULT}")
 
         print(f"{TEXT_COLOR_CYAN}Total time: {datetime.now() - start_time}{TEXT_COLOR_DEFAULT}")
 
 
 if __name__ == "__main__":
-    parser = build_parser("yacat")
-
-    parser.add_argument("--number-of-providers", dest="number_of_providers", type=int, default=3)
-    parser.add_argument("mask")
-    parser.add_argument("hash")
-
-    args = parser.parse_args()
+    args = arg_parser.parse_args()
 
     # This is only required when running on Windows with Python prior to 3.8:
     windows_event_loop_fix()